From 25ab03e5d3615739ca01b2913169cb2dc7ba5d37 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 15 Dec 2023 17:50:16 +0800 Subject: [PATCH 01/94] refact: improve info logging msg in mmProcessWriteMsg --- source/dnode/mnode/impl/src/mndSync.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index f46f33ac22..585d8ea8e7 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -180,7 +180,7 @@ int32_t mndProcessWriteMsg(SMnode *pMnode, SRpcMsg *pMsg, SFsmCbMeta *pMeta) { goto _OUT; } - mInfo("trans:%d, is proposed, saved:%d code:0x%x, apply index:%" PRId64 " term:%" PRIu64 " config:%" PRId64 + mInfo("trans:%d, process sync proposal, saved:%d code:0x%x, apply index:%" PRId64 " term:%" PRIu64 " config:%" PRId64 " role:%s raw:%p sec:%d seq:%" PRId64, transId, pMgmt->transId, pMeta->code, pMeta->index, pMeta->term, pMeta->lastConfigIndex, syncStr(pMeta->state), pRaw, pMgmt->transSec, pMgmt->transSeq); From ea6e78cbaaa8d433a79d16024a8bd1937fb03461 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 18 Dec 2023 16:33:44 +0800 Subject: [PATCH 02/94] fix(stream): add async call restart, instead of sync wait. --- include/dnode/vnode/tqCommon.h | 1 + include/libs/stream/tstream.h | 12 ++++--- source/dnode/snode/src/snode.c | 2 +- source/dnode/vnode/src/tq/tq.c | 3 +- source/dnode/vnode/src/tqCommon/tqCommon.c | 42 ++++++++++++++++------ source/libs/stream/src/streamMeta.c | 8 +++-- source/libs/stream/src/streamStart.c | 3 ++ 7 files changed, 52 insertions(+), 19 deletions(-) diff --git a/include/dnode/vnode/tqCommon.h b/include/dnode/vnode/tqCommon.h index 75dafcdbff..32ac38cee6 100644 --- a/include/dnode/vnode/tqCommon.h +++ b/include/dnode/vnode/tqCommon.h @@ -32,5 +32,6 @@ int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLeader); int32_t startStreamTasks(SStreamMeta* pMeta); int32_t resetStreamTaskStatus(SStreamMeta* pMeta); +int32_t tqStartTaskCompleteCallback(SStreamMeta* pMeta); #endif // TDENGINE_TQ_COMMON_H diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 6730d211df..812b57b01f 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -460,14 +460,18 @@ struct SStreamTask { char reserve[256]; }; +typedef int32_t (*startComplete_fn_t)(struct SStreamMeta*); + typedef struct STaskStartInfo { int64_t startTs; int64_t readyTs; int32_t tasksWillRestart; - int32_t taskStarting; // restart flag, sentinel to guard the restart procedure. - SHashObj* pReadyTaskSet; // tasks that are all ready for running stream processing - SHashObj* pFailedTaskSet; // tasks that are done the check downstream process, may be successful or failed + int32_t taskStarting; // restart flag, sentinel to guard the restart procedure. + SHashObj* pReadyTaskSet; // tasks that are all ready for running stream processing + SHashObj* pFailedTaskSet; // tasks that are done the check downstream process, may be successful or failed int64_t elapsedTime; + int32_t restartCount; // restart task counter + startComplete_fn_t completeFn; // complete callback function } STaskStartInfo; typedef struct STaskUpdateInfo { @@ -827,7 +831,7 @@ int32_t streamProcessScanHistoryFinishRsp(SStreamTask* pTask); // stream task meta void streamMetaInit(); void streamMetaCleanup(); -SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandFunc, int32_t vgId, int64_t stage); +SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandFunc, int32_t vgId, int64_t stage, startComplete_fn_t fn); void streamMetaClose(SStreamMeta* streamMeta); int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask); // save to stream meta store int32_t streamMetaRemoveTask(SStreamMeta* pMeta, STaskId* pKey); diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 380be1dd38..432f80824a 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -127,7 +127,7 @@ SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) { } pSnode->msgCb = pOption->msgCb; - pSnode->pMeta = streamMetaOpen(path, pSnode, (FTaskExpand *)sndExpandTask, SNODE_HANDLE, taosGetTimestampMs()); + pSnode->pMeta = streamMetaOpen(path, pSnode, (FTaskExpand *)sndExpandTask, SNODE_HANDLE, taosGetTimestampMs(), tqStartTaskCompleteCallback); if (pSnode->pMeta == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto FAIL; diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 138c58b45f..76049d21b1 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -96,7 +96,8 @@ int32_t tqInitialize(STQ* pTq) { return -1; } - pTq->pStreamMeta = streamMetaOpen(pTq->path, pTq, (FTaskExpand*)tqExpandTask, pTq->pVnode->config.vgId, -1); + int32_t vgId = TD_VID(pTq->pVnode); + pTq->pStreamMeta = streamMetaOpen(pTq->path, pTq, (FTaskExpand*)tqExpandTask, vgId, -1, tqStartTaskCompleteCallback); if (pTq->pStreamMeta == NULL) { return -1; } diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index 480ed7fd38..004eb240b3 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -699,8 +699,8 @@ int32_t startStreamTasks(SStreamMeta* pMeta) { } int32_t resetStreamTaskStatus(SStreamMeta* pMeta) { - int32_t vgId = pMeta->vgId; - int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); + int32_t vgId = pMeta->vgId; + int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); tqDebug("vgId:%d reset all %d stream task(s) status to be uninit", vgId, numOfTasks); if (numOfTasks == 0) { @@ -723,16 +723,17 @@ static int32_t restartStreamTasks(SStreamMeta* pMeta, bool isLeader) { int32_t code = 0; int64_t st = taosGetTimestampMs(); - while(1) { - int32_t startVal = atomic_val_compare_exchange_32(&pMeta->startInfo.taskStarting, 0, 1); - if (startVal == 0) { - break; - } - - tqDebug("vgId:%d in start stream tasks procedure, wait for 500ms and recheck", vgId); - taosMsleep(500); + streamMetaWLock(pMeta); + if (pMeta->startInfo.taskStarting == 1) { + pMeta->startInfo.restartCount += 1; + tqDebug("vgId:%d in start tasks procedure, inc restartCounter by 1, %d", vgId, pMeta->startInfo.restartCount); + streamMetaWUnLock(pMeta); + return TSDB_CODE_SUCCESS; } + pMeta->startInfo.taskStarting = 1; + streamMetaWUnLock(pMeta); + terrno = 0; tqInfo("vgId:%d tasks are all updated and stopped, restart all tasks, triggered by transId:%d", vgId, pMeta->updateInfo.transId); @@ -791,7 +792,7 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead char* p = NULL; if (streamTaskReadyToRun(pTask, &p)) { tqDebug("vgId:%d s-task:%s start to process block from inputQ, next checked ver:%" PRId64, vgId, pTask->id.idStr, - pTask->chkInfo.nextProcessVer); + pTask->chkInfo.nextProcessVer); streamExecTask(pTask); } else { int8_t status = streamTaskSetSchedStatusInactive(pTask); @@ -808,4 +809,23 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead } } +int32_t tqStartTaskCompleteCallback(SStreamMeta* pMeta) { + STaskStartInfo* pStartInfo = &pMeta->startInfo; + taosWLockLatch(&pMeta->lock); + if (pStartInfo->restartCount > 0) { + pStartInfo->restartCount -= 1; + + ASSERT(pStartInfo->taskStarting == 0); + tqDebug("vgId:%d role:%d need to restart all tasks again, restartCounter:%d", pMeta->vgId, pMeta->role, + pStartInfo->restartCount); + + taosWUnLockLatch(&pMeta->lock); + restartStreamTasks(pMeta, (pMeta->role == NODE_ROLE_LEADER)); + } else { + taosWUnLockLatch(&pMeta->lock); + tqDebug("vgId:%d start all tasks completed", pMeta->vgId); + } + + return TSDB_CODE_SUCCESS; +} \ No newline at end of file diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 23cb6f5a35..8868a5719c 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -277,7 +277,9 @@ int32_t streamTaskSetDb(SStreamMeta* pMeta, void* arg, char* key) { stDebug("s-task:0x%x set backend %p", pTask->id.taskId, pBackend); return 0; } -SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandFunc, int32_t vgId, int64_t stage) { + +SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandFunc, int32_t vgId, int64_t stage, + startComplete_fn_t fn) { int32_t code = -1; SStreamMeta* pMeta = taosMemoryCalloc(1, sizeof(SStreamMeta)); if (pMeta == NULL) { @@ -343,6 +345,7 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF pMeta->stage = stage; pMeta->role = (vgId == SNODE_HANDLE) ? NODE_ROLE_LEADER : NODE_ROLE_UNINIT; + pMeta->startInfo.completeFn = fn; pMeta->pTaskDbUnique = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); // pMeta->chkpId = streamGetLatestCheckpointId(pMeta); @@ -1258,8 +1261,9 @@ void streamMetaResetStartInfo(STaskStartInfo* pStartInfo) { taosHashClear(pStartInfo->pFailedTaskSet); pStartInfo->tasksWillRestart = 0; pStartInfo->readyTs = 0; + // reset the sentinel flag value to be 0 - atomic_store_32(&pStartInfo->taskStarting, 0); + pStartInfo->taskStarting = 0; } void streamMetaRLock(SStreamMeta* pMeta) { diff --git a/source/libs/stream/src/streamStart.c b/source/libs/stream/src/streamStart.c index 53f87591e8..dd713290f3 100644 --- a/source/libs/stream/src/streamStart.c +++ b/source/libs/stream/src/streamStart.c @@ -1083,6 +1083,7 @@ int32_t streamMetaUpdateTaskDownstreamStatus(SStreamMeta* pMeta, int64_t streamI int64_t endTs, bool ready) { STaskStartInfo* pStartInfo = &pMeta->startInfo; STaskId id = {.streamId = streamId, .taskId = taskId}; + bool restart = true; streamMetaWLock(pMeta); SHashObj* pDst = ready ? pStartInfo->pReadyTaskSet : pStartInfo->pFailedTaskSet; @@ -1106,6 +1107,8 @@ int32_t streamMetaUpdateTaskDownstreamStatus(SStreamMeta* pMeta, int64_t streamI displayStatusInfo(pMeta, pStartInfo->pReadyTaskSet, true); displayStatusInfo(pMeta, pStartInfo->pFailedTaskSet, false); streamMetaResetStartInfo(pStartInfo); + + pStartInfo->completeFn(pMeta); } else { stDebug("vgId:%d recv check down results:%d, total:%d", pMeta->vgId, numOfRecv, numOfTotal); } From e734569de07850ef02c7077fd95b702debb44a1b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 18 Dec 2023 18:42:57 +0800 Subject: [PATCH 03/94] fix(stream): fix dead lock. --- source/dnode/vnode/src/tqCommon/tqCommon.c | 6 +++--- source/libs/stream/src/streamMeta.c | 12 ------------ source/libs/stream/src/streamStart.c | 3 ++- 3 files changed, 5 insertions(+), 16 deletions(-) diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index 004eb240b3..a42f81edab 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -811,7 +811,7 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead int32_t tqStartTaskCompleteCallback(SStreamMeta* pMeta) { STaskStartInfo* pStartInfo = &pMeta->startInfo; - taosWLockLatch(&pMeta->lock); + streamMetaWLock(pMeta); if (pStartInfo->restartCount > 0) { pStartInfo->restartCount -= 1; @@ -820,10 +820,10 @@ int32_t tqStartTaskCompleteCallback(SStreamMeta* pMeta) { tqDebug("vgId:%d role:%d need to restart all tasks again, restartCounter:%d", pMeta->vgId, pMeta->role, pStartInfo->restartCount); - taosWUnLockLatch(&pMeta->lock); + streamMetaWUnLock(pMeta); restartStreamTasks(pMeta, (pMeta->role == NODE_ROLE_LEADER)); } else { - taosWUnLockLatch(&pMeta->lock); + streamMetaWUnLock(pMeta); tqDebug("vgId:%d start all tasks completed", pMeta->vgId); } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 8868a5719c..e94c1b7596 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -348,18 +348,6 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF pMeta->startInfo.completeFn = fn; pMeta->pTaskDbUnique = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); - // pMeta->chkpId = streamGetLatestCheckpointId(pMeta); - // pMeta->streamBackend = streamBackendInit(pMeta->path, pMeta->chkpId); - // while (pMeta->streamBackend == NULL) { - // qError("vgId:%d failed to init stream backend", pMeta->vgId); - // taosMsleep(2 * 1000); - // qInfo("vgId:%d retry to init stream backend", pMeta->vgId); - // pMeta->streamBackend = streamBackendInit(pMeta->path, pMeta->chkpId); - // if (pMeta->streamBackend == NULL) { - // } - // } - // pMeta->streamBackendRid = taosAddRef(streamBackendId, pMeta->streamBackend); - pMeta->numOfPausedTasks = 0; pMeta->numOfStreamTasks = 0; stInfo("vgId:%d open stream meta successfully, latest checkpoint:%" PRId64 ", stage:%" PRId64, vgId, pMeta->chkpId, diff --git a/source/libs/stream/src/streamStart.c b/source/libs/stream/src/streamStart.c index dd713290f3..8bd7984d29 100644 --- a/source/libs/stream/src/streamStart.c +++ b/source/libs/stream/src/streamStart.c @@ -1107,12 +1107,13 @@ int32_t streamMetaUpdateTaskDownstreamStatus(SStreamMeta* pMeta, int64_t streamI displayStatusInfo(pMeta, pStartInfo->pReadyTaskSet, true); displayStatusInfo(pMeta, pStartInfo->pFailedTaskSet, false); streamMetaResetStartInfo(pStartInfo); + streamMetaWUnLock(pMeta); pStartInfo->completeFn(pMeta); } else { + streamMetaWUnLock(pMeta); stDebug("vgId:%d recv check down results:%d, total:%d", pMeta->vgId, numOfRecv, numOfTotal); } - streamMetaWUnLock(pMeta); return TSDB_CODE_SUCCESS; } From 4c13dee7216a24fccff8f8fa0936b7cbcab087dc Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 19 Dec 2023 12:00:36 +0800 Subject: [PATCH 04/94] enh: protect processing of snap preparation rsp with sender buf mutex --- source/libs/sync/src/syncSnapshot.c | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index f060e9da13..2277b70c8f 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -1087,7 +1087,10 @@ static int32_t syncSnapSenderExchgSnapInfo(SSyncNode *pSyncNode, SSyncSnapshotSe // sender static int32_t syncNodeOnSnapshotPrepRsp(SSyncNode *pSyncNode, SSyncSnapshotSender *pSender, SyncSnapshotRsp *pMsg) { + int32_t code = -1; SSnapshot snapshot = {0}; + + taosThreadMutexLock(&pSender->pSndBuf->mutex); pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot); // prepare @@ -1103,20 +1106,24 @@ static int32_t syncNodeOnSnapshotPrepRsp(SSyncNode *pSyncNode, SSyncSnapshotSend // start reader if (pMsg->payloadType == TDMT_SYNC_PREP_SNAPSHOT_REPLY) { if (syncSnapSenderExchgSnapInfo(pSyncNode, pSender, pMsg) != 0) { - return -1; + goto _out; } } - int32_t code = pSyncNode->pFsm->FpSnapshotStartRead(pSyncNode->pFsm, &pSender->snapshotParam, &pSender->pReader); + code = pSyncNode->pFsm->FpSnapshotStartRead(pSyncNode->pFsm, &pSender->snapshotParam, &pSender->pReader); if (code != 0) { sSError(pSender, "prepare snapshot failed since %s", terrstr()); - return -1; + goto _out; } // update next index syncIndexMgrSetIndex(pSyncNode->pNextIndex, &pMsg->srcId, snapshot.lastApplyIndex + 1); - return snapshotSend(pSender); + code = snapshotSend(pSender); + +_out: + taosThreadMutexUnlock(&pSender->pSndBuf->mutex); + return code; } static int32_t snapshotSenderSignatureCmp(SSyncSnapshotSender *pSender, SyncSnapshotRsp *pMsg) { From fee198f9ada4cdf223810f3c28512f11867e0348 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 19 Dec 2023 16:21:11 +0800 Subject: [PATCH 05/94] refactor: do some internal refactor. --- include/dnode/vnode/tqCommon.h | 2 +- source/dnode/snode/src/snode.c | 2 +- source/dnode/vnode/src/tqCommon/tqCommon.c | 6 +++--- source/dnode/vnode/src/vnd/vnodeSync.c | 2 +- source/libs/stream/src/streamMeta.c | 4 ++++ source/libs/stream/src/streamStart.c | 1 - source/os/test/osTests.cpp | 2 +- 7 files changed, 11 insertions(+), 8 deletions(-) diff --git a/include/dnode/vnode/tqCommon.h b/include/dnode/vnode/tqCommon.h index 32ac38cee6..0ad3023cbe 100644 --- a/include/dnode/vnode/tqCommon.h +++ b/include/dnode/vnode/tqCommon.h @@ -31,7 +31,7 @@ int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, int64_t sversion, char* int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen); int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLeader); int32_t startStreamTasks(SStreamMeta* pMeta); -int32_t resetStreamTaskStatus(SStreamMeta* pMeta); +int32_t tqStreamTaskResetStatus(SStreamMeta* pMeta); int32_t tqStartTaskCompleteCallback(SStreamMeta* pMeta); #endif // TDENGINE_TQ_COMMON_H diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 853dc9d1aa..31d923169a 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -150,7 +150,7 @@ FAIL: } int32_t sndInit(SSnode * pSnode) { - resetStreamTaskStatus(pSnode->pMeta); + tqStreamTaskResetStatus(pSnode->pMeta); startStreamTasks(pSnode->pMeta); return 0; } diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index bfae691dbf..011ce674ec 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -218,7 +218,7 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM if (vnodeIsRoleLeader(pTq->pVnode) && !tsDisableStream) { tqInfo("vgId:%d start all stream tasks after all being updated", vgId); - resetStreamTaskStatus(pTq->pStreamMeta); + tqStreamTaskResetStatus(pTq->pStreamMeta); tqStartStreamTaskAsync(pTq, false); } else { tqInfo("vgId:%d, follower node not start stream tasks", vgId); @@ -703,7 +703,7 @@ int32_t startStreamTasks(SStreamMeta* pMeta) { return code; } -int32_t resetStreamTaskStatus(SStreamMeta* pMeta) { +int32_t tqStreamTaskResetStatus(SStreamMeta* pMeta) { int32_t vgId = pMeta->vgId; int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); @@ -763,7 +763,7 @@ static int32_t restartStreamTasks(SStreamMeta* pMeta, bool isLeader) { } if (isLeader && !tsDisableStream) { - resetStreamTaskStatus(pMeta); + tqStreamTaskResetStatus(pMeta); streamMetaWUnLock(pMeta); tqInfo("vgId:%d restart all stream tasks after all tasks being updated", vgId); diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 5871a60c9e..48811b2925 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -570,7 +570,7 @@ static void vnodeRestoreFinish(const SSyncFSM *pFsm, const SyncIndex commitIdx) vInfo("vgId:%d, sync restore finished, not launch stream tasks, since stream tasks are disabled", vgId); } else { vInfo("vgId:%d sync restore finished, start to launch stream tasks", pVnode->config.vgId); - resetStreamTaskStatus(pVnode->pTq->pStreamMeta); + tqStreamTaskResetStatus(pVnode->pTq->pStreamMeta); tqStreamTaskStartAsync(pMeta, &pVnode->msgCb, false); } } else { diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 59178336d8..c263968828 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -1270,18 +1270,22 @@ void streamMetaRLock(SStreamMeta* pMeta) { stTrace("vgId:%d meta-rlock", pMeta->vgId); taosRLockLatch(&pMeta->lock); } + void streamMetaRUnLock(SStreamMeta* pMeta) { stTrace("vgId:%d meta-runlock", pMeta->vgId); taosRUnLockLatch(&pMeta->lock); } + void streamMetaWLock(SStreamMeta* pMeta) { stTrace("vgId:%d meta-wlock", pMeta->vgId); taosWLockLatch(&pMeta->lock); } + void streamMetaWUnLock(SStreamMeta* pMeta) { stTrace("vgId:%d meta-wunlock", pMeta->vgId); taosWUnLockLatch(&pMeta->lock); } + static void execHelper(struct SSchedMsg* pSchedMsg) { __async_exec_fn_t execFn = (__async_exec_fn_t)pSchedMsg->ahandle; int32_t code = execFn(pSchedMsg->thandle); diff --git a/source/libs/stream/src/streamStart.c b/source/libs/stream/src/streamStart.c index 8bd7984d29..0ae85024c0 100644 --- a/source/libs/stream/src/streamStart.c +++ b/source/libs/stream/src/streamStart.c @@ -1083,7 +1083,6 @@ int32_t streamMetaUpdateTaskDownstreamStatus(SStreamMeta* pMeta, int64_t streamI int64_t endTs, bool ready) { STaskStartInfo* pStartInfo = &pMeta->startInfo; STaskId id = {.streamId = streamId, .taskId = taskId}; - bool restart = true; streamMetaWLock(pMeta); SHashObj* pDst = ready ? pStartInfo->pReadyTaskSet : pStartInfo->pFailedTaskSet; diff --git a/source/os/test/osTests.cpp b/source/os/test/osTests.cpp index 16660a9477..4d9ad9b5bd 100644 --- a/source/os/test/osTests.cpp +++ b/source/os/test/osTests.cpp @@ -72,7 +72,7 @@ TEST(osTest, osSystem) { const int sysLen = 64; char osSysName[sysLen]; int ret = taosGetOsReleaseName(osSysName, NULL, NULL, sysLen); - printf("os systeme name:%s\n", osSysName); + printf("os system name:%s\n", osSysName); ASSERT_EQ(ret, 0); } From 92045f54856713bff0ef6801f8cc4bc412afce69 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 19 Dec 2023 18:34:30 +0800 Subject: [PATCH 06/94] fix(stream): add snode check, and handle the task status reset from checkpoint when it is in the snode. --- include/dnode/vnode/tqCommon.h | 1 + source/dnode/mgmt/mgmt_snode/src/smHandle.c | 2 +- source/dnode/mnode/impl/src/mndStream.c | 127 ++++++++++++++------ source/dnode/snode/src/snode.c | 2 + source/dnode/vnode/src/tq/tq.c | 21 +--- source/dnode/vnode/src/tqCommon/tqCommon.c | 22 ++++ 6 files changed, 120 insertions(+), 55 deletions(-) diff --git a/include/dnode/vnode/tqCommon.h b/include/dnode/vnode/tqCommon.h index 0ad3023cbe..da9dcf1c04 100644 --- a/include/dnode/vnode/tqCommon.h +++ b/include/dnode/vnode/tqCommon.h @@ -33,5 +33,6 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead int32_t startStreamTasks(SStreamMeta* pMeta); int32_t tqStreamTaskResetStatus(SStreamMeta* pMeta); int32_t tqStartTaskCompleteCallback(SStreamMeta* pMeta); +int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, SRpcMsg* pMsg); #endif // TDENGINE_TQ_COMMON_H diff --git a/source/dnode/mgmt/mgmt_snode/src/smHandle.c b/source/dnode/mgmt/mgmt_snode/src/smHandle.c index 6de29f8513..444739e461 100644 --- a/source/dnode/mgmt/mgmt_snode/src/smHandle.c +++ b/source/dnode/mgmt/mgmt_snode/src/smHandle.c @@ -90,7 +90,7 @@ SArray *smGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_SCAN_HISTORY_FINISH, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_SCAN_HISTORY_FINISH_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECKPOINT_READY, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; - + if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_RESET, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER; code = 0; _OVER: diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 62840f7e1f..b8ecf66686 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -685,6 +685,40 @@ _OVER: return -1; } +static int32_t extractNodeEpset(SMnode *pMnode, SEpSet *pEpSet, bool* hasEpset, int32_t taskId, int32_t nodeId) { + *hasEpset = false; + + if (nodeId == SNODE_HANDLE) { + SSnodeObj *pObj = NULL; + void *pIter = NULL; + + pIter = sdbFetch(pMnode->pSdb, SDB_SNODE, pIter, (void **)&pObj); + if (pIter != NULL) { + addEpIntoEpSet(pEpSet, pObj->pDnode->fqdn, pObj->pDnode->port); + sdbRelease(pMnode->pSdb, pObj); + sdbCancelFetch(pMnode->pSdb, pIter); + *hasEpset = true; + return TSDB_CODE_SUCCESS; + } else { + mError("failed to acquire snode epset"); + return TSDB_CODE_INVALID_PARA; + } + } else { + SVgObj *pVgObj = mndAcquireVgroup(pMnode, nodeId); + if (pVgObj != NULL) { + SEpSet epset = mndGetVgroupEpset(pMnode, pVgObj); + mndReleaseVgroup(pMnode, pVgObj); + + epsetAssign(pEpSet, &epset); + *hasEpset = true; + return TSDB_CODE_SUCCESS; + } else { + mDebug("orphaned task:0x%x need to be dropped, nodeId:%d, no redo action", taskId, nodeId); + return TSDB_CODE_SUCCESS; + } + } +} + static int32_t mndPersistTaskDropReq(SMnode *pMnode, STrans *pTrans, SStreamTask *pTask) { SVDropStreamTaskReq *pReq = taosMemoryCalloc(1, sizeof(SVDropStreamTaskReq)); if (pReq == NULL) { @@ -698,28 +732,17 @@ static int32_t mndPersistTaskDropReq(SMnode *pMnode, STrans *pTrans, SStreamTask STransAction action = {0}; SEpSet epset = {0}; - if (pTask->info.nodeId == SNODE_HANDLE) { - SSnodeObj *pObj = NULL; - void *pIter = NULL; - while (1) { - pIter = sdbFetch(pMnode->pSdb, SDB_SNODE, pIter, (void **)&pObj); - if (pIter == NULL) { - break; - } + bool hasEpset = false; - addEpIntoEpSet(&epset, pObj->pDnode->fqdn, pObj->pDnode->port); - sdbRelease(pMnode->pSdb, pObj); - } - } else { - SVgObj *pVgObj = mndAcquireVgroup(pMnode, pTask->info.nodeId); - if (pVgObj != NULL) { - epset = mndGetVgroupEpset(pMnode, pVgObj); - mndReleaseVgroup(pMnode, pVgObj); - } else { - mDebug("orphaned task:0x%x need to be dropped, nodeId:%d, no redo action", pTask->id.taskId, pTask->info.nodeId); - taosMemoryFree(pReq); - return 0; - } + int32_t code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + return -1; + } + + // no valid epset, return directly without redoAction + if (!hasEpset) { + return TSDB_CODE_SUCCESS; } // The epset of nodeId of this task may have been expired now, let's use the newest epset from mnode. @@ -1776,9 +1799,20 @@ static int32_t mndPauseStreamTask(SMnode *pMnode, STrans *pTrans, SStreamTask *p pReq->taskId = pTask->id.taskId; pReq->streamId = pTask->id.streamId; - SVgObj *pVgObj = mndAcquireVgroup(pMnode, pTask->info.nodeId); - SEpSet epset = mndGetVgroupEpset(pMnode, pVgObj); - mndReleaseVgroup(pMnode, pVgObj); + SEpSet epset; + bool hasEpset = false; + int32_t code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + taosMemoryFree(pReq); + return -1; + } + + // no valid epset, return directly without redoAction + if (!hasEpset) { + taosMemoryFree(pReq); + return TSDB_CODE_SUCCESS; + } STransAction action = {0}; initTransAction(&action, pReq, sizeof(SVPauseStreamTaskReq), TDMT_STREAM_TASK_PAUSE, &epset, 0); @@ -1920,17 +1954,25 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) { static int32_t mndResumeStreamTask(STrans *pTrans, SMnode *pMnode, SStreamTask *pTask, int8_t igUntreated) { SVResumeStreamTaskReq *pReq = taosMemoryCalloc(1, sizeof(SVResumeStreamTaskReq)); if (pReq == NULL) { + mError("failed to malloc in resume stream, size:%" PRIzu ", code:%s", sizeof(SVResumeStreamTaskReq), + tstrerror(TSDB_CODE_OUT_OF_MEMORY)); terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } + pReq->head.vgId = htonl(pTask->info.nodeId); pReq->taskId = pTask->id.taskId; pReq->streamId = pTask->id.streamId; pReq->igUntreated = igUntreated; - SVgObj *pVgObj = mndAcquireVgroup(pMnode, pTask->info.nodeId); - SEpSet epset = mndGetVgroupEpset(pMnode, pVgObj); - mndReleaseVgroup(pMnode, pVgObj); + SEpSet epset; + bool hasEpset = false; + int32_t code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + taosMemoryFree(pReq); + return -1; + } STransAction action = {0}; initTransAction(&action, pReq, sizeof(SVResumeStreamTaskReq), TDMT_STREAM_TASK_RESUME, &epset, 0); @@ -2208,6 +2250,8 @@ static SVgroupChangeInfo mndFindChangedNodeInfo(SMnode *pMnode, const SArray *pP epsetAssign(&updateInfo.newEp, &pCurrent->epset); taosArrayPush(info.pUpdateNodeList, &updateInfo); } + + // todo handle the snode info if (pCurrent->nodeId != SNODE_HANDLE) { SVgObj *pVgroup = mndAcquireVgroup(pMnode, pCurrent->nodeId); taosHashPut(info.pDBMap, pVgroup->dbName, strlen(pVgroup->dbName), NULL, 0); @@ -2704,9 +2748,18 @@ int32_t createStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream) { pReq->taskId = pTask->id.taskId; pReq->streamId = pTask->id.streamId; - SVgObj *pVgObj = mndAcquireVgroup(pMnode, pTask->info.nodeId); - SEpSet epset = mndGetVgroupEpset(pMnode, pVgObj); - mndReleaseVgroup(pMnode, pVgObj); + SEpSet epset; + bool hasEpset = false; + int32_t code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId); + if (code != TSDB_CODE_SUCCESS) { + taosMemoryFree(pReq); + continue; + } + + if (!hasEpset) { + taosMemoryFree(pReq); + continue; + } STransAction action = {0}; initTransAction(&action, pReq, sizeof(SVResetStreamTaskReq), TDMT_VND_STREAM_TASK_RESET, &epset, 0); @@ -2878,11 +2931,17 @@ static int32_t mndDropRelatedFillhistoryTask(SMnode *pMnode, STaskStatusEntry *p mDebug("build and send drop related fill-history task for task:0x%x", pTask->id.taskId); - SVgObj *pVgObj = mndAcquireVgroup(pMnode, pTask->info.nodeId); - SEpSet epset = mndGetVgroupEpset(pMnode, pVgObj); - mndReleaseVgroup(pMnode, pVgObj); + SEpSet epset; + bool hasEpset = false; + int32_t code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + if (hasEpset) { + tmsgSendReq(&epset, &msg); + } - tmsgSendReq(&epset, &msg); return TSDB_CODE_SUCCESS; } diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 31d923169a..f0fe662025 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -205,6 +205,8 @@ int32_t sndProcessWriteMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg *pRsp) { return tqStreamTaskProcessDropReq(pSnode->pMeta, pMsg->pCont, pMsg->contLen); case TDMT_VND_STREAM_TASK_UPDATE: return tqStreamTaskProcessUpdateReq(pSnode->pMeta, &pSnode->msgCb, pMsg, true); + case TDMT_VND_STREAM_TASK_RESET: + return tqStreamTaskProcessTaskResetReq(pSnode->pMeta, pMsg); default: ASSERT(0); } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 76049d21b1..c33becf5b9 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1336,26 +1336,7 @@ int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg) { } int32_t tqProcessTaskResetReq(STQ* pTq, SRpcMsg* pMsg) { - SVPauseStreamTaskReq* pReq = (SVPauseStreamTaskReq*)pMsg->pCont; - - SStreamMeta* pMeta = pTq->pStreamMeta; - SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId); - if (pTask == NULL) { - tqError("vgId:%d process task-reset req, failed to acquire task:0x%x, it may have been dropped already", - pMeta->vgId, pReq->taskId); - return TSDB_CODE_SUCCESS; - } - - tqDebug("s-task:%s receive task-reset msg from mnode, reset status and ready for data processing", pTask->id.idStr); - - // clear flag set during do checkpoint, and open inputQ for all upstream tasks - if (streamTaskGetStatus(pTask, NULL) == TASK_STATUS__CK) { - streamTaskClearCheckInfo(pTask, true); - streamTaskSetStatusReady(pTask); - } - - streamMetaReleaseTask(pMeta, pTask); - return TSDB_CODE_SUCCESS; + return tqStreamTaskProcessTaskResetReq(pTq->pStreamMeta, pMsg); } int32_t tqProcessTaskDropHTask(STQ* pTq, SRpcMsg* pMsg) { diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index 011ce674ec..0d932e6e72 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -832,5 +832,27 @@ int32_t tqStartTaskCompleteCallback(SStreamMeta* pMeta) { tqDebug("vgId:%d start all tasks completed", pMeta->vgId); } + return TSDB_CODE_SUCCESS; +} + +int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { + SVPauseStreamTaskReq* pReq = (SVPauseStreamTaskReq*)pMsg->pCont; + + SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->taskId); + if (pTask == NULL) { + tqError("vgId:%d process task-reset req, failed to acquire task:0x%x, it may have been dropped already", + pMeta->vgId, pReq->taskId); + return TSDB_CODE_SUCCESS; + } + + tqDebug("s-task:%s receive task-reset msg from mnode, reset status and ready for data processing", pTask->id.idStr); + + // clear flag set during do checkpoint, and open inputQ for all upstream tasks + if (streamTaskGetStatus(pTask, NULL) == TASK_STATUS__CK) { + streamTaskClearCheckInfo(pTask, true); + streamTaskSetStatusReady(pTask); + } + + streamMetaReleaseTask(pMeta, pTask); return TSDB_CODE_SUCCESS; } \ No newline at end of file From 2de0f6e9713fb56c04cfa34ba505cfe3e2cf8d3d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 19 Dec 2023 18:56:15 +0800 Subject: [PATCH 07/94] test(stream): lock the update procedure. --- source/dnode/vnode/src/tqCommon/tqCommon.c | 63 +++------------------- 1 file changed, 8 insertions(+), 55 deletions(-) diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index 0d932e6e72..c39e653596 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -106,14 +106,15 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM return rsp.code; } - streamMetaWUnLock(pMeta); +// streamMetaWUnLock(pMeta); + // todo for test purpose // the following two functions should not be executed within the scope of meta lock to avoid deadlock streamTaskUpdateEpsetInfo(pTask, req.pNodeList); streamTaskResetStatus(pTask); // continue after lock the meta again - streamMetaWLock(pMeta); +// streamMetaWLock(pMeta); SStreamTask** ppHTask = NULL; if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { @@ -166,65 +167,17 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM streamMetaWUnLock(pMeta); } else { if (!restored) { - tqDebug("vgId:%d vnode restore not completed, not restart the tasks, clear the start after nodeUpdate flag", - vgId); + tqDebug("vgId:%d vnode restore not completed, not start the tasks, clear the start after nodeUpdate flag", vgId); pMeta->startInfo.tasksWillRestart = 0; streamMetaWUnLock(pMeta); } else { tqDebug("vgId:%d all %d task(s) nodeEp updated and closed", vgId, numOfTasks); -#if 1 +#if 0 + // for test purpose, to trigger the leader election + taosMSleep(5000); +#endif tqStreamTaskStartAsync(pMeta, cb, true); streamMetaWUnLock(pMeta); -#else - streamMetaWUnLock(pMeta); - - // For debug purpose. - // the following procedure consume many CPU resource, result in the re-election of leader - // with high probability. So we employ it as a test case for the stream processing framework, with - // checkpoint/restart/nodeUpdate etc. - while (1) { - int32_t startVal = atomic_val_compare_exchange_32(&pMeta->startInfo.taskStarting, 0, 1); - if (startVal == 0) { - break; - } - - tqDebug("vgId:%d in start stream tasks procedure, wait for 500ms and recheck", vgId); - taosMsleep(500); - } - - while (streamMetaTaskInTimer(pMeta)) { - tqDebug("vgId:%d some tasks in timer, wait for 100ms and recheck", pMeta->vgId); - taosMsleep(100); - } - - streamMetaWLock(pMeta); - - int32_t code = streamMetaReopen(pMeta); - if (code != 0) { - tqError("vgId:%d failed to reopen stream meta", vgId); - streamMetaWUnLock(pMeta); - taosArrayDestroy(req.pNodeList); - return -1; - } - - streamMetaInitBackend(pMeta); - - if (streamMetaLoadAllTasks(pTq->pStreamMeta) < 0) { - tqError("vgId:%d failed to load stream tasks", vgId); - streamMetaWUnLock(pMeta); - taosArrayDestroy(req.pNodeList); - return -1; - } - - if (vnodeIsRoleLeader(pTq->pVnode) && !tsDisableStream) { - tqInfo("vgId:%d start all stream tasks after all being updated", vgId); - tqStreamTaskResetStatus(pTq->pStreamMeta); - tqStartStreamTaskAsync(pTq, false); - } else { - tqInfo("vgId:%d, follower node not start stream tasks", vgId); - } - streamMetaWUnLock(pMeta); -#endif } } From f7ab8e99b503f6f20285d712f2f3362a008b2a65 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 19 Dec 2023 19:51:51 +0800 Subject: [PATCH 08/94] enh: demarcate top and bottom halves of task trans execution to ensure a consistent order --- source/dnode/mnode/impl/inc/mndTrans.h | 2 +- source/dnode/mnode/impl/src/mndSync.c | 8 +- source/dnode/mnode/impl/src/mndTrans.c | 148 ++++++++++++------------- 3 files changed, 74 insertions(+), 84 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndTrans.h b/source/dnode/mnode/impl/inc/mndTrans.h index 04544da80e..1bd39a2299 100644 --- a/source/dnode/mnode/impl/inc/mndTrans.h +++ b/source/dnode/mnode/impl/inc/mndTrans.h @@ -97,7 +97,7 @@ SSdbRaw *mndTransEncode(STrans *pTrans); SSdbRow *mndTransDecode(SSdbRaw *pRaw); void mndTransDropData(STrans *pTrans); -bool mndTransPerformPrepareStage(SMnode *pMnode, STrans *pTrans); +bool mndTransPerformPrepareStage(SMnode *pMnode, STrans *pTrans, bool topHalf); #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 585d8ea8e7..1ba6c51d4c 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -208,15 +208,11 @@ int32_t mndProcessWriteMsg(SMnode *pMnode, SRpcMsg *pMsg, SFsmCbMeta *pMeta) { } if (pTrans->stage == TRN_STAGE_PREPARE) { - bool continueExec = mndTransPerformPrepareStage(pMnode, pTrans); + bool continueExec = mndTransPerformPrepareStage(pMnode, pTrans, false); if (!continueExec) goto _OUT; } - if (pTrans->id != pMgmt->transId) { - mInfo("trans:%d, execute in mnode which not leader or sync timeout, createTime:%" PRId64 " saved trans:%d", - pTrans->id, pTrans->createdTime, pMgmt->transId); - mndTransRefresh(pMnode, pTrans); - } + mndTransRefresh(pMnode, pTrans); sdbSetApplyInfo(pMnode->pSdb, pMeta->index, pMeta->term, pMeta->lastConfigIndex); sdbWriteFile(pMnode->pSdb, tsMndSdbWriteDelta); diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 7749decf91..49b483114a 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -36,21 +36,25 @@ static int32_t mndTransAppendLog(SArray *pArray, SSdbRaw *pRaw); static int32_t mndTransAppendAction(SArray *pArray, STransAction *pAction); static void mndTransDropLogs(SArray *pArray); static void mndTransDropActions(SArray *pArray); -static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pArray); -static int32_t mndTransExecuteRedoLogs(SMnode *pMnode, STrans *pTrans); -static int32_t mndTransExecuteUndoLogs(SMnode *pMnode, STrans *pTrans); -static int32_t mndTransExecuteRedoActions(SMnode *pMnode, STrans *pTrans); -static int32_t mndTransExecuteUndoActions(SMnode *pMnode, STrans *pTrans); -static int32_t mndTransExecuteCommitActions(SMnode *pMnode, STrans *pTrans); -static bool mndTransPerformRedoLogStage(SMnode *pMnode, STrans *pTrans); -static bool mndTransPerformRedoActionStage(SMnode *pMnode, STrans *pTrans); -static bool mndTransPerformUndoLogStage(SMnode *pMnode, STrans *pTrans); -static bool mndTransPerformUndoActionStage(SMnode *pMnode, STrans *pTrans); -static bool mndTransPerformCommitActionStage(SMnode *pMnode, STrans *pTrans); -static bool mndTransPerformCommitStage(SMnode *pMnode, STrans *pTrans); -static bool mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans); -static bool mndTransPerformFinishStage(SMnode *pMnode, STrans *pTrans); -static bool mndCannotExecuteTransAction(SMnode *pMnode) { return !pMnode->deploy && !mndIsLeader(pMnode); } + +static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pArray, bool topHalf); +static int32_t mndTransExecuteRedoLogs(SMnode *pMnode, STrans *pTrans, bool topHalf); +static int32_t mndTransExecuteUndoLogs(SMnode *pMnode, STrans *pTrans, bool topHalf); +static int32_t mndTransExecuteRedoActions(SMnode *pMnode, STrans *pTrans, bool topHalf); +static int32_t mndTransExecuteUndoActions(SMnode *pMnode, STrans *pTrans, bool topHalf); +static int32_t mndTransExecuteCommitActions(SMnode *pMnode, STrans *pTrans, bool topHalf); +static bool mndTransPerformRedoLogStage(SMnode *pMnode, STrans *pTrans, bool topHalf); +static bool mndTransPerformRedoActionStage(SMnode *pMnode, STrans *pTrans, bool topHalf); +static bool mndTransPerformUndoLogStage(SMnode *pMnode, STrans *pTrans, bool topHalf); +static bool mndTransPerformUndoActionStage(SMnode *pMnode, STrans *pTrans, bool topHalf); +static bool mndTransPerformCommitActionStage(SMnode *pMnode, STrans *pTrans, bool topHalf); +static bool mndTransPerformCommitStage(SMnode *pMnode, STrans *pTrans, bool topHalf); +static bool mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans, bool topHalf); +static bool mndTransPerformFinishStage(SMnode *pMnode, STrans *pTrans, bool topHalf); + +static bool mndCannotExecuteTransAction(SMnode *pMnode, bool topHalf) { + return (!pMnode->deploy && !mndIsLeader(pMnode)) || !topHalf; +} static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans); static int32_t mndProcessTransTimer(SRpcMsg *pReq); @@ -1090,8 +1094,9 @@ static void mndTransResetActions(SMnode *pMnode, STrans *pTrans, SArray *pArray) } } -static int32_t mndTransWriteSingleLog(SMnode *pMnode, STrans *pTrans, STransAction *pAction) { +static int32_t mndTransWriteSingleLog(SMnode *pMnode, STrans *pTrans, STransAction *pAction, bool topHalf) { if (pAction->rawWritten) return 0; + if (topHalf) return -1; int32_t code = sdbWriteWithoutFree(pMnode->pSdb, pAction->pRaw); if (code == 0 || terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { @@ -1112,9 +1117,9 @@ static int32_t mndTransWriteSingleLog(SMnode *pMnode, STrans *pTrans, STransActi return code; } -static int32_t mndTransSendSingleMsg(SMnode *pMnode, STrans *pTrans, STransAction *pAction) { - if (pAction->msgSent) return 0; - if (mndCannotExecuteTransAction(pMnode)) return -1; +static int32_t mndTransSendSingleMsg(SMnode *pMnode, STrans *pTrans, STransAction *pAction, bool topHalf) { + if (pAction->msgSent) return 0; + if (mndCannotExecuteTransAction(pMnode, topHalf)) return -1; int64_t signature = pTrans->id; signature = (signature << 32); @@ -1159,7 +1164,8 @@ static int32_t mndTransSendSingleMsg(SMnode *pMnode, STrans *pTrans, STransActio return code; } -static int32_t mndTransExecNullMsg(SMnode *pMnode, STrans *pTrans, STransAction *pAction) { +static int32_t mndTransExecNullMsg(SMnode *pMnode, STrans *pTrans, STransAction *pAction, bool topHalf) { + if (!topHalf) return -1; pAction->rawWritten = 0; pAction->errCode = 0; mInfo("trans:%d, %s:%d confirm action executed", pTrans->id, mndTransStr(pAction->stage), pAction->id); @@ -1168,34 +1174,34 @@ static int32_t mndTransExecNullMsg(SMnode *pMnode, STrans *pTrans, STransAction return 0; } -static int32_t mndTransExecSingleAction(SMnode *pMnode, STrans *pTrans, STransAction *pAction) { +static int32_t mndTransExecSingleAction(SMnode *pMnode, STrans *pTrans, STransAction *pAction, bool topHalf) { if (pAction->actionType == TRANS_ACTION_RAW) { - return mndTransWriteSingleLog(pMnode, pTrans, pAction); + return mndTransWriteSingleLog(pMnode, pTrans, pAction, topHalf); } else if (pAction->actionType == TRANS_ACTION_MSG) { - return mndTransSendSingleMsg(pMnode, pTrans, pAction); + return mndTransSendSingleMsg(pMnode, pTrans, pAction, topHalf); } else { - return mndTransExecNullMsg(pMnode, pTrans, pAction); + return mndTransExecNullMsg(pMnode, pTrans, pAction, topHalf); } } -static int32_t mndTransExecSingleActions(SMnode *pMnode, STrans *pTrans, SArray *pArray) { +static int32_t mndTransExecSingleActions(SMnode *pMnode, STrans *pTrans, SArray *pArray, bool topHalf) { int32_t numOfActions = taosArrayGetSize(pArray); int32_t code = 0; for (int32_t action = 0; action < numOfActions; ++action) { STransAction *pAction = taosArrayGet(pArray, action); - code = mndTransExecSingleAction(pMnode, pTrans, pAction); + code = mndTransExecSingleAction(pMnode, pTrans, pAction, topHalf); if (code != 0) break; } return code; } -static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pArray) { +static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pArray, bool topHalf) { int32_t numOfActions = taosArrayGetSize(pArray); if (numOfActions == 0) return 0; - if (mndTransExecSingleActions(pMnode, pTrans, pArray) != 0) { + if (mndTransExecSingleActions(pMnode, pTrans, pArray, topHalf) != 0) { return -1; } @@ -1248,31 +1254,31 @@ static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pA } } -static int32_t mndTransExecuteRedoActions(SMnode *pMnode, STrans *pTrans) { - int32_t code = mndTransExecuteActions(pMnode, pTrans, pTrans->redoActions); +static int32_t mndTransExecuteRedoActions(SMnode *pMnode, STrans *pTrans, bool topHalf) { + int32_t code = mndTransExecuteActions(pMnode, pTrans, pTrans->redoActions, topHalf); if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("failed to execute redoActions since:%s, code:0x%x", terrstr(), terrno); } return code; } -static int32_t mndTransExecuteUndoActions(SMnode *pMnode, STrans *pTrans) { - int32_t code = mndTransExecuteActions(pMnode, pTrans, pTrans->undoActions); +static int32_t mndTransExecuteUndoActions(SMnode *pMnode, STrans *pTrans, bool topHalf) { + int32_t code = mndTransExecuteActions(pMnode, pTrans, pTrans->undoActions, topHalf); if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("failed to execute undoActions since %s", terrstr()); } return code; } -static int32_t mndTransExecuteCommitActions(SMnode *pMnode, STrans *pTrans) { - int32_t code = mndTransExecuteActions(pMnode, pTrans, pTrans->commitActions); +static int32_t mndTransExecuteCommitActions(SMnode *pMnode, STrans *pTrans, bool topHalf) { + int32_t code = mndTransExecuteActions(pMnode, pTrans, pTrans->commitActions, topHalf); if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { mError("failed to execute commitActions since %s", terrstr()); } return code; } -static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans) { +static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans, bool topHalf) { int32_t code = 0; int32_t numOfActions = taosArrayGetSize(pTrans->redoActions); if (numOfActions == 0) return code; @@ -1289,7 +1295,7 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans) for (int32_t action = pTrans->redoActionPos; action < numOfActions; ++action) { STransAction *pAction = taosArrayGet(pTrans->redoActions, pTrans->redoActionPos); - code = mndTransExecSingleAction(pMnode, pTrans, pAction); + code = mndTransExecSingleAction(pMnode, pTrans, pAction, topHalf); if (code == 0) { if (pAction->msgSent) { if (pAction->msgReceived) { @@ -1317,14 +1323,16 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans) } mndSetTransLastAction(pTrans, pAction); - if (mndCannotExecuteTransAction(pMnode)) break; + if (mndCannotExecuteTransAction(pMnode, topHalf)) break; if (code == 0) { pTrans->code = 0; pTrans->redoActionPos++; mInfo("trans:%d, %s:%d is executed and need sync to other mnodes", pTrans->id, mndTransStr(pAction->stage), pAction->id); + taosThreadMutexUnlock(&pTrans->mutex); code = mndTransSync(pMnode, pTrans); + taosThreadMutexLock(&pTrans->mutex); if (code != 0) { pTrans->redoActionPos--; pTrans->code = terrno; @@ -1357,7 +1365,7 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans) return code; } -bool mndTransPerformPrepareStage(SMnode *pMnode, STrans *pTrans) { +bool mndTransPerformPrepareStage(SMnode *pMnode, STrans *pTrans, bool topHalf) { bool continueExec = true; int32_t code = 0; @@ -1368,7 +1376,7 @@ bool mndTransPerformPrepareStage(SMnode *pMnode, STrans *pTrans) { for (int32_t action = 0; action < numOfActions; ++action) { STransAction *pAction = taosArrayGet(pTrans->prepareActions, action); - code = mndTransExecSingleAction(pMnode, pTrans, pAction); + code = mndTransExecSingleAction(pMnode, pTrans, pAction, topHalf); if (code != 0) { mError("trans:%d, failed to execute prepare action:%d, numOfActions:%d", pTrans->id, action, numOfActions); return false; @@ -1381,17 +1389,17 @@ _OVER: return continueExec; } -static bool mndTransPerformRedoActionStage(SMnode *pMnode, STrans *pTrans) { +static bool mndTransPerformRedoActionStage(SMnode *pMnode, STrans *pTrans, bool topHalf) { bool continueExec = true; int32_t code = 0; if (pTrans->exec == TRN_EXEC_SERIAL) { - code = mndTransExecuteRedoActionsSerial(pMnode, pTrans); + code = mndTransExecuteRedoActionsSerial(pMnode, pTrans, topHalf); } else { - code = mndTransExecuteRedoActions(pMnode, pTrans); + code = mndTransExecuteRedoActions(pMnode, pTrans, topHalf); } - if (mndCannotExecuteTransAction(pMnode)) return false; + if (mndCannotExecuteTransAction(pMnode, topHalf)) return false; terrno = code; if (code == 0) { @@ -1431,8 +1439,8 @@ static bool mndTransPerformRedoActionStage(SMnode *pMnode, STrans *pTrans) { return continueExec; } -static bool mndTransPerformCommitStage(SMnode *pMnode, STrans *pTrans) { - if (mndCannotExecuteTransAction(pMnode)) return false; +static bool mndTransPerformCommitStage(SMnode *pMnode, STrans *pTrans, bool topHalf) { + if (mndCannotExecuteTransAction(pMnode, topHalf)) return false; bool continueExec = true; int32_t code = mndTransCommit(pMnode, pTrans); @@ -1452,9 +1460,9 @@ static bool mndTransPerformCommitStage(SMnode *pMnode, STrans *pTrans) { return continueExec; } -static bool mndTransPerformCommitActionStage(SMnode *pMnode, STrans *pTrans) { +static bool mndTransPerformCommitActionStage(SMnode *pMnode, STrans *pTrans, bool topHalf) { bool continueExec = true; - int32_t code = mndTransExecuteCommitActions(pMnode, pTrans); + int32_t code = mndTransExecuteCommitActions(pMnode, pTrans, topHalf); if (code == 0) { pTrans->code = 0; @@ -1471,9 +1479,9 @@ static bool mndTransPerformCommitActionStage(SMnode *pMnode, STrans *pTrans) { return continueExec; } -static bool mndTransPerformUndoActionStage(SMnode *pMnode, STrans *pTrans) { +static bool mndTransPerformUndoActionStage(SMnode *pMnode, STrans *pTrans, bool topHalf) { bool continueExec = true; - int32_t code = mndTransExecuteUndoActions(pMnode, pTrans); + int32_t code = mndTransExecuteUndoActions(pMnode, pTrans, topHalf); if (code == 0) { pTrans->stage = TRN_STAGE_PRE_FINISH; @@ -1491,8 +1499,8 @@ static bool mndTransPerformUndoActionStage(SMnode *pMnode, STrans *pTrans) { return continueExec; } -static bool mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans) { - if (mndCannotExecuteTransAction(pMnode)) return false; +static bool mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans, bool topHalf) { + if (mndCannotExecuteTransAction(pMnode, topHalf)) return false; bool continueExec = true; int32_t code = mndTransRollback(pMnode, pTrans); @@ -1510,8 +1518,8 @@ static bool mndTransPerformRollbackStage(SMnode *pMnode, STrans *pTrans) { return continueExec; } -static bool mndTransPerformPreFinishStage(SMnode *pMnode, STrans *pTrans) { - if (mndCannotExecuteTransAction(pMnode)) return false; +static bool mndTransPerformPreFinishStage(SMnode *pMnode, STrans *pTrans, bool topHalf) { + if (mndCannotExecuteTransAction(pMnode, topHalf)) return false; bool continueExec = true; int32_t code = mndTransPreFinish(pMnode, pTrans); @@ -1529,8 +1537,9 @@ static bool mndTransPerformPreFinishStage(SMnode *pMnode, STrans *pTrans) { return continueExec; } -static bool mndTransPerformFinishStage(SMnode *pMnode, STrans *pTrans) { +static bool mndTransPerformFinishStage(SMnode *pMnode, STrans *pTrans, bool topHalf) { bool continueExec = false; + if (topHalf) return continueExec; SSdbRaw *pRaw = mndTransEncode(pTrans); if (pRaw == NULL) { @@ -1558,43 +1567,28 @@ void mndTransExecuteImp(SMnode *pMnode, STrans *pTrans, bool topHalf) { pTrans->lastExecTime = taosGetTimestampMs(); switch (pTrans->stage) { case TRN_STAGE_PREPARE: - continueExec = mndTransPerformPrepareStage(pMnode, pTrans); + continueExec = mndTransPerformPrepareStage(pMnode, pTrans, topHalf); break; case TRN_STAGE_REDO_ACTION: - continueExec = mndTransPerformRedoActionStage(pMnode, pTrans); + continueExec = mndTransPerformRedoActionStage(pMnode, pTrans, topHalf); break; case TRN_STAGE_COMMIT: - if (topHalf) { - continueExec = mndTransPerformCommitStage(pMnode, pTrans); - } else { - mInfo("trans:%d, can not commit since not leader", pTrans->id); - continueExec = false; - } + continueExec = mndTransPerformCommitStage(pMnode, pTrans, topHalf); break; case TRN_STAGE_COMMIT_ACTION: - continueExec = mndTransPerformCommitActionStage(pMnode, pTrans); + continueExec = mndTransPerformCommitActionStage(pMnode, pTrans, topHalf); break; case TRN_STAGE_ROLLBACK: - if (topHalf) { - continueExec = mndTransPerformRollbackStage(pMnode, pTrans); - } else { - mInfo("trans:%d, can not rollback since not leader", pTrans->id); - continueExec = false; - } + continueExec = mndTransPerformRollbackStage(pMnode, pTrans, topHalf); break; case TRN_STAGE_UNDO_ACTION: - continueExec = mndTransPerformUndoActionStage(pMnode, pTrans); + continueExec = mndTransPerformUndoActionStage(pMnode, pTrans, topHalf); break; case TRN_STAGE_PRE_FINISH: - if (topHalf) { - continueExec = mndTransPerformPreFinishStage(pMnode, pTrans); - } else { - mInfo("trans:%d, can not pre-finish since not leader", pTrans->id); - continueExec = false; - } + continueExec = mndTransPerformPreFinishStage(pMnode, pTrans, topHalf); break; case TRN_STAGE_FINISH: - continueExec = mndTransPerformFinishStage(pMnode, pTrans); + continueExec = mndTransPerformFinishStage(pMnode, pTrans, topHalf); break; default: continueExec = false; From e06f355143d5be866c226080a8b6e9122cd9c3ad Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 19 Dec 2023 20:16:33 +0800 Subject: [PATCH 09/94] enh: improve logging msg of mndPostMgmtCode --- source/dnode/mnode/impl/src/mndSync.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 1ba6c51d4c..00cbcbce6c 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -230,6 +230,7 @@ static int32_t mndPostMgmtCode(SMnode *pMnode, int32_t code) { goto _OUT; } + int32_t transId = pMgmt->transId; pMgmt->transId = 0; pMgmt->transSec = 0; pMgmt->transSeq = 0; @@ -237,9 +238,9 @@ static int32_t mndPostMgmtCode(SMnode *pMnode, int32_t code) { tsem_post(&pMgmt->syncSem); if (pMgmt->errCode != 0) { - mError("trans:%d, failed to propose since %s, post sem", pMgmt->transId, tstrerror(pMgmt->errCode)); + mError("trans:%d, failed to propose since %s, post sem", transId, tstrerror(pMgmt->errCode)); } else { - mInfo("trans:%d, is proposed and post sem, seq:%" PRId64, pMgmt->transId, pMgmt->transSeq); + mInfo("trans:%d, is proposed and post sem, seq:%" PRId64, transId, pMgmt->transSeq); } _OUT: From 541967e99ebf6fd85ec60064abff3e1bc08e3acf Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 20 Dec 2023 10:24:03 +0800 Subject: [PATCH 10/94] refactor: do some internal refactor. --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 42 +++++++++++++++---------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index fd179dba2e..773ce9e513 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -1360,7 +1360,8 @@ static bool tryCopyDistinctRowFromFileBlock(STsdbReader* pReader, SBlockData* pB static bool nextRowFromSttBlocks(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo, SVersionRange* pVerRange) { - int32_t step = ASCENDING_TRAVERSE(pSttBlockReader->order) ? 1 : -1; + int32_t order = pSttBlockReader->order; + int32_t step = ASCENDING_TRAVERSE(order) ? 1 : -1; while (1) { bool hasVal = tMergeTreeNext(&pSttBlockReader->mergeTree); @@ -1377,10 +1378,12 @@ static bool nextRowFromSttBlocks(SSttBlockReader* pSttBlockReader, STableBlockSc pSttBlockReader->currentKey = key; pScanInfo->sttKeyInfo.nextProcKey = key; - if (!hasBeenDropped(pScanInfo->delSkyline, &pScanInfo->sttBlockDelIndex, key, ver, pSttBlockReader->order, - pVerRange)) { - pScanInfo->sttKeyInfo.status = STT_FILE_HAS_DATA; - return true; + if (pScanInfo->delSkyline != NULL && TARRAY_SIZE(pScanInfo->delSkyline) > 0) { + bool dropped = hasBeenDropped(pScanInfo->delSkyline, &pScanInfo->sttBlockDelIndex, key, ver, order, pVerRange); + if (!dropped) { + pScanInfo->sttKeyInfo.status = STT_FILE_HAS_DATA; + return true; + } } } } @@ -2054,9 +2057,12 @@ static bool isValidFileBlockRow(SBlockData* pBlockData, SFileBlockDumpInfo* pDum return false; } - if (hasBeenDropped(pBlockScanInfo->delSkyline, &pBlockScanInfo->fileDelIndex, ts, ver, pReader->info.order, - &pReader->info.verRange)) { - return false; + if (pBlockScanInfo->delSkyline != NULL && TARRAY_SIZE(pBlockScanInfo->delSkyline) > 0) { + bool deleted = hasBeenDropped(pBlockScanInfo->delSkyline, &pBlockScanInfo->fileDelIndex, ts, ver, + pReader->info.order, &pReader->info.verRange); + if (deleted) { + return false; + } } return true; @@ -3219,7 +3225,7 @@ SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, int8_ bool hasBeenDropped(const SArray* pDelList, int32_t* index, int64_t key, int64_t ver, int32_t order, SVersionRange* pVerRange) { - if (pDelList == NULL || (taosArrayGetSize(pDelList) == 0)) { + if (pDelList == NULL || (TARRAY_SIZE(pDelList) == 0)) { return false; } @@ -3327,16 +3333,18 @@ TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* p TSDBROW* pRow = tsdbTbDataIterGet(pIter->iter); TSDBKEY key = TSDBROW_KEY(pRow); - + int32_t order = pReader->info.order; if (outOfTimeWindow(key.ts, &pReader->info.window)) { pIter->hasVal = false; return NULL; } // it is a valid data version - if ((key.version <= pReader->info.verRange.maxVer && key.version >= pReader->info.verRange.minVer) && - (!hasBeenDropped(pDelList, &pIter->index, key.ts, key.version, pReader->info.order, &pReader->info.verRange))) { - return pRow; + if ((key.version <= pReader->info.verRange.maxVer && key.version >= pReader->info.verRange.minVer)) { + bool dropped = hasBeenDropped(pDelList, &pIter->index, key.ts, key.version, order, &pReader->info.verRange); + if (!dropped) { + return pRow; + } } while (1) { @@ -3353,9 +3361,11 @@ TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* p return NULL; } - if (key.version <= pReader->info.verRange.maxVer && key.version >= pReader->info.verRange.minVer && - (!hasBeenDropped(pDelList, &pIter->index, key.ts, key.version, pReader->info.order, &pReader->info.verRange))) { - return pRow; + if (key.version <= pReader->info.verRange.maxVer && key.version >= pReader->info.verRange.minVer) { + bool dropped = hasBeenDropped(pDelList, &pIter->index, key.ts, key.version, order, &pReader->info.verRange); + if (!dropped) { + return pRow; + } } } } From 81d8b2434f8b5c8e4a2c36be00115d3e0d25672b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 20 Dec 2023 10:25:31 +0800 Subject: [PATCH 11/94] refactor: do some internal refactor. --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 773ce9e513..751569bf90 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -3340,7 +3340,8 @@ TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* p } // it is a valid data version - if ((key.version <= pReader->info.verRange.maxVer && key.version >= pReader->info.verRange.minVer)) { + if ((key.version <= pReader->info.verRange.maxVer && key.version >= pReader->info.verRange.minVer) && + pDelList != NULL && TARRAY_SIZE(pDelList) > 0) { bool dropped = hasBeenDropped(pDelList, &pIter->index, key.ts, key.version, order, &pReader->info.verRange); if (!dropped) { return pRow; @@ -3361,7 +3362,8 @@ TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* p return NULL; } - if (key.version <= pReader->info.verRange.maxVer && key.version >= pReader->info.verRange.minVer) { + if (key.version <= pReader->info.verRange.maxVer && key.version >= pReader->info.verRange.minVer && + pDelList != NULL && TARRAY_SIZE(pDelList) > 0) { bool dropped = hasBeenDropped(pDelList, &pIter->index, key.ts, key.version, order, &pReader->info.verRange); if (!dropped) { return pRow; From 94b260a1f9431d21a4776d1c7e078de13781368e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 20 Dec 2023 10:29:04 +0800 Subject: [PATCH 12/94] refactor: do some internal refactor. --- source/libs/stream/src/streamStart.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/source/libs/stream/src/streamStart.c b/source/libs/stream/src/streamStart.c index 743661bb59..68380e4ba4 100644 --- a/source/libs/stream/src/streamStart.c +++ b/source/libs/stream/src/streamStart.c @@ -209,7 +209,11 @@ void streamTaskCheckDownstream(SStreamTask* pTask) { int32_t numOfVgs = taosArrayGetSize(vgInfo); pTask->notReadyTasks = numOfVgs; - pTask->checkReqIds = taosArrayInit(numOfVgs, sizeof(int64_t)); + if (pTask->checkReqIds == NULL) { + pTask->checkReqIds = taosArrayInit(numOfVgs, sizeof(int64_t)); + } else { + taosArrayClear(pTask->checkReqIds); + } stDebug("s-task:%s check %d downstream tasks, ver:%" PRId64 "-%" PRId64 " window:%" PRId64 "-%" PRId64, pTask->id.idStr, numOfVgs, pRange->range.minVer, pRange->range.maxVer, pWindow->skey, pWindow->ekey); @@ -436,8 +440,7 @@ int32_t streamProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRs ASSERT(left >= 0); if (left == 0) { - taosArrayDestroy(pTask->checkReqIds); - pTask->checkReqIds = NULL; + pTask->checkReqIds = taosArrayDestroy(pTask->checkReqIds);; doProcessDownstreamReadyRsp(pTask); } else { From d25a323a4c1e801999e3c197fbbf85811f622829 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 20 Dec 2023 13:55:12 +0800 Subject: [PATCH 13/94] fix(stream): record the launch failure. --- include/libs/stream/tstream.h | 1 + source/dnode/vnode/src/tq/tq.c | 1 + source/dnode/vnode/src/tqCommon/tqCommon.c | 4 +++- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 866bb4dc29..e38a677c8b 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -324,6 +324,7 @@ typedef struct SStreamStatus { int8_t keepTaskStatus; bool appendTranstateBlock; // has append the transfer state data block already, todo: remove it int32_t timerActive; // timer is active + int8_t allowedAddInTimer; // allowed to add into timer int32_t inScanHistorySentinel; } SStreamStatus; diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index c33becf5b9..7e2bd1c5e0 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1071,6 +1071,7 @@ int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) { tqScanWal(pTq); return 0; } + int32_t code = tqStreamTaskProcessRunReq(pTq->pStreamMeta, pMsg, vnodeIsRoleLeader(pTq->pVnode)); if(code == 0 && taskId > 0){ tqScanWalAsync(pTq, false); diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index c39e653596..2201f0744e 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -621,6 +621,8 @@ int32_t startStreamTasks(SStreamMeta* pMeta) { SStreamTaskId* pTaskId = taosArrayGet(pTaskList, i); SStreamTask* pTask = streamMetaAcquireTask(pMeta, pTaskId->streamId, pTaskId->taskId); if (pTask == NULL) { + streamMetaUpdateTaskDownstreamStatus(pMeta, pTaskId->streamId, pTaskId->taskId, 0, + taosGetTimestampMs(), false); continue; } @@ -637,7 +639,7 @@ int32_t startStreamTasks(SStreamMeta* pMeta) { streamLaunchFillHistoryTask(pTask); } - streamMetaUpdateTaskDownstreamStatus(pTask->pMeta, pTask->id.streamId, pTask->id.taskId, pTask->execInfo.init, + streamMetaUpdateTaskDownstreamStatus(pMeta, pTask->id.streamId, pTask->id.taskId, pTask->execInfo.init, pTask->execInfo.start, true); streamMetaReleaseTask(pMeta, pTask); continue; From 1a8583887f471ca7e87128a7c6c4e17517d8d8a1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 20 Dec 2023 15:35:43 +0800 Subject: [PATCH 14/94] fix(stream): remove clear backendcache. --- source/dnode/vnode/src/tqCommon/tqCommon.c | 5 +++-- source/libs/stream/src/streamMeta.c | 4 ++-- source/libs/stream/src/streamStart.c | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index 2201f0744e..0864afdf2c 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -686,7 +686,8 @@ static int32_t restartStreamTasks(SStreamMeta* pMeta, bool isLeader) { streamMetaWLock(pMeta); if (pMeta->startInfo.taskStarting == 1) { pMeta->startInfo.restartCount += 1; - tqDebug("vgId:%d in start tasks procedure, inc restartCounter by 1, %d", vgId, pMeta->startInfo.restartCount); + tqDebug("vgId:%d in start tasks procedure, inc restartCounter by 1, remaining restart:%d", vgId, + pMeta->startInfo.restartCount); streamMetaWUnLock(pMeta); return TSDB_CODE_SUCCESS; } @@ -720,7 +721,7 @@ static int32_t restartStreamTasks(SStreamMeta* pMeta, bool isLeader) { if (isLeader && !tsDisableStream) { tqStreamTaskResetStatus(pMeta); streamMetaWUnLock(pMeta); - tqInfo("vgId:%d restart all stream tasks after all tasks being updated", vgId); + tqInfo("vgId:%d start all stream tasks after reload tasks from disk", vgId); startStreamTasks(pMeta); } else { diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 2bbc1607b3..b67b81b611 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -266,11 +266,12 @@ int32_t streamTaskSetDb(SStreamMeta* pMeta, void* arg, char* key) { if (pBackend == NULL) { taosThreadMutexUnlock(&pMeta->backendMutex); taosMsleep(1000); - stDebug("backed holded by other task, restart later, path: %s, key: %s", pMeta->path, key); + stDebug("backend held by other task, restart later, path:%s, key:%s", pMeta->path, key); } else { taosThreadMutexUnlock(&pMeta->backendMutex); break; } + taosThreadMutexLock(&pMeta->backendMutex); pBackend = taskDbOpen(pMeta->path, key, chkpId); } @@ -448,7 +449,6 @@ void streamMetaClear(SStreamMeta* pMeta) { taosRemoveRef(streamBackendId, pMeta->streamBackendRid); taosHashClear(pMeta->pTasksMap); - taosHashClear(pMeta->pTaskDbUnique); taosArrayClear(pMeta->pTaskList); taosArrayClear(pMeta->chkpSaved); diff --git a/source/libs/stream/src/streamStart.c b/source/libs/stream/src/streamStart.c index 68380e4ba4..5ce7668048 100644 --- a/source/libs/stream/src/streamStart.c +++ b/source/libs/stream/src/streamStart.c @@ -1113,7 +1113,7 @@ int32_t streamMetaUpdateTaskDownstreamStatus(SStreamMeta* pMeta, int64_t streamI pStartInfo->completeFn(pMeta); } else { streamMetaWUnLock(pMeta); - stDebug("vgId:%d recv check down results:%d, total:%d", pMeta->vgId, numOfRecv, numOfTotal); + stDebug("vgId:%d recv check downstream results:%d, total:%d", pMeta->vgId, numOfRecv, numOfTotal); } return TSDB_CODE_SUCCESS; From 43fb7cf14ade07c3b4d6ed3baf4f71e538662ca1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 20 Dec 2023 16:04:20 +0800 Subject: [PATCH 15/94] other: add log info. --- source/libs/stream/src/streamDispatch.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 5fb7db233f..f0ee03810b 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -372,8 +372,8 @@ static int32_t doBuildDispatchMsg(SStreamTask* pTask, const SStreamDataBlock* pD pTask->msgInfo.pData = pReqs; } - stDebug("s-task:%s build dispatch msg success, msgId:%d, stage:%" PRId64, pTask->id.idStr, pTask->execInfo.dispatch, - pTask->pMeta->stage); + stDebug("s-task:%s build dispatch msg success, msgId:%d, stage:%"PRId64" %p", pTask->id.idStr, pTask->execInfo.dispatch, + pTask->pMeta->stage, pTask->msgInfo.pData); return code; } @@ -1081,6 +1081,7 @@ int32_t streamNotifyUpstreamContinue(SStreamTask* pTask) { // this message has been sent successfully, let's try next one. static int32_t handleDispatchSuccessRsp(SStreamTask* pTask, int32_t downstreamId) { + stDebug("s-task:%s destroy dispatch msg:%p", pTask->id.idStr, pTask->msgInfo.pData); destroyDispatchMsg(pTask->msgInfo.pData, getNumOfDispatchBranch(pTask)); bool delayDispatch = (pTask->msgInfo.dispatchMsgType == STREAM_INPUT__CHECKPOINT_TRIGGER); From 849aaf8d4fb70b73c03bbd85f9e068d0b2d74aa1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 20 Dec 2023 18:07:53 +0800 Subject: [PATCH 16/94] fix(stream): avoid dead lock. --- source/libs/stream/src/streamMeta.c | 1 + source/libs/stream/src/streamStart.c | 21 ++++++++++++++------- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index b67b81b611..d6e1286093 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -1215,6 +1215,7 @@ bool streamMetaTaskInTimer(SStreamMeta* pMeta) { SStreamTask* pTask = *(SStreamTask**)pIter; if (pTask->status.timerActive >= 1) { + stDebug("s-task:%s in timer, blocking tasks in vgId:%d restart", pTask->id.idStr, pMeta->vgId); inTimer = true; } } diff --git a/source/libs/stream/src/streamStart.c b/source/libs/stream/src/streamStart.c index 5ce7668048..94f635a480 100644 --- a/source/libs/stream/src/streamStart.c +++ b/source/libs/stream/src/streamStart.c @@ -48,9 +48,13 @@ static void tryLaunchHistoryTask(void* param, void* tmrId); static void doProcessDownstreamReadyRsp(SStreamTask* pTask); int32_t streamTaskSetReady(SStreamTask* pTask) { - char* p = NULL; - int32_t numOfDowns = streamTaskGetNumOfDownstream(pTask); - ETaskStatus status = streamTaskGetStatus(pTask, &p); + SStreamMeta* pMeta = pTask->pMeta; + SStreamTaskId id = pTask->id; + int64_t initTs = pTask->execInfo.init; + int64_t startTs = pTask->execInfo.start; + char* p = NULL; + int32_t numOfDowns = streamTaskGetNumOfDownstream(pTask); + ETaskStatus status = streamTaskGetStatus(pTask, &p); if ((status == TASK_STATUS__SCAN_HISTORY || status == TASK_STATUS__STREAM_SCAN_HISTORY) && pTask->info.taskLevel != TASK_LEVEL__SOURCE) { @@ -67,8 +71,12 @@ int32_t streamTaskSetReady(SStreamTask* pTask) { stDebug("s-task:%s all %d downstream ready, init completed, elapsed time:%" PRId64 "ms, task status:%s", pTask->id.idStr, numOfDowns, el, p); - streamMetaUpdateTaskDownstreamStatus(pTask->pMeta, pTask->id.streamId, pTask->id.taskId, pTask->execInfo.init, - pTask->execInfo.start, true); + taosThreadMutexUnlock(&pTask->lock); + + // to avoid deadlock + streamMetaUpdateTaskDownstreamStatus(pMeta, id.streamId, id.taskId, initTs, startTs, true); + + taosThreadMutexLock(&pTask->lock); return TSDB_CODE_SUCCESS; } @@ -461,8 +469,7 @@ int32_t streamProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRs if (pRsp->status == TASK_UPSTREAM_NEW_STAGE) { stError("s-task:%s vgId:%d self vnode-transfer/leader-change/restart detected, old stage:%" PRId64 ", current stage:%" PRId64 - ", " - "not check wait for downstream task nodeUpdate, and all tasks restart", + ", not check wait for downstream task nodeUpdate, and all tasks restart", id, pRsp->upstreamNodeId, pRsp->oldStage, pTask->pMeta->stage); addIntoNodeUpdateList(pTask, pRsp->upstreamNodeId); } else { From 49062431150be43ac0704c00c12cbe2b36bdfc76 Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Wed, 20 Dec 2023 18:32:23 +0800 Subject: [PATCH 17/94] fix: dynamic cfg resetlog --- source/common/src/tglobal.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index fd88098b03..fdac3882c3 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -1432,6 +1432,13 @@ static int32_t taosCfgSetOption(OptionNameAndVar *pOptions, int32_t optionSize, static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, char *name) { terrno = TSDB_CODE_SUCCESS; + if (strcasecmp(name, "resetlog") == 0) { + // trigger, no item in cfg + taosResetLog(); + cfgDumpCfg(tsCfg, 0, false); + return 0; + } + SConfigItem *pItem = cfgGetItem(pCfg, name); if (!pItem || (pItem->dynScope & CFG_DYN_SERVER) == 0) { uError("failed to config:%s, not support", name); @@ -1445,12 +1452,6 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, char *name) { return 0; } - if (strcasecmp(name, "resetlog") == 0) { - taosResetLog(); - cfgDumpCfg(tsCfg, 0, false); - return 0; - } - { // 'bool/int32_t/int64_t/float/double' variables with general modification function static OptionNameAndVar debugOptions[] = { {"dDebugFlag", &dDebugFlag}, {"vDebugFlag", &vDebugFlag}, {"mDebugFlag", &mDebugFlag}, @@ -1774,4 +1775,4 @@ void taosSetAllDebugFlag(int32_t flag, bool rewrite) { uInfo("all debug flag are set to %d", flag); } -int8_t taosGranted() { return atomic_load_8(&tsGrant); } \ No newline at end of file +int8_t taosGranted() { return atomic_load_8(&tsGrant); } From 64eee4bbd2f799f1cb843d0b944fbbeef9863d77 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 20 Dec 2023 19:06:32 +0800 Subject: [PATCH 18/94] fix(stream): add conflict check for nodeUpdate --- source/dnode/mnode/impl/src/mndStream.c | 30 ++++++++++++++++---- source/dnode/mnode/impl/src/mndStreamTrans.c | 4 +-- source/libs/stream/src/streamStart.c | 2 +- 3 files changed, 27 insertions(+), 9 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index b8ecf66686..0e593a5109 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -2330,12 +2330,28 @@ static SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady) { } static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChangeInfo) { - SSdb *pSdb = pMnode->pSdb; - - // check all streams that involved this vnode should update the epset info + SSdb *pSdb = pMnode->pSdb; SStreamObj *pStream = NULL; - void * pIter = NULL; - STrans * pTrans = NULL; + void *pIter = NULL; + STrans *pTrans = NULL; + + // conflict check for nodeUpdate trans, here we randomly chose one stream to add into the trans pool + while(1) { + pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream); + if (pIter == NULL) { + break; + } + + bool conflict = streamTransConflictOtherTrans(pMnode, pStream->uid, MND_STREAM_TASK_UPDATE_NAME, true); + sdbRelease(pSdb, pStream); + + if (conflict) { + mWarn("nodeUpdate trans in progress, current nodeUpdate ignored"); + sdbCancelFetch(pSdb, pIter); + return -1; + } + } + while (1) { pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream); @@ -2351,6 +2367,8 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange sdbCancelFetch(pSdb, pIter); return terrno; } + + mndStreamRegisterTrans(pTrans, MND_STREAM_TASK_RESET_NAME, pStream->uid); } void *p = taosHashGet(pChangeInfo->pDBMap, pStream->targetDb, strlen(pStream->targetDb)); @@ -2597,7 +2615,7 @@ static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg) { code = mndProcessVgroupChange(pMnode, &changeInfo); - // keep the new vnode snapshot + // keep the new vnode snapshot if success if (code == TSDB_CODE_SUCCESS || code == TSDB_CODE_ACTION_IN_PROGRESS) { mDebug("create trans successfully, update cached node list"); taosArrayDestroy(execInfo.pNodeList); diff --git a/source/dnode/mnode/impl/src/mndStreamTrans.c b/source/dnode/mnode/impl/src/mndStreamTrans.c index 0db3cb0a8a..a7d4703c8c 100644 --- a/source/dnode/mnode/impl/src/mndStreamTrans.c +++ b/source/dnode/mnode/impl/src/mndStreamTrans.c @@ -94,8 +94,8 @@ bool streamTransConflictOtherTrans(SMnode* pMnode, int64_t streamUid, const char tInfo.name); return true; } - } else if ((strcmp(tInfo.name, MND_STREAM_CREATE_NAME) == 0) || - (strcmp(tInfo.name, MND_STREAM_DROP_NAME) == 0)) { + } else if ((strcmp(tInfo.name, MND_STREAM_CREATE_NAME) == 0) || (strcmp(tInfo.name, MND_STREAM_DROP_NAME) == 0) || + (strcmp(tInfo.name, MND_STREAM_TASK_RESET_NAME) == 0)) { mWarn("conflict with other transId:%d streamUid:%" PRIx64 ", trans:%s", tInfo.transId, tInfo.streamUid, tInfo.name); return true; diff --git a/source/libs/stream/src/streamStart.c b/source/libs/stream/src/streamStart.c index 94f635a480..d94aa80369 100644 --- a/source/libs/stream/src/streamStart.c +++ b/source/libs/stream/src/streamStart.c @@ -73,7 +73,7 @@ int32_t streamTaskSetReady(SStreamTask* pTask) { taosThreadMutexUnlock(&pTask->lock); - // to avoid deadlock + // todo: fix it, to avoid deadlock in: tqStreamTaskProcessUpdateReq streamMetaUpdateTaskDownstreamStatus(pMeta, id.streamId, id.taskId, initTs, startTs, true); taosThreadMutexLock(&pTask->lock); From ad5d78a6ca95d5fdb503bdbed9d0c7a392e700b2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 20 Dec 2023 19:12:47 +0800 Subject: [PATCH 19/94] fix(stream): fix deadlock. --- source/dnode/mnode/impl/src/mndStream.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 0e593a5109..d5bb8b2b92 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -2342,7 +2342,7 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange break; } - bool conflict = streamTransConflictOtherTrans(pMnode, pStream->uid, MND_STREAM_TASK_UPDATE_NAME, true); + bool conflict = streamTransConflictOtherTrans(pMnode, pStream->uid, MND_STREAM_TASK_UPDATE_NAME, false); sdbRelease(pSdb, pStream); if (conflict) { From 32ce4b6a4ce9452f6fd4b14bfb1fc59295850932 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 20 Dec 2023 23:12:59 +0800 Subject: [PATCH 20/94] fix(stream): fix race condition. --- source/libs/stream/src/streamStart.c | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/source/libs/stream/src/streamStart.c b/source/libs/stream/src/streamStart.c index d94aa80369..d62990f181 100644 --- a/source/libs/stream/src/streamStart.c +++ b/source/libs/stream/src/streamStart.c @@ -48,10 +48,6 @@ static void tryLaunchHistoryTask(void* param, void* tmrId); static void doProcessDownstreamReadyRsp(SStreamTask* pTask); int32_t streamTaskSetReady(SStreamTask* pTask) { - SStreamMeta* pMeta = pTask->pMeta; - SStreamTaskId id = pTask->id; - int64_t initTs = pTask->execInfo.init; - int64_t startTs = pTask->execInfo.start; char* p = NULL; int32_t numOfDowns = streamTaskGetNumOfDownstream(pTask); ETaskStatus status = streamTaskGetStatus(pTask, &p); @@ -70,13 +66,6 @@ int32_t streamTaskSetReady(SStreamTask* pTask) { int64_t el = (pTask->execInfo.start - pTask->execInfo.init); stDebug("s-task:%s all %d downstream ready, init completed, elapsed time:%" PRId64 "ms, task status:%s", pTask->id.idStr, numOfDowns, el, p); - - taosThreadMutexUnlock(&pTask->lock); - - // todo: fix it, to avoid deadlock in: tqStreamTaskProcessUpdateReq - streamMetaUpdateTaskDownstreamStatus(pMeta, id.streamId, id.taskId, initTs, startTs, true); - - taosThreadMutexLock(&pTask->lock); return TSDB_CODE_SUCCESS; } @@ -392,6 +381,10 @@ void doProcessDownstreamReadyRsp(SStreamTask* pTask) { } streamTaskOnHandleEventSuccess(pTask->status.pSM, event); + + int64_t initTs = pTask->execInfo.init; + int64_t startTs = pTask->execInfo.start; + streamMetaUpdateTaskDownstreamStatus(pTask->pMeta, pTask->id.streamId, pTask->id.taskId, initTs, startTs, true); } static void addIntoNodeUpdateList(SStreamTask* pTask, int32_t nodeId) { From 572368703dc0108090c572ebee03a13ec5f0e0e3 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 21 Dec 2023 09:33:56 +0800 Subject: [PATCH 21/94] enh: ensure mndTransProcessRsp invoked in mnode-write thread --- source/dnode/mgmt/mgmt_mnode/src/mmHandle.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index d6de406987..0fb246e945 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -108,11 +108,11 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_SNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_VNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_VNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_DND_CONFIG_DNODE_RSP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_DND_ALTER_MNODE_TYPE_RSP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_DND_ALTER_VNODE_TYPE_RSP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_DND_CHECK_VNODE_LEARNER_CATCHUP_RSP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_SYNC_CONFIG_CHANGE_RSP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_CONFIG_DNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_ALTER_MNODE_TYPE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_ALTER_VNODE_TYPE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_CHECK_VNODE_LEARNER_CATCHUP_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_SYNC_CONFIG_CHANGE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CONNECT, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_ACCT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; From 8a8ddf56997ccfc5db8133217b3ebb241a689ee8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 21 Dec 2023 10:30:44 +0800 Subject: [PATCH 22/94] fix(stream): record the checkpoint failure when nodeEp changed. --- source/libs/stream/inc/streamInt.h | 12 +----------- source/libs/stream/src/streamCheckpoint.c | 5 ++--- source/libs/stream/src/streamStart.c | 15 +++++++++++++++ 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h index f709741b57..341706ff3b 100644 --- a/source/libs/stream/inc/streamInt.h +++ b/source/libs/stream/inc/streamInt.h @@ -116,6 +116,7 @@ int32_t streamSendCheckMsg(SStreamTask* pTask, const SStreamTaskCheckReq* pReq, int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t srcTaskId, int32_t index, int64_t checkpointId); int32_t streamTaskSendCheckpointReadyMsg(SStreamTask* pTask); int32_t streamTaskSendCheckpointSourceRsp(SStreamTask* pTask); +void streamTaskSetCheckpointFailedId(SStreamTask* pTask); int32_t streamTaskGetNumOfDownstream(const SStreamTask* pTask); int32_t streamTaskInitTokenBucket(STokenBucket* pBucket, int32_t numCap, int32_t numRate, float quotaRate, const char*); STaskId streamTaskExtractKey(const SStreamTask* pTask); @@ -137,17 +138,6 @@ int32_t streamAddEndScanHistoryMsg(SStreamTask* pTask, SRpcHandleInfo* pRpcInfo, int32_t streamNotifyUpstreamContinue(SStreamTask* pTask); int32_t streamTransferStateToStreamTask(SStreamTask* pTask); -// <<<<<<< HEAD -// void streamClearChkptReadyMsg(SStreamTask* pTask); - -// int32_t streamTaskInitTokenBucket(STokenBucket* pBucket, int32_t numCap, int32_t numRate, float quotaRate, const -// char*); STaskId streamTaskExtractKey(const SStreamTask* pTask); void streamTaskInitForLaunchHTask(SHistoryTaskInfo* -// pInfo); void streamTaskSetRetryInfoForLaunch(SHistoryTaskInfo* pInfo); - -// void streamMetaResetStartInfo(STaskStartInfo* pMeta); - -// ======= -// >>>>>>> 3.0 SStreamQueue* streamQueueOpen(int64_t cap); void streamQueueClose(SStreamQueue* pQueue, int32_t taskId); void streamQueueProcessSuccess(SStreamQueue* queue); diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index cf0682f037..4a99d54de1 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -341,9 +341,8 @@ int32_t streamSaveTaskCheckpointInfo(SStreamTask* p, int64_t checkpointId) { return code; } -void streamTaskSetFailedId(SStreamTask* pTask) { +void streamTaskSetCheckpointFailedId(SStreamTask* pTask) { pTask->chkInfo.failedId = pTask->chkInfo.checkpointingId; - pTask->chkInfo.checkpointId = pTask->chkInfo.checkpointingId; } int32_t getChkpMeta(char* id, char* path, SArray* list) { @@ -485,7 +484,7 @@ int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) { code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_CHECKPOINT_DONE); taosThreadMutexUnlock(&pTask->lock); - streamTaskSetFailedId(pTask); + streamTaskSetCheckpointFailedId(pTask); stDebug("s-task:%s clear checkpoint flag since gen checkpoint failed, checkpointId:%" PRId64, pTask->id.idStr, ckId); } diff --git a/source/libs/stream/src/streamStart.c b/source/libs/stream/src/streamStart.c index d62990f181..3bef6adf57 100644 --- a/source/libs/stream/src/streamStart.c +++ b/source/libs/stream/src/streamStart.c @@ -313,9 +313,24 @@ int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_ stError("s-task:%s receive check msg from upstream task:0x%x(vgId:%d), new stage received:%" PRId64 ", prev:%" PRId64, id, upstreamTaskId, vgId, stage, pInfo->stage); + // record the checkpoint failure id and sent to mnode + taosThreadMutexLock(&pTask->lock); + ETaskStatus status = streamTaskGetStatus(pTask, NULL); + if (status == TASK_STATUS__CK) { + streamTaskSetCheckpointFailedId(pTask); + } + taosThreadMutexUnlock(&pTask->lock); } if (pInfo->stage != stage) { + + taosThreadMutexLock(&pTask->lock); + ETaskStatus status = streamTaskGetStatus(pTask, NULL); + if (status == TASK_STATUS__CK) { + streamTaskSetCheckpointFailedId(pTask); + } + taosThreadMutexUnlock(&pTask->lock); + return TASK_UPSTREAM_NEW_STAGE; } else if (pTask->status.downstreamReady != 1) { stDebug("s-task:%s vgId:%d leader:%d, downstream not ready", id, vgId, (pTask->pMeta->role == NODE_ROLE_LEADER)); From 71f6d9f06fd57b95ce938f5796d1d5698779f611 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 21 Dec 2023 11:52:45 +0800 Subject: [PATCH 23/94] fix(stream): handle the case that recvs drop msg twice. --- source/dnode/vnode/src/tq/tq.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 7e2bd1c5e0..02c9b61432 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1340,6 +1340,7 @@ int32_t tqProcessTaskResetReq(STQ* pTq, SRpcMsg* pMsg) { return tqStreamTaskProcessTaskResetReq(pTq->pStreamMeta, pMsg); } +// NOTE: here we may receive this message more than once, so need to handle this case int32_t tqProcessTaskDropHTask(STQ* pTq, SRpcMsg* pMsg) { SVDropHTaskReq* pReq = (SVDropHTaskReq*)pMsg->pCont; @@ -1358,14 +1359,17 @@ int32_t tqProcessTaskDropHTask(STQ* pTq, SRpcMsg* pMsg) { return TSDB_CODE_SUCCESS; } + taosThreadMutexLock(&pTask->lock); ETaskStatus status = streamTaskGetStatus(pTask, NULL); - ASSERT(status == TASK_STATUS__STREAM_SCAN_HISTORY); - - streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_SCANHIST_DONE); + if (status == TASK_STATUS__STREAM_SCAN_HISTORY) { + streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_SCANHIST_DONE); + } SStreamTaskId id = {.streamId = pTask->hTaskInfo.id.streamId, .taskId = pTask->hTaskInfo.id.taskId}; streamBuildAndSendDropTaskMsg(pTask->pMsgCb, pMeta->vgId, &id); + taosThreadMutexUnlock(&pTask->lock); + // clear the scheduler status streamTaskSetSchedStatusInactive(pTask); tqDebug("s-task:%s set scheduler status:%d after drop fill-history task", pTask->id.idStr, pTask->status.schedStatus); From 2acc8423c5263fe57fc205a9e2f26447926e56e0 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 21 Dec 2023 16:00:50 +0800 Subject: [PATCH 24/94] fix(stream): send msg to mnode before closing all tasks. --- include/libs/stream/tstream.h | 3 + source/dnode/vnode/src/tq/tqStreamTask.c | 13 ++- source/dnode/vnode/src/tq/tqUtil.c | 6 ++ source/libs/stream/src/streamMeta.c | 128 +++++++++++++++-------- 4 files changed, 101 insertions(+), 49 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index e38a677c8b..9b96a41516 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -494,6 +494,7 @@ typedef struct SStreamMeta { int32_t vgId; int64_t stage; int32_t role; + bool sendMsgBeforeClosing; // send hb to mnode before close all tasks when switch to follower. STaskStartInfo startInfo; SRWLatch lock; int32_t walScanCounter; @@ -782,6 +783,8 @@ int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_ int64_t* oldStage); int32_t streamTaskUpdateEpsetInfo(SStreamTask* pTask, SArray* pNodeList); void streamTaskResetUpstreamStageInfo(SStreamTask* pTask); +SArray* streamMetaSendMsgBeforeCloseTasks(SStreamMeta* pMeta); + bool streamTaskAllUpstreamClosed(SStreamTask* pTask); bool streamTaskSetSchedStatusWait(SStreamTask* pTask); int8_t streamTaskSetSchedStatusActive(SStreamTask* pTask); diff --git a/source/dnode/vnode/src/tq/tqStreamTask.c b/source/dnode/vnode/src/tq/tqStreamTask.c index 1b0a76e81c..99cc3a36ea 100644 --- a/source/dnode/vnode/src/tq/tqStreamTask.c +++ b/source/dnode/vnode/src/tq/tqStreamTask.c @@ -126,17 +126,16 @@ int32_t tqScanWalAsync(STQ* pTq, bool ckPause) { int32_t tqStopStreamTasks(STQ* pTq) { SStreamMeta* pMeta = pTq->pStreamMeta; int32_t vgId = TD_VID(pTq->pVnode); - int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); + int32_t num = taosArrayGetSize(pMeta->pTaskList); - tqDebug("vgId:%d stop all %d stream task(s)", vgId, numOfTasks); - if (numOfTasks == 0) { + tqDebug("vgId:%d stop all %d stream task(s)", vgId, num); + if (num == 0) { return TSDB_CODE_SUCCESS; } - SArray* pTaskList = NULL; - streamMetaWLock(pMeta); - pTaskList = taosArrayDup(pMeta->pTaskList, NULL); - streamMetaWUnLock(pMeta); + // send hb msg to mnode before closing all tasks. + SArray* pTaskList = streamMetaSendMsgBeforeCloseTasks(pMeta); + int32_t numOfTasks = taosArrayGetSize(pTaskList); for (int32_t i = 0; i < numOfTasks; ++i) { SStreamTaskId* pTaskId = taosArrayGet(pTaskList, i); diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 110cf79b4e..d3ea1f19e5 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -40,6 +40,12 @@ void tqUpdateNodeStage(STQ* pTq, bool isLeader) { int64_t stage = pMeta->stage; pMeta->stage = state.term; + + // mark the sign to send msg before close all tasks + if ((!isLeader) && (pMeta->role == NODE_ROLE_LEADER)) { + pMeta->sendMsgBeforeClosing = true; + } + pMeta->role = (isLeader)? NODE_ROLE_LEADER:NODE_ROLE_FOLLOWER; if (isLeader) { tqInfo("vgId:%d update meta stage:%" PRId64 ", prev:%" PRId64 " leader:%d, start to send Hb", pMeta->vgId, diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index d6e1286093..355e17db9a 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -1051,43 +1051,7 @@ static void addUpdateNodeIntoHbMsg(SStreamTask* pTask, SStreamHbMsg* pMsg) { taosThreadMutexUnlock(&pTask->lock); } -void metaHbToMnode(void* param, void* tmrId) { - int64_t rid = *(int64_t*)param; - - SStreamMeta* pMeta = taosAcquireRef(streamMetaId, rid); - if (pMeta == NULL) { - return; - } - - // need to stop, stop now - if (pMeta->pHbInfo->stopFlag == STREAM_META_WILL_STOP) { - pMeta->pHbInfo->stopFlag = STREAM_META_OK_TO_STOP; - stDebug("vgId:%d jump out of meta timer", pMeta->vgId); - taosReleaseRef(streamMetaId, rid); - return; - } - - // not leader not send msg - if (pMeta->role != NODE_ROLE_LEADER) { - stInfo("vgId:%d role:%d not leader not send hb to mnode", pMeta->vgId, pMeta->role); - taosReleaseRef(streamMetaId, rid); - pMeta->pHbInfo->hbStart = 0; - return; - } - - // set the hb start time - if (pMeta->pHbInfo->hbStart == 0) { - pMeta->pHbInfo->hbStart = taosGetTimestampMs(); - } - - if (!waitForEnoughDuration(pMeta->pHbInfo)) { - taosTmrReset(metaHbToMnode, META_HB_CHECK_INTERVAL, param, streamTimer, &pMeta->pHbInfo->hbTmr); - taosReleaseRef(streamMetaId, rid); - return; - } - - stDebug("vgId:%d build stream task hb, leader:%d", pMeta->vgId, (pMeta->role == NODE_ROLE_LEADER)); - +static int32_t metaHeartbeatToMnodeImpl(SStreamMeta* pMeta) { SStreamHbMsg hbMsg = {0}; SEpSet epset = {0}; bool hasMnodeEpset = false; @@ -1181,23 +1145,62 @@ void metaHbToMnode(void* param, void* tmrId) { } tEncoderClear(&encoder); - SRpcMsg msg = { - .info.noResp = 1, - }; + SRpcMsg msg = {.info.noResp = 1}; initRpcMsg(&msg, TDMT_MND_STREAM_HEARTBEAT, buf, tlen); pMeta->pHbInfo->hbCount += 1; - stDebug("vgId:%d build and send hb to mnode, numOfTasks:%d total:%d", pMeta->vgId, hbMsg.numOfTasks, pMeta->pHbInfo->hbCount); + tmsgSendReq(&epset, &msg); } else { stDebug("vgId:%d no tasks and no mnd epset, not send stream hb to mnode", pMeta->vgId); } -_end: + _end: streamMetaClearHbMsg(&hbMsg); taosArrayDestroy(pIdList); + return TSDB_CODE_SUCCESS; +} + +void metaHbToMnode(void* param, void* tmrId) { + int64_t rid = *(int64_t*)param; + + SStreamMeta* pMeta = taosAcquireRef(streamMetaId, rid); + if (pMeta == NULL) { + return; + } + + // need to stop, stop now + if (pMeta->pHbInfo->stopFlag == STREAM_META_WILL_STOP) { + pMeta->pHbInfo->stopFlag = STREAM_META_OK_TO_STOP; + stDebug("vgId:%d jump out of meta timer", pMeta->vgId); + taosReleaseRef(streamMetaId, rid); + return; + } + + // not leader not send msg + if (pMeta->role != NODE_ROLE_LEADER) { + stInfo("vgId:%d role:%d not leader not send hb to mnode", pMeta->vgId, pMeta->role); + taosReleaseRef(streamMetaId, rid); + pMeta->pHbInfo->hbStart = 0; + return; + } + + // set the hb start time + if (pMeta->pHbInfo->hbStart == 0) { + pMeta->pHbInfo->hbStart = taosGetTimestampMs(); + } + + if (!waitForEnoughDuration(pMeta->pHbInfo)) { + taosTmrReset(metaHbToMnode, META_HB_CHECK_INTERVAL, param, streamTimer, &pMeta->pHbInfo->hbTmr); + taosReleaseRef(streamMetaId, rid); + return; + } + + stDebug("vgId:%d build stream task hb, leader:%d", pMeta->vgId, (pMeta->role == NODE_ROLE_LEADER)); + metaHeartbeatToMnodeImpl(pMeta); + taosTmrReset(metaHbToMnode, META_HB_CHECK_INTERVAL, param, streamTimer, &pMeta->pHbInfo->hbTmr); taosReleaseRef(streamMetaId, rid); } @@ -1320,3 +1323,44 @@ int32_t streamMetaAsyncExec(SStreamMeta* pMeta, __stream_async_exec_fn_t fn, voi schedMsg.msg = code; return taosScheduleTask(pMeta->qHandle, &schedMsg); } + +SArray* streamMetaSendMsgBeforeCloseTasks(SStreamMeta* pMeta) { + SArray* pTaskList = NULL; + bool sendMsg = false; + + streamMetaWLock(pMeta); + pTaskList = taosArrayDup(pMeta->pTaskList, NULL); + sendMsg = pMeta->sendMsgBeforeClosing; + streamMetaWUnLock(pMeta); + + if (!sendMsg) { + stDebug("vgId:%d no need to send msg to mnode before closing tasks", pMeta->vgId); + return pTaskList; + } + + stDebug("vgId:%d send msg to mnode before closing all tasks", pMeta->vgId); + + // send hb msg to mnode before closing all tasks. + int32_t numOfTasks = taosArrayGetSize(pTaskList); + for (int32_t i = 0; i < numOfTasks; ++i) { + SStreamTaskId* pTaskId = taosArrayGet(pTaskList, i); + SStreamTask* pTask = streamMetaAcquireTask(pMeta, pTaskId->streamId, pTaskId->taskId); + if (pTask == NULL) { + continue; + } + + taosThreadMutexLock(&pTask->lock); + ETaskStatus s = streamTaskGetStatus(pTask, NULL); + if (s == TASK_STATUS__CK) { + streamTaskSetCheckpointFailedId(pTask); + stDebug("s-task:%s mark the checkpoint:%"PRId64" failed", pTask->id.idStr, pTask->chkInfo.checkpointingId); + } + + taosThreadMutexUnlock(&pTask->lock); + streamMetaReleaseTask(pMeta, pTask); + } + + metaHeartbeatToMnodeImpl(pMeta); + pMeta->sendMsgBeforeClosing = false; + return pTaskList; +} From 4137a44fd0f8df409bbcebe1360377a425de0df2 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 21 Dec 2023 17:49:20 +0800 Subject: [PATCH 25/94] enh: protect syncNodeRestore with log buffer mutex --- source/libs/sync/src/syncMain.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index b8740a2858..6f3b3fdf98 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1162,9 +1162,12 @@ int32_t syncNodeRestore(SSyncNode* pSyncNode) { ASSERTS(pSyncNode->pLogStore != NULL, "log store not created"); ASSERTS(pSyncNode->pLogBuf != NULL, "ring log buffer not created"); + taosThreadMutexLock(&pSyncNode->pLogBuf->mutex); SyncIndex lastVer = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore); SyncIndex commitIndex = pSyncNode->pLogStore->syncLogCommitIndex(pSyncNode->pLogStore); SyncIndex endIndex = pSyncNode->pLogBuf->endIndex; + taosThreadMutexUnlock(&pSyncNode->pLogBuf->mutex); + if (lastVer != -1 && endIndex != lastVer + 1) { terrno = TSDB_CODE_WAL_LOG_INCOMPLETE; sError("vgId:%d, failed to restore sync node since %s. expected lastLogIndex:%" PRId64 ", lastVer:%" PRId64 "", From f564e3f7bfb77ac7ab63416afce20c647b10bf18 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Thu, 21 Dec 2023 18:43:40 +0800 Subject: [PATCH 26/94] test: modify test of last_cache_scan.py --- tests/pytest/util/sql.py | 96 ++++++++++++++++---- tests/system-test/2-query/last_cache_scan.py | 38 +------- 2 files changed, 80 insertions(+), 54 deletions(-) diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index 99d166ee33..5ed985187a 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -78,6 +78,26 @@ class TDSql: self.cursor.execute(s) time.sleep(2) + def execute(self, sql, queryTimes=30, show=False): + self.sql = sql + if show: + tdLog.info(sql) + i=1 + while i <= queryTimes: + try: + self.affectedRows = self.cursor.execute(sql) + return self.affectedRows + except Exception as e: + tdLog.notice("Try to execute sql again, query times: %d "%i) + if i == queryTimes: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, sql, repr(e)) + tdLog.notice("%s(%d) failed: sql:%s, %s" % args) + raise Exception(repr(e)) + i+=1 + time.sleep(1) + pass + def error(self, sql, expectedErrno = None, expectErrInfo = None, fullMatched = True): caller = inspect.getframeinfo(inspect.stack()[1][0]) expectErrNotOccured = True @@ -158,6 +178,62 @@ class TDSql: time.sleep(1) pass + def query_success_failed(self, sql, row_tag=None, queryTimes=10, count_expected_res=None, expectErrInfo = None, fullMatched = True): + self.sql = sql + i=1 + while i <= queryTimes: + try: + self.cursor.execute(sql) + self.queryResult = self.cursor.fetchall() + self.queryRows = len(self.queryResult) + self.queryCols = len(self.cursor.description) + + if count_expected_res is not None: + counter = 0 + while count_expected_res != self.queryResult[0][0]: + self.cursor.execute(sql) + self.queryResult = self.cursor.fetchall() + if counter < queryTimes: + counter += 0.5 + time.sleep(0.5) + else: + return False + + tdLog.info("query is success") + time.sleep(1) + continue + except Exception as e: + tdLog.notice("Try to query again, query times: %d "%i) + caller = inspect.getframeinfo(inspect.stack()[1][0]) + if i < queryTimes: + error_info = repr(e) + self.error_info = ','.join(error_info[error_info.index('(')+1:-1].split(",")[:-1]).replace("'","") + self.queryRows = 0 + self.queryCols = 0 + self.queryResult = None + + if fullMatched: + if expectErrInfo != None: + if expectErrInfo == self.error_info: + tdLog.info("sql:%s, expected expectErrInfo '%s' occured" % (sql, expectErrInfo)) + else: + tdLog.exit("%s(%d) failed: sql:%s, expectErrInfo '%s' occured, but not expected errno '%s'" % (caller.filename, caller.lineno, sql, self.error_info, expectErrInfo)) + else: + if expectErrInfo != None: + if expectErrInfo in self.error_info: + tdLog.info("sql:%s, expected expectErrInfo '%s' occured" % (sql, expectErrInfo)) + else: + tdLog.exit("%s(%d) failed: sql:%s, expectErrInfo %s occured, but not expected errno '%s'" % (caller.filename, caller.lineno, sql, self.error_info, expectErrInfo)) + + return self.error_info + elif i == queryTimes: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, sql, repr(e)) + tdLog.notice("%s(%d) failed: sql:%s, %s" % args) + raise Exception(repr(e)) + i+=1 + time.sleep(1) + pass def is_err_sql(self, sql): err_flag = True @@ -471,25 +547,7 @@ class TDSql: time.sleep(1) continue - def execute(self, sql, queryTimes=30, show=False): - self.sql = sql - if show: - tdLog.info(sql) - i=1 - while i <= queryTimes: - try: - self.affectedRows = self.cursor.execute(sql) - return self.affectedRows - except Exception as e: - tdLog.notice("Try to execute sql again, query times: %d "%i) - if i == queryTimes: - caller = inspect.getframeinfo(inspect.stack()[1][0]) - args = (caller.filename, caller.lineno, sql, repr(e)) - tdLog.notice("%s(%d) failed: sql:%s, %s" % args) - raise Exception(repr(e)) - i+=1 - time.sleep(1) - pass + def checkAffectedRows(self, expectAffectedRows): if self.affectedRows != expectAffectedRows: diff --git a/tests/system-test/2-query/last_cache_scan.py b/tests/system-test/2-query/last_cache_scan.py index 01795f6eef..0457475ee7 100644 --- a/tests/system-test/2-query/last_cache_scan.py +++ b/tests/system-test/2-query/last_cache_scan.py @@ -339,44 +339,12 @@ class TDTestCase: tdSql.checkData(0, 0, '999') p = subprocess.run(["taos", '-s', "alter table test.meters drop column c1; alter table test.meters add column c12 int"]) p.check_returncode() - tdSql.query("select last(c1) from meters", queryTimes=1) - tdSql.checkData(0, 0, None) - tdSql.query('select last(*) from meters', queryTimes=1) - print(str(tdSql.queryResult)) - tdSql.checkData(0, 1, None) - tdSql.query('select last(c1), c1, ts from meters', queryTimes=1) - tdSql.checkRows(1) - tdSql.checkData(0, 0, None) - tdSql.checkData(0, 1, None) - tdSql.checkData(0, 2, None) - - try: - tdSql.query('select ts, last(c1), c1, ts, c1 from meters', queryTimes=1) - except Exception as e: - if str(e).count('Invalid column name') == 1: - print('column has been dropped, the cache has been updated: %s' % (str(e))) - return - else: - raise - tdSql.checkRows(1) - tdSql.checkCols(5) - tdSql.checkData(0, 0, None) - tdSql.checkData(0, 1, None) - tdSql.checkData(0, 2, None) - tdSql.checkData(0, 3, None) - tdSql.checkData(0, 4, None) - - try: - tdSql.query('select last(c1), last(c2), last(c3) from meters', queryTimes=1) - except Exception as e: - if str(e).count('Invalid column name') == 1: - print('column has been dropped, the cache has been updated: %s' % (str(e))) - return - else: - raise + tdSql.query_success_failed("select ts, last(c1), c1, ts, c1 from meters", queryTimes=10, expectErrInfo="Invalid column name: c1") + tdSql.query('select last(c12), c12, ts from meters', queryTimes=1) tdSql.checkRows(1) tdSql.checkCols(3) tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) def test_cache_scan_with_drop_column(self): tdSql.query('select last(*) from meters') From 84583580e25dd55d787bd524b14ee521b8da7d0b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 21 Dec 2023 19:12:49 +0800 Subject: [PATCH 27/94] fix(stream): transfer the checkpoint trans id. --- include/libs/stream/tstream.h | 4 +- source/dnode/mnode/impl/inc/mndStream.h | 7 ++- source/dnode/mnode/impl/src/mndStream.c | 61 ++++++++++++----------- source/libs/stream/src/streamCheckpoint.c | 7 ++- source/libs/stream/src/streamMeta.c | 8 ++- source/libs/stream/src/streamTask.c | 1 + 6 files changed, 51 insertions(+), 37 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 9b96a41516..9137965849 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -314,6 +314,7 @@ typedef struct SCheckpointInfo { int32_t checkpointNotReadyTasks; bool dispatchCheckpointTrigger; int64_t msgVer; + int32_t transId; } SCheckpointInfo; typedef struct SStreamStatus { @@ -635,6 +636,7 @@ typedef struct { int32_t nodeId; SEpSet mgmtEps; int32_t mnodeId; + int32_t transId; int64_t expireTime; } SStreamCheckpointSourceReq; @@ -677,8 +679,8 @@ typedef struct STaskStatusEntry { int64_t verStart; // start version in WAL, only valid for source task int64_t verEnd; // end version in WAL, only valid for source task int64_t processedVer; // only valid for source task - int32_t relatedHTask; // has related fill-history task int64_t activeCheckpointId; // current active checkpoint id + int32_t chkpointTransId; // checkpoint trans id bool checkpointFailed; // denote if the checkpoint is failed or not bool inputQChanging; // inputQ is changing or not int64_t inputQUnchangeCounter; diff --git a/source/dnode/mnode/impl/inc/mndStream.h b/source/dnode/mnode/impl/inc/mndStream.h index b0f6273acc..4d06b16297 100644 --- a/source/dnode/mnode/impl/inc/mndStream.h +++ b/source/dnode/mnode/impl/inc/mndStream.h @@ -44,12 +44,11 @@ typedef struct SStreamTransMgmt { } SStreamTransMgmt; typedef struct SStreamExecInfo { - SArray * pNodeList; + SArray *pNodeList; int64_t ts; // snapshot ts SStreamTransMgmt transMgmt; - int64_t activeCheckpoint; // active check point id - SHashObj * pTaskMap; - SArray * pTaskList; + SHashObj *pTaskMap; + SArray *pTaskList; TdThreadMutex lock; } SStreamExecInfo; diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index d5bb8b2b92..b04a707728 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -62,7 +62,7 @@ static void mndCancelGetNextStreamTask(SMnode *pMnode, void *pIter); static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq); static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq); static int32_t mndBuildStreamCheckpointSourceReq2(void **pBuf, int32_t *pLen, int32_t nodeId, int64_t checkpointId, - int64_t streamId, int32_t taskId); + int64_t streamId, int32_t taskId, int32_t transId); static int32_t mndProcessNodeCheck(SRpcMsg *pReq); static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg); static SArray *extractNodeListFromStream(SMnode *pMnode); @@ -997,13 +997,14 @@ static int32_t mndProcessStreamRemainChkptTmr(SRpcMsg *pReq) { } static int32_t mndBuildStreamCheckpointSourceReq2(void **pBuf, int32_t *pLen, int32_t nodeId, int64_t checkpointId, - int64_t streamId, int32_t taskId) { + int64_t streamId, int32_t taskId, int32_t transId) { SStreamCheckpointSourceReq req = {0}; req.checkpointId = checkpointId; req.nodeId = nodeId; req.expireTime = -1; req.streamId = streamId; // pTask->id.streamId; req.taskId = taskId; // pTask->id.taskId; + req.transId = transId; int32_t code; int32_t blen; @@ -1093,7 +1094,7 @@ static int32_t mndProcessStreamCheckpointTrans(SMnode *pMnode, SStreamObj *pStre void * buf; int32_t tlen; if (mndBuildStreamCheckpointSourceReq2(&buf, &tlen, pTask->info.nodeId, checkpointId, pTask->id.streamId, - pTask->id.taskId) < 0) { + pTask->id.taskId, pTrans->id) < 0) { mndReleaseVgroup(pMnode, pVgObj); taosWUnLockLatch(&pStream->lock); goto _ERR; @@ -1162,7 +1163,7 @@ static int32_t mndAddStreamCheckpointToTrans(STrans *pTrans, SStreamObj *pStream void * buf; int32_t tlen; if (mndBuildStreamCheckpointSourceReq2(&buf, &tlen, pTask->info.nodeId, chkptId, pTask->id.streamId, - pTask->id.taskId) < 0) { + pTask->id.taskId, pTrans->id) < 0) { mndReleaseVgroup(pMnode, pVgObj); taosWUnLockLatch(&pStream->lock); return -1; @@ -2837,39 +2838,36 @@ int32_t killActiveCheckpointTrans(SMnode *pMnode, const char *pDBName, size_t le return TSDB_CODE_SUCCESS; } -static int32_t mndResetStatusFromCheckpoint(SMnode *pMnode, int32_t transId) { +static int32_t mndResetStatusFromCheckpoint(SMnode *pMnode, int64_t streamId, int32_t transId) { + int32_t code = TSDB_CODE_SUCCESS; + STrans *pTrans = mndAcquireTrans(pMnode, transId); if (pTrans != NULL) { mInfo("kill checkpoint transId:%d to reset task status", transId); mndKillTrans(pMnode, pTrans); mndReleaseTrans(pMnode, pTrans); + } else { + mError("failed to acquire checkpoint trans:%d", transId); } - // set all tasks status to be normal, refactor later to be stream level, instead of vnode level. - SSdb * pSdb = pMnode->pSdb; - SStreamObj *pStream = NULL; - void * pIter = NULL; - while (1) { - pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream); - if (pIter == NULL) { - break; - } - + SStreamObj *pStream = mndGetStreamObj(pMnode, streamId); + if (pStream == NULL) { + code = TSDB_CODE_STREAM_TASK_NOT_EXIST; + mError("failed to acquire the streamObj:0x%" PRIx64 " to reset checkpoint, may have been dropped", pStream->uid); + } else { bool conflict = streamTransConflictOtherTrans(pMnode, pStream->uid, MND_STREAM_TASK_RESET_NAME, false); if (conflict) { - mError("stream:%s other trans exists in DB:%s & %s failed to start reset-status trans", pStream->name, - pStream->sourceDb, pStream->targetDb); - continue; - } - - mDebug("stream:%s (0x%" PRIx64 ") reset checkpoint procedure, create reset trans", pStream->name, pStream->uid); - int32_t code = createStreamResetStatusTrans(pMnode, pStream); - if (code != TSDB_CODE_SUCCESS) { - sdbCancelFetch(pSdb, pIter); - return code; + mError("stream:%s other trans exists in DB:%s, dstTable:%s failed to start reset-status trans", pStream->name, + pStream->sourceDb, pStream->targetSTbName); + } else { + mDebug("stream:%s (0x%" PRIx64 ") reset checkpoint procedure, transId:%d, create reset trans", pStream->name, + pStream->uid, transId); + code = createStreamResetStatusTrans(pMnode, pStream); + mndReleaseStream(pMnode, pStream); } } - return 0; + + return code; } static SStreamTask *mndGetStreamTask(STaskId *pId, SStreamObj *pStream) { @@ -3007,6 +3005,7 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { bool checkpointFailed = false; int64_t activeCheckpointId = 0; + int64_t streamId = 0; SDecoder decoder = {0}; tDecoderInit(&decoder, pReq->pCont, pReq->contLen); @@ -3049,7 +3048,9 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { if (pTaskEntry->stage != p->stage && pTaskEntry->stage != -1) { updateStageInfo(pTaskEntry, p->stage); - if (pTaskEntry->nodeId == SNODE_HANDLE) snodeChanged = true; + if (pTaskEntry->nodeId == SNODE_HANDLE) { + snodeChanged = true; + } } else { // task is idle for more than 50 sec. if (fabs(pTaskEntry->inputQUsed - p->inputQUsed) <= DBL_EPSILON) { @@ -3073,6 +3074,7 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { if (p->checkpointFailed) { checkpointFailed = p->checkpointFailed; + streamId = p->id.streamId; } } } @@ -3111,9 +3113,8 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { if (allReady || snodeChanged) { // if the execInfo.activeCheckpoint == 0, the checkpoint is restoring from wal - mInfo("checkpointId:%" PRId64 " failed, issue task-reset trans to reset all tasks status", - execInfo.activeCheckpoint); - mndResetStatusFromCheckpoint(pMnode, activeCheckpointId); + mInfo("checkpointId:%" PRId64 " failed, issue task-reset trans to reset all tasks status", activeCheckpointId); + mndResetStatusFromCheckpoint(pMnode, streamId, activeCheckpointId); } else { mInfo("not all vgroups are ready, wait for next HB from stream tasks"); } diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index 4a99d54de1..2a766f21ec 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -25,6 +25,7 @@ typedef struct { SStreamTask* pTask; } SAsyncUploadArg; + int32_t tEncodeStreamCheckpointSourceReq(SEncoder* pEncoder, const SStreamCheckpointSourceReq* pReq) { if (tStartEncode(pEncoder) < 0) return -1; if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1; @@ -34,6 +35,7 @@ int32_t tEncodeStreamCheckpointSourceReq(SEncoder* pEncoder, const SStreamCheckp if (tEncodeSEpSet(pEncoder, &pReq->mgmtEps) < 0) return -1; if (tEncodeI32(pEncoder, pReq->mnodeId) < 0) return -1; if (tEncodeI64(pEncoder, pReq->expireTime) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->transId) < 0) return -1; tEndEncode(pEncoder); return pEncoder->pos; } @@ -47,6 +49,7 @@ int32_t tDecodeStreamCheckpointSourceReq(SDecoder* pDecoder, SStreamCheckpointSo if (tDecodeSEpSet(pDecoder, &pReq->mgmtEps) < 0) return -1; if (tDecodeI32(pDecoder, &pReq->mnodeId) < 0) return -1; if (tDecodeI64(pDecoder, &pReq->expireTime) < 0) return -1; + if (tDecodeI32(pDecoder, &pReq->transId) < 0) return -1; tEndDecode(pDecoder); return 0; } @@ -149,6 +152,7 @@ int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSo // 1. set task status to be prepared for check point, no data are allowed to put into inputQ. streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_GEN_CHECKPOINT); + pTask->chkInfo.transId = pReq->transId; pTask->chkInfo.checkpointingId = pReq->checkpointId; pTask->chkInfo.checkpointNotReadyTasks = streamTaskGetNumOfDownstream(pTask); pTask->chkInfo.startTs = taosGetTimestampMs(); @@ -273,8 +277,9 @@ void streamTaskClearCheckInfo(SStreamTask* pTask, bool clearChkpReadyMsg) { pTask->chkInfo.failedId = 0; pTask->chkInfo.startTs = 0; // clear the recorded start time pTask->chkInfo.checkpointNotReadyTasks = 0; - // pTask->chkInfo.checkpointAlignCnt = 0; + pTask->chkInfo.transId = 0; pTask->chkInfo.dispatchCheckpointTrigger = false; + streamTaskOpenAllUpstreamInput(pTask); // open inputQ for all upstream tasks if (clearChkpReadyMsg) { streamClearChkptReadyMsg(pTask); diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 355e17db9a..d1df74b8d3 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -940,6 +940,7 @@ int32_t tEncodeStreamHbMsg(SEncoder* pEncoder, const SStreamHbMsg* pReq) { if (tEncodeI64(pEncoder, ps->verEnd) < 0) return -1; if (tEncodeI64(pEncoder, ps->activeCheckpointId) < 0) return -1; if (tEncodeI8(pEncoder, ps->checkpointFailed) < 0) return -1; + if (tEncodeI32(pEncoder, ps->chkpointTransId) < 0) return -1; } int32_t numOfVgs = taosArrayGetSize(pReq->pUpdateNodes); @@ -978,6 +979,7 @@ int32_t tDecodeStreamHbMsg(SDecoder* pDecoder, SStreamHbMsg* pReq) { if (tDecodeI64(pDecoder, &entry.verEnd) < 0) return -1; if (tDecodeI64(pDecoder, &entry.activeCheckpointId) < 0) return -1; if (tDecodeI8(pDecoder, (int8_t*)&entry.checkpointFailed) < 0) return -1; + if (tDecodeI32(pDecoder, &entry.chkpointTransId) < 0) return -1; entry.id.taskId = taskId; taosArrayPush(pReq->pTaskStatus, &entry); @@ -1103,6 +1105,7 @@ static int32_t metaHeartbeatToMnodeImpl(SStreamMeta* pMeta) { if ((*pTask)->chkInfo.checkpointingId != 0) { entry.checkpointFailed = ((*pTask)->chkInfo.failedId >= (*pTask)->chkInfo.checkpointingId); entry.activeCheckpointId = (*pTask)->chkInfo.checkpointingId; + entry.chkpointTransId = (*pTask)->chkInfo.transId; } if ((*pTask)->exec.pWalReader != NULL) { @@ -1350,10 +1353,13 @@ SArray* streamMetaSendMsgBeforeCloseTasks(SStreamMeta* pMeta) { } taosThreadMutexLock(&pTask->lock); - ETaskStatus s = streamTaskGetStatus(pTask, NULL); + char* p = NULL; + ETaskStatus s = streamTaskGetStatus(pTask, &p); if (s == TASK_STATUS__CK) { streamTaskSetCheckpointFailedId(pTask); stDebug("s-task:%s mark the checkpoint:%"PRId64" failed", pTask->id.idStr, pTask->chkInfo.checkpointingId); + } else { + stDebug("s-task:%s status:%s not reset the checkpoint", pTask->id.idStr, p); } taosThreadMutexUnlock(&pTask->lock); diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 335f9d27d5..4d343a484d 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -796,5 +796,6 @@ void streamTaskStatusCopy(STaskStatusEntry* pDst, const STaskStatusEntry* pSrc) pDst->sinkDataSize = pSrc->sinkDataSize; pDst->activeCheckpointId = pSrc->activeCheckpointId; pDst->checkpointFailed = pSrc->checkpointFailed; + pDst->chkpointTransId = pSrc->chkpointTransId; } From 7466fcc39838935ef1a3782260b297774ac971f0 Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Thu, 21 Dec 2023 19:52:40 +0800 Subject: [PATCH 28/94] fix: no more rm old log in resetlog --- source/util/src/tlog.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 184e18fc67..e113a95fcc 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -337,14 +337,10 @@ static int32_t taosOpenNewLogFile() { } void taosResetLog() { - char lastName[LOG_FILE_NAME_LEN + 20]; - sprintf(lastName, "%s.%d", tsLogObj.logName, tsLogObj.flag); - // force create a new log file tsLogObj.lines = tsNumOfLogLines + 10; taosOpenNewLogFile(); - (void)taosRemoveFile(lastName); uInfo("=================================="); uInfo(" reset log file "); From 0fab9a1827da20da21d12856c89329f7868e1f4a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 22 Dec 2023 00:29:49 +0800 Subject: [PATCH 29/94] other: fix syntax error. --- source/dnode/mnode/impl/src/mndStream.c | 2 ++ source/dnode/mnode/impl/src/mndStreamTrans.c | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index b04a707728..3ba6280f59 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -3006,6 +3006,7 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { bool checkpointFailed = false; int64_t activeCheckpointId = 0; int64_t streamId = 0; + int32_t transId = 0; SDecoder decoder = {0}; tDecoderInit(&decoder, pReq->pCont, pReq->contLen); @@ -3075,6 +3076,7 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { if (p->checkpointFailed) { checkpointFailed = p->checkpointFailed; streamId = p->id.streamId; + transId = p->chkpointTransId; } } } diff --git a/source/dnode/mnode/impl/src/mndStreamTrans.c b/source/dnode/mnode/impl/src/mndStreamTrans.c index a7d4703c8c..b8a1c3d4ae 100644 --- a/source/dnode/mnode/impl/src/mndStreamTrans.c +++ b/source/dnode/mnode/impl/src/mndStreamTrans.c @@ -93,6 +93,8 @@ bool streamTransConflictOtherTrans(SMnode* pMnode, int64_t streamUid, const char mWarn("conflict with other transId:%d streamUid:%" PRIx64 ", trans:%s", tInfo.transId, tInfo.streamUid, tInfo.name); return true; + } else { + mDebug("not conflict with checkpoint trans, name:%s, continue create trans", pTransName); } } else if ((strcmp(tInfo.name, MND_STREAM_CREATE_NAME) == 0) || (strcmp(tInfo.name, MND_STREAM_DROP_NAME) == 0) || (strcmp(tInfo.name, MND_STREAM_TASK_RESET_NAME) == 0)) { @@ -100,6 +102,8 @@ bool streamTransConflictOtherTrans(SMnode* pMnode, int64_t streamUid, const char tInfo.name); return true; } + } else { + mDebug("stream:0x%"PRIx64" no conflict trans existed, continue create trans", streamUid); } if (lock) { From c9475060de3d834be1153e4dc6584b73441c070f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 22 Dec 2023 09:34:49 +0800 Subject: [PATCH 30/94] fix(stream): fix race condition. --- source/dnode/mnode/impl/src/mndStreamTrans.c | 4 ++-- source/dnode/vnode/src/tq/tqStreamTask.c | 7 ++++++- source/dnode/vnode/src/tq/tqUtil.c | 8 ++++++-- source/libs/stream/src/streamMeta.c | 9 ++------- 4 files changed, 16 insertions(+), 12 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStreamTrans.c b/source/dnode/mnode/impl/src/mndStreamTrans.c index b8a1c3d4ae..8f94843500 100644 --- a/source/dnode/mnode/impl/src/mndStreamTrans.c +++ b/source/dnode/mnode/impl/src/mndStreamTrans.c @@ -90,7 +90,7 @@ bool streamTransConflictOtherTrans(SMnode* pMnode, int64_t streamUid, const char if (strcmp(tInfo.name, MND_STREAM_CHECKPOINT_NAME) == 0) { if (strcmp(pTransName, MND_STREAM_DROP_NAME) != 0) { - mWarn("conflict with other transId:%d streamUid:%" PRIx64 ", trans:%s", tInfo.transId, tInfo.streamUid, + mWarn("conflict with other transId:%d streamUid:0x%" PRIx64 ", trans:%s", tInfo.transId, tInfo.streamUid, tInfo.name); return true; } else { @@ -98,7 +98,7 @@ bool streamTransConflictOtherTrans(SMnode* pMnode, int64_t streamUid, const char } } else if ((strcmp(tInfo.name, MND_STREAM_CREATE_NAME) == 0) || (strcmp(tInfo.name, MND_STREAM_DROP_NAME) == 0) || (strcmp(tInfo.name, MND_STREAM_TASK_RESET_NAME) == 0)) { - mWarn("conflict with other transId:%d streamUid:%" PRIx64 ", trans:%s", tInfo.transId, tInfo.streamUid, + mWarn("conflict with other transId:%d streamUid:0x%" PRIx64 ", trans:%s", tInfo.transId, tInfo.streamUid, tInfo.name); return true; } diff --git a/source/dnode/vnode/src/tq/tqStreamTask.c b/source/dnode/vnode/src/tq/tqStreamTask.c index 99cc3a36ea..4fc1c6cc01 100644 --- a/source/dnode/vnode/src/tq/tqStreamTask.c +++ b/source/dnode/vnode/src/tq/tqStreamTask.c @@ -126,10 +126,13 @@ int32_t tqScanWalAsync(STQ* pTq, bool ckPause) { int32_t tqStopStreamTasks(STQ* pTq) { SStreamMeta* pMeta = pTq->pStreamMeta; int32_t vgId = TD_VID(pTq->pVnode); - int32_t num = taosArrayGetSize(pMeta->pTaskList); + streamMetaRLock(pMeta); + + int32_t num = taosArrayGetSize(pMeta->pTaskList); tqDebug("vgId:%d stop all %d stream task(s)", vgId, num); if (num == 0) { + streamMetaRUnLock(pMeta); return TSDB_CODE_SUCCESS; } @@ -149,6 +152,8 @@ int32_t tqStopStreamTasks(STQ* pTq) { } taosArrayDestroy(pTaskList); + + streamMetaRUnLock(pMeta); return 0; } diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index d3ea1f19e5..64c733d63d 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -39,6 +39,8 @@ void tqUpdateNodeStage(STQ* pTq, bool isLeader) { SStreamMeta* pMeta = pTq->pStreamMeta; int64_t stage = pMeta->stage; + streamMetaWLock(pMeta); + pMeta->stage = state.term; // mark the sign to send msg before close all tasks @@ -52,9 +54,11 @@ void tqUpdateNodeStage(STQ* pTq, bool isLeader) { state.term, stage, isLeader); streamMetaStartHb(pMeta); } else { - tqInfo("vgId:%d update meta stage:%" PRId64 " prev:%" PRId64 " leader:%d", pMeta->vgId, state.term, stage, - isLeader); + tqInfo("vgId:%d update meta stage:%" PRId64 " prev:%" PRId64 " leader:%d sendMsg beforeClosing:%d", pMeta->vgId, + state.term, stage, isLeader, pMeta->sendMsgBeforeClosing); } + + streamMetaWUnLock(pMeta); } static int32_t tqInitTaosxRsp(STaosxRsp* pRsp, STqOffsetVal pOffset) { diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index d1df74b8d3..3504c97398 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -1328,14 +1328,9 @@ int32_t streamMetaAsyncExec(SStreamMeta* pMeta, __stream_async_exec_fn_t fn, voi } SArray* streamMetaSendMsgBeforeCloseTasks(SStreamMeta* pMeta) { - SArray* pTaskList = NULL; - bool sendMsg = false; - - streamMetaWLock(pMeta); - pTaskList = taosArrayDup(pMeta->pTaskList, NULL); - sendMsg = pMeta->sendMsgBeforeClosing; - streamMetaWUnLock(pMeta); + SArray* pTaskList = taosArrayDup(pMeta->pTaskList, NULL); + bool sendMsg = pMeta->sendMsgBeforeClosing; if (!sendMsg) { stDebug("vgId:%d no need to send msg to mnode before closing tasks", pMeta->vgId); return pTaskList; From 7dd6cb422774719c692ec477febd99fac147b27d Mon Sep 17 00:00:00 2001 From: slzhou Date: Fri, 22 Dec 2023 13:17:25 +0800 Subject: [PATCH 31/94] fix: move bi to community version --- source/libs/parser/CMakeLists.txt | 3 - source/libs/parser/src/parTranslater.c | 194 +++++++++++++++++++++++-- 2 files changed, 183 insertions(+), 14 deletions(-) diff --git a/source/libs/parser/CMakeLists.txt b/source/libs/parser/CMakeLists.txt index 4e4a4def1d..c5ee1a00c4 100644 --- a/source/libs/parser/CMakeLists.txt +++ b/source/libs/parser/CMakeLists.txt @@ -4,9 +4,6 @@ IF (TD_ENTERPRISE) LIST(APPEND PARSER_SRC ${TD_ENTERPRISE_DIR}/src/plugins/view/src/parserView.c) ENDIF () -IF (TD_BI_SUPPORT) - LIST(APPEND PARSER_SRC ${TD_ENTERPRISE_DIR}/src/plugins/bi/src/biRewriteQuery.c) -ENDIF () add_library(parser STATIC ${PARSER_SRC}) target_include_directories( parser diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 3bb24566c2..0c41bc48b5 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1104,11 +1104,192 @@ static EDealRes translateColumnUseAlias(STranslateContext* pCxt, SColumnNode** p return DEAL_RES_CONTINUE; } -#ifndef TD_ENTERPRISE +static void biMakeAliasNameInMD5(char* pExprStr, int32_t len, char* pAlias) { + T_MD5_CTX ctx; + tMD5Init(&ctx); + tMD5Update(&ctx, pExprStr, len); + tMD5Final(&ctx); + char* p = pAlias; + for (uint8_t i = 0; i < tListLen(ctx.digest); ++i) { + sprintf(p, "%02x", ctx.digest[i]); + p += 2; + } +} + +static SNode* biMakeTbnameProjectAstNode(char* funcName, char* tableAlias) { + SValueNode* valNode = NULL; + if (tableAlias != NULL) { + SValueNode* n = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); + n->literal = tstrdup(tableAlias); + n->node.resType.type = TSDB_DATA_TYPE_BINARY; + n->node.resType.bytes = strlen(n->literal); + n->isDuration = false; + n->translate = false; + valNode = n; + } + + SFunctionNode* tbNameFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); + tstrncpy(tbNameFunc->functionName, "tbname", TSDB_FUNC_NAME_LEN); + if (valNode != NULL) { + nodesListMakeAppend(&tbNameFunc->pParameterList, (SNode*)valNode); + } + snprintf(tbNameFunc->node.userAlias, sizeof(tbNameFunc->node.userAlias), + (tableAlias)? "%s.tbname" : "%stbname", + (tableAlias)? tableAlias : ""); + strncpy(tbNameFunc->node.aliasName, tbNameFunc->functionName, TSDB_COL_NAME_LEN); + + if (funcName == NULL) { + return (SNode*)tbNameFunc; + } else { + SFunctionNode* multiResFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); + tstrncpy(multiResFunc->functionName, funcName, TSDB_FUNC_NAME_LEN); + nodesListMakeAppend(&multiResFunc->pParameterList, (SNode*)tbNameFunc); + + if (tsKeepColumnName) { + snprintf(multiResFunc->node.userAlias, sizeof(tbNameFunc->node.userAlias), + (tableAlias)? "%s.tbname" : "%stbname", + (tableAlias)? tableAlias : ""); + strcpy(multiResFunc->node.aliasName, tbNameFunc->functionName); + } else { + snprintf(multiResFunc->node.userAlias, sizeof(multiResFunc->node.userAlias), + tableAlias? "%s(%s.tbname)" : "%s(%stbname)", funcName, + tableAlias? tableAlias: ""); + biMakeAliasNameInMD5(multiResFunc->node.userAlias, strlen(multiResFunc->node.userAlias), multiResFunc->node.aliasName); + } + + return (SNode*)multiResFunc; + } +} + +static int32_t biRewriteSelectFuncParamStar(STranslateContext* pCxt, SSelectStmt* pSelect, SNode* pNode, SListCell* pSelectListCell) { + SNodeList* pTbnameNodeList = nodesMakeList(); + + SFunctionNode* pFunc = (SFunctionNode*)pNode; + if (strcasecmp(pFunc->functionName, "last") == 0 || + strcasecmp(pFunc->functionName, "last_row") == 0 || + strcasecmp(pFunc->functionName, "first") == 0) { + SNodeList* pParams = pFunc->pParameterList; + SNode* pPara = NULL; + FOREACH(pPara, pParams) { + if (nodesIsStar(pPara)) { + SArray* pTables = taosArrayGetP(pCxt->pNsLevel, pCxt->currLevel); + size_t n = taosArrayGetSize(pTables); + for (int32_t i = 0; i < n; ++i) { + STableNode* pTable = taosArrayGetP(pTables, i); + if (nodeType(pTable) == QUERY_NODE_REAL_TABLE && ((SRealTableNode*)pTable)->pMeta != NULL && + ((SRealTableNode*)pTable)->pMeta->tableType == TSDB_SUPER_TABLE) { + SNode* pTbnameNode = biMakeTbnameProjectAstNode(pFunc->functionName, NULL); + nodesListAppend(pTbnameNodeList, pTbnameNode); + } + } + if (LIST_LENGTH(pTbnameNodeList) > 0) { + nodesListInsertListAfterPos(pSelect->pProjectionList, pSelectListCell, pTbnameNodeList); + } + } else if (nodesIsTableStar(pPara)) { + char* pTableAlias = ((SColumnNode*)pPara)->tableAlias; + STableNode* pTable = NULL; + int32_t code = findTable(pCxt, pTableAlias, &pTable); + if (TSDB_CODE_SUCCESS == code && nodeType(pTable) == QUERY_NODE_REAL_TABLE && + ((SRealTableNode*)pTable)->pMeta != NULL && + ((SRealTableNode*)pTable)->pMeta->tableType == TSDB_SUPER_TABLE) { + SNode* pTbnameNode = biMakeTbnameProjectAstNode(pFunc->functionName, pTableAlias); + nodesListAppend(pTbnameNodeList, pTbnameNode); + } + if (LIST_LENGTH(pTbnameNodeList) > 0) { + nodesListInsertListAfterPos(pSelect->pProjectionList, pSelectListCell, pTbnameNodeList); + } + } + } + } + return TSDB_CODE_SUCCESS; +} + +// after translate from +// before translate select list +int32_t biRewriteSelectStar(STranslateContext* pCxt, SSelectStmt* pSelect) { + SNode* pNode = NULL; + SNodeList* pTbnameNodeList = nodesMakeList(); + WHERE_EACH(pNode, pSelect->pProjectionList) { + if (nodesIsStar(pNode)) { + SArray* pTables = taosArrayGetP(pCxt->pNsLevel, pCxt->currLevel); + size_t n = taosArrayGetSize(pTables); + for (int32_t i = 0; i < n; ++i) { + STableNode* pTable = taosArrayGetP(pTables, i); + if (nodeType(pTable) == QUERY_NODE_REAL_TABLE && + ((SRealTableNode*)pTable)->pMeta != NULL && + ((SRealTableNode*)pTable)->pMeta->tableType == TSDB_SUPER_TABLE) { + SNode* pTbnameNode = biMakeTbnameProjectAstNode(NULL, NULL); + nodesListAppend(pTbnameNodeList, pTbnameNode); + } + } + if (LIST_LENGTH(pTbnameNodeList) > 0) { + nodesListInsertListAfterPos(pSelect->pProjectionList, cell, pTbnameNodeList); + } + } else if (nodesIsTableStar(pNode)) { + char* pTableAlias = ((SColumnNode*)pNode)->tableAlias; + STableNode* pTable = NULL; + int32_t code = findTable(pCxt, pTableAlias, &pTable); + if (TSDB_CODE_SUCCESS == code && + nodeType(pTable) == QUERY_NODE_REAL_TABLE && + ((SRealTableNode*)pTable)->pMeta != NULL && + ((SRealTableNode*)pTable)->pMeta->tableType == TSDB_SUPER_TABLE) { + SNode* pTbnameNode = biMakeTbnameProjectAstNode(NULL, pTableAlias); + nodesListAppend(pTbnameNodeList, pTbnameNode); + } + if (LIST_LENGTH(pTbnameNodeList) > 0) { + nodesListInsertListAfterPos(pSelect->pProjectionList, cell, pTbnameNodeList); + } + } else if (nodeType(pNode) == QUERY_NODE_FUNCTION) { + biRewriteSelectFuncParamStar(pCxt, pSelect, pNode, cell); + } + WHERE_NEXT; + } + + return TSDB_CODE_SUCCESS; +} + bool biRewriteToTbnameFunc(STranslateContext* pCxt, SNode** ppNode) { + SColumnNode* pCol = (SColumnNode*)(*ppNode); + if ((strcasecmp(pCol->colName, "tbname") == 0) && + ((SSelectStmt*)pCxt->pCurrStmt)->pFromTable && + QUERY_NODE_REAL_TABLE == nodeType(((SSelectStmt*)pCxt->pCurrStmt)->pFromTable)) { + SFunctionNode* tbnameFuncNode = NULL; + tbnameFuncNode = (SFunctionNode*)biMakeTbnameProjectAstNode(NULL, (pCol->tableAlias[0]!='\0') ? pCol->tableAlias : NULL); + tbnameFuncNode->node.resType = pCol->node.resType; + strcpy(tbnameFuncNode->node.aliasName, pCol->node.aliasName); + strcpy(tbnameFuncNode->node.userAlias, pCol->node.userAlias); + + nodesDestroyNode(*ppNode); + *ppNode = (SNode*)tbnameFuncNode; + return true; + } + return false; } -#endif + +int32_t biCheckCreateTableTbnameCol(STranslateContext* pCxt, SCreateTableStmt* pStmt) { + if (pStmt->pTags) { + SNode* pNode = NULL; + FOREACH(pNode, pStmt->pTags) { + SColumnDefNode* pTag = (SColumnDefNode*)pNode; + if (strcasecmp(pTag->colName, "tbname") == 0) { + int32_t code = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAG_NAME, "tbname can not used for tags in BI mode"); + return code; + } + } + } + if (pStmt->pCols) { + SNode* pNode = NULL; + FOREACH(pNode, pStmt->pCols) { + SColumnDefNode* pCol = (SColumnDefNode*)pNode; + if (strcasecmp(pCol->colName, "tbname") == 0) { + int32_t code = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, "tbname can not used for columns in BI mode"); + return code; + } + } + } + return TSDB_CODE_SUCCESS; +} static EDealRes translateColumn(STranslateContext* pCxt, SColumnNode** pCol) { if (NULL == pCxt->pCurrStmt || @@ -3178,10 +3359,6 @@ static int32_t createTags(STranslateContext* pCxt, SNodeList** pOutput) { return TSDB_CODE_SUCCESS; } -#ifndef TD_ENTERPRISE -int32_t biRewriteSelectStar(STranslateContext* pCxt, SSelectStmt* pSelect) { return TSDB_CODE_SUCCESS; } -#endif - static int32_t translateStar(STranslateContext* pCxt, SSelectStmt* pSelect) { SNode* pNode = NULL; WHERE_EACH(pNode, pSelect->pProjectionList) { @@ -5743,11 +5920,6 @@ static int32_t checkTableDeleteMarkOption(STranslateContext* pCxt, STableOptions return code; } -#ifndef TD_ENTERPRISE -int32_t biCheckCreateTableTbnameCol(STranslateContext* pCxt, SCreateTableStmt* pStmt) { - return TSDB_CODE_SUCCESS; -} -#endif static int32_t checkCreateTable(STranslateContext* pCxt, SCreateTableStmt* pStmt, bool createStable) { if (NULL != strchr(pStmt->tableName, '.')) { From 90898a16e0393da66938541560585d8e3679d6ea Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 22 Dec 2023 13:57:15 +0800 Subject: [PATCH 32/94] refactor: do some internal refactor. --- source/dnode/vnode/src/tq/tqStreamTask.c | 4 ++-- source/libs/stream/src/streamMeta.c | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqStreamTask.c b/source/dnode/vnode/src/tq/tqStreamTask.c index 4fc1c6cc01..fde5082212 100644 --- a/source/dnode/vnode/src/tq/tqStreamTask.c +++ b/source/dnode/vnode/src/tq/tqStreamTask.c @@ -141,8 +141,8 @@ int32_t tqStopStreamTasks(STQ* pTq) { int32_t numOfTasks = taosArrayGetSize(pTaskList); for (int32_t i = 0; i < numOfTasks; ++i) { - SStreamTaskId* pTaskId = taosArrayGet(pTaskList, i); - SStreamTask* pTask = streamMetaAcquireTask(pMeta, pTaskId->streamId, pTaskId->taskId); + SStreamTaskId* pTaskId = taosArrayGet(pTaskList, i); + SStreamTask* pTask = streamMetaAcquireTask(pMeta, pTaskId->streamId, pTaskId->taskId); if (pTask == NULL) { continue; } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 3504c97398..151605493a 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -1210,7 +1210,7 @@ void metaHbToMnode(void* param, void* tmrId) { bool streamMetaTaskInTimer(SStreamMeta* pMeta) { bool inTimer = false; - streamMetaWLock(pMeta); + streamMetaRLock(pMeta); void* pIter = NULL; while (1) { @@ -1226,7 +1226,7 @@ bool streamMetaTaskInTimer(SStreamMeta* pMeta) { } } - streamMetaWUnLock(pMeta); + streamMetaRUnLock(pMeta); return inTimer; } @@ -1303,6 +1303,7 @@ void streamMetaRUnLock(SStreamMeta* pMeta) { void streamMetaWLock(SStreamMeta* pMeta) { stTrace("vgId:%d meta-wlock", pMeta->vgId); taosWLockLatch(&pMeta->lock); + stTrace("vgId:%d meta-wlock completed", pMeta->vgId); } void streamMetaWUnLock(SStreamMeta* pMeta) { From e91afa7df2b948cfa3a69edf5a75e9fa3c1ded19 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 22 Dec 2023 14:51:04 +0800 Subject: [PATCH 33/94] fix(stream): fix deadlock. --- include/libs/stream/tstream.h | 1 + source/dnode/vnode/src/tq/tqStreamTask.c | 3 ++- source/libs/stream/src/streamMeta.c | 25 +++++++++++++----------- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 9137965849..9772c7ecf3 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -845,6 +845,7 @@ int32_t streamMetaRemoveTask(SStreamMeta* pMeta, STaskId* pKey); int32_t streamMetaRegisterTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask, bool* pAdded); int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId); int32_t streamMetaGetNumOfTasks(SStreamMeta* pMeta); +SStreamTask* streamMetaAcquireTaskNoLock(SStreamMeta* pMeta, int64_t streamId, int32_t taskId); SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId); void streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask); void streamMetaClear(SStreamMeta* pMeta); diff --git a/source/dnode/vnode/src/tq/tqStreamTask.c b/source/dnode/vnode/src/tq/tqStreamTask.c index fde5082212..63e71f8ee8 100644 --- a/source/dnode/vnode/src/tq/tqStreamTask.c +++ b/source/dnode/vnode/src/tq/tqStreamTask.c @@ -123,6 +123,7 @@ int32_t tqScanWalAsync(STQ* pTq, bool ckPause) { return 0; } +// todo: createMsg to invoke this function in stream threads, to avoid blocking the syn thread int32_t tqStopStreamTasks(STQ* pTq) { SStreamMeta* pMeta = pTq->pStreamMeta; int32_t vgId = TD_VID(pTq->pVnode); @@ -142,7 +143,7 @@ int32_t tqStopStreamTasks(STQ* pTq) { for (int32_t i = 0; i < numOfTasks; ++i) { SStreamTaskId* pTaskId = taosArrayGet(pTaskList, i); - SStreamTask* pTask = streamMetaAcquireTask(pMeta, pTaskId->streamId, pTaskId->taskId); + SStreamTask* pTask = streamMetaAcquireTaskNoLock(pMeta, pTaskId->streamId, pTaskId->taskId); if (pTask == NULL) { continue; } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 151605493a..686cf841e1 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -607,22 +607,23 @@ int32_t streamMetaGetNumOfTasks(SStreamMeta* pMeta) { return (int32_t)size; } -SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId) { - streamMetaRLock(pMeta); - +SStreamTask* streamMetaAcquireTaskNoLock(SStreamMeta* pMeta, int64_t streamId, int32_t taskId) { STaskId id = {.streamId = streamId, .taskId = taskId}; SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); - if (ppTask != NULL) { - if (!streamTaskShouldStop(*ppTask)) { - int32_t ref = atomic_add_fetch_32(&(*ppTask)->refCnt, 1); - streamMetaRUnLock(pMeta); - stTrace("s-task:%s acquire task, ref:%d", (*ppTask)->id.idStr, ref); - return *ppTask; - } + if (ppTask == NULL || streamTaskShouldStop(*ppTask)) { + return NULL; } + int32_t ref = atomic_add_fetch_32(&(*ppTask)->refCnt, 1); + stTrace("s-task:%s acquire task, ref:%d", (*ppTask)->id.idStr, ref); + return *ppTask; +} + +SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId) { + streamMetaRLock(pMeta); + SStreamTask* p = streamMetaAcquireTaskNoLock(pMeta, streamId, taskId); streamMetaRUnLock(pMeta); - return NULL; + return p; } void streamMetaReleaseTask(SStreamMeta* UNUSED_PARAM(pMeta), SStreamTask* pTask) { @@ -1293,6 +1294,7 @@ void streamMetaResetStartInfo(STaskStartInfo* pStartInfo) { void streamMetaRLock(SStreamMeta* pMeta) { stTrace("vgId:%d meta-rlock", pMeta->vgId); taosRLockLatch(&pMeta->lock); + ASSERT(pMeta->lock != 0x40000001); } void streamMetaRUnLock(SStreamMeta* pMeta) { @@ -1303,6 +1305,7 @@ void streamMetaRUnLock(SStreamMeta* pMeta) { void streamMetaWLock(SStreamMeta* pMeta) { stTrace("vgId:%d meta-wlock", pMeta->vgId); taosWLockLatch(&pMeta->lock); + ASSERT(pMeta->lock != 0x40000001); stTrace("vgId:%d meta-wlock completed", pMeta->vgId); } From dce0a6c74a3218a98f0c3146ebd43e47b3ef92e0 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 22 Dec 2023 15:24:37 +0800 Subject: [PATCH 34/94] fix(stream): remove invalid assert --- source/libs/stream/src/streamMeta.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 686cf841e1..b82021abf6 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -1294,7 +1294,6 @@ void streamMetaResetStartInfo(STaskStartInfo* pStartInfo) { void streamMetaRLock(SStreamMeta* pMeta) { stTrace("vgId:%d meta-rlock", pMeta->vgId); taosRLockLatch(&pMeta->lock); - ASSERT(pMeta->lock != 0x40000001); } void streamMetaRUnLock(SStreamMeta* pMeta) { From 2de6d6c81d996ad8663721a93bbdd2c8d49339f5 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 22 Dec 2023 16:06:59 +0800 Subject: [PATCH 35/94] enh: add TSDB_CODE_MND_TRANS_CTX_SWITCH --- include/util/taoserror.h | 1 + source/dnode/mnode/impl/src/mndSync.c | 2 +- source/dnode/mnode/impl/src/mndTrans.c | 15 ++++++++++----- source/util/src/terror.c | 1 + 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 06e6b8d041..a4f56cf066 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -355,6 +355,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_TRANS_NETWORK_UNAVAILL TAOS_DEF_ERROR_CODE(0, 0x03D5) #define TSDB_CODE_MND_LAST_TRANS_NOT_FINISHED TAOS_DEF_ERROR_CODE(0, 0x03D6) //internal #define TSDB_CODE_MND_TRANS_SYNC_TIMEOUT TAOS_DEF_ERROR_CODE(0, 0x03D7) +#define TSDB_CODE_MND_TRANS_CTX_SWITCH TAOS_DEF_ERROR_CODE(0, 0x03D8) #define TSDB_CODE_MND_TRANS_UNKNOW_ERROR TAOS_DEF_ERROR_CODE(0, 0x03DF) // mnode-mq diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 00cbcbce6c..0fc8dad420 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -539,7 +539,7 @@ int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw, int32_t transId) { taosThreadMutexLock(&pMgmt->lock); pMgmt->errCode = 0; - if (pMgmt->transId != 0 /* && pMgmt->transId != transId*/) { + if (pMgmt->transId != 0) { mError("trans:%d, can't be proposed since trans:%d already waiting for confirm", transId, pMgmt->transId); taosThreadMutexUnlock(&pMgmt->lock); rpcFreeCont(req.pCont); diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 49b483114a..9e478f3aa5 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -1096,7 +1096,7 @@ static void mndTransResetActions(SMnode *pMnode, STrans *pTrans, SArray *pArray) static int32_t mndTransWriteSingleLog(SMnode *pMnode, STrans *pTrans, STransAction *pAction, bool topHalf) { if (pAction->rawWritten) return 0; - if (topHalf) return -1; + if (topHalf) return TSDB_CODE_MND_TRANS_CTX_SWITCH; int32_t code = sdbWriteWithoutFree(pMnode->pSdb, pAction->pRaw); if (code == 0 || terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { @@ -1119,7 +1119,7 @@ static int32_t mndTransWriteSingleLog(SMnode *pMnode, STrans *pTrans, STransActi static int32_t mndTransSendSingleMsg(SMnode *pMnode, STrans *pTrans, STransAction *pAction, bool topHalf) { if (pAction->msgSent) return 0; - if (mndCannotExecuteTransAction(pMnode, topHalf)) return -1; + if (mndCannotExecuteTransAction(pMnode, topHalf)) return TSDB_CODE_MND_TRANS_CTX_SWITCH; int64_t signature = pTrans->id; signature = (signature << 32); @@ -1165,7 +1165,7 @@ static int32_t mndTransSendSingleMsg(SMnode *pMnode, STrans *pTrans, STransActio } static int32_t mndTransExecNullMsg(SMnode *pMnode, STrans *pTrans, STransAction *pAction, bool topHalf) { - if (!topHalf) return -1; + if (!topHalf) return TSDB_CODE_MND_TRANS_CTX_SWITCH; pAction->rawWritten = 0; pAction->errCode = 0; mInfo("trans:%d, %s:%d confirm action executed", pTrans->id, mndTransStr(pAction->stage), pAction->id); @@ -1191,7 +1191,11 @@ static int32_t mndTransExecSingleActions(SMnode *pMnode, STrans *pTrans, SArray for (int32_t action = 0; action < numOfActions; ++action) { STransAction *pAction = taosArrayGet(pArray, action); code = mndTransExecSingleAction(pMnode, pTrans, pAction, topHalf); - if (code != 0) break; + if (code != 0) { + mInfo("trans:%d, action:%d not executed since %s. numOfActions:%d", pTrans->id, action, tstrerror(code), + numOfActions); + break; + } } return code; @@ -1199,9 +1203,10 @@ static int32_t mndTransExecSingleActions(SMnode *pMnode, STrans *pTrans, SArray static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pArray, bool topHalf) { int32_t numOfActions = taosArrayGetSize(pArray); + int32_t code = 0; if (numOfActions == 0) return 0; - if (mndTransExecSingleActions(pMnode, pTrans, pArray, topHalf) != 0) { + if ((code = mndTransExecSingleActions(pMnode, pTrans, pArray, topHalf)) != 0) { return -1; } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index dc5f44cf43..91e4c09334 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -286,6 +286,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_CLOG_IS_NULL, "Transaction commitlog TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_NETWORK_UNAVAILL, "Unable to establish connection While execute transaction and will continue in the background") TAOS_DEFINE_ERROR(TSDB_CODE_MND_LAST_TRANS_NOT_FINISHED, "Last Transaction not finished") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_SYNC_TIMEOUT, "Sync timeout While execute transaction and will continue in the background") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_CTX_SWITCH, "Transaction context switch") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TRANS_UNKNOW_ERROR, "Unknown transaction error") // mnode-mq From 7ce545bcd417b4d63531fc3a54b76e9bffeb2aeb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 22 Dec 2023 16:14:49 +0800 Subject: [PATCH 36/94] refactor: refactor the lock type. --- include/libs/stream/tstream.h | 2 +- source/dnode/vnode/src/sma/smaRollup.c | 20 ++++++++++---------- source/dnode/vnode/src/tq/tqStreamTaskSnap.c | 12 ++++++------ source/libs/stream/src/streamMeta.c | 9 ++++----- 4 files changed, 21 insertions(+), 22 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 9772c7ecf3..e7ac0e99c5 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -497,7 +497,7 @@ typedef struct SStreamMeta { int32_t role; bool sendMsgBeforeClosing; // send hb to mnode before close all tasks when switch to follower. STaskStartInfo startInfo; - SRWLatch lock; + TdThreadRwlock lock; int32_t walScanCounter; void* streamBackend; int64_t streamBackendRid; diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index abe4c3f2fc..4963a0aa3b 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -240,23 +240,23 @@ int32_t tdFetchTbUidList(SSma *pSma, STbUidStore **ppStore, tb_uid_t suid, tb_ui static void tdRSmaTaskInit(SStreamMeta *pMeta, SRSmaInfoItem *pItem, SStreamTaskId *pId) { STaskId id = {.streamId = pId->streamId, .taskId = pId->taskId}; - taosRLockLatch(&pMeta->lock); + streamMetaRLock(pMeta); SStreamTask **ppTask = (SStreamTask **)taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); if (ppTask && *ppTask) { pItem->submitReqVer = (*ppTask)->chkInfo.checkpointVer; pItem->fetchResultVer = (*ppTask)->info.triggerParam; } - taosRUnLockLatch(&pMeta->lock); + streamMetaRUnLock(pMeta); } static void tdRSmaTaskRemove(SStreamMeta *pMeta, int64_t streamId, int32_t taskId) { streamMetaUnregisterTask(pMeta, streamId, taskId); - taosWLockLatch(&pMeta->lock); + streamMetaWLock(pMeta); int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta); if (streamMetaCommit(pMeta) < 0) { // persist to disk } - taosWUnLockLatch(&pMeta->lock); + streamMetaWUnLock(pMeta); smaDebug("vgId:%d, rsma task:%" PRIi64 ",%d dropped, remain tasks:%d", pMeta->vgId, streamId, taskId, numOfTasks); } @@ -1301,14 +1301,14 @@ _checkpoint: checkpointBuilt = true; } - taosWLockLatch(&pMeta->lock); + streamMetaWLock(pMeta); if (streamMetaSaveTask(pMeta, pTask)) { - taosWUnLockLatch(&pMeta->lock); + streamMetaWUnLock(pMeta); code = terrno ? terrno : TSDB_CODE_OUT_OF_MEMORY; taosHashCancelIterate(pInfoHash, infoHash); TSDB_CHECK_CODE(code, lino, _exit); } - taosWUnLockLatch(&pMeta->lock); + streamMetaWUnLock(pMeta); smaDebug("vgId:%d, rsma commit, succeed to commit task:%p, submitReqVer:%" PRIi64 ", fetchResultVer:%" PRIi64 ", table:%" PRIi64 ", level:%d", TD_VID(pVnode), pTask, pItem->submitReqVer, pItem->fetchResultVer, pRSmaInfo->suid, i + 1); @@ -1316,13 +1316,13 @@ _checkpoint: } } if (pMeta) { - taosWLockLatch(&pMeta->lock); + streamMetaWLock(pMeta); if (streamMetaCommit(pMeta)) { - taosWUnLockLatch(&pMeta->lock); + streamMetaWUnLock(pMeta); code = terrno ? terrno : TSDB_CODE_OUT_OF_MEMORY; TSDB_CHECK_CODE(code, lino, _exit); } - taosWUnLockLatch(&pMeta->lock); + streamMetaWUnLock(pMeta); } if (checkpointBuilt) { smaInfo("vgId:%d, rsma commit, succeed to commit checkpoint:%" PRIi64, TD_VID(pVnode), checkpointId); diff --git a/source/dnode/vnode/src/tq/tqStreamTaskSnap.c b/source/dnode/vnode/src/tq/tqStreamTaskSnap.c index f966e90b9a..dda5173ad9 100644 --- a/source/dnode/vnode/src/tq/tqStreamTaskSnap.c +++ b/source/dnode/vnode/src/tq/tqStreamTaskSnap.c @@ -196,7 +196,7 @@ int32_t streamTaskSnapWriterClose(SStreamTaskWriter* pWriter, int8_t rollback) { int32_t code = 0; STQ* pTq = pWriter->pTq; - taosWLockLatch(&pTq->pStreamMeta->lock); + streamMetaWLock(pTq->pStreamMeta); tqDebug("vgId:%d, vnode stream-task snapshot writer closed", TD_VID(pTq->pVnode)); if (rollback) { tdbAbort(pTq->pStreamMeta->db, pTq->pStreamMeta->txn); @@ -212,14 +212,14 @@ int32_t streamTaskSnapWriterClose(SStreamTaskWriter* pWriter, int8_t rollback) { taosMemoryFree(pWriter); goto _err; } - taosWUnLockLatch(&pTq->pStreamMeta->lock); + streamMetaWUnLock(pTq->pStreamMeta); taosMemoryFree(pWriter); return code; _err: tqError("vgId:%d, vnode stream-task snapshot writer failed to close since %s", TD_VID(pWriter->pTq->pVnode), tstrerror(code)); - taosWUnLockLatch(&pTq->pStreamMeta->lock); + streamMetaWUnLock(pTq->pStreamMeta); return code; } @@ -240,13 +240,13 @@ int32_t streamTaskSnapWrite(SStreamTaskWriter* pWriter, uint8_t* pData, uint32_t tDecoderClear(&decoder); int64_t key[2] = {taskId.streamId, taskId.taskId}; - taosWLockLatch(&pTq->pStreamMeta->lock); + streamMetaWLock(pTq->pStreamMeta); if (tdbTbUpsert(pTq->pStreamMeta->pTaskDb, key, sizeof(int64_t) << 1, (uint8_t*)pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr), pTq->pStreamMeta->txn) < 0) { - taosWUnLockLatch(&pTq->pStreamMeta->lock); + streamMetaWUnLock(pTq->pStreamMeta); return -1; } - taosWUnLockLatch(&pTq->pStreamMeta->lock); + streamMetaWUnLock(pTq->pStreamMeta); } else if (pHdr->type == SNAP_DATA_STREAM_TASK_CHECKPOINT) { // do nothing } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index b82021abf6..665288cf7b 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -1293,24 +1293,23 @@ void streamMetaResetStartInfo(STaskStartInfo* pStartInfo) { void streamMetaRLock(SStreamMeta* pMeta) { stTrace("vgId:%d meta-rlock", pMeta->vgId); - taosRLockLatch(&pMeta->lock); + taosThreadRwlockRdlock(&pMeta->lock); } void streamMetaRUnLock(SStreamMeta* pMeta) { stTrace("vgId:%d meta-runlock", pMeta->vgId); - taosRUnLockLatch(&pMeta->lock); + taosThreadRwlockUnlock(&pMeta->lock); } void streamMetaWLock(SStreamMeta* pMeta) { stTrace("vgId:%d meta-wlock", pMeta->vgId); - taosWLockLatch(&pMeta->lock); - ASSERT(pMeta->lock != 0x40000001); + taosThreadRwlockWrlock(&pMeta->lock); stTrace("vgId:%d meta-wlock completed", pMeta->vgId); } void streamMetaWUnLock(SStreamMeta* pMeta) { stTrace("vgId:%d meta-wunlock", pMeta->vgId); - taosWUnLockLatch(&pMeta->lock); + taosThreadRwlockUnlock(&pMeta->lock); } static void execHelper(struct SSchedMsg* pSchedMsg) { From 212c1ebb3c139a10881c15cd3e6bb8af7375b3f8 Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Thu, 21 Dec 2023 19:52:40 +0800 Subject: [PATCH 37/94] udpate test case test_hot_refresh_configurations --- source/util/src/tlog.c | 4 ---- .../system-test/0-others/test_hot_refresh_configurations.py | 6 ++++++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 184e18fc67..e113a95fcc 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -337,14 +337,10 @@ static int32_t taosOpenNewLogFile() { } void taosResetLog() { - char lastName[LOG_FILE_NAME_LEN + 20]; - sprintf(lastName, "%s.%d", tsLogObj.logName, tsLogObj.flag); - // force create a new log file tsLogObj.lines = tsNumOfLogLines + 10; taosOpenNewLogFile(); - (void)taosRemoveFile(lastName); uInfo("=================================="); uInfo(" reset log file "); diff --git a/tests/system-test/0-others/test_hot_refresh_configurations.py b/tests/system-test/0-others/test_hot_refresh_configurations.py index cbde8c060e..59222281ae 100644 --- a/tests/system-test/0-others/test_hot_refresh_configurations.py +++ b/tests/system-test/0-others/test_hot_refresh_configurations.py @@ -215,6 +215,12 @@ class TDTestCase: self.svr_check(item["name"], item["alias"], item["except_values"], True) else: raise Exception(f"unknown key: {key}") + # reset log + path = os.sep.join([tdDnodes.getDnodesRootDir(), "dnode1", "log", "taosdlog.*"]) + tdSql.execute("alter all dnodes 'resetlog';") + r = subprocess.Popen("cat {} | grep 'reset log file'".format(path), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = r.communicate() + assert('reset log file' in stdout.decode()) def stop(self): tdSql.close() From 536c002ae42a1f8254bbb0928f9559e9708766d9 Mon Sep 17 00:00:00 2001 From: chenhaoran Date: Fri, 22 Dec 2023 16:48:57 +0800 Subject: [PATCH 38/94] test: modify test of last_cache_scan.py --- tests/pytest/util/sql.py | 9 ++-- tests/system-test/2-query/last_cache_scan.py | 53 ++++++++++---------- 2 files changed, 31 insertions(+), 31 deletions(-) diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index 5ed985187a..f8a49983e1 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -128,7 +128,7 @@ class TDSql: if expectErrInfo == self.error_info: tdLog.info("sql:%s, expected expectErrInfo '%s' occured" % (sql, expectErrInfo)) else: - tdLog.exit("%s(%d) failed: sql:%s, expectErrInfo '%s' occured, but not expected errno '%s'" % (caller.filename, caller.lineno, sql, self.error_info, expectErrInfo)) + tdLog.exit("%s(%d) failed: sql:%s, expectErrInfo '%s' occured, but not expected expectErrInfo '%s'" % (caller.filename, caller.lineno, sql, self.error_info, expectErrInfo)) else: if expectedErrno != None: if expectedErrno in self.errno: @@ -140,7 +140,7 @@ class TDSql: if expectErrInfo in self.error_info: tdLog.info("sql:%s, expected expectErrInfo '%s' occured" % (sql, expectErrInfo)) else: - tdLog.exit("%s(%d) failed: sql:%s, expectErrInfo %s occured, but not expected errno '%s'" % (caller.filename, caller.lineno, sql, self.error_info, expectErrInfo)) + tdLog.exit("%s(%d) failed: sql:%s, expectErrInfo %s occured, but not expected expectErrInfo '%s'" % (caller.filename, caller.lineno, sql, self.error_info, expectErrInfo)) return self.error_info @@ -207,6 +207,7 @@ class TDSql: caller = inspect.getframeinfo(inspect.stack()[1][0]) if i < queryTimes: error_info = repr(e) + print(error_info) self.error_info = ','.join(error_info[error_info.index('(')+1:-1].split(",")[:-1]).replace("'","") self.queryRows = 0 self.queryCols = 0 @@ -217,13 +218,13 @@ class TDSql: if expectErrInfo == self.error_info: tdLog.info("sql:%s, expected expectErrInfo '%s' occured" % (sql, expectErrInfo)) else: - tdLog.exit("%s(%d) failed: sql:%s, expectErrInfo '%s' occured, but not expected errno '%s'" % (caller.filename, caller.lineno, sql, self.error_info, expectErrInfo)) + tdLog.exit("%s(%d) failed: sql:%s, expectErrInfo '%s' occured, but not expected expectErrInfo '%s'" % (caller.filename, caller.lineno, sql, self.error_info, expectErrInfo)) else: if expectErrInfo != None: if expectErrInfo in self.error_info: tdLog.info("sql:%s, expected expectErrInfo '%s' occured" % (sql, expectErrInfo)) else: - tdLog.exit("%s(%d) failed: sql:%s, expectErrInfo %s occured, but not expected errno '%s'" % (caller.filename, caller.lineno, sql, self.error_info, expectErrInfo)) + tdLog.exit("%s(%d) failed: sql:%s, expectErrInfo %s occured, but not expected expectErrInfo '%s'" % (caller.filename, caller.lineno, sql, self.error_info, expectErrInfo)) return self.error_info elif i == queryTimes: diff --git a/tests/system-test/2-query/last_cache_scan.py b/tests/system-test/2-query/last_cache_scan.py index 0457475ee7..3504ca62d3 100644 --- a/tests/system-test/2-query/last_cache_scan.py +++ b/tests/system-test/2-query/last_cache_scan.py @@ -28,7 +28,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), False) + tdSql.init(conn.cursor(), True) def create_database(self,tsql, dbName,dropFlag=1,vgroups=2,replica=1, duration:str='1d'): if dropFlag == 1: @@ -369,49 +369,48 @@ class TDTestCase: tdSql.checkData(0, 10, None) def test_cache_scan_last_row_with_drop_column2(self): - tdSql.query('select last_row(c1) from meters') + tdSql.query('select last_row(c2) from meters') print(str(tdSql.queryResult)) tdSql.checkCols(1) - p = subprocess.run(["taos", '-s', "alter table test.meters drop column c1; alter table test.meters add column c11 int"]) + p = subprocess.run(["taos", '-s', "alter table test.meters drop column c2; alter table test.meters add column c1 int"]) p.check_returncode() - tdSql.query('select last_row(c1) from meters', queryTimes=1) - print(str(tdSql.queryResult)) - tdSql.checkCols(1) + tdSql.query_success_failed("select ts, last_row(c2), c12, ts, c12 from meters", queryTimes=10, expectErrInfo="Invalid column name: c2") + tdSql.query('select last(c1), c1, ts from meters', queryTimes=1) + tdSql.checkRows(1) + tdSql.checkCols(3) tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) def test_cache_scan_last_row_with_partition_by(self): tdSql.query('select last(c1) from meters partition by t1') print(str(tdSql.queryResult)) tdSql.checkCols(1) - tdSql.checkRows(5) - p = subprocess.run(["taos", '-s', "alter table test.meters drop column c1; alter table test.meters add column c11 int"]) + tdSql.checkRows(2) + p = subprocess.run(["taos", '-s', "alter table test.meters drop column c1; alter table test.meters add column c2 int"]) p.check_returncode() - tdSql.query('select last_row(c1) from meters partition by t1', queryTimes=1) + tdSql.query_success_failed('select last(c1) from meters partition by t1', queryTimes=10, expectErrInfo="Invalid column name: c1") + tdSql.query('select last(c2), c2, ts from meters', queryTimes=1) print(str(tdSql.queryResult)) - tdSql.checkCols(1) - tdSql.checkRows(5) + tdSql.checkRows(1) + tdSql.checkCols(3) tdSql.checkData(0, 0, None) - tdSql.checkData(1, 0, None) - tdSql.checkData(2, 0, None) - tdSql.checkData(3, 0, None) - tdSql.checkData(4, 0, None) + tdSql.checkData(0, 1, None) + def test_cache_scan_last_row_with_partition_by_tbname(self): - tdSql.query('select last(c1) from meters partition by tbname', queryTimes=1) + tdSql.query('select last(c2) from meters partition by tbname') print(str(tdSql.queryResult)) tdSql.checkCols(1) tdSql.checkRows(10) - p = subprocess.run(["taos", '-s', "alter table test.meters drop column c1; alter table test.meters add column c11 int"]) + p = subprocess.run(["taos", '-s', "alter table test.meters drop column c2; alter table test.meters add column c1 int"]) p.check_returncode() - tdSql.query('select last_row(c1) from meters partition by tbname', queryTimes=1) + tdSql.query_success_failed('select last_row(c2) from meters partition by tbname', queryTimes=10, expectErrInfo="Invalid column name: c2") + tdSql.query('select last(c1), c1, ts from meters', queryTimes=1) print(str(tdSql.queryResult)) - tdSql.checkCols(1) - tdSql.checkRows(10) + tdSql.checkRows(1) + tdSql.checkCols(3) tdSql.checkData(0, 0, None) - tdSql.checkData(1, 0, None) - tdSql.checkData(2, 0, None) - tdSql.checkData(3, 0, None) - tdSql.checkData(4, 0, None) + tdSql.checkData(0, 1, None) @@ -423,9 +422,9 @@ class TDTestCase: self.test_cache_scan_with_drop_and_add_column2() #self.test_cache_scan_with_drop_column() #self.test_cache_scan_last_row_with_drop_column() - #self.test_cache_scan_last_row_with_drop_column2() - #self.test_cache_scan_last_row_with_partition_by() - #self.test_cache_scan_last_row_with_partition_by_tbname() + self.test_cache_scan_last_row_with_drop_column2() + self.test_cache_scan_last_row_with_partition_by() + self.test_cache_scan_last_row_with_partition_by_tbname() def stop(self): tdSql.close() From a71fabb24fdcdfc105d04ee05a1a8e376fe9b292 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Fri, 22 Dec 2023 17:50:55 +0800 Subject: [PATCH 39/94] coverage: remove tfunctionInt.c --- source/libs/function/inc/tfunctionInt.h | 2 -- source/libs/function/src/tfunctionInt.c | 30 ------------------------- 2 files changed, 32 deletions(-) diff --git a/source/libs/function/inc/tfunctionInt.h b/source/libs/function/inc/tfunctionInt.h index 821544106f..b4c48abf37 100644 --- a/source/libs/function/inc/tfunctionInt.h +++ b/source/libs/function/inc/tfunctionInt.h @@ -28,8 +28,6 @@ extern "C" { #include "tudf.h" #include "tvariant.h" -bool topbot_datablock_filter(SqlFunctionCtx *pCtx, const char *minval, const char *maxval); - /** * the numOfRes should be kept, since it may be used later * and allow the ResultInfo to be re initialized diff --git a/source/libs/function/src/tfunctionInt.c b/source/libs/function/src/tfunctionInt.c index 3da5d63fa1..80b869b2d2 100644 --- a/source/libs/function/src/tfunctionInt.c +++ b/source/libs/function/src/tfunctionInt.c @@ -27,33 +27,3 @@ #include "tpercentile.h" #include "ttszip.h" #include "tudf.h" - -int32_t getNumOfResult(SqlFunctionCtx* pCtx, int32_t num, SSDataBlock* pResBlock) { - int32_t maxRows = 0; - - for (int32_t j = 0; j < num; ++j) { - SResultRowEntryInfo* pResInfo = GET_RES_INFO(&pCtx[j]); - if (pResInfo != NULL && maxRows < pResInfo->numOfRes) { - maxRows = pResInfo->numOfRes; - } - } - - blockDataEnsureCapacity(pResBlock, maxRows); - for (int32_t i = 0; i < num; ++i) { - SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, i); - - SResultRowEntryInfo* pResInfo = GET_RES_INFO(&pCtx[i]); - if (pResInfo->numOfRes == 0) { - for (int32_t j = 0; j < pResInfo->numOfRes; ++j) { - colDataSetVal(pCol, j, NULL, true); // TODO add set null data api - } - } else { - for (int32_t j = 0; j < pResInfo->numOfRes; ++j) { - colDataSetVal(pCol, j, GET_ROWCELL_INTERBUF(pResInfo), false); - } - } - } - - pResBlock->info.rows = maxRows; - return maxRows; -} From b175a4b7b3cc7db8400e00547d421d898e486d0f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 22 Dec 2023 17:56:16 +0800 Subject: [PATCH 40/94] refactor(stream): async stop tasks, and do some internal refactor. --- include/dnode/vnode/tqCommon.h | 1 - include/libs/stream/tstream.h | 11 +- source/dnode/mnode/impl/src/mndStream.c | 2 +- source/dnode/snode/src/snode.c | 2 +- source/dnode/vnode/src/inc/tq.h | 3 - source/dnode/vnode/src/inc/vnodeInt.h | 1 + source/dnode/vnode/src/tq/tq.c | 1 + source/dnode/vnode/src/tq/tqStreamTask.c | 43 +++----- source/dnode/vnode/src/tq/tqUtil.c | 26 +---- source/dnode/vnode/src/tqCommon/tqCommon.c | 67 +----------- source/dnode/vnode/src/vnd/vnodeSync.c | 2 +- source/libs/stream/src/streamMeta.c | 116 +++++++++++++++++++++ 12 files changed, 150 insertions(+), 125 deletions(-) diff --git a/include/dnode/vnode/tqCommon.h b/include/dnode/vnode/tqCommon.h index da9dcf1c04..da2325f006 100644 --- a/include/dnode/vnode/tqCommon.h +++ b/include/dnode/vnode/tqCommon.h @@ -30,7 +30,6 @@ int32_t tqStreamTaskProcessCheckpointReadyMsg(SStreamMeta* pMeta, SRpcMsg* pMsg) int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, int64_t sversion, char* msg, int32_t msgLen, bool isLeader, bool restored); int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen); int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLeader); -int32_t startStreamTasks(SStreamMeta* pMeta); int32_t tqStreamTaskResetStatus(SStreamMeta* pMeta); int32_t tqStartTaskCompleteCallback(SStreamMeta* pMeta); int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, SRpcMsg* pMsg); diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index e7ac0e99c5..2d2db5c1dc 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -53,6 +53,7 @@ extern "C" { #define STREAM_EXEC_EXTRACT_DATA_IN_WAL_ID (-1) #define STREAM_EXEC_START_ALL_TASKS_ID (-2) #define STREAM_EXEC_RESTART_ALL_TASKS_ID (-3) +#define STREAM_EXEC_STOP_ALL_TASKS_ID (-4) typedef struct SStreamTask SStreamTask; typedef struct SStreamQueue SStreamQueue; @@ -785,8 +786,6 @@ int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_ int64_t* oldStage); int32_t streamTaskUpdateEpsetInfo(SStreamTask* pTask, SArray* pNodeList); void streamTaskResetUpstreamStageInfo(SStreamTask* pTask); -SArray* streamMetaSendMsgBeforeCloseTasks(SStreamMeta* pMeta); - bool streamTaskAllUpstreamClosed(SStreamTask* pTask); bool streamTaskSetSchedStatusWait(SStreamTask* pTask); int8_t streamTaskSetSchedStatusActive(SStreamTask* pTask); @@ -821,6 +820,7 @@ int32_t streamTaskReleaseState(SStreamTask* pTask); int32_t streamTaskReloadState(SStreamTask* pTask); void streamTaskCloseUpstreamInput(SStreamTask* pTask, int32_t taskId); void streamTaskOpenAllUpstreamInput(SStreamTask* pTask); +int32_t streamTaskSetDb(SStreamMeta* pMeta, void* pTask, char* key); void streamTaskStatusInit(STaskStatusEntry* pEntry, const SStreamTask* pTask); void streamTaskStatusCopy(STaskStatusEntry* pDst, const STaskStatusEntry* pSrc); @@ -851,10 +851,8 @@ void streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask); void streamMetaClear(SStreamMeta* pMeta); void streamMetaInitBackend(SStreamMeta* pMeta); int32_t streamMetaCommit(SStreamMeta* pMeta); -int32_t streamMetaLoadAllTasks(SStreamMeta* pMeta); int64_t streamMetaGetLatestCheckpointId(SStreamMeta* pMeta); void streamMetaNotifyClose(SStreamMeta* pMeta); -int32_t streamTaskSetDb(SStreamMeta* pMeta, void* pTask, char* key); void streamMetaStartHb(SStreamMeta* pMeta); bool streamMetaTaskInTimer(SStreamMeta* pMeta); int32_t streamMetaUpdateTaskDownstreamStatus(SStreamMeta* pMeta, int64_t streamId, int32_t taskId, int64_t startTs, @@ -864,6 +862,11 @@ void streamMetaRUnLock(SStreamMeta* pMeta); void streamMetaWLock(SStreamMeta* pMeta); void streamMetaWUnLock(SStreamMeta* pMeta); void streamMetaResetStartInfo(STaskStartInfo* pMeta); +SArray* streamMetaSendMsgBeforeCloseTasks(SStreamMeta* pMeta); +void streamMetaUpdateStageRole(SStreamMeta* pMeta, int64_t stage, bool isLeader); +int32_t streamMetaLoadAllTasks(SStreamMeta* pMeta); +int32_t streamMetaStartAllTasks(SStreamMeta* pMeta); +int32_t streamMetaStopAllTasks(SStreamMeta* pMeta); // checkpoint int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSourceReq* pReq); diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 3ba6280f59..25f51b85df 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -3116,7 +3116,7 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { if (allReady || snodeChanged) { // if the execInfo.activeCheckpoint == 0, the checkpoint is restoring from wal mInfo("checkpointId:%" PRId64 " failed, issue task-reset trans to reset all tasks status", activeCheckpointId); - mndResetStatusFromCheckpoint(pMnode, streamId, activeCheckpointId); + mndResetStatusFromCheckpoint(pMnode, streamId, transId); } else { mInfo("not all vgroups are ready, wait for next HB from stream tasks"); } diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index f0fe662025..2225a07d3a 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -151,7 +151,7 @@ FAIL: int32_t sndInit(SSnode * pSnode) { tqStreamTaskResetStatus(pSnode->pMeta); - startStreamTasks(pSnode->pMeta); + streamMetaStartAllTasks(pSnode->pMeta); return 0; } diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index cf57623a43..0ef29fcb3a 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -152,9 +152,6 @@ void tqSinkDataIntoDstTable(SStreamTask* pTask, void* vnode, void* data); char* tqOffsetBuildFName(const char* path, int32_t fVer); int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname); -// tqStream -int32_t tqStopStreamTasks(STQ* pTq); - // tq util int32_t extractDelDataBlock(const void* pData, int32_t len, int64_t ver, void** pRefBlock, int32_t type); int32_t tqExtractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index f3b495675d..84f51827ba 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -232,6 +232,7 @@ int tqPushMsg(STQ*, tmsg_t msgType); int tqRegisterPushHandle(STQ* pTq, void* handle, SRpcMsg* pMsg); int tqUnregisterPushHandle(STQ* pTq, void* pHandle); int tqScanWalAsync(STQ* pTq, bool ckPause); +int32_t tqStopStreamTasksAsync(STQ* pTq); int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp); int32_t tqProcessTaskCheckpointReadyMsg(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 02c9b61432..e8e88e52f7 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1076,6 +1076,7 @@ int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) { if(code == 0 && taskId > 0){ tqScanWalAsync(pTq, false); } + return code; } diff --git a/source/dnode/vnode/src/tq/tqStreamTask.c b/source/dnode/vnode/src/tq/tqStreamTask.c index 63e71f8ee8..2bb2609ae9 100644 --- a/source/dnode/vnode/src/tq/tqStreamTask.c +++ b/source/dnode/vnode/src/tq/tqStreamTask.c @@ -109,8 +109,8 @@ int32_t tqScanWalAsync(STQ* pTq, bool ckPause) { return -1; } - tqDebug("vgId:%d create msg to start wal scan to launch stream tasks, numOfTasks:%d, restored:%d", vgId, numOfTasks, - alreadyRestored); + tqDebug("vgId:%d create msg to start wal scan to launch stream tasks, numOfTasks:%d, vnd restored:%d", vgId, + numOfTasks, alreadyRestored); pRunReq->head.vgId = vgId; pRunReq->streamId = 0; @@ -123,38 +123,25 @@ int32_t tqScanWalAsync(STQ* pTq, bool ckPause) { return 0; } -// todo: createMsg to invoke this function in stream threads, to avoid blocking the syn thread -int32_t tqStopStreamTasks(STQ* pTq) { +int32_t tqStopStreamTasksAsync(STQ* pTq) { SStreamMeta* pMeta = pTq->pStreamMeta; - int32_t vgId = TD_VID(pTq->pVnode); + int32_t vgId = pMeta->vgId; - streamMetaRLock(pMeta); - - int32_t num = taosArrayGetSize(pMeta->pTaskList); - tqDebug("vgId:%d stop all %d stream task(s)", vgId, num); - if (num == 0) { - streamMetaRUnLock(pMeta); - return TSDB_CODE_SUCCESS; + SStreamTaskRunReq* pRunReq = rpcMallocCont(sizeof(SStreamTaskRunReq)); + if (pRunReq == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + tqError("vgId:%d failed to create msg to stop tasks, code:%s", vgId, terrstr()); + return -1; } - // send hb msg to mnode before closing all tasks. - SArray* pTaskList = streamMetaSendMsgBeforeCloseTasks(pMeta); - int32_t numOfTasks = taosArrayGetSize(pTaskList); + tqDebug("vgId:%d create msg to stop tasks", vgId); - for (int32_t i = 0; i < numOfTasks; ++i) { - SStreamTaskId* pTaskId = taosArrayGet(pTaskList, i); - SStreamTask* pTask = streamMetaAcquireTaskNoLock(pMeta, pTaskId->streamId, pTaskId->taskId); - if (pTask == NULL) { - continue; - } + pRunReq->head.vgId = vgId; + pRunReq->streamId = 0; + pRunReq->taskId = STREAM_EXEC_STOP_ALL_TASKS_ID; - streamTaskStop(pTask); - streamMetaReleaseTask(pMeta, pTask); - } - - taosArrayDestroy(pTaskList); - - streamMetaRUnLock(pMeta); + SRpcMsg msg = {.msgType = TDMT_STREAM_TASK_RUN, .pCont = pRunReq, .contLen = sizeof(SStreamTaskRunReq)}; + tmsgPutToQueue(&pTq->pVnode->msgCb, STREAM_QUEUE, &msg); return 0; } diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 64c733d63d..d18455d221 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -35,30 +35,8 @@ int32_t tqInitDataRsp(SMqDataRsp* pRsp, STqOffsetVal pOffset) { } void tqUpdateNodeStage(STQ* pTq, bool isLeader) { - SSyncState state = syncGetState(pTq->pVnode->sync); - SStreamMeta* pMeta = pTq->pStreamMeta; - int64_t stage = pMeta->stage; - - streamMetaWLock(pMeta); - - pMeta->stage = state.term; - - // mark the sign to send msg before close all tasks - if ((!isLeader) && (pMeta->role == NODE_ROLE_LEADER)) { - pMeta->sendMsgBeforeClosing = true; - } - - pMeta->role = (isLeader)? NODE_ROLE_LEADER:NODE_ROLE_FOLLOWER; - if (isLeader) { - tqInfo("vgId:%d update meta stage:%" PRId64 ", prev:%" PRId64 " leader:%d, start to send Hb", pMeta->vgId, - state.term, stage, isLeader); - streamMetaStartHb(pMeta); - } else { - tqInfo("vgId:%d update meta stage:%" PRId64 " prev:%" PRId64 " leader:%d sendMsg beforeClosing:%d", pMeta->vgId, - state.term, stage, isLeader, pMeta->sendMsgBeforeClosing); - } - - streamMetaWUnLock(pMeta); + SSyncState state = syncGetState(pTq->pVnode->sync); + streamMetaUpdateStageRole(pTq->pStreamMeta, state.term, isLeader); } static int32_t tqInitTaosxRsp(STaosxRsp* pRsp, STqOffsetVal pOffset) { diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index 0864afdf2c..af9363e363 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -598,66 +598,6 @@ int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen return 0; } -int32_t startStreamTasks(SStreamMeta* pMeta) { - int32_t code = TSDB_CODE_SUCCESS; - int32_t vgId = pMeta->vgId; - - int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); - tqDebug("vgId:%d start to check all %d stream task(s) downstream status", vgId, numOfTasks); - if (numOfTasks == 0) { - return TSDB_CODE_SUCCESS; - } - - SArray* pTaskList = NULL; - streamMetaWLock(pMeta); - pTaskList = taosArrayDup(pMeta->pTaskList, NULL); - taosHashClear(pMeta->startInfo.pReadyTaskSet); - taosHashClear(pMeta->startInfo.pFailedTaskSet); - pMeta->startInfo.startTs = taosGetTimestampMs(); - streamMetaWUnLock(pMeta); - - // broadcast the check downstream tasks msg - for (int32_t i = 0; i < numOfTasks; ++i) { - SStreamTaskId* pTaskId = taosArrayGet(pTaskList, i); - SStreamTask* pTask = streamMetaAcquireTask(pMeta, pTaskId->streamId, pTaskId->taskId); - if (pTask == NULL) { - streamMetaUpdateTaskDownstreamStatus(pMeta, pTaskId->streamId, pTaskId->taskId, 0, - taosGetTimestampMs(), false); - continue; - } - - // fill-history task can only be launched by related stream tasks. - if (pTask->info.fillHistory == 1) { - streamMetaReleaseTask(pMeta, pTask); - continue; - } - - if (pTask->status.downstreamReady == 1) { - if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { - tqDebug("s-task:%s downstream ready, no need to check downstream, check only related fill-history task", - pTask->id.idStr); - streamLaunchFillHistoryTask(pTask); - } - - streamMetaUpdateTaskDownstreamStatus(pMeta, pTask->id.streamId, pTask->id.taskId, pTask->execInfo.init, - pTask->execInfo.start, true); - streamMetaReleaseTask(pMeta, pTask); - continue; - } - - EStreamTaskEvent event = (HAS_RELATED_FILLHISTORY_TASK(pTask)) ? TASK_EVENT_INIT_STREAM_SCANHIST : TASK_EVENT_INIT; - int32_t ret = streamTaskHandleEvent(pTask->status.pSM, event); - if (ret != TSDB_CODE_SUCCESS) { - code = ret; - } - - streamMetaReleaseTask(pMeta, pTask); - } - - taosArrayDestroy(pTaskList); - return code; -} - int32_t tqStreamTaskResetStatus(SStreamMeta* pMeta) { int32_t vgId = pMeta->vgId; int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); @@ -723,7 +663,7 @@ static int32_t restartStreamTasks(SStreamMeta* pMeta, bool isLeader) { streamMetaWUnLock(pMeta); tqInfo("vgId:%d start all stream tasks after reload tasks from disk", vgId); - startStreamTasks(pMeta); + streamMetaStartAllTasks(pMeta); } else { streamMetaResetStartInfo(&pMeta->startInfo); streamMetaWUnLock(pMeta); @@ -741,11 +681,14 @@ int32_t tqStreamTaskProcessRunReq(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLead int32_t vgId = pMeta->vgId; if (taskId == STREAM_EXEC_START_ALL_TASKS_ID) { - startStreamTasks(pMeta); + streamMetaStartAllTasks(pMeta); return 0; } else if (taskId == STREAM_EXEC_RESTART_ALL_TASKS_ID) { restartStreamTasks(pMeta, isLeader); return 0; + } else if (taskId == STREAM_EXEC_STOP_ALL_TASKS_ID) { + streamMetaStopAllTasks(pMeta); + return 0; } SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, taskId); diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 48811b2925..048092131d 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -594,7 +594,7 @@ static void vnodeBecomeFollower(const SSyncFSM *pFsm) { if (pVnode->pTq) { tqUpdateNodeStage(pVnode->pTq, false); - tqStopStreamTasks(pVnode->pTq); + tqStopStreamTasksAsync(pVnode->pTq); } } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 665288cf7b..0702a908ab 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -1367,3 +1367,119 @@ SArray* streamMetaSendMsgBeforeCloseTasks(SStreamMeta* pMeta) { pMeta->sendMsgBeforeClosing = false; return pTaskList; } + +void streamMetaUpdateStageRole(SStreamMeta* pMeta, int64_t stage, bool isLeader) { + streamMetaWLock(pMeta); + + int64_t prevStage = pMeta->stage; + pMeta->stage = stage; + + // mark the sign to send msg before close all tasks + if ((!isLeader) && (pMeta->role == NODE_ROLE_LEADER)) { + pMeta->sendMsgBeforeClosing = true; + } + + pMeta->role = (isLeader)? NODE_ROLE_LEADER:NODE_ROLE_FOLLOWER; + + if (isLeader) { + stInfo("vgId:%d update meta stage:%" PRId64 ", prev:%" PRId64 " leader:%d, start to send Hb", pMeta->vgId, + prevStage, stage, isLeader); + streamMetaStartHb(pMeta); + } else { + stInfo("vgId:%d update meta stage:%" PRId64 " prev:%" PRId64 " leader:%d sendMsg beforeClosing:%d", pMeta->vgId, + prevStage, stage, isLeader, pMeta->sendMsgBeforeClosing); + } + + streamMetaWUnLock(pMeta); +} + +int32_t streamMetaStopAllTasks(SStreamMeta* pMeta) { + streamMetaRLock(pMeta); + + int32_t num = taosArrayGetSize(pMeta->pTaskList); + stDebug("vgId:%d stop all %d stream task(s)", pMeta->vgId, num); + if (num == 0) { + streamMetaRUnLock(pMeta); + return TSDB_CODE_SUCCESS; + } + + // send hb msg to mnode before closing all tasks. + SArray* pTaskList = streamMetaSendMsgBeforeCloseTasks(pMeta); + int32_t numOfTasks = taosArrayGetSize(pTaskList); + + for (int32_t i = 0; i < numOfTasks; ++i) { + SStreamTaskId* pTaskId = taosArrayGet(pTaskList, i); + SStreamTask* pTask = streamMetaAcquireTaskNoLock(pMeta, pTaskId->streamId, pTaskId->taskId); + if (pTask == NULL) { + continue; + } + + streamTaskStop(pTask); + streamMetaReleaseTask(pMeta, pTask); + } + + taosArrayDestroy(pTaskList); + + streamMetaRUnLock(pMeta); + return 0; +} + +int32_t streamMetaStartAllTasks(SStreamMeta* pMeta) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t vgId = pMeta->vgId; + + int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); + stDebug("vgId:%d start to check all %d stream task(s) downstream status", vgId, numOfTasks); + if (numOfTasks == 0) { + return TSDB_CODE_SUCCESS; + } + + SArray* pTaskList = NULL; + streamMetaWLock(pMeta); + pTaskList = taosArrayDup(pMeta->pTaskList, NULL); + taosHashClear(pMeta->startInfo.pReadyTaskSet); + taosHashClear(pMeta->startInfo.pFailedTaskSet); + pMeta->startInfo.startTs = taosGetTimestampMs(); + streamMetaWUnLock(pMeta); + + // broadcast the check downstream tasks msg + for (int32_t i = 0; i < numOfTasks; ++i) { + SStreamTaskId* pTaskId = taosArrayGet(pTaskList, i); + SStreamTask* pTask = streamMetaAcquireTask(pMeta, pTaskId->streamId, pTaskId->taskId); + if (pTask == NULL) { + streamMetaUpdateTaskDownstreamStatus(pMeta, pTaskId->streamId, pTaskId->taskId, 0, + taosGetTimestampMs(), false); + continue; + } + + // fill-history task can only be launched by related stream tasks. + if (pTask->info.fillHistory == 1) { + streamMetaReleaseTask(pMeta, pTask); + continue; + } + + if (pTask->status.downstreamReady == 1) { + if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { + stDebug("s-task:%s downstream ready, no need to check downstream, check only related fill-history task", + pTask->id.idStr); + streamLaunchFillHistoryTask(pTask); + } + + streamMetaUpdateTaskDownstreamStatus(pMeta, pTask->id.streamId, pTask->id.taskId, pTask->execInfo.init, + pTask->execInfo.start, true); + streamMetaReleaseTask(pMeta, pTask); + continue; + } + + EStreamTaskEvent event = (HAS_RELATED_FILLHISTORY_TASK(pTask)) ? TASK_EVENT_INIT_STREAM_SCANHIST : TASK_EVENT_INIT; + int32_t ret = streamTaskHandleEvent(pTask->status.pSM, event); + if (ret != TSDB_CODE_SUCCESS) { + code = ret; + } + + streamMetaReleaseTask(pMeta, pTask); + } + + taosArrayDestroy(pTaskList); + return code; +} \ No newline at end of file From 80604ec8e29c83313da061d9b2d868d558158aaa Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 22 Dec 2023 23:31:51 +0800 Subject: [PATCH 41/94] fix(stream): init rwlock. --- source/libs/stream/src/streamMeta.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 0702a908ab..292bd8cb38 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -375,6 +375,7 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF stage); pMeta->rid = taosAddRef(streamMetaId, pMeta); + taosThreadRwlockInit(&pMeta->lock, NULL); int64_t* pRid = taosMemoryMalloc(sizeof(int64_t)); memcpy(pRid, &pMeta->rid, sizeof(pMeta->rid)); From 66bdace9eb70058bd80476a9185d090edb1ee980 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 23 Dec 2023 00:09:55 +0800 Subject: [PATCH 42/94] fix(stream): check rsp. --- source/libs/stream/src/streamMeta.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 292bd8cb38..eebde87b28 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -1299,7 +1299,12 @@ void streamMetaRLock(SStreamMeta* pMeta) { void streamMetaRUnLock(SStreamMeta* pMeta) { stTrace("vgId:%d meta-runlock", pMeta->vgId); - taosThreadRwlockUnlock(&pMeta->lock); + int32_t code = taosThreadRwlockUnlock(&pMeta->lock); + if (code != TSDB_CODE_SUCCESS) { + stError("vgId:%d meta-runlock failed, code:%d", pMeta->vgId, code); + } else { + stDebug("vgId:%d meta-runlock completed", pMeta->vgId); + } } void streamMetaWLock(SStreamMeta* pMeta) { From 5c1ff76082d393fae04374e09e679004a07dc7ad Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 23 Dec 2023 00:58:52 +0800 Subject: [PATCH 43/94] fix(stream): set rwlock attr. --- source/libs/stream/src/streamMeta.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index eebde87b28..25f0fbf991 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -375,7 +375,13 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF stage); pMeta->rid = taosAddRef(streamMetaId, pMeta); - taosThreadRwlockInit(&pMeta->lock, NULL); + + TdThreadRwlockAttr attr; + taosThreadRwlockAttrInit(&attr); + + pthread_rwlockattr_setkind_np(&attr, PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP); + taosThreadRwlockInit(&pMeta->lock, &attr); + taosThreadRwlockAttrDestroy(&attr); int64_t* pRid = taosMemoryMalloc(sizeof(int64_t)); memcpy(pRid, &pMeta->rid, sizeof(pMeta->rid)); @@ -1386,6 +1392,7 @@ void streamMetaUpdateStageRole(SStreamMeta* pMeta, int64_t stage, bool isLeader) } pMeta->role = (isLeader)? NODE_ROLE_LEADER:NODE_ROLE_FOLLOWER; + streamMetaWUnLock(pMeta); if (isLeader) { stInfo("vgId:%d update meta stage:%" PRId64 ", prev:%" PRId64 " leader:%d, start to send Hb", pMeta->vgId, @@ -1395,8 +1402,6 @@ void streamMetaUpdateStageRole(SStreamMeta* pMeta, int64_t stage, bool isLeader) stInfo("vgId:%d update meta stage:%" PRId64 " prev:%" PRId64 " leader:%d sendMsg beforeClosing:%d", pMeta->vgId, prevStage, stage, isLeader, pMeta->sendMsgBeforeClosing); } - - streamMetaWUnLock(pMeta); } int32_t streamMetaStopAllTasks(SStreamMeta* pMeta) { From dc451984b5ae4acc77cf4c21479df75d3c054023 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 23 Dec 2023 12:59:46 +0800 Subject: [PATCH 44/94] feat: add level and disk param on test.py --- tests/army/frame/caseBase.py | 25 ++ tests/army/frame/common.py | 2 +- tests/army/frame/dnodes-default.py | 502 ---------------------- tests/army/frame/dnodes-no-random-fail.py | 500 --------------------- tests/army/frame/dnodes-random-fail.py | 497 --------------------- tests/army/frame/{ => server}/cluster.py | 10 +- tests/army/frame/{ => server}/dnodes.py | 23 +- tests/army/frame/sql.py | 3 + tests/army/frame/srvCtl.py | 27 ++ tests/army/test.py | 21 +- 10 files changed, 96 insertions(+), 1514 deletions(-) create mode 100644 tests/army/frame/caseBase.py delete mode 100644 tests/army/frame/dnodes-default.py delete mode 100644 tests/army/frame/dnodes-no-random-fail.py delete mode 100644 tests/army/frame/dnodes-random-fail.py rename tests/army/frame/{ => server}/cluster.py (91%) rename tests/army/frame/{ => server}/dnodes.py (98%) create mode 100644 tests/army/frame/srvCtl.py diff --git a/tests/army/frame/caseBase.py b/tests/army/frame/caseBase.py new file mode 100644 index 0000000000..abaa92a51c --- /dev/null +++ b/tests/army/frame/caseBase.py @@ -0,0 +1,25 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +import time +import datetime + +class caseBase: + def __init__(self): + # record server information + self.dnodeNum = 0 + self.mnodeNum = 0 + self.mLevel = 0 + self.mLevelDisk = 0 diff --git a/tests/army/frame/common.py b/tests/army/frame/common.py index 93059ff078..b8704080c0 100644 --- a/tests/army/frame/common.py +++ b/tests/army/frame/common.py @@ -23,7 +23,7 @@ import taos from frame.log import * from frame.sql import * from frame.cases import * -from frame.dnodes import * +from TDengine.tests.army.frame.server.dnodes import * from frame.common import * from frame.constant import * from dataclasses import dataclass,field diff --git a/tests/army/frame/dnodes-default.py b/tests/army/frame/dnodes-default.py deleted file mode 100644 index a1d1698b00..0000000000 --- a/tests/army/frame/dnodes-default.py +++ /dev/null @@ -1,502 +0,0 @@ -################################################################### -# Copyright (c) 2016 by TAOS Technologies, Inc. -# All rights reserved. -# -# This file is proprietary and confidential to TAOS Technologies. -# No part of this file may be reproduced, stored, transmitted, -# disclosed or used in any form or by any means other than as -# expressly provided by the written permission from Jianhui Tao -# -################################################################### - -# -*- coding: utf-8 -*- - -import sys -import os -import os.path -import subprocess -from frame.log import * - - -class TDSimClient: - def __init__(self): - self.testCluster = False - - self.cfgDict = { - "numOfLogLines": "100000000", - "numOfThreadsPerCore": "2.0", - "locale": "en_US.UTF-8", - "charset": "UTF-8", - "asyncLog": "0", - "minTablesPerVnode": "4", - "maxTablesPerVnode": "1000", - "tableIncStepPerVnode": "10000", - "maxVgroupsPerDb": "1000", - "sdbDebugFlag": "143", - "rpcDebugFlag": "135", - "tmrDebugFlag": "131", - "cDebugFlag": "135", - "udebugFlag": "135", - "jnidebugFlag": "135", - "qdebugFlag": "135", - "telemetryReporting": "0", - } - def init(self, path): - self.__init__() - self.path = path - - def getLogDir(self): - self.logDir = os.path.join(self.path,"sim","psim","log") - return self.logDir - - def getCfgDir(self): - self.cfgDir = os.path.join(self.path,"sim","psim","cfg") - return self.cfgDir - - def setTestCluster(self, value): - self.testCluster = value - - def addExtraCfg(self, option, value): - self.cfgDict.update({option: value}) - - def cfg(self, option, value): - cmd = "echo %s %s >> %s" % (option, value, self.cfgPath) - if os.system(cmd) != 0: - tdLog.exit(cmd) - - def deploy(self): - self.logDir = os.path.join(self.path,"sim","psim","log") - self.cfgDir = os.path.join(self.path,"sim","psim","cfg") - self.cfgPath = os.path.join(self.path,"sim","psim","cfg","taos.cfg") - - cmd = "rm -rf " + self.logDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "mkdir -p " + self.logDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "rm -rf " + self.cfgDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "mkdir -p " + self.cfgDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "touch " + self.cfgPath - if os.system(cmd) != 0: - tdLog.exit(cmd) - - if self.testCluster: - self.cfg("masterIp", "192.168.0.1") - self.cfg("secondIp", "192.168.0.2") - self.cfg("logDir", self.logDir) - - for key, value in self.cfgDict.items(): - self.cfg(key, value) - - tdLog.debug("psim is deployed and configured by %s" % (self.cfgPath)) - - -class TDDnode: - def __init__(self, index): - self.index = index - self.running = 0 - self.deployed = 0 - self.testCluster = False - self.valgrind = 0 - - def init(self, path): - self.path = path - - def setTestCluster(self, value): - self.testCluster = value - - def setValgrind(self, value): - self.valgrind = value - - def getDataSize(self): - totalSize = 0 - - if (self.deployed == 1): - for dirpath, dirnames, filenames in os.walk(self.dataDir): - for f in filenames: - fp = os.path.join(dirpath, f) - - if not os.path.islink(fp): - totalSize = totalSize + os.path.getsize(fp) - - return totalSize - - def deploy(self): - self.logDir = os.path.join(self.path,"sim","dnode%d" % self.index, "log") - self.dataDir = os.path.join(self.path,"sim","dnode%d" % self.index, "data") - self.cfgDir = os.path.join(self.path,"sim","dnode%d" % self.index, "cfg") - self.cfgPath = os.path.join(self.path,"sim","dnode%d" % self.index, "cfg","taos.cfg") - - cmd = "rm -rf " + self.dataDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "rm -rf " + self.logDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "rm -rf " + self.cfgDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "mkdir -p " + self.dataDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "mkdir -p " + self.logDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "mkdir -p " + self.cfgDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "touch " + self.cfgPath - if os.system(cmd) != 0: - tdLog.exit(cmd) - - if self.testCluster: - self.startIP() - - if self.testCluster: - self.cfg("masterIp", "192.168.0.1") - self.cfg("secondIp", "192.168.0.2") - self.cfg("publicIp", "192.168.0.%d" % (self.index)) - self.cfg("internalIp", "192.168.0.%d" % (self.index)) - self.cfg("privateIp", "192.168.0.%d" % (self.index)) - self.cfg("dataDir", self.dataDir) - self.cfg("logDir", self.logDir) - self.cfg("numOfLogLines", "100000000") - self.cfg("mnodeEqualVnodeNum", "0") - self.cfg("walLevel", "2") - self.cfg("fsync", "1000") - self.cfg("statusInterval", "1") - self.cfg("numOfMnodes", "3") - self.cfg("numOfThreadsPerCore", "2.0") - self.cfg("monitor", "0") - self.cfg("maxVnodeConnections", "30000") - self.cfg("maxMgmtConnections", "30000") - self.cfg("maxMeterConnections", "30000") - self.cfg("maxShellConns", "30000") - self.cfg("locale", "en_US.UTF-8") - self.cfg("charset", "UTF-8") - self.cfg("asyncLog", "0") - self.cfg("anyIp", "0") - self.cfg("dDebugFlag", "135") - self.cfg("mDebugFlag", "135") - self.cfg("sdbDebugFlag", "135") - self.cfg("rpcDebugFlag", "135") - self.cfg("tmrDebugFlag", "131") - self.cfg("cDebugFlag", "135") - self.cfg("httpDebugFlag", "135") - self.cfg("monitorDebugFlag", "135") - self.cfg("udebugFlag", "135") - self.cfg("jnidebugFlag", "135") - self.cfg("qdebugFlag", "135") - self.deployed = 1 - tdLog.debug( - "dnode:%d is deployed and configured by %s" % - (self.index, self.cfgPath)) - - def getBuildPath(self): - selfPath = os.path.dirname(os.path.realpath(__file__)) - - if ("community" in selfPath): - projPath = selfPath[:selfPath.find("community")] - else: - projPath = selfPath[:selfPath.find("tests")] - - for root, dirs, files in os.walk(projPath): - if ("taosd" in files): - rootRealPath = os.path.dirname(os.path.realpath(root)) - if ("packaging" not in rootRealPath): - buildPath = root[:len(root)-len("/build/bin")] - break - return buildPath - - def start(self): - buildPath = self.getBuildPath() - - if (buildPath == ""): - tdLog.exit("taosd not found!") - else: - tdLog.info("taosd found in %s" % buildPath) - - binPath = buildPath + "/build/bin/taosd" - - if self.deployed == 0: - tdLog.exit("dnode:%d is not deployed" % (self.index)) - - if self.valgrind == 0: - cmd = "nohup %s -c %s > /dev/null 2>&1 & " % ( - binPath, self.cfgDir) - else: - valgrindCmdline = "valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes" - - cmd = "nohup %s %s -c %s 2>&1 & " % ( - valgrindCmdline, binPath, self.cfgDir) - - print(cmd) - - if os.system(cmd) != 0: - tdLog.exit(cmd) - self.running = 1 - tdLog.debug("dnode:%d is running with %s " % (self.index, cmd)) - - tdLog.debug("wait 5 seconds for the dnode:%d to start." % (self.index)) - time.sleep(5) - - def stop(self): - if self.valgrind == 0: - toBeKilled = "taosd" - else: - toBeKilled = "valgrind.bin" - - if self.running != 0: - psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - while(processID): - killCmd = "kill -INT %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - for port in range(6030, 6041): - fuserCmd = "fuser -k -n tcp %d" % port - os.system(fuserCmd) - if self.valgrind: - time.sleep(2) - - self.running = 0 - tdLog.debug("dnode:%d is stopped by kill -INT" % (self.index)) - - def forcestop(self): - if self.valgrind == 0: - toBeKilled = "taosd" - else: - toBeKilled = "valgrind.bin" - - if self.running != 0: - psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - while(processID): - killCmd = "kill -KILL %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - for port in range(6030, 6041): - fuserCmd = "fuser -k -n tcp %d" % port - os.system(fuserCmd) - if self.valgrind: - time.sleep(2) - - self.running = 0 - tdLog.debug("dnode:%d is stopped by kill -KILL" % (self.index)) - - def startIP(self): - cmd = "sudo ifconfig lo:%d 192.168.0.%d up" % (self.index, self.index) - if os.system(cmd) != 0: - tdLog.exit(cmd) - - def stopIP(self): - cmd = "sudo ifconfig lo:%d 192.168.0.%d down" % ( - self.index, self.index) - if os.system(cmd) != 0: - tdLog.exit(cmd) - - def cfg(self, option, value): - cmd = "echo %s %s >> %s" % (option, value, self.cfgPath) - if os.system(cmd) != 0: - tdLog.exit(cmd) - - def getDnodeRootDir(self, index): - dnodeRootDir = os.path.join(self.path,"sim","psim","dnode%d" % index) - return dnodeRootDir - - def getDnodesRootDir(self): - dnodesRootDir = os.path.join(self.path,"sim","psim") - return dnodesRootDir - - -class TDDnodes: - def __init__(self): - self.dnodes = [] - self.dnodes.append(TDDnode(1)) - self.dnodes.append(TDDnode(2)) - self.dnodes.append(TDDnode(3)) - self.dnodes.append(TDDnode(4)) - self.dnodes.append(TDDnode(5)) - self.dnodes.append(TDDnode(6)) - self.dnodes.append(TDDnode(7)) - self.dnodes.append(TDDnode(8)) - self.dnodes.append(TDDnode(9)) - self.dnodes.append(TDDnode(10)) - self.simDeployed = False - - def init(self, path): - psCmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'" - processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") - while(processID): - killCmd = "kill -TERM %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'" - processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") - while(processID): - killCmd = "kill -TERM %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - binPath = os.path.dirname(os.path.realpath(__file__)) - binPath = binPath + "/../../../debug/" - tdLog.debug("binPath %s" % (binPath)) - binPath = os.path.realpath(binPath) - tdLog.debug("binPath real path %s" % (binPath)) - - # cmd = "sudo cp %s/build/lib/libtaos.so /usr/local/lib/taos/" % (binPath) - # tdLog.debug(cmd) - # os.system(cmd) - - # cmd = "sudo cp %s/build/bin/taos /usr/local/bin/taos/" % (binPath) - # if os.system(cmd) != 0 : - # tdLog.exit(cmd) - # tdLog.debug("execute %s" % (cmd)) - - # cmd = "sudo cp %s/build/bin/taosd /usr/local/bin/taos/" % (binPath) - # if os.system(cmd) != 0 : - # tdLog.exit(cmd) - # tdLog.debug("execute %s" % (cmd)) - - if path == "": - # self.path = os.path.expanduser('~') - self.path = os.path.abspath(binPath + "../../") - else: - self.path = os.path.realpath(path) - - for i in range(len(self.dnodes)): - self.dnodes[i].init(self.path) - - self.sim = TDSimClient() - self.sim.init(self.path) - - def setTestCluster(self, value): - self.testCluster = value - - def setValgrind(self, value): - self.valgrind = value - - def deploy(self, index): - self.sim.setTestCluster(self.testCluster) - - if (self.simDeployed == False): - self.sim.deploy() - self.simDeployed = True - - self.check(index) - self.dnodes[index - 1].setTestCluster(self.testCluster) - self.dnodes[index - 1].setValgrind(self.valgrind) - self.dnodes[index - 1].deploy() - - def cfg(self, index, option, value): - self.check(index) - self.dnodes[index - 1].cfg(option, value) - - def start(self, index): - self.check(index) - self.dnodes[index - 1].start() - - def stop(self, index): - self.check(index) - self.dnodes[index - 1].stop() - - def getDataSize(self, index): - self.check(index) - return self.dnodes[index - 1].getDataSize() - - def forcestop(self, index): - self.check(index) - self.dnodes[index - 1].forcestop() - - def startIP(self, index): - self.check(index) - - if self.testCluster: - self.dnodes[index - 1].startIP() - - def stopIP(self, index): - self.check(index) - - if self.dnodes[index - 1].testCluster: - self.dnodes[index - 1].stopIP() - - def check(self, index): - if index < 1 or index > 10: - tdLog.exit("index:%d should on a scale of [1, 10]" % (index)) - - def stopAll(self): - tdLog.info("stop all dnodes") - for i in range(len(self.dnodes)): - self.dnodes[i].stop() - - psCmd = "ps -ef | grep -w taosd | grep 'root' | grep -v grep | awk '{print $2}'" - processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") - if processID: - cmd = "sudo systemctl stop taosd" - os.system(cmd) - # if os.system(cmd) != 0 : - # tdLog.exit(cmd) - psCmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'" - processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") - while(processID): - killCmd = "kill -TERM %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'" - processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") - while(processID): - killCmd = "kill -TERM %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - # if os.system(cmd) != 0 : - # tdLog.exit(cmd) - - def getDnodesRootDir(self): - dnodesRootDir = "%s/sim" % (self.path) - return dnodesRootDir - - def getSimCfgPath(self): - return self.sim.getCfgDir() - - def getSimLogPath(self): - return self.sim.getLogDir() - - def addSimExtraCfg(self, option, value): - self.sim.addExtraCfg(option, value) - - -tdDnodes = TDDnodes() diff --git a/tests/army/frame/dnodes-no-random-fail.py b/tests/army/frame/dnodes-no-random-fail.py deleted file mode 100644 index 3b3083396b..0000000000 --- a/tests/army/frame/dnodes-no-random-fail.py +++ /dev/null @@ -1,500 +0,0 @@ -################################################################### -# Copyright (c) 2016 by TAOS Technologies, Inc. -# All rights reserved. -# -# This file is proprietary and confidential to TAOS Technologies. -# No part of this file may be reproduced, stored, transmitted, -# disclosed or used in any form or by any means other than as -# expressly provided by the written permission from Jianhui Tao -# -################################################################### - -# -*- coding: utf-8 -*- - -import sys -import os -import os.path -import subprocess -from frame.log import * - - -class TDSimClient: - def __init__(self): - self.testCluster = False - - self.cfgDict = { - "numOfLogLines": "100000000", - "numOfThreadsPerCore": "2.0", - "locale": "en_US.UTF-8", - "charset": "UTF-8", - "asyncLog": "0", - "anyIp": "0", - "sdbDebugFlag": "135", - "rpcDebugFlag": "135", - "tmrDebugFlag": "131", - "cDebugFlag": "135", - "udebugFlag": "135", - "jnidebugFlag": "135", - "qdebugFlag": "135", - "telemetryReporting": "0", - } - - def init(self, path): - self.__init__() - self.path = path - - def getLogDir(self): - self.logDir = os.path.join(self.path,"sim","psim","log") - return self.logDir - - def getCfgDir(self): - self.cfgDir = os.path.join(self.path,"sim","psim","cfg") - return self.cfgDir - - def setTestCluster(self, value): - self.testCluster = value - - def addExtraCfg(self, option, value): - self.cfgDict.update({option: value}) - - def cfg(self, option, value): - cmd = "echo %s %s >> %s" % (option, value, self.cfgPath) - if os.system(cmd) != 0: - tdLog.exit(cmd) - - def deploy(self): - self.logDir = os.path.join(self.path,"sim","psim","log") - self.cfgDir = os.path.join(self.path,"sim","psim","cfg") - self.cfgPath = os.path.join(self.path,"sim","psim","cfg","taos.cfg") - - cmd = "rm -rf " + self.logDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "mkdir -p " + self.logDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "rm -rf " + self.cfgDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "mkdir -p " + self.cfgDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "touch " + self.cfgPath - if os.system(cmd) != 0: - tdLog.exit(cmd) - - if self.testCluster: - self.cfg("masterIp", "192.168.0.1") - self.cfg("secondIp", "192.168.0.2") - self.cfg("logDir", self.logDir) - - for key, value in self.cfgDict.items(): - self.cfg(key, value) - - tdLog.debug("psim is deployed and configured by %s" % (self.cfgPath)) - - -class TDDnode: - def __init__(self, index): - self.index = index - self.running = 0 - self.deployed = 0 - self.testCluster = False - self.valgrind = 0 - - def init(self, path): - self.path = path - - def setTestCluster(self, value): - self.testCluster = value - - def setValgrind(self, value): - self.valgrind = value - - def getDataSize(self): - totalSize = 0 - - if (self.deployed == 1): - for dirpath, dirnames, filenames in os.walk(self.dataDir): - for f in filenames: - fp = os.path.join(dirpath, f) - - if not os.path.islink(fp): - totalSize = totalSize + os.path.getsize(fp) - - return totalSize - - def deploy(self): - self.logDir = os.path.join(self.path,"sim","dnode%d" % self.index, "log") - self.dataDir = os.path.join(self.path,"sim","dnode%d" % self.index, "data") - self.cfgDir = os.path.join(self.path,"sim","dnode%d" % self.index, "cfg") - self.cfgPath = os.path.join(self.path,"sim","dnode%d" % self.index, "cfg","taos.cfg") - - cmd = "rm -rf " + self.dataDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "rm -rf " + self.logDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "rm -rf " + self.cfgDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "mkdir -p " + self.dataDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "mkdir -p " + self.logDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "mkdir -p " + self.cfgDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "touch " + self.cfgPath - if os.system(cmd) != 0: - tdLog.exit(cmd) - - if self.testCluster: - self.startIP() - - if self.testCluster: - self.cfg("masterIp", "192.168.0.1") - self.cfg("secondIp", "192.168.0.2") - self.cfg("publicIp", "192.168.0.%d" % (self.index)) - self.cfg("internalIp", "192.168.0.%d" % (self.index)) - self.cfg("privateIp", "192.168.0.%d" % (self.index)) - self.cfg("dataDir", self.dataDir) - self.cfg("logDir", self.logDir) - self.cfg("numOfLogLines", "100000000") - self.cfg("mnodeEqualVnodeNum", "0") - self.cfg("walLevel", "2") - self.cfg("fsync", "1000") - self.cfg("statusInterval", "1") - self.cfg("numOfMnodes", "3") - self.cfg("numOfThreadsPerCore", "2.0") - self.cfg("monitor", "0") - self.cfg("maxVnodeConnections", "30000") - self.cfg("maxMgmtConnections", "30000") - self.cfg("maxMeterConnections", "30000") - self.cfg("maxShellConns", "30000") - self.cfg("locale", "en_US.UTF-8") - self.cfg("charset", "UTF-8") - self.cfg("asyncLog", "0") - self.cfg("anyIp", "0") - self.cfg("dDebugFlag", "135") - self.cfg("mDebugFlag", "135") - self.cfg("sdbDebugFlag", "135") - self.cfg("rpcDebugFlag", "135") - self.cfg("tmrDebugFlag", "131") - self.cfg("cDebugFlag", "135") - self.cfg("httpDebugFlag", "135") - self.cfg("monitorDebugFlag", "135") - self.cfg("udebugFlag", "135") - self.cfg("jnidebugFlag", "135") - self.cfg("qdebugFlag", "135") - self.deployed = 1 - tdLog.debug( - "dnode:%d is deployed and configured by %s" % - (self.index, self.cfgPath)) - - def getBuildPath(self): - selfPath = os.path.dirname(os.path.realpath(__file__)) - - if ("community" in selfPath): - projPath = selfPath[:selfPath.find("community")] - else: - projPath = selfPath[:selfPath.find("tests")] - - for root, dirs, files in os.walk(projPath): - if ("taosd" in files): - rootRealPath = os.path.dirname(os.path.realpath(root)) - if ("packaging" not in rootRealPath): - buildPath = root[:len(root)-len("/build/bin")] - break - return buildPath - - def start(self): - buildPath = self.getBuildPath() - - if (buildPath == ""): - tdLog.exit("taosd not found!") - else: - tdLog.info("taosd found in %s" % buildPath) - - binPath = buildPath + "/build/bin/taosd" - - if self.deployed == 0: - tdLog.exit("dnode:%d is not deployed" % (self.index)) - - if self.valgrind == 0: - cmd = "nohup %s -c %s --random-file-fail-factor 0 > /dev/null 2>&1 & " % ( - binPath, self.cfgDir) - else: - valgrindCmdline = "valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes" - - cmd = "nohup %s %s -c %s 2>&1 & " % ( - valgrindCmdline, binPath, self.cfgDir) - - print(cmd) - - if os.system(cmd) != 0: - tdLog.exit(cmd) - self.running = 1 - tdLog.debug("dnode:%d is running with %s " % (self.index, cmd)) - - tdLog.debug("wait 5 seconds for the dnode:%d to start." % (self.index)) - time.sleep(5) - - def stop(self): - if self.valgrind == 0: - toBeKilled = "taosd" - else: - toBeKilled = "valgrind.bin" - - if self.running != 0: - psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - while(processID): - killCmd = "kill -INT %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - for port in range(6030, 6041): - fuserCmd = "fuser -k -n tcp %d" % port - os.system(fuserCmd) - if self.valgrind: - time.sleep(2) - - self.running = 0 - tdLog.debug("dnode:%d is stopped by kill -INT" % (self.index)) - - def forcestop(self): - if self.valgrind == 0: - toBeKilled = "taosd" - else: - toBeKilled = "valgrind.bin" - - if self.running != 0: - psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - while(processID): - killCmd = "kill -KILL %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - for port in range(6030, 6041): - fuserCmd = "fuser -k -n tcp %d" % port - os.system(fuserCmd) - if self.valgrind: - time.sleep(2) - - self.running = 0 - tdLog.debug("dnode:%d is stopped by kill -KILL" % (self.index)) - - def startIP(self): - cmd = "sudo ifconfig lo:%d 192.168.0.%d up" % (self.index, self.index) - if os.system(cmd) != 0: - tdLog.exit(cmd) - - def stopIP(self): - cmd = "sudo ifconfig lo:%d 192.168.0.%d down" % ( - self.index, self.index) - if os.system(cmd) != 0: - tdLog.exit(cmd) - - def cfg(self, option, value): - cmd = "echo %s %s >> %s" % (option, value, self.cfgPath) - if os.system(cmd) != 0: - tdLog.exit(cmd) - - def getDnodeRootDir(self, index): - dnodeRootDir = os.path.join(self.path,"sim","psim","dnode%d" % index) - return dnodeRootDir - - def getDnodesRootDir(self): - dnodesRootDir = os.path.join(self.path,"sim","psim") - return dnodesRootDir - - -class TDDnodes: - def __init__(self): - self.dnodes = [] - self.dnodes.append(TDDnode(1)) - self.dnodes.append(TDDnode(2)) - self.dnodes.append(TDDnode(3)) - self.dnodes.append(TDDnode(4)) - self.dnodes.append(TDDnode(5)) - self.dnodes.append(TDDnode(6)) - self.dnodes.append(TDDnode(7)) - self.dnodes.append(TDDnode(8)) - self.dnodes.append(TDDnode(9)) - self.dnodes.append(TDDnode(10)) - self.simDeployed = False - - def init(self, path): - psCmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'" - processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") - while(processID): - killCmd = "kill -TERM %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'" - processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") - while(processID): - killCmd = "kill -TERM %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - binPath = os.path.dirname(os.path.realpath(__file__)) - binPath = binPath + "/../../../debug/" - tdLog.debug("binPath %s" % (binPath)) - binPath = os.path.realpath(binPath) - tdLog.debug("binPath real path %s" % (binPath)) - - # cmd = "sudo cp %s/build/lib/libtaos.so /usr/local/lib/taos/" % (binPath) - # tdLog.debug(cmd) - # os.system(cmd) - - # cmd = "sudo cp %s/build/bin/taos /usr/local/bin/taos/" % (binPath) - # if os.system(cmd) != 0 : - # tdLog.exit(cmd) - # tdLog.debug("execute %s" % (cmd)) - - # cmd = "sudo cp %s/build/bin/taosd /usr/local/bin/taos/" % (binPath) - # if os.system(cmd) != 0 : - # tdLog.exit(cmd) - # tdLog.debug("execute %s" % (cmd)) - - if path == "": - # self.path = os.path.expanduser('~') - self.path = os.path.abspath(binPath + "../../") - else: - self.path = os.path.realpath(path) - - for i in range(len(self.dnodes)): - self.dnodes[i].init(self.path) - - self.sim = TDSimClient() - self.sim.init(self.path) - - def setTestCluster(self, value): - self.testCluster = value - - def setValgrind(self, value): - self.valgrind = value - - def deploy(self, index): - self.sim.setTestCluster(self.testCluster) - - if (self.simDeployed == False): - self.sim.deploy() - self.simDeployed = True - - self.check(index) - self.dnodes[index - 1].setTestCluster(self.testCluster) - self.dnodes[index - 1].setValgrind(self.valgrind) - self.dnodes[index - 1].deploy() - - def cfg(self, index, option, value): - self.check(index) - self.dnodes[index - 1].cfg(option, value) - - def start(self, index): - self.check(index) - self.dnodes[index - 1].start() - - def stop(self, index): - self.check(index) - self.dnodes[index - 1].stop() - - def getDataSize(self, index): - self.check(index) - return self.dnodes[index - 1].getDataSize() - - def forcestop(self, index): - self.check(index) - self.dnodes[index - 1].forcestop() - - def startIP(self, index): - self.check(index) - - if self.testCluster: - self.dnodes[index - 1].startIP() - - def stopIP(self, index): - self.check(index) - - if self.dnodes[index - 1].testCluster: - self.dnodes[index - 1].stopIP() - - def check(self, index): - if index < 1 or index > 10: - tdLog.exit("index:%d should on a scale of [1, 10]" % (index)) - - def stopAll(self): - tdLog.info("stop all dnodes") - for i in range(len(self.dnodes)): - self.dnodes[i].stop() - - psCmd = "ps -ef | grep -w taosd | grep 'root' | grep -v grep | awk '{print $2}'" - processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") - if processID: - cmd = "sudo systemctl stop taosd" - os.system(cmd) - # if os.system(cmd) != 0 : - # tdLog.exit(cmd) - psCmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'" - processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") - while(processID): - killCmd = "kill -TERM %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'" - processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") - while(processID): - killCmd = "kill -TERM %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - # if os.system(cmd) != 0 : - # tdLog.exit(cmd) - - def getDnodesRootDir(self): - dnodesRootDir = "%s/sim" % (self.path) - return dnodesRootDir - - def getSimCfgPath(self): - return self.sim.getCfgDir() - - def getSimLogPath(self): - return self.sim.getLogDir() - - def addSimExtraCfg(self, option, value): - self.sim.addExtraCfg(option, value) - - -tdDnodes = TDDnodes() diff --git a/tests/army/frame/dnodes-random-fail.py b/tests/army/frame/dnodes-random-fail.py deleted file mode 100644 index 794adfd7ed..0000000000 --- a/tests/army/frame/dnodes-random-fail.py +++ /dev/null @@ -1,497 +0,0 @@ -################################################################### -# Copyright (c) 2016 by TAOS Technologies, Inc. -# All rights reserved. -# -# This file is proprietary and confidential to TAOS Technologies. -# No part of this file may be reproduced, stored, transmitted, -# disclosed or used in any form or by any means other than as -# expressly provided by the written permission from Jianhui Tao -# -################################################################### - -# -*- coding: utf-8 -*- - -import sys -import os -import os.path -import subprocess -from frame.log import * - - -class TDSimClient: - def __init__(self): - self.testCluster = False - - self.cfgDict = { - "numOfLogLines": "100000000", - "locale": "en_US.UTF-8", - "charset": "UTF-8", - "asyncLog": "0", - "rpcDebugFlag": "135", - "tmrDebugFlag": "131", - "cDebugFlag": "135", - "udebugFlag": "135", - "jnidebugFlag": "135", - "qdebugFlag": "135", - "telemetryReporting": "0", - } - - def init(self, path): - self.__init__() - self.path = path - - def getLogDir(self): - self.logDir = os.path.join(self.path,"sim","psim","log") - return self.logDir - - def getCfgDir(self): - self.cfgDir = os.path.join(self.path,"sim","psim","cfg") - return self.cfgDir - - def setTestCluster(self, value): - self.testCluster = value - - def addExtraCfg(self, option, value): - self.cfgDict.update({option: value}) - - def cfg(self, option, value): - cmd = "echo %s %s >> %s" % (option, value, self.cfgPath) - if os.system(cmd) != 0: - tdLog.exit(cmd) - - def deploy(self): - self.logDir = os.path.join(self.path,"sim","psim","log") - self.cfgDir = os.path.join(self.path,"sim","psim","cfg") - self.cfgPath = os.path.join(self.path,"sim","psim","cfg","taos.cfg") - - cmd = "rm -rf " + self.logDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "mkdir -p " + self.logDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "rm -rf " + self.cfgDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "mkdir -p " + self.cfgDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "touch " + self.cfgPath - if os.system(cmd) != 0: - tdLog.exit(cmd) - - if self.testCluster: - self.cfg("masterIp", "192.168.0.1") - self.cfg("secondIp", "192.168.0.2") - self.cfg("logDir", self.logDir) - - for key, value in self.cfgDict.items(): - self.cfg(key, value) - - tdLog.debug("psim is deployed and configured by %s" % (self.cfgPath)) - - -class TDDnode: - def __init__(self, index): - self.index = index - self.running = 0 - self.deployed = 0 - self.testCluster = False - self.valgrind = 0 - - def init(self, path): - self.path = path - - def setTestCluster(self, value): - self.testCluster = value - - def setValgrind(self, value): - self.valgrind = value - - def getDataSize(self): - totalSize = 0 - - if (self.deployed == 1): - for dirpath, dirnames, filenames in os.walk(self.dataDir): - for f in filenames: - fp = os.path.join(dirpath, f) - - if not os.path.islink(fp): - totalSize = totalSize + os.path.getsize(fp) - - return totalSize - - def deploy(self): - self.logDir = os.path.join(self.path,"sim","dnode%d" % self.index, "log") - self.dataDir = os.path.join(self.path,"sim","dnode%d" % self.index, "data") - self.cfgDir = os.path.join(self.path,"sim","dnode%d" % self.index, "cfg") - self.cfgPath = os.path.join(self.path,"sim","dnode%d" % self.index, "cfg","taos.cfg") - - cmd = "rm -rf " + self.dataDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "rm -rf " + self.logDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "rm -rf " + self.cfgDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "mkdir -p " + self.dataDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "mkdir -p " + self.logDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "mkdir -p " + self.cfgDir - if os.system(cmd) != 0: - tdLog.exit(cmd) - - cmd = "touch " + self.cfgPath - if os.system(cmd) != 0: - tdLog.exit(cmd) - - if self.testCluster: - self.startIP() - - if self.testCluster: - self.cfg("masterIp", "192.168.0.1") - self.cfg("secondIp", "192.168.0.2") - self.cfg("publicIp", "192.168.0.%d" % (self.index)) - self.cfg("internalIp", "192.168.0.%d" % (self.index)) - self.cfg("privateIp", "192.168.0.%d" % (self.index)) - self.cfg("dataDir", self.dataDir) - self.cfg("logDir", self.logDir) - self.cfg("numOfLogLines", "100000000") - self.cfg("mnodeEqualVnodeNum", "0") - self.cfg("walLevel", "2") - self.cfg("fsync", "1000") - self.cfg("statusInterval", "1") - self.cfg("numOfMnodes", "3") - self.cfg("numOfThreadsPerCore", "2.0") - self.cfg("monitor", "0") - self.cfg("maxVnodeConnections", "30000") - self.cfg("maxMgmtConnections", "30000") - self.cfg("maxMeterConnections", "30000") - self.cfg("maxShellConns", "30000") - self.cfg("locale", "en_US.UTF-8") - self.cfg("charset", "UTF-8") - self.cfg("asyncLog", "0") - self.cfg("anyIp", "0") - self.cfg("dDebugFlag", "135") - self.cfg("mDebugFlag", "135") - self.cfg("sdbDebugFlag", "135") - self.cfg("rpcDebugFlag", "135") - self.cfg("tmrDebugFlag", "131") - self.cfg("cDebugFlag", "135") - self.cfg("httpDebugFlag", "135") - self.cfg("monitorDebugFlag", "135") - self.cfg("udebugFlag", "135") - self.cfg("jnidebugFlag", "135") - self.cfg("qdebugFlag", "135") - self.deployed = 1 - tdLog.debug( - "dnode:%d is deployed and configured by %s" % - (self.index, self.cfgPath)) - - def getBuildPath(self): - selfPath = os.path.dirname(os.path.realpath(__file__)) - - if ("community" in selfPath): - projPath = selfPath[:selfPath.find("community")] - else: - projPath = selfPath[:selfPath.find("tests")] - - for root, dirs, files in os.walk(projPath): - if ("taosd" in files): - rootRealPath = os.path.dirname(os.path.realpath(root)) - if ("packaging" not in rootRealPath): - buildPath = root[:len(root)-len("/build/bin")] - break - return buildPath - - def start(self): - buildPath = self.getBuildPath() - - if (buildPath == ""): - tdLog.exit("taosd not found!") - else: - tdLog.info("taosd found in %s" % buildPath) - - binPath = buildPath + "/build/bin/taosd" - - if self.deployed == 0: - tdLog.exit("dnode:%d is not deployed" % (self.index)) - - if self.valgrind == 0: - cmd = "nohup %s -c %s --alloc-random-fail --random-file-fail-factor 5 > /dev/null 2>&1 & " % ( - binPath, self.cfgDir) - else: - valgrindCmdline = "valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes" - - cmd = "nohup %s %s -c %s 2>&1 & " % ( - valgrindCmdline, binPath, self.cfgDir) - - print(cmd) - - if os.system(cmd) != 0: - tdLog.exit(cmd) - self.running = 1 - tdLog.debug("dnode:%d is running with %s " % (self.index, cmd)) - - tdLog.debug("wait 5 seconds for the dnode:%d to start." % (self.index)) - time.sleep(5) - - def stop(self): - if self.valgrind == 0: - toBeKilled = "taosd" - else: - toBeKilled = "valgrind.bin" - - if self.running != 0: - psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - while(processID): - killCmd = "kill -INT %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - for port in range(6030, 6041): - fuserCmd = "fuser -k -n tcp %d" % port - os.system(fuserCmd) - if self.valgrind: - time.sleep(2) - - self.running = 0 - tdLog.debug("dnode:%d is stopped by kill -INT" % (self.index)) - - def forcestop(self): - if self.valgrind == 0: - toBeKilled = "taosd" - else: - toBeKilled = "valgrind.bin" - - if self.running != 0: - psCmd = "ps -ef|grep -w %s| grep -v grep | awk '{print $2}'" % toBeKilled - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - while(processID): - killCmd = "kill -KILL %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - for port in range(6030, 6041): - fuserCmd = "fuser -k -n tcp %d" % port - os.system(fuserCmd) - if self.valgrind: - time.sleep(2) - - self.running = 0 - tdLog.debug("dnode:%d is stopped by kill -KILL" % (self.index)) - - def startIP(self): - cmd = "sudo ifconfig lo:%d 192.168.0.%d up" % (self.index, self.index) - if os.system(cmd) != 0: - tdLog.exit(cmd) - - def stopIP(self): - cmd = "sudo ifconfig lo:%d 192.168.0.%d down" % ( - self.index, self.index) - if os.system(cmd) != 0: - tdLog.exit(cmd) - - def cfg(self, option, value): - cmd = "echo %s %s >> %s" % (option, value, self.cfgPath) - if os.system(cmd) != 0: - tdLog.exit(cmd) - - def getDnodeRootDir(self, index): - dnodeRootDir = os.path.join(self.path,"sim","psim","dnode%d" % index) - return dnodeRootDir - - def getDnodesRootDir(self): - dnodesRootDir = os.path.join(self.path,"sim","psim") - return dnodesRootDir - - -class TDDnodes: - def __init__(self): - self.dnodes = [] - self.dnodes.append(TDDnode(1)) - self.dnodes.append(TDDnode(2)) - self.dnodes.append(TDDnode(3)) - self.dnodes.append(TDDnode(4)) - self.dnodes.append(TDDnode(5)) - self.dnodes.append(TDDnode(6)) - self.dnodes.append(TDDnode(7)) - self.dnodes.append(TDDnode(8)) - self.dnodes.append(TDDnode(9)) - self.dnodes.append(TDDnode(10)) - self.simDeployed = False - - def init(self, path): - psCmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'" - processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") - while(processID): - killCmd = "kill -TERM %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'" - processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") - while(processID): - killCmd = "kill -TERM %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - binPath = os.path.dirname(os.path.realpath(__file__)) - binPath = binPath + "/../../../debug/" - tdLog.debug("binPath %s" % (binPath)) - binPath = os.path.realpath(binPath) - tdLog.debug("binPath real path %s" % (binPath)) - - # cmd = "sudo cp %s/build/lib/libtaos.so /usr/local/lib/taos/" % (binPath) - # tdLog.debug(cmd) - # os.system(cmd) - - # cmd = "sudo cp %s/build/bin/taos /usr/local/bin/taos/" % (binPath) - # if os.system(cmd) != 0 : - # tdLog.exit(cmd) - # tdLog.debug("execute %s" % (cmd)) - - # cmd = "sudo cp %s/build/bin/taosd /usr/local/bin/taos/" % (binPath) - # if os.system(cmd) != 0 : - # tdLog.exit(cmd) - # tdLog.debug("execute %s" % (cmd)) - - if path == "": - # self.path = os.path.expanduser('~') - self.path = os.path.abspath(binPath + "../../") - else: - self.path = os.path.realpath(path) - - for i in range(len(self.dnodes)): - self.dnodes[i].init(self.path) - - self.sim = TDSimClient() - self.sim.init(self.path) - - def setTestCluster(self, value): - self.testCluster = value - - def setValgrind(self, value): - self.valgrind = value - - def deploy(self, index): - self.sim.setTestCluster(self.testCluster) - - if (self.simDeployed == False): - self.sim.deploy() - self.simDeployed = True - - self.check(index) - self.dnodes[index - 1].setTestCluster(self.testCluster) - self.dnodes[index - 1].setValgrind(self.valgrind) - self.dnodes[index - 1].deploy() - - def cfg(self, index, option, value): - self.check(index) - self.dnodes[index - 1].cfg(option, value) - - def start(self, index): - self.check(index) - self.dnodes[index - 1].start() - - def stop(self, index): - self.check(index) - self.dnodes[index - 1].stop() - - def getDataSize(self, index): - self.check(index) - return self.dnodes[index - 1].getDataSize() - - def forcestop(self, index): - self.check(index) - self.dnodes[index - 1].forcestop() - - def startIP(self, index): - self.check(index) - - if self.testCluster: - self.dnodes[index - 1].startIP() - - def stopIP(self, index): - self.check(index) - - if self.dnodes[index - 1].testCluster: - self.dnodes[index - 1].stopIP() - - def check(self, index): - if index < 1 or index > 10: - tdLog.exit("index:%d should on a scale of [1, 10]" % (index)) - - def stopAll(self): - tdLog.info("stop all dnodes") - for i in range(len(self.dnodes)): - self.dnodes[i].stop() - - psCmd = "ps -ef | grep -w taosd | grep 'root' | grep -v grep | awk '{print $2}'" - processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") - if processID: - cmd = "sudo systemctl stop taosd" - os.system(cmd) - # if os.system(cmd) != 0 : - # tdLog.exit(cmd) - psCmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'" - processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") - while(processID): - killCmd = "kill -TERM %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - psCmd = "ps -ef|grep -w valgrind.bin| grep -v grep | awk '{print $2}'" - processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") - while(processID): - killCmd = "kill -TERM %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - # if os.system(cmd) != 0 : - # tdLog.exit(cmd) - - def getDnodesRootDir(self): - dnodesRootDir = "%s/sim" % (self.path) - return dnodesRootDir - - def getSimCfgPath(self): - return self.sim.getCfgDir() - - def getSimLogPath(self): - return self.sim.getLogDir() - - def addSimExtraCfg(self, option, value): - self.sim.addExtraCfg(option, value) - - -tdDnodes = TDDnodes() diff --git a/tests/army/frame/cluster.py b/tests/army/frame/server/cluster.py similarity index 91% rename from tests/army/frame/cluster.py rename to tests/army/frame/server/cluster.py index 4da53840c0..ade8ac39a2 100644 --- a/tests/army/frame/cluster.py +++ b/tests/army/frame/server/cluster.py @@ -8,7 +8,7 @@ import socket from frame.log import * from frame.sql import * from frame.cases import * -from frame.dnodes import * +from frame.server.dnodes import * from frame.common import * class ClusterDnodes(TDDnodes): @@ -35,17 +35,21 @@ class ConfigureyCluster: self.startPort = 6030 self.portStep = 100 self.mnodeNums = 0 + self.level = 0 + self.disk = 0 - def configure_cluster(self ,dnodeNums=5,mnodeNums=0,independentMnode=True,startPort=6030,portStep=100,hostname="%s"%hostname): + def configure_cluster(self ,dnodeNums=5, mnodeNums=0, independentMnode=True, startPort=6030, portStep=100, hostname="%s"%hostname, level=1, disk=1): self.startPort=int(startPort) self.portStep=int(portStep) self.hostname=hostname self.dnodeNums = int(dnodeNums) self.mnodeNums = int(mnodeNums) + self.level = int(level) + self.disk = int(disk) self.dnodes = [] startPort_sec = int(startPort+portStep) for num in range(1, (self.dnodeNums+1)): - dnode = TDDnode(num) + dnode = TDDnode(num, self.level, self.disk) dnode.addExtraCfg("firstEp", f"{hostname}:{self.startPort}") dnode.addExtraCfg("fqdn", f"{hostname}") dnode.addExtraCfg("serverPort", f"{self.startPort + (num-1)*self.portStep}") diff --git a/tests/army/frame/dnodes.py b/tests/army/frame/server/dnodes.py similarity index 98% rename from tests/army/frame/dnodes.py rename to tests/army/frame/server/dnodes.py index 5581f29a57..7eee6f4bb6 100644 --- a/tests/army/frame/dnodes.py +++ b/tests/army/frame/server/dnodes.py @@ -115,8 +115,11 @@ class TDSimClient: class TDDnode: - def __init__(self, index): + def __init__(self, index, level, disk): self.index = index + self.level = level + self.disk = disk + self.dataDir = [] self.running = 0 self.deployed = 0 self.testCluster = False @@ -210,13 +213,18 @@ class TDDnode: def deploy(self, *updatecfgDict): self.logDir = os.path.join(self.path,"sim","dnode%d" % self.index, "log") - self.dataDir = os.path.join(self.path,"sim","dnode%d" % self.index, "data") + simPath = os.path.join(self.path, "sim", "dnode%d" % self.index) + for i in range(1, self.level): + for j in range(1, self.disk): + eDir = os.path.join(simPath, f"data{i}{j}") + self.dataDir.append(eDir) self.cfgDir = os.path.join(self.path,"sim","dnode%d" % self.index, "cfg") self.cfgPath = os.path.join(self.path,"sim","dnode%d" % self.index, "cfg","taos.cfg") - - cmd = "rm -rf " + self.dataDir - if os.system(cmd) != 0: - tdLog.exit(cmd) + + for eDir in self.dataDir: + cmd = "rm -rf " + eDir + if os.system(cmd) != 0: + tdLog.exit(cmd) cmd = "rm -rf " + self.logDir if os.system(cmd) != 0: @@ -229,7 +237,8 @@ class TDDnode: # cmd = "mkdir -p " + self.dataDir # if os.system(cmd) != 0: # tdLog.exit(cmd) - os.makedirs(self.dataDir) + for eDir in self.dataDir: + os.makedirs(self.dataDir) # cmd = "mkdir -p " + self.logDir # if os.system(cmd) != 0: diff --git a/tests/army/frame/sql.py b/tests/army/frame/sql.py index d7dce2bc3e..409dc6c96c 100644 --- a/tests/army/frame/sql.py +++ b/tests/army/frame/sql.py @@ -630,6 +630,7 @@ class TDSql: tdLog.exit("taosd state is %d != expect:%d" %args) pass + ''' def haveFile(self, dir, state): if os.path.exists(dir) and os.path.isdir(dir): if not os.listdir(dir): @@ -644,6 +645,7 @@ class TDSql: tdLog.exit("dir: %s is not empty, expect: empty" %dir) else: tdLog.exit("dir: %s doesn't exist" %dir) + def createDir(self, dir): if os.path.exists(dir): shrmtree(dir) @@ -651,5 +653,6 @@ class TDSql: os.makedirs( dir, 755 ) tdLog.info("dir: %s is created" %dir) pass +''' tdSql = TDSql() diff --git a/tests/army/frame/srvCtl.py b/tests/army/frame/srvCtl.py new file mode 100644 index 0000000000..e7ef08cde9 --- /dev/null +++ b/tests/army/frame/srvCtl.py @@ -0,0 +1,27 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +import time +import datetime + +class srvCtl: + def __init__(self): + # record server information + self.dnodeNum = 0 + self.mnodeNum = 0 + self.mLevel = 0 + self.mLevelDisk = 0 + +sc = srvCtl() \ No newline at end of file diff --git a/tests/army/test.py b/tests/army/test.py index cc114e0d16..ae873fb9d6 100644 --- a/tests/army/test.py +++ b/tests/army/test.py @@ -28,9 +28,9 @@ import importlib import toml from frame.log import * -from frame.dnodes import * +from frame.server.dnodes import * +from frame.server.cluster import * from frame.cases import * -from frame.cluster import * from frame.taosadapter import * import taos @@ -111,6 +111,9 @@ if __name__ == "__main__": asan = False independentMnode = False previousCluster = False + level = 1 + disk = 1 + opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:N:M:Q:C:RWD:n:i:aP', [ 'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'restart', 'updateCfgDict', 'killv', 'execCmd','dnodeNums','mnodeNums','queryPolicy','createDnodeNums','restful','websocket','adaptercfgupdate','replicaVar','independentMnode','previous']) for key, value in opts: @@ -139,6 +142,8 @@ if __name__ == "__main__": tdLog.printNoPrefix('-i independentMnode Mnode') tdLog.printNoPrefix('-a address sanitizer mode') tdLog.printNoPrefix('-P run case with [P]revious cluster, do not create new cluster to run case.') + tdLog.printNoPrefix('-L set multiple level number. range 1 ~ 3') + tdLog.printNoPrefix('-D set disk number on each level. range 1 ~ 10') sys.exit(0) @@ -226,6 +231,12 @@ if __name__ == "__main__": if key in ['-P', '--previous']: previousCluster = True + if key in ['-L', '--level']: + level = value + + if key in ['-D', '--disk']: + disk = value + # # do exeCmd command # @@ -391,7 +402,7 @@ if __name__ == "__main__": tdLog.exit(f"alter queryPolicy to {queryPolicy} failed") else : tdLog.debug("create an cluster with %s nodes and make %s dnode as independent mnode"%(dnodeNums,mnodeNums)) - dnodeslist = cluster.configure_cluster(dnodeNums=dnodeNums, mnodeNums=mnodeNums, independentMnode=independentMnode) + dnodeslist = cluster.configure_cluster(dnodeNums=dnodeNums, mnodeNums=mnodeNums, independentMnode=independentMnode, level=level, disk=disk) tdDnodes = ClusterDnodes(dnodeslist) tdDnodes.init(deployPath, masterIp) tdDnodes.setTestCluster(testCluster) @@ -498,6 +509,7 @@ if __name__ == "__main__": else: tdLog.info("not need to query") else: + # except windows tdDnodes.setKillValgrind(killValgrind) tdDnodes.init(deployPath, masterIp) tdDnodes.setTestCluster(testCluster) @@ -574,7 +586,8 @@ if __name__ == "__main__": # dnode > 1 cluster tdLog.debug("create an cluster with %s nodes and make %s dnode as independent mnode"%(dnodeNums,mnodeNums)) print(independentMnode,"independentMnode valuse") - dnodeslist = cluster.configure_cluster(dnodeNums=dnodeNums, mnodeNums=mnodeNums, independentMnode=independentMnode) + # create dnode list + dnodeslist = cluster.configure_cluster(dnodeNums=dnodeNums, mnodeNums=mnodeNums, independentMnode=independentMnode, level=level, disk=disk) tdDnodes = ClusterDnodes(dnodeslist) tdDnodes.init(deployPath, masterIp) tdDnodes.setTestCluster(testCluster) From 9e7a4b8a868df69614f3f7d474cf3a09f39b7799 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 23 Dec 2023 13:09:02 +0800 Subject: [PATCH 45/94] feat: test.py add -L --level -D --disk options --- tests/army/test.py | 2 +- tests/parallel_test/cases.task | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/army/test.py b/tests/army/test.py index ae873fb9d6..25cb9b383d 100644 --- a/tests/army/test.py +++ b/tests/army/test.py @@ -218,7 +218,7 @@ if __name__ == "__main__": if key in ['-a', '--asan']: asan = True - if key in ['-D', '--adaptercfgupdate']: + if key in ['-U', '--adaptercfgupdate']: try: adaptercfgupdate = eval(base64.b64decode(value.encode()).decode()) except: diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index bcdd143cfc..19a44767a7 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -6,7 +6,7 @@ ,,y,unit-test,bash test.sh #army-test -,,y,army,./pytest.sh python3 ./test.py -f empty.py +,,y,army,./pytest.sh python3 ./test.py -f enterprise/multi-level/mlevel_basic.py -N 3 -L 3 -D 2 #system test ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/stream_basic.py From 8a714db17df323425fb4ca85b5edefda625a3cc4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 23 Dec 2023 15:33:50 +0800 Subject: [PATCH 46/94] fix: fix deadlock. --- source/libs/stream/src/streamMeta.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 25f0fbf991..7217a49ff8 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -1356,7 +1356,7 @@ SArray* streamMetaSendMsgBeforeCloseTasks(SStreamMeta* pMeta) { int32_t numOfTasks = taosArrayGetSize(pTaskList); for (int32_t i = 0; i < numOfTasks; ++i) { SStreamTaskId* pTaskId = taosArrayGet(pTaskList, i); - SStreamTask* pTask = streamMetaAcquireTask(pMeta, pTaskId->streamId, pTaskId->taskId); + SStreamTask* pTask = streamMetaAcquireTaskNoLock(pMeta, pTaskId->streamId, pTaskId->taskId); if (pTask == NULL) { continue; } From 266ac30d96336204f4ebe040faf624d01dcebb37 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 23 Dec 2023 16:28:39 +0800 Subject: [PATCH 47/94] feat: support -L -D for test.py submit --- tests/army/frame/common.py | 2 +- tests/army/frame/server/dnodes.py | 23 +++++++++++++++++------ tests/army/test.py | 7 ++++--- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/tests/army/frame/common.py b/tests/army/frame/common.py index b8704080c0..5cdb3f9f46 100644 --- a/tests/army/frame/common.py +++ b/tests/army/frame/common.py @@ -23,7 +23,7 @@ import taos from frame.log import * from frame.sql import * from frame.cases import * -from TDengine.tests.army.frame.server.dnodes import * +from frame.server.dnodes import * from frame.common import * from frame.constant import * from dataclasses import dataclass,field diff --git a/tests/army/frame/server/dnodes.py b/tests/army/frame/server/dnodes.py index 7eee6f4bb6..8368a1573e 100644 --- a/tests/army/frame/server/dnodes.py +++ b/tests/army/frame/server/dnodes.py @@ -115,7 +115,7 @@ class TDSimClient: class TDDnode: - def __init__(self, index, level, disk): + def __init__(self, index=1, level=1, disk=1): self.index = index self.level = level self.disk = disk @@ -212,12 +212,19 @@ class TDDnode: self.remote_conn.run("python3 ./test.py %s -d %s -e %s"%(valgrindStr,remoteCfgDictStr,execCmdStr)) def deploy(self, *updatecfgDict): + # logDir self.logDir = os.path.join(self.path,"sim","dnode%d" % self.index, "log") + # dataDir simPath = os.path.join(self.path, "sim", "dnode%d" % self.index) - for i in range(1, self.level): - for j in range(1, self.disk): + primary = 1 + for i in range(self.level): + for j in range(self.disk): eDir = os.path.join(simPath, f"data{i}{j}") - self.dataDir.append(eDir) + self.dataDir.append(f"{eDir} {i} {primary}") + if primary == 1: + primary = 0 + + # taos.cfg self.cfgDir = os.path.join(self.path,"sim","dnode%d" % self.index, "cfg") self.cfgPath = os.path.join(self.path,"sim","dnode%d" % self.index, "cfg","taos.cfg") @@ -238,7 +245,7 @@ class TDDnode: # if os.system(cmd) != 0: # tdLog.exit(cmd) for eDir in self.dataDir: - os.makedirs(self.dataDir) + os.makedirs(eDir.split(' ')[0]) # cmd = "mkdir -p " + self.logDir # if os.system(cmd) != 0: @@ -284,7 +291,11 @@ class TDDnode: self.addExtraCfg(key, value) if (self.remoteIP == ""): for key, value in self.cfgDict.items(): - self.cfg(key, value) + if type(value) == list: + for v in value: + self.cfg(key, v) + else: + self.cfg(key, value) else: self.remoteExec(self.cfgDict, "tdDnodes.deploy(%d,updateCfgDict)"%self.index) diff --git a/tests/army/test.py b/tests/army/test.py index 25cb9b383d..83d065b24b 100644 --- a/tests/army/test.py +++ b/tests/army/test.py @@ -114,8 +114,9 @@ if __name__ == "__main__": level = 1 disk = 1 - opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:N:M:Q:C:RWD:n:i:aP', [ - 'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'restart', 'updateCfgDict', 'killv', 'execCmd','dnodeNums','mnodeNums','queryPolicy','createDnodeNums','restful','websocket','adaptercfgupdate','replicaVar','independentMnode','previous']) + opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:N:M:Q:C:RWU:n:i:aP:L:D:', [ + 'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'restart', 'updateCfgDict', 'killv', 'execCmd','dnodeNums','mnodeNums', + 'queryPolicy','createDnodeNums','restful','websocket','adaptercfgupdate','replicaVar','independentMnode',"asan",'previous','level','disk']) for key, value in opts: if key in ['-h', '--help']: tdLog.printNoPrefix( @@ -137,7 +138,7 @@ if __name__ == "__main__": tdLog.printNoPrefix('-C create Dnode Numbers in one cluster') tdLog.printNoPrefix('-R restful realization form') tdLog.printNoPrefix('-W websocket connection') - tdLog.printNoPrefix('-D taosadapter update cfg dict ') + tdLog.printNoPrefix('-U taosadapter update cfg dict ') tdLog.printNoPrefix('-n the number of replicas') tdLog.printNoPrefix('-i independentMnode Mnode') tdLog.printNoPrefix('-a address sanitizer mode') From c61203b55a98707c12675358c2e6e56b175395e3 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 23 Dec 2023 16:37:21 +0800 Subject: [PATCH 48/94] fix: fix deadlock. --- source/libs/stream/src/streamMeta.c | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 7217a49ff8..46e3dded11 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -1065,28 +1065,16 @@ static int32_t metaHeartbeatToMnodeImpl(SStreamMeta* pMeta) { SStreamHbMsg hbMsg = {0}; SEpSet epset = {0}; bool hasMnodeEpset = false; - int64_t stage = 0; + int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta); - streamMetaRLock(pMeta); - - int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta); hbMsg.vgId = pMeta->vgId; - stage = pMeta->stage; - - SArray* pIdList = taosArrayDup(pMeta->pTaskList, NULL); - - streamMetaRUnLock(pMeta); - hbMsg.pTaskStatus = taosArrayInit(numOfTasks, sizeof(STaskStatusEntry)); hbMsg.pUpdateNodes = taosArrayInit(numOfTasks, sizeof(int32_t)); for (int32_t i = 0; i < numOfTasks; ++i) { - STaskId* pId = taosArrayGet(pIdList, i); + STaskId* pId = taosArrayGet(pMeta->pTaskList, i); - streamMetaRLock(pMeta); SStreamTask** pTask = taosHashGet(pMeta->pTasksMap, pId, sizeof(*pId)); - streamMetaRUnLock(pMeta); - if (pTask == NULL) { continue; } @@ -1100,7 +1088,7 @@ static int32_t metaHeartbeatToMnodeImpl(SStreamMeta* pMeta) { .id = *pId, .status = streamTaskGetStatus(*pTask, NULL), .nodeId = hbMsg.vgId, - .stage = stage, + .stage = pMeta->stage, .inputQUsed = SIZE_IN_MiB(streamQueueGetItemSize((*pTask)->inputq.queue)), }; @@ -1170,7 +1158,6 @@ static int32_t metaHeartbeatToMnodeImpl(SStreamMeta* pMeta) { _end: streamMetaClearHbMsg(&hbMsg); - taosArrayDestroy(pIdList); return TSDB_CODE_SUCCESS; } @@ -1210,7 +1197,9 @@ void metaHbToMnode(void* param, void* tmrId) { } stDebug("vgId:%d build stream task hb, leader:%d", pMeta->vgId, (pMeta->role == NODE_ROLE_LEADER)); + streamMetaRLock(pMeta); metaHeartbeatToMnodeImpl(pMeta); + streamMetaRUnLock(pMeta); taosTmrReset(metaHbToMnode, META_HB_CHECK_INTERVAL, param, streamTimer, &pMeta->pHbInfo->hbTmr); taosReleaseRef(streamMetaId, rid); From 4b6bd14295954abf828efd2a78bcdd04512b1d17 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 23 Dec 2023 16:39:13 +0800 Subject: [PATCH 49/94] fix: if level=1 disk=1 save old data folder --- tests/army/frame/server/dnodes.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/tests/army/frame/server/dnodes.py b/tests/army/frame/server/dnodes.py index 8368a1573e..1e51fbc3be 100644 --- a/tests/army/frame/server/dnodes.py +++ b/tests/army/frame/server/dnodes.py @@ -217,12 +217,16 @@ class TDDnode: # dataDir simPath = os.path.join(self.path, "sim", "dnode%d" % self.index) primary = 1 - for i in range(self.level): - for j in range(self.disk): - eDir = os.path.join(simPath, f"data{i}{j}") - self.dataDir.append(f"{eDir} {i} {primary}") - if primary == 1: - primary = 0 + if self.level == 1 and self.disk == 1: + eDir = os.path.join(simPath, "data") + self.dataDir.append(eDir) + else: + for i in range(self.level): + for j in range(self.disk): + eDir = os.path.join(simPath, f"data{i}{j}") + self.dataDir.append(f"{eDir} {i} {primary}") + if primary == 1: + primary = 0 # taos.cfg self.cfgDir = os.path.join(self.path,"sim","dnode%d" % self.index, "cfg") From 25f798e42f24c90917192ce30c705053d156d43b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 23 Dec 2023 23:29:30 +0800 Subject: [PATCH 50/94] fix(stream): record the start failure tasks --- source/dnode/vnode/src/tqCommon/tqCommon.c | 2 -- source/libs/stream/src/streamMeta.c | 20 ++++++++++++++------ source/libs/stream/src/streamStart.c | 7 ++++--- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index af9363e363..f576345f64 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -661,8 +661,6 @@ static int32_t restartStreamTasks(SStreamMeta* pMeta, bool isLeader) { if (isLeader && !tsDisableStream) { tqStreamTaskResetStatus(pMeta); streamMetaWUnLock(pMeta); - tqInfo("vgId:%d start all stream tasks after reload tasks from disk", vgId); - streamMetaStartAllTasks(pMeta); } else { streamMetaResetStartInfo(&pMeta->startInfo); diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 46e3dded11..a46853fc26 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -1429,8 +1429,9 @@ int32_t streamMetaStartAllTasks(SStreamMeta* pMeta) { int32_t vgId = pMeta->vgId; int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); - stDebug("vgId:%d start to check all %d stream task(s) downstream status", vgId, numOfTasks); + stInfo("vgId:%d start to check all %d stream task(s) downstream status", vgId, numOfTasks); if (numOfTasks == 0) { + stInfo("vgId:%d start tasks completed", pMeta->vgId); return TSDB_CODE_SUCCESS; } @@ -1443,16 +1444,20 @@ int32_t streamMetaStartAllTasks(SStreamMeta* pMeta) { streamMetaWUnLock(pMeta); // broadcast the check downstream tasks msg + int64_t now = taosGetTimestampMs(); + for (int32_t i = 0; i < numOfTasks; ++i) { SStreamTaskId* pTaskId = taosArrayGet(pTaskList, i); - SStreamTask* pTask = streamMetaAcquireTask(pMeta, pTaskId->streamId, pTaskId->taskId); + + SStreamTask* pTask = streamMetaAcquireTask(pMeta, pTaskId->streamId, pTaskId->taskId); if (pTask == NULL) { - streamMetaUpdateTaskDownstreamStatus(pMeta, pTaskId->streamId, pTaskId->taskId, 0, - taosGetTimestampMs(), false); + stError("vgId:%d failed to acquire task:0x%x during start tasks", pMeta->vgId, pTaskId->taskId); + streamMetaUpdateTaskDownstreamStatus(pMeta, pTaskId->streamId, pTaskId->taskId, 0, now, false); continue; } // fill-history task can only be launched by related stream tasks. + STaskExecStatisInfo* pInfo = &pTask->execInfo; if (pTask->info.fillHistory == 1) { streamMetaReleaseTask(pMeta, pTask); continue; @@ -1465,8 +1470,7 @@ int32_t streamMetaStartAllTasks(SStreamMeta* pMeta) { streamLaunchFillHistoryTask(pTask); } - streamMetaUpdateTaskDownstreamStatus(pMeta, pTask->id.streamId, pTask->id.taskId, pTask->execInfo.init, - pTask->execInfo.start, true); + streamMetaUpdateTaskDownstreamStatus(pMeta, pTaskId->streamId, pTaskId->taskId, pInfo->init, pInfo->start, true); streamMetaReleaseTask(pMeta, pTask); continue; } @@ -1474,12 +1478,16 @@ int32_t streamMetaStartAllTasks(SStreamMeta* pMeta) { EStreamTaskEvent event = (HAS_RELATED_FILLHISTORY_TASK(pTask)) ? TASK_EVENT_INIT_STREAM_SCANHIST : TASK_EVENT_INIT; int32_t ret = streamTaskHandleEvent(pTask->status.pSM, event); if (ret != TSDB_CODE_SUCCESS) { + stError("vgId:%d failed to handle event:%d", pMeta->vgId, event); code = ret; + + streamMetaUpdateTaskDownstreamStatus(pMeta, pTaskId->streamId, pTaskId->taskId, pInfo->init, pInfo->start, false); } streamMetaReleaseTask(pMeta, pTask); } + stInfo("vgId:%d start tasks completed", pMeta->vgId); taosArrayDestroy(pTaskList); return code; } \ No newline at end of file diff --git a/source/libs/stream/src/streamStart.c b/source/libs/stream/src/streamStart.c index 3bef6adf57..267dadf109 100644 --- a/source/libs/stream/src/streamStart.c +++ b/source/libs/stream/src/streamStart.c @@ -1114,9 +1114,9 @@ int32_t streamMetaUpdateTaskDownstreamStatus(SStreamMeta* pMeta, int64_t streamI pStartInfo->readyTs = taosGetTimestampMs(); pStartInfo->elapsedTime = (pStartInfo->startTs != 0) ? pStartInfo->readyTs - pStartInfo->startTs : 0; - stDebug("vgId:%d all %d task(s) check downstream completed, last completed task:0x%x startTs:%" PRId64 + stDebug("vgId:%d all %d task(s) check downstream completed, last completed task:0x%x (succ:%d) startTs:%" PRId64 ", readyTs:%" PRId64 " total elapsed time:%.2fs", - pMeta->vgId, numOfTotal, taskId, pStartInfo->startTs, pStartInfo->readyTs, + pMeta->vgId, numOfTotal, taskId, ready, pStartInfo->startTs, pStartInfo->readyTs, pStartInfo->elapsedTime / 1000.0); // print the initialization elapsed time and info @@ -1128,7 +1128,8 @@ int32_t streamMetaUpdateTaskDownstreamStatus(SStreamMeta* pMeta, int64_t streamI pStartInfo->completeFn(pMeta); } else { streamMetaWUnLock(pMeta); - stDebug("vgId:%d recv check downstream results:%d, total:%d", pMeta->vgId, numOfRecv, numOfTotal); + stDebug("vgId:%d recv check downstream results, s-task:0x%x succ:%d, received:%d, total:%d", pMeta->vgId, taskId, + ready, numOfRecv, numOfTotal); } return TSDB_CODE_SUCCESS; From 30498a40285565dfdfa7f4afcfac29906162619c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 24 Dec 2023 02:24:46 +0800 Subject: [PATCH 51/94] refactor: do some internal refactor. --- source/dnode/vnode/src/tq/tq.c | 5 +++-- source/libs/stream/src/streamMeta.c | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index e8e88e52f7..c8cc4f9d54 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1259,10 +1259,11 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) if (pTask->status.downstreamReady != 1) { pTask->chkInfo.failedId = req.checkpointId; // record the latest failed checkpoint id pTask->chkInfo.checkpointingId = req.checkpointId; + pTask->chkInfo.transId = req.transId; tqError("s-task:%s not ready for checkpoint, since downstream not ready, ignore this checkpoint:%" PRId64 - ", set it failure", - pTask->id.idStr, req.checkpointId); + ", transId:%d set it failed", + pTask->id.idStr, req.checkpointId, req.transId); streamMetaReleaseTask(pMeta, pTask); SRpcMsg rsp = {0}; diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index a46853fc26..9e20a2997c 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -1102,6 +1102,7 @@ static int32_t metaHeartbeatToMnodeImpl(SStreamMeta* pMeta) { entry.checkpointFailed = ((*pTask)->chkInfo.failedId >= (*pTask)->chkInfo.checkpointingId); entry.activeCheckpointId = (*pTask)->chkInfo.checkpointingId; entry.chkpointTransId = (*pTask)->chkInfo.transId; + stInfo("s-task:%s send kill checkpoint trans info, transId:%d", (*pTask)->id.idStr, (*pTask)->chkInfo.transId); } if ((*pTask)->exec.pWalReader != NULL) { From bf895fadcb144cb31951cb54f3e0874290f8c8d2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 24 Dec 2023 03:05:29 +0800 Subject: [PATCH 52/94] fix(stream): fix invalid read. --- source/libs/stream/src/streamDispatch.c | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index f0ee03810b..bfa269b0d5 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -352,6 +352,7 @@ static int32_t doBuildDispatchMsg(SStreamTask* pTask, const SStreamDataBlock* pD return code; } + // it's a new vnode to receive dispatch msg, so add one if (pReqs[j].blockNum == 0) { atomic_add_fetch_32(&pTask->outputInfo.shuffleDispatcher.waitingRspCnt, 1); } @@ -372,8 +373,15 @@ static int32_t doBuildDispatchMsg(SStreamTask* pTask, const SStreamDataBlock* pD pTask->msgInfo.pData = pReqs; } - stDebug("s-task:%s build dispatch msg success, msgId:%d, stage:%"PRId64" %p", pTask->id.idStr, pTask->execInfo.dispatch, - pTask->pMeta->stage, pTask->msgInfo.pData); + if (pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH) { + stDebug("s-task:%s build dispatch msg success, msgId:%d, stage:%" PRId64 " %p", pTask->id.idStr, + pTask->execInfo.dispatch, pTask->pMeta->stage, pTask->msgInfo.pData); + } else { + stDebug("s-task:%s build dispatch msg success, msgId:%d, stage:%" PRId64 " dstVgNum:%d %p", pTask->id.idStr, + pTask->execInfo.dispatch, pTask->pMeta->stage, pTask->outputInfo.shuffleDispatcher.waitingRspCnt, + pTask->msgInfo.pData); + } + return code; } @@ -395,9 +403,11 @@ static int32_t sendDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pDispatch SArray* vgInfo = pTask->outputInfo.shuffleDispatcher.dbInfo.pVgroupInfos; int32_t numOfVgroups = taosArrayGetSize(vgInfo); - stDebug("s-task:%s (child taskId:%d) start to shuffle-dispatch blocks to %d vgroup(s), msgId:%d", id, - pTask->info.selfChildId, numOfVgroups, msgId); + int32_t actualVgroups = pTask->outputInfo.shuffleDispatcher.waitingRspCnt; + stDebug("s-task:%s (child taskId:%d) start to shuffle-dispatch blocks to %d/%d vgroup(s), msgId:%d", id, + pTask->info.selfChildId, actualVgroups, numOfVgroups, msgId); + int32_t numOfSend = 0; for (int32_t i = 0; i < numOfVgroups; i++) { if (pDispatchMsg[i].blockNum > 0) { SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, i); @@ -408,6 +418,11 @@ static int32_t sendDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pDispatch if (code < 0) { break; } + + // no need to try remain, all already send. + if (++numOfSend == actualVgroups) { + break; + } } } From 287c36185223c5276d00eeb42d614b2ec150b72c Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Sun, 24 Dec 2023 18:49:48 +0800 Subject: [PATCH 53/94] adjust test case --- tests/system-test/1-insert/ts-4272.py | 54 ++++++++++----------------- 1 file changed, 20 insertions(+), 34 deletions(-) diff --git a/tests/system-test/1-insert/ts-4272.py b/tests/system-test/1-insert/ts-4272.py index bb81305eb3..496433664b 100644 --- a/tests/system-test/1-insert/ts-4272.py +++ b/tests/system-test/1-insert/ts-4272.py @@ -26,12 +26,10 @@ class TDTestCase: self.file1 = f"{self.testcasePath}/b.csv" self.file2 = f"{self.testcasePath}/c.csv" - #os.system("rm -rf %s/b.csv" %self.testcasePath) tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), logSql) def check_count(self, rows, records): - tdSql.execute(f"use {self.db};") tdSql.query(f"select tbname,count(*) from {self.stable0} group by tbname order by tbname;") tdSql.checkRows(rows) for i in range(rows): @@ -39,13 +37,6 @@ class TDTestCase: def reset_tb(self): # create database and tables - # os.system("taos -s 'drop database if exists d1;'") - # os.system("taos -s 'create database d1;use d1;create stable meters (ts timestamp, current float, voltage int, phase float) tags (location binary(64), groupId int);'") - # os.system(f"taos -s 'use d1;create table d2001 using meters(groupId) tags(5);'") - # res = os.system(f"taos -s 'use d1;create table d2002 using meters(groupId) tags(6);'") - # if (0 != res): - # tdLog.exit(f"create tb error") - tdSql.execute(f"drop database if exists {self.db};") tdSql.execute(f"create database {self.db};") tdSql.execute(f"use {self.db};") @@ -54,13 +45,14 @@ class TDTestCase: tdSql.execute(f"create table {self.tb2} {self.tag2};") tdSql.execute(f"create stable {self.stable1} (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp , q_int_null int , q_bigint_null bigint , q_smallint_null smallint , q_tinyint_null tinyint, q_float_null float , q_double_null double , q_bool_null bool , q_binary_null binary(20) , q_nchar_null nchar(20) , q_ts_null timestamp) tags(loc nchar(100) , t_int int , t_bigint bigint , t_smallint smallint , t_tinyint tinyint, t_bool bool , t_binary binary(100) , t_nchar nchar(100) ,t_float float , t_double double , t_ts timestamp);") tdSql.execute(f"create stable {self.stable2} (ts timestamp , q_int int , q_bigint bigint , q_smallint smallint , q_tinyint tinyint , q_float float , q_double double , q_bool bool , q_binary binary(100) , q_nchar nchar(100) , q_ts timestamp , q_int_null int , q_bigint_null bigint , q_smallint_null smallint , q_tinyint_null tinyint, q_float_null float , q_double_null double , q_bool_null bool , q_binary_null binary(20) , q_nchar_null nchar(20) , q_ts_null timestamp) tags(loc nchar(100) , t_int int , t_bigint bigint , t_smallint smallint , t_tinyint tinyint, t_bool bool , t_binary binary(100) , t_nchar nchar(100) ,t_float float , t_double double , t_ts timestamp);") + tdSql.execute(f"create table {self.stable1}_1 using {self.stable1}(t_int) tags(1);") + tdSql.execute(f"create table {self.stable2}_1 using {self.stable2}(t_int) tags(2);") def test(self, sql): sql = f"use {self.db};" + sql - res = os.system(f'taos -s "{sql}"') - # if (0 != res): - # tdLog.exit(f"taos sql error") - + os.system(f'taos -s "{sql}"') + # print(f'{sql}\n') + # tdSql.execute(sql, 1) def check(self): # same table, auto create + create @@ -95,13 +87,8 @@ class TDTestCase: sql = f"insert into {self.tb1} file '{self.file1}' {self.tb2} file '{self.file2}';" self.test(sql) - # bigNum = 1010000 - # self.check_count(5, [2100, 2100, bigNum, bigNum, bigNum]) + self.check_count(2, [2010000, 1000000]) - result = os.popen("taos -s 'select count(*) from %s.%s'" %(self.db, self.tb1)) - res = result.read() - if (f"OK" in res): - tdLog.info(f"check count success") def make_csv(self, filepath, once, qtime, startts): f = open(filepath, 'w') @@ -118,10 +105,8 @@ class TDTestCase: def test_mix(self): #forbid use both value and file in one insert - result = os.popen(f"insert into {self.tb1} file '{self.file2}' {self.tb2} values('2021-07-13 14:06:34.630', 10.2, 219, 0.32);") - res = result.read() - if (f"error" in res): - tdLog.info(f"forbid success") + self.make_csv(self.file2, 10, 10, self.ts) + tdSql.error(f"insert into {self.tb1} file '{self.file2}' {self.tb2} values('2021-07-13 14:06:34.630', 10.2, 219, 0.32);") def test_bigcsv(self): # prepare csv @@ -144,7 +129,6 @@ class TDTestCase: self.test(sql) print("end insert to table") - #tdSql.execute(f"use d1;") tdSql.query(f"select tbname,count(*) from {self.stable0} group by tbname order by tbname;") tdSql.checkRows(2) tdSql.checkData(0, 1, rowNum1) @@ -160,7 +144,7 @@ class TDTestCase: ts = startts + offset rows = [] for i in range(once): - rows.append([table_name, ts + i, offset + i, 'NULL']) + rows.append([f"\'{table_name}\'", ts + i, offset + i, 'NULL']) writer.writerows(rows) f.close() print(datetime.now(), filepath, " ready!") @@ -171,22 +155,22 @@ class TDTestCase: once = 10000 qtime1 = 101 qtime2 = 100 - # rowNum1 = qtime1 * once - # rowNum2 = qtime2 * once child_1 = f"{self.stable1}_1" child_2 = f"{self.stable2}_1" self.make_stable_csv(self.file1, once, qtime1, self.ts - 86400000, child_1) self.make_stable_csv(self.file2, once, qtime2, self.ts, child_2) print("end stable_csv data prepare") - - # insert create child table of stable + sql = f"insert into {self.db}.{self.stable1}(tbname,ts,q_int,q_binary) file '{self.file1}' {self.db}.{self.stable2}(tbname,ts,q_int,q_binary) file '{self.file2}';" self.test(sql) print("end insert to stable") - #tdSql.execute(f"insert into {self.db}.{child_1}(ts, q_int) values(now, 1);") - tdSql.query(f"select tbname,count(*) from {self.stable1} group by tbname order by tbname;") - tdSql.checkRows(0) + tdSql.query(f"select tbname,count(*) from {self.stable1} group by tbname;") + tdSql.checkRows(1) + tdSql.checkData(0, 1, qtime1 * once) + tdSql.query(f"select tbname,count(*) from {self.stable2} group by tbname;") + tdSql.checkRows(1) + tdSql.checkData(0, 1, qtime2 * once) print("check stable success") def run(self): @@ -194,12 +178,14 @@ class TDTestCase: self.reset_tb() self.test_stable_csv() self.test_bigcsv() - self.test_mix() self.check() + self.test_mix() + os.system(f"rm -rf {self.file1}") + os.system(f"rm -rf {self.file2}") tdSql.close() def stop(self): tdLog.success(f"{__file__} successfully executed") tdCases.addLinux(__file__, TDTestCase()) -tdCases.addWindows(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file From 29818803174c08f2e620969ea5c6cc7d3a1f6826 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Sun, 24 Dec 2023 18:56:17 +0800 Subject: [PATCH 54/94] adjust --- tests/system-test/1-insert/ts-4272.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/1-insert/ts-4272.py b/tests/system-test/1-insert/ts-4272.py index 496433664b..f64f0556bf 100644 --- a/tests/system-test/1-insert/ts-4272.py +++ b/tests/system-test/1-insert/ts-4272.py @@ -188,4 +188,4 @@ class TDTestCase: tdLog.success(f"{__file__} successfully executed") tdCases.addLinux(__file__, TDTestCase()) -tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file +tdCases.addWindows(__file__, TDTestCase()) From cd85dae3fddc45f58eb29c85832013462c6c806b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 24 Dec 2023 23:30:00 +0800 Subject: [PATCH 55/94] fix(stream): fix the dead lock. --- source/libs/stream/inc/streamInt.h | 2 -- source/libs/stream/src/streamCheckpoint.c | 1 + source/libs/stream/src/streamMeta.c | 5 ++++- source/libs/stream/src/streamStart.c | 16 ++++++++++++---- 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h index 341706ff3b..9b5134d449 100644 --- a/source/libs/stream/inc/streamInt.h +++ b/source/libs/stream/inc/streamInt.h @@ -109,8 +109,6 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) int32_t tEncodeStreamRetrieveReq(SEncoder* pEncoder, const SStreamRetrieveReq* pReq); int32_t streamSaveTaskCheckpointInfo(SStreamTask* p, int64_t checkpointId); -int32_t streamTaskBuildCheckpoint(SStreamTask* pTask); -int32_t streamSaveAllTaskStatus(SStreamMeta* pMeta, int64_t checkpointId); int32_t streamSendCheckMsg(SStreamTask* pTask, const SStreamTaskCheckReq* pReq, int32_t nodeId, SEpSet* pEpSet); int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t srcTaskId, int32_t index, int64_t checkpointId); diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index 2a766f21ec..b54adb0f96 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -114,6 +114,7 @@ static int32_t streamAlignCheckpoint(SStreamTask* pTask) { return atomic_sub_fetch_32(&pTask->chkInfo.downstreamAlignNum, 1); } +// todo handle down the transId of checkpoint to sink/agg tasks. static int32_t appendCheckpointIntoInputQ(SStreamTask* pTask, int32_t checkpointType) { SStreamDataBlock* pChkpoint = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, sizeof(SSDataBlock)); if (pChkpoint == NULL) { diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 9e20a2997c..8b2330fda0 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -1102,7 +1102,10 @@ static int32_t metaHeartbeatToMnodeImpl(SStreamMeta* pMeta) { entry.checkpointFailed = ((*pTask)->chkInfo.failedId >= (*pTask)->chkInfo.checkpointingId); entry.activeCheckpointId = (*pTask)->chkInfo.checkpointingId; entry.chkpointTransId = (*pTask)->chkInfo.transId; - stInfo("s-task:%s send kill checkpoint trans info, transId:%d", (*pTask)->id.idStr, (*pTask)->chkInfo.transId); + + if (entry.checkpointFailed) { + stInfo("s-task:%s send kill checkpoint trans info, transId:%d", (*pTask)->id.idStr, (*pTask)->chkInfo.transId); + } } if ((*pTask)->exec.pWalReader != NULL) { diff --git a/source/libs/stream/src/streamStart.c b/source/libs/stream/src/streamStart.c index 267dadf109..ea5e2edc09 100644 --- a/source/libs/stream/src/streamStart.c +++ b/source/libs/stream/src/streamStart.c @@ -379,10 +379,11 @@ int32_t streamTaskOnScanhistoryTaskReady(SStreamTask* pTask) { stDebug("s-task:%s enter into scan-history data stage, status:%s", id, p); streamTaskStartScanHistory(pTask); - // start the related fill-history task, when current task is ready - if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { - streamLaunchFillHistoryTask(pTask); - } + // NOTE: there will be an deadlock if launch fill history here. +// // start the related fill-history task, when current task is ready +// if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { +// streamLaunchFillHistoryTask(pTask); +// } return TSDB_CODE_SUCCESS; } @@ -400,6 +401,13 @@ void doProcessDownstreamReadyRsp(SStreamTask* pTask) { int64_t initTs = pTask->execInfo.init; int64_t startTs = pTask->execInfo.start; streamMetaUpdateTaskDownstreamStatus(pTask->pMeta, pTask->id.streamId, pTask->id.taskId, initTs, startTs, true); + + // start the related fill-history task, when current task is ready + // not invoke in success callback due to the deadlock. + if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { + stDebug("s-task:%s try to launch related fill-history task", pTask->id.idStr); + streamLaunchFillHistoryTask(pTask); + } } static void addIntoNodeUpdateList(SStreamTask* pTask, int32_t nodeId) { From 18dc6ff695bd1b05ac9573b6cd860fc415459dd3 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 25 Dec 2023 09:07:47 +0800 Subject: [PATCH 56/94] enh(cmake/curl): download from github instead of curl.se --- cmake/curl_CMakeLists.txt.in | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmake/curl_CMakeLists.txt.in b/cmake/curl_CMakeLists.txt.in index af7f005dda..197d978fb7 100644 --- a/cmake/curl_CMakeLists.txt.in +++ b/cmake/curl_CMakeLists.txt.in @@ -1,6 +1,7 @@ # curl ExternalProject_Add(curl2 - URL https://curl.se/download/curl-8.2.1.tar.gz + URL https://github.com/curl/curl/releases/download/curl-8_2_1/curl-8.2.1.tar.gz + #URL https://curl.se/download/curl-8.2.1.tar.gz URL_HASH MD5=b25588a43556068be05e1624e0e74d41 DOWNLOAD_NO_PROGRESS 1 DOWNLOAD_DIR "${TD_CONTRIB_DIR}/deps-download" From 4f29500f74509577dea1fb472c12fd34396cf2b0 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 22 Dec 2023 16:26:45 +0800 Subject: [PATCH 57/94] fix: set commit log for mndDropSubByTopic --- source/dnode/mnode/impl/src/mndSubscribe.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 44ee804d22..01d4d1029c 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -1215,7 +1215,7 @@ int32_t mndDropSubByTopic(SMnode *pMnode, STrans *pTrans, const char *topicName) return -1; } - if (mndSetDropSubRedoLogs(pMnode, pTrans, pSub) < 0) { + if (mndSetDropSubCommitLogs(pMnode, pTrans, pSub) < 0) { sdbRelease(pSdb, pSub); sdbCancelFetch(pSdb, pIter); return -1; From 7edfbf8023ece6def03bfe97526012b49f3cd3b5 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Mon, 25 Dec 2023 09:44:55 +0800 Subject: [PATCH 58/94] fix: set commit log for mndAddCompactDetailToTran --- source/dnode/mnode/impl/src/mndCompactDetail.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndCompactDetail.c b/source/dnode/mnode/impl/src/mndCompactDetail.c index a1c0e95c20..6b1dd78093 100644 --- a/source/dnode/mnode/impl/src/mndCompactDetail.c +++ b/source/dnode/mnode/impl/src/mndCompactDetail.c @@ -290,11 +290,11 @@ int32_t mndAddCompactDetailToTran(SMnode *pMnode, STrans *pTrans, SCompactObj* p SSdbRaw *pVgRaw = mndCompactDetailActionEncode(&compactDetail); if (pVgRaw == NULL) return -1; - if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) { + if (mndTransAppendCommitlog(pTrans, pVgRaw) != 0) { sdbFreeRaw(pVgRaw); return -1; } (void)sdbSetRawStatus(pVgRaw, SDB_STATUS_READY); return 0; -} \ No newline at end of file +} From 00fac852e8e22cb5c05d7e6405f1f5040af3be7d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 25 Dec 2023 09:55:24 +0800 Subject: [PATCH 59/94] fix(tsdb): fix invalid delete check. --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 26 +++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 751569bf90..5dafe91d3d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -2058,9 +2058,9 @@ static bool isValidFileBlockRow(SBlockData* pBlockData, SFileBlockDumpInfo* pDum } if (pBlockScanInfo->delSkyline != NULL && TARRAY_SIZE(pBlockScanInfo->delSkyline) > 0) { - bool deleted = hasBeenDropped(pBlockScanInfo->delSkyline, &pBlockScanInfo->fileDelIndex, ts, ver, + bool dropped = hasBeenDropped(pBlockScanInfo->delSkyline, &pBlockScanInfo->fileDelIndex, ts, ver, pReader->info.order, &pReader->info.verRange); - if (deleted) { + if (dropped) { return false; } } @@ -3340,11 +3340,14 @@ TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* p } // it is a valid data version - if ((key.version <= pReader->info.verRange.maxVer && key.version >= pReader->info.verRange.minVer) && - pDelList != NULL && TARRAY_SIZE(pDelList) > 0) { - bool dropped = hasBeenDropped(pDelList, &pIter->index, key.ts, key.version, order, &pReader->info.verRange); - if (!dropped) { + if (key.version <= pReader->info.verRange.maxVer && key.version >= pReader->info.verRange.minVer) { + if (pDelList == NULL || TARRAY_SIZE(pDelList) == 0) { return pRow; + } else { + bool dropped = hasBeenDropped(pDelList, &pIter->index, key.ts, key.version, order, &pReader->info.verRange); + if (!dropped) { + return pRow; + } } } @@ -3362,11 +3365,14 @@ TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* p return NULL; } - if (key.version <= pReader->info.verRange.maxVer && key.version >= pReader->info.verRange.minVer && - pDelList != NULL && TARRAY_SIZE(pDelList) > 0) { - bool dropped = hasBeenDropped(pDelList, &pIter->index, key.ts, key.version, order, &pReader->info.verRange); - if (!dropped) { + if (key.version <= pReader->info.verRange.maxVer && key.version >= pReader->info.verRange.minVer) { + if (pDelList == NULL || TARRAY_SIZE(pDelList) == 0) { return pRow; + } else { + bool dropped = hasBeenDropped(pDelList, &pIter->index, key.ts, key.version, order, &pReader->info.verRange); + if (!dropped) { + return pRow; + } } } } From 1beb3347faa5c9e348887de01178f708687bffab Mon Sep 17 00:00:00 2001 From: slzhou Date: Mon, 25 Dec 2023 10:18:19 +0800 Subject: [PATCH 60/94] fix: change compact rsp first column name --- source/libs/parser/src/parTranslater.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 3bb24566c2..16e0332ee9 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -8769,7 +8769,7 @@ static int32_t extractShowVariablesResultSchema(int32_t* numOfCols, SSchema** pS (*pSchema)[0].type = TSDB_DATA_TYPE_BINARY; (*pSchema)[0].bytes = TSDB_CONFIG_OPTION_LEN; - strcpy((*pSchema)[0].name, "name"); + strcpy((*pSchema)[0].name, "result"); (*pSchema)[1].type = TSDB_DATA_TYPE_BINARY; (*pSchema)[1].bytes = TSDB_CONFIG_VALUE_LEN; From 12bbf50c8f48022bef32088fd9c67a75c60ccac0 Mon Sep 17 00:00:00 2001 From: Bob Liu Date: Mon, 25 Dec 2023 10:29:15 +0800 Subject: [PATCH 61/94] adjust --- tests/system-test/1-insert/ts-4272.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/system-test/1-insert/ts-4272.py b/tests/system-test/1-insert/ts-4272.py index f64f0556bf..f2bacd0c2b 100644 --- a/tests/system-test/1-insert/ts-4272.py +++ b/tests/system-test/1-insert/ts-4272.py @@ -49,10 +49,10 @@ class TDTestCase: tdSql.execute(f"create table {self.stable2}_1 using {self.stable2}(t_int) tags(2);") def test(self, sql): - sql = f"use {self.db};" + sql - os.system(f'taos -s "{sql}"') - # print(f'{sql}\n') - # tdSql.execute(sql, 1) + # sql = f"use {self.db};" + sql + # os.system(f'taos -s "{sql}"') + print(f'{sql}\n') + tdSql.execute(sql, 1) def check(self): # same table, auto create + create From e5087f6e23ae34c69c9df9f942b5bc8acc490c07 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 25 Dec 2023 10:40:36 +0800 Subject: [PATCH 62/94] fix(tsdb): fix invalid delete check. --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 5dafe91d3d..5ad067c113 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -1379,11 +1379,13 @@ static bool nextRowFromSttBlocks(SSttBlockReader* pSttBlockReader, STableBlockSc pScanInfo->sttKeyInfo.nextProcKey = key; if (pScanInfo->delSkyline != NULL && TARRAY_SIZE(pScanInfo->delSkyline) > 0) { - bool dropped = hasBeenDropped(pScanInfo->delSkyline, &pScanInfo->sttBlockDelIndex, key, ver, order, pVerRange); - if (!dropped) { + if (!hasBeenDropped(pScanInfo->delSkyline, &pScanInfo->sttBlockDelIndex, key, ver, order, pVerRange)) { pScanInfo->sttKeyInfo.status = STT_FILE_HAS_DATA; return true; } + } else { + pScanInfo->sttKeyInfo.status = STT_FILE_HAS_DATA; + return true; } } } From 60977d56bd2f6756542e5eb1d773cb2b04976b1f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 25 Dec 2023 12:47:16 +0800 Subject: [PATCH 63/94] refactor: set attribute only on Linux --- source/libs/stream/src/streamMeta.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 3ee51ffd03..d09d370a36 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -376,12 +376,16 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF pMeta->rid = taosAddRef(streamMetaId, pMeta); + // set the attribute when running on Linux OS +#if defined LINUX TdThreadRwlockAttr attr; taosThreadRwlockAttrInit(&attr); pthread_rwlockattr_setkind_np(&attr, PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP); taosThreadRwlockInit(&pMeta->lock, &attr); + taosThreadRwlockAttrDestroy(&attr); +#endif int64_t* pRid = taosMemoryMalloc(sizeof(int64_t)); memcpy(pRid, &pMeta->rid, sizeof(pMeta->rid)); From f567c19d05acd296799aa0d34551f5745f5177bf Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Mon, 25 Dec 2023 15:59:29 +0800 Subject: [PATCH 64/94] fix: add new-frame public file --- .../enterprise/multi-level/mlevel_basic.json | 58 +++++++++++++ .../enterprise/multi-level/mlevel_basic.py | 61 +++++++++++--- tests/army/frame/caseBase.py | 83 ++++++++++++++++++- tests/army/frame/eos.py | 35 ++++++++ tests/army/frame/epath.py | 53 ++++++++++++ tests/army/frame/etime.py | 23 +++++ tests/army/frame/etool.py | 40 +++++++++ tests/army/frame/pathFinding.py | 83 ------------------- tests/army/frame/sql.py | 13 ++- 9 files changed, 352 insertions(+), 97 deletions(-) create mode 100644 tests/army/enterprise/multi-level/mlevel_basic.json create mode 100644 tests/army/frame/eos.py create mode 100644 tests/army/frame/epath.py create mode 100644 tests/army/frame/etime.py create mode 100644 tests/army/frame/etool.py delete mode 100644 tests/army/frame/pathFinding.py diff --git a/tests/army/enterprise/multi-level/mlevel_basic.json b/tests/army/enterprise/multi-level/mlevel_basic.json new file mode 100644 index 0000000000..7df5b1caa8 --- /dev/null +++ b/tests/army/enterprise/multi-level/mlevel_basic.json @@ -0,0 +1,58 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "connection_pool_size": 8, + "num_of_records_per_req": 2000, + "thread_count": 2, + "create_table_thread_count": 1, + "confirm_parameter_prompt": "no", + "databases": [ + { + "dbinfo": { + "name": "db", + "drop": "yes", + "vgroups": 2, + "replica": 1, + "duration":"1d", + "keep": "3d,6d,30d" + }, + "super_tables": [ + { + "name": "stb", + "child_table_exists": "no", + "childtable_count": 2, + "insert_rows": 1000000, + "childtable_prefix": "d", + "insert_mode": "taosc", + "timestamp_step": 1000, + "start_timestamp":"now-10d", + "columns": [ + { "type": "bool", "name": "bc"}, + { "type": "float", "name": "fc" }, + { "type": "double", "name": "dc"}, + { "type": "tinyint", "name": "ti", "values":["1"]}, + { "type": "smallint", "name": "si" }, + { "type": "int", "name": "ic" }, + { "type": "bigint", "name": "bi" }, + { "type": "utinyint", "name": "uti"}, + { "type": "usmallint", "name": "usi"}, + { "type": "uint", "name": "ui" }, + { "type": "ubigint", "name": "ubi"}, + { "type": "binary", "name": "bin", "len": 32}, + { "type": "nchar", "name": "nch", "len": 64} + ], + "tags": [ + {"type": "tinyint", "name": "groupid","max": 10,"min": 1}, + {"name": "location","type": "binary", "len": 16, "values": + ["San Francisco", "Los Angles", "San Diego", "San Jose", "Palo Alto", "Campbell", "Mountain View","Sunnyvale", "Santa Clara", "Cupertino"] + } + ] + } + ] + } + ] +} diff --git a/tests/army/enterprise/multi-level/mlevel_basic.py b/tests/army/enterprise/multi-level/mlevel_basic.py index e3a3f57c74..a12b1b59eb 100644 --- a/tests/army/enterprise/multi-level/mlevel_basic.py +++ b/tests/army/enterprise/multi-level/mlevel_basic.py @@ -15,26 +15,65 @@ import sys import time import taos +import frame + from frame.log import * from frame.cases import * from frame.sql import * +from frame.caseBase import * + + + +class TDTestCase(TBase): + + + def insertData(self): + tdLog.info(f"insert data.") + # taosBenchmark run + json = frame.etool.curFile(__file__, "mlevel_basic.json") + frame.etool.runBenchmark(json=json) + + # set insert data information + self.childtable_count = 2 + self.insert_rows = 1000000 + self.timestamp_step = 1000 + + def doAction(self): + tdLog.info(f"trim database.") + self.trimDb() + + def saveData(self): + tdLog.info(f"check correct.") + + def checkSaveCorrent(self): + tdLog.info(f"check correct.") + + def checkCorrect(self): + tdLog.info(f"check correct.") -class TDTestCase: - # init - def init(self, conn, logSql, replicaVar=1): - self.replicaVar = int(replicaVar) - tdLog.debug(f"start to excute {__file__}") - tdSql.init(conn.cursor(), True) # run def run(self): - # check two db query result same - tdLog.info(f"hello world.") + tdLog.debug(f"start to excute {__file__}") + + # insert data + self.insertData() + + # check insert data correct + self.checkInsertCorrect() + + # save + self.snapshotAgg() + + # do action + self.doAction() + + # check save agg result correct + self.checkAggCorrect() - # stop - def stop(self): - tdSql.close() tdLog.success(f"{__file__} successfully executed") + + tdCases.addLinux(__file__, TDTestCase()) tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/army/frame/caseBase.py b/tests/army/frame/caseBase.py index abaa92a51c..7b371cd822 100644 --- a/tests/army/frame/caseBase.py +++ b/tests/army/frame/caseBase.py @@ -16,10 +16,89 @@ import os import time import datetime -class caseBase: - def __init__(self): +from log import * +from sql import * + +# test case base +class DbBase: + +# +# frame call +# + + # init + def init(self, conn, logSql, replicaVar=1): + # save param + self.replicaVar = int(replicaVar) + tdSql.init(conn.cursor(), True) + # record server information self.dnodeNum = 0 self.mnodeNum = 0 self.mLevel = 0 self.mLevelDisk = 0 + + # test case information + self.db = "db" + self.stb = "stb" + + # variant in taosBenchmark json + self.childtable_count = 2 + self.insert_rows = 1000000 + self.timestamp_step = 1000 + + # sql + self.sqlSum = f"select sum(ic) from {self.stb}" + self.sqlMax = f"select max(ic) from {self.stb}" + self.sqlMin = f"select min(ic) from {self.stb}" + self.sqlAvg = f"select avg(ic) from {self.stb}" + + + # stop + def stop(self): + tdSql.close() + + +# +# db action +# + + def trimDb(self): + tdSql.execute(f"trim database {self.db}") + + def compactDb(self): + tdSql.execute(f"compact database {self.db}") + + +# +# check db correct +# + + # basic + def checkInsertCorrect(self): + # check count + sql = f"select count(*) from {self.stb}" + tdSql.checkAgg(sql, self.childtable_count * self.insert_rows) + + # check child table count + sql = f" select count(*) from (select count(*) as cnt , tbname from {self.stb} group by tbname) where cnt = {self.insert_rows} " + tdSql.checkAgg(sql, self.childtable_count) + + # check step + sql = f"select count(*) from (select diff(ts) as dif from {self.stb} group by tbname) where dif != {self.timestamp_step}" + tdSql.checkAgg(sql, 0) + + # save agg result + def snapshotAgg(self): + + self.sum = tdSql.getFirstValue(self.sqlSum) + self.avg = tdSql.getFirstValue(self.sqlAvg) + self.min = tdSql.getFirstValue(self.sqlMin) + self.max = tdSql.getFirstValue(self.sqlMax) + + # check agg + def checkAggCorrect(self): + tdSql.checkAgg(self.sqlSum, self.sum) + tdSql.checkAgg(self.sqlAvg, self.avg) + tdSql.checkAgg(self.sqlMin, self.min) + tdSql.checkAgg(self.sqlMax, self.max) diff --git a/tests/army/frame/eos.py b/tests/army/frame/eos.py new file mode 100644 index 0000000000..2d8559f3eb --- /dev/null +++ b/tests/army/frame/eos.py @@ -0,0 +1,35 @@ +################################################################### +# Copyright (c) 2023 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +# +# about system funciton extension +# + +import sys +import os +import time +import datetime + +# if windows platform return True +def isWin(): + return sys.platform.system().lower() == 'windows' + +# wait util execute file finished +def exe(file): + return os.system(file) + +# execute file and return immediately +def exeNoWait(file): + print("exe no wait") + + diff --git a/tests/army/frame/epath.py b/tests/army/frame/epath.py new file mode 100644 index 0000000000..644e448fb7 --- /dev/null +++ b/tests/army/frame/epath.py @@ -0,0 +1,53 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +# +# about path function extension +# + +import os +from frame.log import * + +# build/bin path +binPath = "" + +def binPath(self): + global binPath + + if binPath != "": + return binPath + + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community/tests" in selfPath): + projPath = selfPath[:selfPath.find("community/tests")] + else: + projPath = selfPath[:selfPath.find("TDengine/tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + # check + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info(f"taosd found in {buildPath}") + # return + binPath = buildPath + "/build/bin/" + return binPath + +def binFile(self, file): + return binPath() + file \ No newline at end of file diff --git a/tests/army/frame/etime.py b/tests/army/frame/etime.py new file mode 100644 index 0000000000..2ee223e122 --- /dev/null +++ b/tests/army/frame/etime.py @@ -0,0 +1,23 @@ +################################################################### +# Copyright (c) 2023 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +# +# about tools funciton extension +# + +import sys +import os +import time +import datetime + + diff --git a/tests/army/frame/etool.py b/tests/army/frame/etool.py new file mode 100644 index 0000000000..5ba29ee1c3 --- /dev/null +++ b/tests/army/frame/etool.py @@ -0,0 +1,40 @@ +################################################################### +# Copyright (c) 2023 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +# +# about system funciton extension +# + +import sys +import os +import time +import datetime +import epath +import eos + +# run taosBenchmark with command or json file mode +def runBenchmark(command = "", json = "") : + # get taosBenchmark path + bmFile = epath.binFile("taosBenchmark") + if eos.isWin(): + bmFile += ".exe" + + # run + if command != "": + eos.exe(bmFile + " " + command) + if json != "": + eos.exe(bmFile + " -f " + json) + +# get current directory file name +def curFile(fullPath, file): + return os.path.dirname(fullPath) + "/" + file diff --git a/tests/army/frame/pathFinding.py b/tests/army/frame/pathFinding.py deleted file mode 100644 index df03f0ed68..0000000000 --- a/tests/army/frame/pathFinding.py +++ /dev/null @@ -1,83 +0,0 @@ -################################################################### -# Copyright (c) 2016 by TAOS Technologies, Inc. -# All rights reserved. -# -# This file is proprietary and confidential to TAOS Technologies. -# No part of this file may be reproduced, stored, transmitted, -# disclosed or used in any form or by any means other than as -# expressly provided by the written permission from Jianhui Tao -# -################################################################### - -# -*- coding: utf-8 -*- - - -import os -from frame.log import * - - - -class TDFindPath: - """This class is for finding path within TDengine - """ - def __init__(self): - self.file = "" - - - def init(self, file): - """[summary] - - Args: - file (str): the file location you want to start the query. Generally using __file__ - """ - self.file = file - - def getTaosdemoPath(self): - """for finding the path of directory containing taosdemo - - Returns: - str: the path to directory containing taosdemo - """ - selfPath = os.path.dirname(os.path.realpath(self.file)) - - if ("community" in selfPath): - projPath = selfPath[:selfPath.find("community")] - else: - projPath = selfPath[:selfPath.find("tests")] - - for root, dirs, files in os.walk(projPath): - if ("taosd" in files): - rootRealPath = os.path.dirname(os.path.realpath(root)) - if ("packaging" not in rootRealPath): - buildPath = root[:len(root)-len("/build/bin")] - break - if (buildPath == ""): - tdLog.exit("taosd not found!") - else: - tdLog.info(f"taosd found in {buildPath}") - return buildPath + "/build/bin/" - - def getTDenginePath(self): - """for finding the root path of TDengine - - Returns: - str: the root path of TDengine - """ - selfPath = os.path.dirname(os.path.realpath(self.file)) - - if ("community" in selfPath): - projPath = selfPath[:selfPath.find("community")] - else: - projPath = selfPath[:selfPath.find("tests")] - print(projPath) - for root, dirs, files in os.walk(projPath): - if ("sim" in dirs): - print(root) - rootRealPath = os.path.realpath(root) - if (rootRealPath == ""): - tdLog.exit("TDengine not found!") - else: - tdLog.info(f"TDengine found in {rootRealPath}") - return rootRealPath - -tdFindPath = TDFindPath() \ No newline at end of file diff --git a/tests/army/frame/sql.py b/tests/army/frame/sql.py index 409dc6c96c..b3ee1ed5f8 100644 --- a/tests/army/frame/sql.py +++ b/tests/army/frame/sql.py @@ -535,6 +535,16 @@ class TDSql: tdLog.info("%s(%d) failed: sql:%s, elm:%s == expect_elm:%s" % args) raise Exception + # check like select count(*) ... sql + def checkAgg(self, sql, expectCnt): + self.query(sql) + self.checkRowCol(0, 0, expectCnt) + + # get first value + def getFirstValue(self, sql) : + self.query(sql) + return self.getData(0, 0) + def get_times(self, time_str, precision="ms"): caller = inspect.getframeinfo(inspect.stack()[1][0]) if time_str[-1] not in TAOS_TIME_INIT: @@ -602,6 +612,7 @@ class TDSql: if self.cursor.istype(col, "BIGINT UNSIGNED"): return "BIGINT UNSIGNED" + ''' def taosdStatus(self, state): tdLog.sleep(5) pstate = 0 @@ -630,7 +641,7 @@ class TDSql: tdLog.exit("taosd state is %d != expect:%d" %args) pass - ''' + def haveFile(self, dir, state): if os.path.exists(dir) and os.path.isdir(dir): if not os.listdir(dir): From a7c8563c0616859f4fbc6bf8eca8cd96b7c84899 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 25 Dec 2023 16:13:30 +0800 Subject: [PATCH 65/94] fix: remove useless code --- source/dnode/vnode/src/inc/tsdb.h | 38 +- source/dnode/vnode/src/inc/vnodeInt.h | 4 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 1628 ---------------------- source/dnode/vnode/src/tsdb/tsdbFS.c | 313 +---- source/dnode/vnode/src/tsdb/tsdbRead2.c | 119 +- 5 files changed, 70 insertions(+), 2032 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index d347ce30d8..ebaf0851e6 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -256,18 +256,7 @@ void tsdbDelFileName(STsdb *pTsdb, SDelFile *pFile, char fname[]); // tsdbFS.c ============================================================================================== int32_t tsdbFSOpen(STsdb *pTsdb, int8_t rollback); int32_t tsdbFSClose(STsdb *pTsdb); -int32_t tsdbFSCopy(STsdb *pTsdb, STsdbFS *pFS); -void tsdbFSDestroy(STsdbFS *pFS); -int32_t tDFileSetCmprFn(const void *p1, const void *p2); -int32_t tsdbFSCommit(STsdb *pTsdb); -int32_t tsdbFSRollback(STsdb *pTsdb); -int32_t tsdbFSPrepareCommit(STsdb *pTsdb, STsdbFS *pFS); -int32_t tsdbFSRef(STsdb *pTsdb, STsdbFS *pFS); -void tsdbFSUnref(STsdb *pTsdb, STsdbFS *pFS); void tsdbGetCurrentFName(STsdb *pTsdb, char *current, char *current_t); - -int32_t tsdbFSUpsertFSet(STsdbFS *pFS, SDFileSet *pSet); -int32_t tsdbFSUpsertDelFile(STsdbFS *pFS, SDelFile *pDelFile); // tsdbReaderWriter.c ============================================================================================== // SDataFWriter int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pSet); @@ -737,7 +726,6 @@ struct STsdbReadSnap { SMemTable *pIMem; SQueryNode *pINode; TFileSetArray *pfSetArray; - STsdbFS fs; }; struct SDataFWriter { @@ -796,16 +784,16 @@ typedef struct { } SSttTableRowsInfo; typedef struct SSttBlockLoadInfo { - SBlockDataInfo blockData[2]; // buffered block data - SArray *aSttBlk; - int32_t currentLoadBlockIndex; - STSchema *pSchema; - int16_t *colIds; - int32_t numOfCols; - bool checkRemainingRow; // todo: no assign value? - bool isLast; - bool sttBlockLoaded; - SSttTableRowsInfo info; + SBlockDataInfo blockData[2]; // buffered block data + SArray *aSttBlk; + int32_t currentLoadBlockIndex; + STSchema *pSchema; + int16_t *colIds; + int32_t numOfCols; + bool checkRemainingRow; // todo: no assign value? + bool isLast; + bool sttBlockLoaded; + SSttTableRowsInfo info; SSttBlockLoadCostInfo cost; } SSttBlockLoadInfo; @@ -894,15 +882,15 @@ typedef struct { _load_tomb_fn loadTombFn; void *pReader; void *idstr; - bool rspRows; // response the rows in stt-file, if possible + bool rspRows; // response the rows in stt-file, if possible } SMergeTreeConf; typedef struct SSttDataInfoForTable { - SArray* pTimeWindowList; + SArray *pTimeWindowList; int64_t numOfRows; } SSttDataInfoForTable; -int32_t tMergeTreeOpen2(SMergeTree *pMTree, SMergeTreeConf *pConf, SSttDataInfoForTable* pTableInfo); +int32_t tMergeTreeOpen2(SMergeTree *pMTree, SMergeTreeConf *pConf, SSttDataInfoForTable *pTableInfo); void tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter); bool tMergeTreeNext(SMergeTree *pMTree); void tMergeTreePinSttBlock(SMergeTree *pMTree); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 84f51827ba..c1a4754b62 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -95,7 +95,7 @@ typedef struct SQueryNode SQueryNode; #define VNODE_RSMA2_DIR "rsma2" #define VNODE_TQ_STREAM "stream" -#if SUSPEND_RESUME_TEST // only for test purpose +#if SUSPEND_RESUME_TEST // only for test purpose #define VNODE_BUFPOOL_SEGMENTS 1 #else #define VNODE_BUFPOOL_SEGMENTS 3 @@ -216,8 +216,6 @@ int32_t tsdbBegin(STsdb* pTsdb); int32_t tsdbCacheCommit(STsdb* pTsdb); int32_t tsdbCompact(STsdb* pTsdb, SCompactInfo* pInfo); int32_t tsdbRetention(STsdb* tsdb, int64_t now, int32_t sync); -// int32_t tsdbFinishCommit(STsdb* pTsdb); -// int32_t tsdbRollbackCommit(STsdb* pTsdb); int tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq2* pMsg); int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq2* pMsg, SSubmitRsp2* pRsp); int32_t tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitTbData* pSubmitTbData, int32_t* affectedRows); diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 55ae25aee4..37413ef920 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -15,92 +15,6 @@ #include "tsdb.h" -typedef enum { MEMORY_DATA_ITER = 0, STT_DATA_ITER } EDataIterT; - -#define USE_STREAM_COMPRESSION 0 - -typedef struct { - SRBTreeNode n; - SRowInfo r; - EDataIterT type; - union { - struct { - int32_t iTbDataP; - STbDataIter iter; - }; // memory data iter - struct { - int32_t iStt; - SArray *aSttBlk; - int32_t iSttBlk; - SBlockData bData; - int32_t iRow; - }; // stt file data iter - }; -} SDataIter; - -typedef struct { - STsdb *pTsdb; - /* commit data */ - int64_t commitID; - int32_t minutes; - int8_t precision; - int32_t minRow; - int32_t maxRow; - int8_t cmprAlg; - int8_t sttTrigger; - SArray *aTbDataP; // memory - STsdbFS fs; // disk - // -------------- - TSKEY nextKey; // reset by each table commit - int32_t commitFid; - int32_t expLevel; - TSKEY minKey; - TSKEY maxKey; - // commit file data - struct { - SDataFReader *pReader; - SArray *aBlockIdx; // SArray - int32_t iBlockIdx; - SBlockIdx *pBlockIdx; - SMapData mBlock; // SMapData - SBlockData bData; - } dReader; - struct { - SDataIter *pIter; - SRBTree rbt; - SDataIter dataIter; - SDataIter aDataIter[TSDB_STT_TRIGGER_ARRAY_SIZE]; - int8_t toLastOnly; - }; - struct { - SDataFWriter *pWriter; - SArray *aBlockIdx; // SArray - SArray *aSttBlk; // SArray - SMapData mBlock; // SMapData - SBlockData bData; -#if USE_STREAM_COMPRESSION - SDiskDataBuilder *pBuilder; -#else - SBlockData bDatal; -#endif - } dWriter; - SSkmInfo skmTable; - SSkmInfo skmRow; - /* commit del */ - SDelFReader *pDelFReader; - SDelFWriter *pDelFWriter; - SArray *aDelIdx; // SArray - SArray *aDelIdxN; // SArray - SArray *aDelData; // SArray -} SCommitter; - -static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter, SCommitInfo *pInfo); -static int32_t tsdbCommitData(SCommitter *pCommitter); -static int32_t tsdbCommitDel(SCommitter *pCommitter); -static int32_t tsdbCommitCache(SCommitter *pCommitter); -static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno); -static int32_t tsdbNextCommitRow(SCommitter *pCommitter); - int32_t tRowInfoCmprFn(const void *p1, const void *p2) { SRowInfo *pInfo1 = (SRowInfo *)p1; SRowInfo *pInfo2 = (SRowInfo *)p2; @@ -150,1545 +64,3 @@ _exit: } return code; } - -int32_t tsdbPrepareCommit(STsdb *pTsdb) { - taosThreadMutexLock(&pTsdb->mutex); - ASSERT(pTsdb->imem == NULL); - pTsdb->imem = pTsdb->mem; - pTsdb->mem = NULL; - taosThreadMutexUnlock(&pTsdb->mutex); - - return 0; -} - -int32_t tsdbCommit(STsdb *pTsdb, SCommitInfo *pInfo) { - if (!pTsdb) return 0; - - int32_t code = 0; - int32_t lino = 0; - SCommitter commith; - SMemTable *pMemTable = pTsdb->imem; - - // check - if (pMemTable->nRow == 0 && pMemTable->nDel == 0) { - taosThreadMutexLock(&pTsdb->mutex); - pTsdb->imem = NULL; - taosThreadMutexUnlock(&pTsdb->mutex); - - tsdbUnrefMemTable(pMemTable, NULL, true); - goto _exit; - } - - // start commit - code = tsdbStartCommit(pTsdb, &commith, pInfo); - TSDB_CHECK_CODE(code, lino, _exit); - - // commit impl - code = tsdbCommitData(&commith); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbCommitDel(&commith); - TSDB_CHECK_CODE(code, lino, _exit); - - // end commit - code = tsdbEndCommit(&commith, 0); - TSDB_CHECK_CODE(code, lino, _exit); - -_exit: - if (code) { - tsdbEndCommit(&commith, code); - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); - } - return code; -} - -static int32_t tsdbCommitDelStart(SCommitter *pCommitter) { - int32_t code = 0; - int32_t lino = 0; - STsdb *pTsdb = pCommitter->pTsdb; - SMemTable *pMemTable = pTsdb->imem; - - if ((pCommitter->aDelIdx = taosArrayInit(0, sizeof(SDelIdx))) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - - if ((pCommitter->aDelData = taosArrayInit(0, sizeof(SDelData))) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - - if ((pCommitter->aDelIdxN = taosArrayInit(0, sizeof(SDelIdx))) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - - SDelFile *pDelFileR = pCommitter->fs.pDelFile; - if (pDelFileR) { - code = tsdbDelFReaderOpen(&pCommitter->pDelFReader, pDelFileR, pTsdb); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbReadDelIdx(pCommitter->pDelFReader, pCommitter->aDelIdx); - TSDB_CHECK_CODE(code, lino, _exit); - } - - // prepare new - SDelFile wDelFile = {.commitID = pCommitter->commitID, .size = 0, .offset = 0}; - code = tsdbDelFWriterOpen(&pCommitter->pDelFWriter, &wDelFile, pTsdb); - TSDB_CHECK_CODE(code, lino, _exit); - -_exit: - if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); - } else { - tsdbDebug("vgId:%d, commit del start", TD_VID(pTsdb->pVnode)); - } - return code; -} - -static int32_t tsdbCommitTableDel(SCommitter *pCommitter, STbData *pTbData, SDelIdx *pDelIdx) { - int32_t code = 0; - int32_t lino = 0; - SDelData *pDelData; - tb_uid_t suid; - tb_uid_t uid; - - if (pTbData) { - suid = pTbData->suid; - uid = pTbData->uid; - - if (pTbData->pHead == NULL) { - pTbData = NULL; - } - } - - if (pDelIdx) { - suid = pDelIdx->suid; - uid = pDelIdx->uid; - - code = tsdbReadDelDatav1(pCommitter->pDelFReader, pDelIdx, pCommitter->aDelData, INT64_MAX); - TSDB_CHECK_CODE(code, lino, _exit); - } else { - taosArrayClear(pCommitter->aDelData); - } - - if (pTbData == NULL && pDelIdx == NULL) goto _exit; - - SDelIdx delIdx = {.suid = suid, .uid = uid}; - - // memory - pDelData = pTbData ? pTbData->pHead : NULL; - for (; pDelData; pDelData = pDelData->pNext) { - if (taosArrayPush(pCommitter->aDelData, pDelData) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - } - - // write - code = tsdbWriteDelData(pCommitter->pDelFWriter, pCommitter->aDelData, &delIdx); - TSDB_CHECK_CODE(code, lino, _exit); - - // put delIdx - if (taosArrayPush(pCommitter->aDelIdxN, &delIdx) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - -_exit: - if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, - tstrerror(code)); - } - return code; -} - -static int32_t tsdbCommitDelEnd(SCommitter *pCommitter) { - int32_t code = 0; - int32_t lino = 0; - STsdb *pTsdb = pCommitter->pTsdb; - - code = tsdbWriteDelIdx(pCommitter->pDelFWriter, pCommitter->aDelIdxN); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbUpdateDelFileHdr(pCommitter->pDelFWriter); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbFSUpsertDelFile(&pCommitter->fs, &pCommitter->pDelFWriter->fDel); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbDelFWriterClose(&pCommitter->pDelFWriter, 1); - TSDB_CHECK_CODE(code, lino, _exit); - - if (pCommitter->pDelFReader) { - code = tsdbDelFReaderClose(&pCommitter->pDelFReader); - TSDB_CHECK_CODE(code, lino, _exit); - } - - taosArrayDestroy(pCommitter->aDelIdx); - taosArrayDestroy(pCommitter->aDelData); - taosArrayDestroy(pCommitter->aDelIdxN); - -_exit: - if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, - tstrerror(code)); - } - return code; -} - -int32_t tsdbUpdateTableSchema(SMeta *pMeta, int64_t suid, int64_t uid, SSkmInfo *pSkmInfo) { - int32_t code = 0; - int32_t lino = 0; - - if (suid) { - if (pSkmInfo->suid == suid) { - pSkmInfo->uid = uid; - goto _exit; - } - } else { - if (pSkmInfo->uid == uid) goto _exit; - } - - pSkmInfo->suid = suid; - pSkmInfo->uid = uid; - tDestroyTSchema(pSkmInfo->pTSchema); - code = metaGetTbTSchemaEx(pMeta, suid, uid, -1, &pSkmInfo->pTSchema); - TSDB_CHECK_CODE(code, lino, _exit); - -_exit: - return code; -} - -static int32_t tsdbCommitterUpdateRowSchema(SCommitter *pCommitter, int64_t suid, int64_t uid, int32_t sver) { - int32_t code = 0; - int32_t lino = 0; - - if (pCommitter->skmRow.pTSchema) { - if (pCommitter->skmRow.suid == suid) { - if (suid == 0) { - if (pCommitter->skmRow.uid == uid && sver == pCommitter->skmRow.pTSchema->version) goto _exit; - } else { - if (sver == pCommitter->skmRow.pTSchema->version) goto _exit; - } - } - } - - pCommitter->skmRow.suid = suid; - pCommitter->skmRow.uid = uid; - tDestroyTSchema(pCommitter->skmRow.pTSchema); - code = metaGetTbTSchemaEx(pCommitter->pTsdb->pVnode->pMeta, suid, uid, sver, &pCommitter->skmRow.pTSchema); - TSDB_CHECK_CODE(code, lino, _exit); - -_exit: - return code; -} - -static int32_t tsdbCommitterNextTableData(SCommitter *pCommitter) { - int32_t code = 0; - int32_t lino = 0; - - ASSERT(pCommitter->dReader.pBlockIdx); - - pCommitter->dReader.iBlockIdx++; - if (pCommitter->dReader.iBlockIdx < taosArrayGetSize(pCommitter->dReader.aBlockIdx)) { - pCommitter->dReader.pBlockIdx = - (SBlockIdx *)taosArrayGet(pCommitter->dReader.aBlockIdx, pCommitter->dReader.iBlockIdx); - - code = tsdbReadDataBlk(pCommitter->dReader.pReader, pCommitter->dReader.pBlockIdx, &pCommitter->dReader.mBlock); - TSDB_CHECK_CODE(code, lino, _exit); - - ASSERT(pCommitter->dReader.mBlock.nItem > 0); - } else { - pCommitter->dReader.pBlockIdx = NULL; - } - -_exit: - return code; -} - -static int32_t tDataIterCmprFn(const SRBTreeNode *n1, const SRBTreeNode *n2) { - SDataIter *pIter1 = (SDataIter *)((uint8_t *)n1 - offsetof(SDataIter, n)); - SDataIter *pIter2 = (SDataIter *)((uint8_t *)n2 - offsetof(SDataIter, n)); - - return tRowInfoCmprFn(&pIter1->r, &pIter2->r); -} - -static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) { - int32_t code = 0; - int32_t lino = 0; - - pCommitter->pIter = NULL; - tRBTreeCreate(&pCommitter->rbt, tDataIterCmprFn); - - // memory - TSDBKEY tKey = {.ts = pCommitter->minKey, .version = VERSION_MIN}; - SDataIter *pIter = &pCommitter->dataIter; - pIter->type = MEMORY_DATA_ITER; - pIter->iTbDataP = 0; - for (; pIter->iTbDataP < taosArrayGetSize(pCommitter->aTbDataP); pIter->iTbDataP++) { - STbData *pTbData = (STbData *)taosArrayGetP(pCommitter->aTbDataP, pIter->iTbDataP); - tsdbTbDataIterOpen(pTbData, &tKey, 0, &pIter->iter); - TSDBROW *pRow = tsdbTbDataIterGet(&pIter->iter); - if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) { - pCommitter->nextKey = TMIN(pCommitter->nextKey, TSDBROW_TS(pRow)); - pRow = NULL; - } - - if (pRow == NULL) continue; - - pIter->r.suid = pTbData->suid; - pIter->r.uid = pTbData->uid; - pIter->r.row = *pRow; - break; - } - ASSERT(pIter->iTbDataP < taosArrayGetSize(pCommitter->aTbDataP)); - tRBTreePut(&pCommitter->rbt, (SRBTreeNode *)pIter); - - // disk - pCommitter->toLastOnly = 0; - SDataFReader *pReader = pCommitter->dReader.pReader; - if (pReader) { - if (pReader->pSet->nSttF >= pCommitter->sttTrigger) { - int8_t iIter = 0; - for (int32_t iStt = 0; iStt < pReader->pSet->nSttF; iStt++) { - pIter = &pCommitter->aDataIter[iIter]; - pIter->type = STT_DATA_ITER; - pIter->iStt = iStt; - - code = tsdbReadSttBlk(pCommitter->dReader.pReader, iStt, pIter->aSttBlk); - TSDB_CHECK_CODE(code, lino, _exit); - - if (taosArrayGetSize(pIter->aSttBlk) == 0) continue; - - pIter->iSttBlk = 0; - SSttBlk *pSttBlk = (SSttBlk *)taosArrayGet(pIter->aSttBlk, 0); - code = tsdbReadSttBlockEx(pCommitter->dReader.pReader, iStt, pSttBlk, &pIter->bData); - TSDB_CHECK_CODE(code, lino, _exit); - - pIter->iRow = 0; - pIter->r.suid = pIter->bData.suid; - pIter->r.uid = pIter->bData.uid ? pIter->bData.uid : pIter->bData.aUid[0]; - pIter->r.row = tsdbRowFromBlockData(&pIter->bData, 0); - - tRBTreePut(&pCommitter->rbt, (SRBTreeNode *)pIter); - iIter++; - } - } else { - for (int32_t iStt = 0; iStt < pReader->pSet->nSttF; iStt++) { - SSttFile *pSttFile = pReader->pSet->aSttF[iStt]; - if (pSttFile->size > pSttFile->offset) { - pCommitter->toLastOnly = 1; - break; - } - } - } - } - - code = tsdbNextCommitRow(pCommitter); - TSDB_CHECK_CODE(code, lino, _exit); - -_exit: - if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, - tstrerror(code)); - } - return code; -} - -static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { - int32_t code = 0; - int32_t lino = 0; - STsdb *pTsdb = pCommitter->pTsdb; - SDFileSet *pRSet = NULL; - - // memory - pCommitter->commitFid = tsdbKeyFid(pCommitter->nextKey, pCommitter->minutes, pCommitter->precision); - pCommitter->expLevel = tsdbFidLevel(pCommitter->commitFid, &pCommitter->pTsdb->keepCfg, taosGetTimestampSec()); - tsdbFidKeyRange(pCommitter->commitFid, pCommitter->minutes, pCommitter->precision, &pCommitter->minKey, - &pCommitter->maxKey); -#if 0 - ASSERT(pCommitter->minKey <= pCommitter->nextKey && pCommitter->maxKey >= pCommitter->nextKey); -#endif - - pCommitter->nextKey = TSKEY_MAX; - - // Reader - SDFileSet tDFileSet = {.fid = pCommitter->commitFid}; - pRSet = (SDFileSet *)taosArraySearch(pCommitter->fs.aDFileSet, &tDFileSet, tDFileSetCmprFn, TD_EQ); - if (pRSet) { - code = tsdbDataFReaderOpen(&pCommitter->dReader.pReader, pTsdb, pRSet); - TSDB_CHECK_CODE(code, lino, _exit); - - // data - code = tsdbReadBlockIdx(pCommitter->dReader.pReader, pCommitter->dReader.aBlockIdx); - TSDB_CHECK_CODE(code, lino, _exit); - - pCommitter->dReader.iBlockIdx = 0; - if (taosArrayGetSize(pCommitter->dReader.aBlockIdx) > 0) { - pCommitter->dReader.pBlockIdx = (SBlockIdx *)taosArrayGet(pCommitter->dReader.aBlockIdx, 0); - code = tsdbReadDataBlk(pCommitter->dReader.pReader, pCommitter->dReader.pBlockIdx, &pCommitter->dReader.mBlock); - TSDB_CHECK_CODE(code, lino, _exit); - } else { - pCommitter->dReader.pBlockIdx = NULL; - } - tBlockDataReset(&pCommitter->dReader.bData); - } else { - pCommitter->dReader.pBlockIdx = NULL; - } - - // Writer - SHeadFile fHead = {.commitID = pCommitter->commitID}; - SDataFile fData = {.commitID = pCommitter->commitID}; - SSmaFile fSma = {.commitID = pCommitter->commitID}; - SSttFile fStt = {.commitID = pCommitter->commitID}; - SDFileSet wSet = {.fid = pCommitter->commitFid, .pHeadF = &fHead, .pDataF = &fData, .pSmaF = &fSma}; - if (pRSet) { - ASSERT(pRSet->nSttF <= pCommitter->sttTrigger); - fData = *pRSet->pDataF; - fSma = *pRSet->pSmaF; - wSet.diskId = pRSet->diskId; - if (pRSet->nSttF < pCommitter->sttTrigger) { - for (int32_t iStt = 0; iStt < pRSet->nSttF; iStt++) { - wSet.aSttF[iStt] = pRSet->aSttF[iStt]; - } - wSet.nSttF = pRSet->nSttF + 1; - } else { - wSet.nSttF = 1; - } - } else { - SDiskID did = {0}; - if (tfsAllocDisk(pTsdb->pVnode->pTfs, pCommitter->expLevel, &did) < 0) { - code = terrno; - TSDB_CHECK_CODE(code, lino, _exit); - } - code = tfsMkdirRecurAt(pTsdb->pVnode->pTfs, pTsdb->path, did); - TSDB_CHECK_CODE(code, lino, _exit); - wSet.diskId = did; - wSet.nSttF = 1; - } - wSet.aSttF[wSet.nSttF - 1] = &fStt; - code = tsdbDataFWriterOpen(&pCommitter->dWriter.pWriter, pTsdb, &wSet); - TSDB_CHECK_CODE(code, lino, _exit); - - taosArrayClear(pCommitter->dWriter.aBlockIdx); - taosArrayClear(pCommitter->dWriter.aSttBlk); - tMapDataReset(&pCommitter->dWriter.mBlock); - tBlockDataReset(&pCommitter->dWriter.bData); -#if USE_STREAM_COMPRESSION - tDiskDataBuilderClear(pCommitter->dWriter.pBuilder); -#else - tBlockDataReset(&pCommitter->dWriter.bDatal); -#endif - - // open iter - code = tsdbOpenCommitIter(pCommitter); - TSDB_CHECK_CODE(code, lino, _exit); - -_exit: - if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); - } - return code; -} - -int32_t tsdbWriteDataBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SMapData *mDataBlk, int8_t cmprAlg) { - int32_t code = 0; - int32_t lino = 0; - - if (pBlockData->nRow == 0) return code; - - SDataBlk dataBlk; - tDataBlkReset(&dataBlk); - - // info - dataBlk.nRow += pBlockData->nRow; - for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) { - TSDBKEY key = {.ts = pBlockData->aTSKEY[iRow], .version = pBlockData->aVersion[iRow]}; - - if (iRow == 0) { - if (tsdbKeyCmprFn(&dataBlk.minKey, &key) > 0) { - dataBlk.minKey = key; - } - } else { - if (pBlockData->aTSKEY[iRow] == pBlockData->aTSKEY[iRow - 1]) { - dataBlk.hasDup = 1; - } - } - - if (iRow == pBlockData->nRow - 1 && tsdbKeyCmprFn(&dataBlk.maxKey, &key) < 0) { - dataBlk.maxKey = key; - } - - dataBlk.minVer = TMIN(dataBlk.minVer, key.version); - dataBlk.maxVer = TMAX(dataBlk.maxVer, key.version); - } - - // write - dataBlk.nSubBlock++; - code = tsdbWriteBlockData(pWriter, pBlockData, &dataBlk.aSubBlock[dataBlk.nSubBlock - 1], - ((dataBlk.nSubBlock == 1) && !dataBlk.hasDup) ? &dataBlk.smaInfo : NULL, cmprAlg, 0); - TSDB_CHECK_CODE(code, lino, _exit); - - // put SDataBlk - code = tMapDataPutItem(mDataBlk, &dataBlk, tPutDataBlk); - TSDB_CHECK_CODE(code, lino, _exit); - - // clear - tBlockDataClear(pBlockData); - -_exit: - if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, - tstrerror(code)); - } - return code; -} - -int32_t tsdbWriteSttBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SArray *aSttBlk, int8_t cmprAlg) { - int32_t code = 0; - int32_t lino = 0; - SSttBlk sstBlk; - - if (pBlockData->nRow == 0) return code; - - // info - sstBlk.suid = pBlockData->suid; - sstBlk.nRow = pBlockData->nRow; - sstBlk.minKey = TSKEY_MAX; - sstBlk.maxKey = TSKEY_MIN; - sstBlk.minVer = VERSION_MAX; - sstBlk.maxVer = VERSION_MIN; - for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) { - sstBlk.minKey = TMIN(sstBlk.minKey, pBlockData->aTSKEY[iRow]); - sstBlk.maxKey = TMAX(sstBlk.maxKey, pBlockData->aTSKEY[iRow]); - sstBlk.minVer = TMIN(sstBlk.minVer, pBlockData->aVersion[iRow]); - sstBlk.maxVer = TMAX(sstBlk.maxVer, pBlockData->aVersion[iRow]); - } - sstBlk.minUid = pBlockData->uid ? pBlockData->uid : pBlockData->aUid[0]; - sstBlk.maxUid = pBlockData->uid ? pBlockData->uid : pBlockData->aUid[pBlockData->nRow - 1]; - - // write - code = tsdbWriteBlockData(pWriter, pBlockData, &sstBlk.bInfo, NULL, cmprAlg, 1); - TSDB_CHECK_CODE(code, lino, _exit); - - // push SSttBlk - if (taosArrayPush(aSttBlk, &sstBlk) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - - // clear - tBlockDataClear(pBlockData); - -_exit: - if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, - tstrerror(code)); - } - return code; -} - -static int32_t tsdbCommitSttBlk(SDataFWriter *pWriter, SDiskDataBuilder *pBuilder, SArray *aSttBlk) { - int32_t code = 0; - int32_t lino = 0; - - if (pBuilder->nRow == 0) return code; - - // gnrt - const SDiskData *pDiskData; - const SBlkInfo *pBlkInfo; - code = tGnrtDiskData(pBuilder, &pDiskData, &pBlkInfo); - TSDB_CHECK_CODE(code, lino, _exit); - - SSttBlk sttBlk = {.suid = pBuilder->suid, - .minUid = pBlkInfo->minUid, - .maxUid = pBlkInfo->maxUid, - .minKey = pBlkInfo->minKey, - .maxKey = pBlkInfo->maxKey, - .minVer = pBlkInfo->minVer, - .maxVer = pBlkInfo->maxVer, - .nRow = pBuilder->nRow}; - // write - code = tsdbWriteDiskData(pWriter, pDiskData, &sttBlk.bInfo, NULL); - TSDB_CHECK_CODE(code, lino, _exit); - - // push - if (taosArrayPush(aSttBlk, &sttBlk) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - - // clear - tDiskDataBuilderClear(pBuilder); - -_exit: - if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, - tstrerror(code)); - } - return code; -} - -static int32_t tsdbCommitFileDataEnd(SCommitter *pCommitter) { - int32_t code = 0; - int32_t lino = 0; - - // write aBlockIdx - code = tsdbWriteBlockIdx(pCommitter->dWriter.pWriter, pCommitter->dWriter.aBlockIdx); - TSDB_CHECK_CODE(code, lino, _exit); - - // write aSttBlk - code = tsdbWriteSttBlk(pCommitter->dWriter.pWriter, pCommitter->dWriter.aSttBlk); - TSDB_CHECK_CODE(code, lino, _exit); - - // update file header - code = tsdbUpdateDFileSetHeader(pCommitter->dWriter.pWriter); - TSDB_CHECK_CODE(code, lino, _exit); - - // upsert SDFileSet - code = tsdbFSUpsertFSet(&pCommitter->fs, &pCommitter->dWriter.pWriter->wSet); - TSDB_CHECK_CODE(code, lino, _exit); - - // close and sync - code = tsdbDataFWriterClose(&pCommitter->dWriter.pWriter, 1); - TSDB_CHECK_CODE(code, lino, _exit); - - if (pCommitter->dReader.pReader) { - code = tsdbDataFReaderClose(&pCommitter->dReader.pReader); - TSDB_CHECK_CODE(code, lino, _exit); - } - -_exit: - if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, - tstrerror(code)); - } - return code; -} - -static int32_t tsdbMoveCommitData(SCommitter *pCommitter, TABLEID toTable) { - int32_t code = 0; - int32_t lino = 0; - - while (pCommitter->dReader.pBlockIdx && tTABLEIDCmprFn(pCommitter->dReader.pBlockIdx, &toTable) < 0) { - SBlockIdx blockIdx = *pCommitter->dReader.pBlockIdx; - code = tsdbWriteDataBlk(pCommitter->dWriter.pWriter, &pCommitter->dReader.mBlock, &blockIdx); - TSDB_CHECK_CODE(code, lino, _exit); - - if (taosArrayPush(pCommitter->dWriter.aBlockIdx, &blockIdx) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - - code = tsdbCommitterNextTableData(pCommitter); - TSDB_CHECK_CODE(code, lino, _exit); - } - -_exit: - if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, - tstrerror(code)); - } - return code; -} - -static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter); -static int32_t tsdbCommitFileData(SCommitter *pCommitter) { - int32_t code = 0; - int32_t lino = 0; - STsdb *pTsdb = pCommitter->pTsdb; - SMemTable *pMemTable = pTsdb->imem; - - // commit file data start - code = tsdbCommitFileDataStart(pCommitter); - TSDB_CHECK_CODE(code, lino, _exit); - - // impl - code = tsdbCommitFileDataImpl(pCommitter); - TSDB_CHECK_CODE(code, lino, _exit); - - // commit file data end - code = tsdbCommitFileDataEnd(pCommitter); - TSDB_CHECK_CODE(code, lino, _exit); - -_exit: - if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); - tsdbDataFReaderClose(&pCommitter->dReader.pReader); - tsdbDataFWriterClose(&pCommitter->dWriter.pWriter, 0); - } - return code; -} - -// ---------------------------------------------------------------------------- -static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter, SCommitInfo *pInfo) { - int32_t code = 0; - int32_t lino = 0; - - memset(pCommitter, 0, sizeof(*pCommitter)); - ASSERT(pTsdb->imem && "last tsdb commit incomplete"); - - pCommitter->pTsdb = pTsdb; - pCommitter->commitID = pInfo->info.state.commitID; - pCommitter->minutes = pTsdb->keepCfg.days; - pCommitter->precision = pTsdb->keepCfg.precision; - pCommitter->minRow = pInfo->info.config.tsdbCfg.minRows; - pCommitter->maxRow = pInfo->info.config.tsdbCfg.maxRows; - pCommitter->cmprAlg = pInfo->info.config.tsdbCfg.compression; - pCommitter->sttTrigger = pInfo->info.config.sttTrigger; - pCommitter->aTbDataP = tsdbMemTableGetTbDataArray(pTsdb->imem); - if (pCommitter->aTbDataP == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - code = tsdbFSCopy(pTsdb, &pCommitter->fs); - TSDB_CHECK_CODE(code, lino, _exit); - -_exit: - if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); - } - return code; -} - -static int32_t tsdbCommitDataStart(SCommitter *pCommitter) { - int32_t code = 0; - int32_t lino = 0; - - // reader - pCommitter->dReader.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); - if (pCommitter->dReader.aBlockIdx == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - - code = tBlockDataCreate(&pCommitter->dReader.bData); - TSDB_CHECK_CODE(code, lino, _exit); - - // merger - for (int32_t iStt = 0; iStt < TSDB_STT_TRIGGER_ARRAY_SIZE; iStt++) { - SDataIter *pIter = &pCommitter->aDataIter[iStt]; - pIter->aSttBlk = taosArrayInit(0, sizeof(SSttBlk)); - if (pIter->aSttBlk == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - - code = tBlockDataCreate(&pIter->bData); - TSDB_CHECK_CODE(code, lino, _exit); - } - - // writer - pCommitter->dWriter.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); - if (pCommitter->dWriter.aBlockIdx == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - - pCommitter->dWriter.aSttBlk = taosArrayInit(0, sizeof(SSttBlk)); - if (pCommitter->dWriter.aSttBlk == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - - code = tBlockDataCreate(&pCommitter->dWriter.bData); - TSDB_CHECK_CODE(code, lino, _exit); - -#if USE_STREAM_COMPRESSION - code = tDiskDataBuilderCreate(&pCommitter->dWriter.pBuilder); -#else - code = tBlockDataCreate(&pCommitter->dWriter.bDatal); -#endif - TSDB_CHECK_CODE(code, lino, _exit); - -_exit: - if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, - tstrerror(code)); - } - return code; -} - -static void tsdbCommitDataEnd(SCommitter *pCommitter) { - // reader - taosArrayDestroy(pCommitter->dReader.aBlockIdx); - tMapDataClear(&pCommitter->dReader.mBlock); - tBlockDataDestroy(&pCommitter->dReader.bData); - - // merger - for (int32_t iStt = 0; iStt < TSDB_STT_TRIGGER_ARRAY_SIZE; iStt++) { - SDataIter *pIter = &pCommitter->aDataIter[iStt]; - taosArrayDestroy(pIter->aSttBlk); - tBlockDataDestroy(&pIter->bData); - } - - // writer - taosArrayDestroy(pCommitter->dWriter.aBlockIdx); - taosArrayDestroy(pCommitter->dWriter.aSttBlk); - tMapDataClear(&pCommitter->dWriter.mBlock); - tBlockDataDestroy(&pCommitter->dWriter.bData); -#if USE_STREAM_COMPRESSION - tDiskDataBuilderDestroy(pCommitter->dWriter.pBuilder); -#else - tBlockDataDestroy(&pCommitter->dWriter.bDatal); -#endif - tDestroyTSchema(pCommitter->skmTable.pTSchema); - tDestroyTSchema(pCommitter->skmRow.pTSchema); -} - -static int32_t tsdbCommitData(SCommitter *pCommitter) { - int32_t code = 0; - int32_t lino = 0; - - STsdb *pTsdb = pCommitter->pTsdb; - SMemTable *pMemTable = pTsdb->imem; - - // check - if (pMemTable->nRow == 0) goto _exit; - - // start ==================== - code = tsdbCommitDataStart(pCommitter); - TSDB_CHECK_CODE(code, lino, _exit); - - // impl ==================== - pCommitter->nextKey = pMemTable->minKey; - while (pCommitter->nextKey < TSKEY_MAX) { - code = tsdbCommitFileData(pCommitter); - TSDB_CHECK_CODE(code, lino, _exit); - } - - // end ==================== - tsdbCommitDataEnd(pCommitter); - -_exit: - if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); - } - return code; -} - -static int32_t tsdbCommitDel(SCommitter *pCommitter) { - int32_t code = 0; - int32_t lino = 0; - - STsdb *pTsdb = pCommitter->pTsdb; - SMemTable *pMemTable = pTsdb->imem; - - if (pMemTable->nDel == 0) { - goto _exit; - } - - // start - code = tsdbCommitDelStart(pCommitter); - if (code) { - TSDB_CHECK_CODE(code, lino, _exit); - } - - // impl - int32_t iDelIdx = 0; - int32_t nDelIdx = taosArrayGetSize(pCommitter->aDelIdx); - int32_t iTbData = 0; - int32_t nTbData = taosArrayGetSize(pCommitter->aTbDataP); - STbData *pTbData; - SDelIdx *pDelIdx; - - ASSERT(nTbData > 0); - - pTbData = (STbData *)taosArrayGetP(pCommitter->aTbDataP, iTbData); - pDelIdx = (iDelIdx < nDelIdx) ? (SDelIdx *)taosArrayGet(pCommitter->aDelIdx, iDelIdx) : NULL; - while (true) { - if (pTbData == NULL && pDelIdx == NULL) break; - - if (pTbData && pDelIdx) { - int32_t c = tTABLEIDCmprFn(pTbData, pDelIdx); - - if (c == 0) { - goto _commit_mem_and_disk_del; - } else if (c < 0) { - goto _commit_mem_del; - } else { - goto _commit_disk_del; - } - } else if (pTbData) { - goto _commit_mem_del; - } else { - goto _commit_disk_del; - } - - _commit_mem_del: - code = tsdbCommitTableDel(pCommitter, pTbData, NULL); - TSDB_CHECK_CODE(code, lino, _exit); - - iTbData++; - pTbData = (iTbData < nTbData) ? (STbData *)taosArrayGetP(pCommitter->aTbDataP, iTbData) : NULL; - continue; - - _commit_disk_del: - code = tsdbCommitTableDel(pCommitter, NULL, pDelIdx); - TSDB_CHECK_CODE(code, lino, _exit); - - iDelIdx++; - pDelIdx = (iDelIdx < nDelIdx) ? (SDelIdx *)taosArrayGet(pCommitter->aDelIdx, iDelIdx) : NULL; - continue; - - _commit_mem_and_disk_del: - code = tsdbCommitTableDel(pCommitter, pTbData, pDelIdx); - TSDB_CHECK_CODE(code, lino, _exit); - - iTbData++; - pTbData = (iTbData < nTbData) ? (STbData *)taosArrayGetP(pCommitter->aTbDataP, iTbData) : NULL; - iDelIdx++; - pDelIdx = (iDelIdx < nDelIdx) ? (SDelIdx *)taosArrayGet(pCommitter->aDelIdx, iDelIdx) : NULL; - continue; - } - - // end - code = tsdbCommitDelEnd(pCommitter); - TSDB_CHECK_CODE(code, lino, _exit); - -_exit: - if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); - } else { - tsdbDebug("vgId:%d, commit del done, nDel:%" PRId64, TD_VID(pTsdb->pVnode), pMemTable->nDel); - } - return code; -} - -static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) { - int32_t code = 0; - int32_t lino = 0; - STsdb *pTsdb = pCommitter->pTsdb; - - if (eno) { - code = eno; - TSDB_CHECK_CODE(code, lino, _exit); - } else { - code = tsdbFSPrepareCommit(pCommitter->pTsdb, &pCommitter->fs); - TSDB_CHECK_CODE(code, lino, _exit); - } - -_exit: - tsdbFSDestroy(&pCommitter->fs); - taosArrayDestroy(pCommitter->aTbDataP); - pCommitter->aTbDataP = NULL; - if (code || eno) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); - } else { - tsdbInfo("vgId:%d, tsdb end commit", TD_VID(pTsdb->pVnode)); - } - return code; -} - -// ================================================================================ - -static FORCE_INLINE SRowInfo *tsdbGetCommitRow(SCommitter *pCommitter) { - return (pCommitter->pIter) ? &pCommitter->pIter->r : NULL; -} - -static int32_t tsdbNextCommitRow(SCommitter *pCommitter) { - int32_t code = 0; - int32_t lino = 0; - - if (pCommitter->pIter) { - SDataIter *pIter = pCommitter->pIter; - if (pCommitter->pIter->type == MEMORY_DATA_ITER) { // memory - tsdbTbDataIterNext(&pIter->iter); - TSDBROW *pRow = tsdbTbDataIterGet(&pIter->iter); - while (true) { - if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) { - pCommitter->nextKey = TMIN(pCommitter->nextKey, TSDBROW_TS(pRow)); - pRow = NULL; - } - - if (pRow) { - pIter->r.suid = pIter->iter.pTbData->suid; - pIter->r.uid = pIter->iter.pTbData->uid; - pIter->r.row = *pRow; - break; - } - - pIter->iTbDataP++; - if (pIter->iTbDataP < taosArrayGetSize(pCommitter->aTbDataP)) { - STbData *pTbData = (STbData *)taosArrayGetP(pCommitter->aTbDataP, pIter->iTbDataP); - TSDBKEY keyFrom = {.ts = pCommitter->minKey, .version = VERSION_MIN}; - tsdbTbDataIterOpen(pTbData, &keyFrom, 0, &pIter->iter); - pRow = tsdbTbDataIterGet(&pIter->iter); - continue; - } else { - pCommitter->pIter = NULL; - break; - } - } - } else if (pCommitter->pIter->type == STT_DATA_ITER) { // last file - pIter->iRow++; - if (pIter->iRow < pIter->bData.nRow) { - pIter->r.uid = pIter->bData.uid ? pIter->bData.uid : pIter->bData.aUid[pIter->iRow]; - pIter->r.row = tsdbRowFromBlockData(&pIter->bData, pIter->iRow); - } else { - pIter->iSttBlk++; - if (pIter->iSttBlk < taosArrayGetSize(pIter->aSttBlk)) { - SSttBlk *pSttBlk = (SSttBlk *)taosArrayGet(pIter->aSttBlk, pIter->iSttBlk); - - code = tsdbReadSttBlockEx(pCommitter->dReader.pReader, pIter->iStt, pSttBlk, &pIter->bData); - if (code) goto _exit; - - pIter->iRow = 0; - pIter->r.suid = pIter->bData.suid; - pIter->r.uid = pIter->bData.uid ? pIter->bData.uid : pIter->bData.aUid[0]; - pIter->r.row = tsdbRowFromBlockData(&pIter->bData, 0); - } else { - pCommitter->pIter = NULL; - } - } - } else { - ASSERT(0); - } - - // compare with min in RB Tree - pIter = (SDataIter *)tRBTreeMin(&pCommitter->rbt); - if (pCommitter->pIter && pIter) { - int32_t c = tRowInfoCmprFn(&pCommitter->pIter->r, &pIter->r); - if (c > 0) { - tRBTreePut(&pCommitter->rbt, (SRBTreeNode *)pCommitter->pIter); - pCommitter->pIter = NULL; - } else { - ASSERT(c); - } - } - } - - if (pCommitter->pIter == NULL) { - pCommitter->pIter = (SDataIter *)tRBTreeMin(&pCommitter->rbt); - if (pCommitter->pIter) { - tRBTreeDrop(&pCommitter->rbt, (SRBTreeNode *)pCommitter->pIter); - } - } - -_exit: - if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, - tstrerror(code)); - } - return code; -} - -static int32_t tsdbCommitAheadBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) { - int32_t code = 0; - int32_t lino = 0; - - SBlockData *pBlockData = &pCommitter->dWriter.bData; - SRowInfo *pRowInfo = tsdbGetCommitRow(pCommitter); - TABLEID id = {.suid = pRowInfo->suid, .uid = pRowInfo->uid}; - - tBlockDataClear(pBlockData); - while (pRowInfo) { - code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row)); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tBlockDataAppendRow(pBlockData, &pRowInfo->row, pCommitter->skmRow.pTSchema, id.uid); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbNextCommitRow(pCommitter); - TSDB_CHECK_CODE(code, lino, _exit); - - pRowInfo = tsdbGetCommitRow(pCommitter); - if (pRowInfo) { - if (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid) { - pRowInfo = NULL; - } else { - TSDBKEY tKey = TSDBROW_KEY(&pRowInfo->row); - if (tsdbKeyCmprFn(&tKey, &pDataBlk->minKey) >= 0) pRowInfo = NULL; - } - } - - if (pBlockData->nRow >= pCommitter->maxRow) { - code = - tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBlockData, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); - TSDB_CHECK_CODE(code, lino, _exit); - } - } - - code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBlockData, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); - TSDB_CHECK_CODE(code, lino, _exit); - -_exit: - if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, - tstrerror(code)); - } - return code; -} - -static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) { - int32_t code = 0; - int32_t lino = 0; - - SRowInfo *pRowInfo = tsdbGetCommitRow(pCommitter); - TABLEID id = {.suid = pRowInfo->suid, .uid = pRowInfo->uid}; - SBlockData *pBDataR = &pCommitter->dReader.bData; - SBlockData *pBDataW = &pCommitter->dWriter.bData; - - code = tsdbReadDataBlock(pCommitter->dReader.pReader, pDataBlk, pBDataR); - TSDB_CHECK_CODE(code, lino, _exit); - - tBlockDataClear(pBDataW); - int32_t iRow = 0; - TSDBROW row = tsdbRowFromBlockData(pBDataR, 0); - TSDBROW *pRow = &row; - - while (pRow && pRowInfo) { - int32_t c = tsdbRowCmprFn(pRow, &pRowInfo->row); - if (c < 0) { - code = tBlockDataAppendRow(pBDataW, pRow, NULL, id.uid); - TSDB_CHECK_CODE(code, lino, _exit); - - iRow++; - if (iRow < pBDataR->nRow) { - row = tsdbRowFromBlockData(pBDataR, iRow); - } else { - pRow = NULL; - } - } else if (c > 0) { - code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row)); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tBlockDataAppendRow(pBDataW, &pRowInfo->row, pCommitter->skmRow.pTSchema, id.uid); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbNextCommitRow(pCommitter); - TSDB_CHECK_CODE(code, lino, _exit); - - pRowInfo = tsdbGetCommitRow(pCommitter); - if (pRowInfo) { - if (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid) { - pRowInfo = NULL; - } else { - TSDBKEY tKey = TSDBROW_KEY(&pRowInfo->row); - if (tsdbKeyCmprFn(&tKey, &pDataBlk->maxKey) > 0) pRowInfo = NULL; - } - } - } else { - ASSERT(0 && "dup rows not allowed"); - } - - if (pBDataW->nRow >= pCommitter->maxRow) { - code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBDataW, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); - TSDB_CHECK_CODE(code, lino, _exit); - } - } - - while (pRow) { - code = tBlockDataAppendRow(pBDataW, pRow, NULL, id.uid); - TSDB_CHECK_CODE(code, lino, _exit); - - iRow++; - if (iRow < pBDataR->nRow) { - row = tsdbRowFromBlockData(pBDataR, iRow); - } else { - pRow = NULL; - } - - if (pBDataW->nRow >= pCommitter->maxRow) { - code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBDataW, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); - TSDB_CHECK_CODE(code, lino, _exit); - } - } - - code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBDataW, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); - TSDB_CHECK_CODE(code, lino, _exit); - -_exit: - if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, - tstrerror(code)); - } - return code; -} - -static int32_t tsdbMergeTableData(SCommitter *pCommitter, TABLEID id) { - int32_t code = 0; - int32_t lino = 0; - - SBlockIdx *pBlockIdx = pCommitter->dReader.pBlockIdx; - - ASSERT(pBlockIdx == NULL || tTABLEIDCmprFn(pBlockIdx, &id) >= 0); - if (pBlockIdx && pBlockIdx->suid == id.suid && pBlockIdx->uid == id.uid) { - int32_t iBlock = 0; - SDataBlk block; - SDataBlk *pDataBlk = █ - SRowInfo *pRowInfo = tsdbGetCommitRow(pCommitter); - - ASSERT(pRowInfo->suid == id.suid && pRowInfo->uid == id.uid); - - tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pDataBlk, tGetDataBlk); - while (pDataBlk && pRowInfo) { - SDataBlk tBlock = {.minKey = TSDBROW_KEY(&pRowInfo->row), .maxKey = TSDBROW_KEY(&pRowInfo->row)}; - int32_t c = tDataBlkCmprFn(pDataBlk, &tBlock); - - if (c < 0) { - code = tMapDataPutItem(&pCommitter->dWriter.mBlock, pDataBlk, tPutDataBlk); - TSDB_CHECK_CODE(code, lino, _exit); - - iBlock++; - if (iBlock < pCommitter->dReader.mBlock.nItem) { - tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pDataBlk, tGetDataBlk); - } else { - pDataBlk = NULL; - } - } else if (c > 0) { - code = tsdbCommitAheadBlock(pCommitter, pDataBlk); - TSDB_CHECK_CODE(code, lino, _exit); - - pRowInfo = tsdbGetCommitRow(pCommitter); - if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) pRowInfo = NULL; - } else { - code = tsdbCommitMergeBlock(pCommitter, pDataBlk); - TSDB_CHECK_CODE(code, lino, _exit); - - iBlock++; - if (iBlock < pCommitter->dReader.mBlock.nItem) { - tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pDataBlk, tGetDataBlk); - } else { - pDataBlk = NULL; - } - pRowInfo = tsdbGetCommitRow(pCommitter); - if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) pRowInfo = NULL; - } - } - - while (pDataBlk) { - code = tMapDataPutItem(&pCommitter->dWriter.mBlock, pDataBlk, tPutDataBlk); - TSDB_CHECK_CODE(code, lino, _exit); - - iBlock++; - if (iBlock < pCommitter->dReader.mBlock.nItem) { - tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pDataBlk, tGetDataBlk); - } else { - pDataBlk = NULL; - } - } - - code = tsdbCommitterNextTableData(pCommitter); - TSDB_CHECK_CODE(code, lino, _exit); - } - -_exit: - if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, - tstrerror(code)); - } - return code; -} - -static int32_t tsdbInitSttBlockBuilderIfNeed(SCommitter *pCommitter, TABLEID id) { - int32_t code = 0; - int32_t lino = 0; - -#if USE_STREAM_COMPRESSION - SDiskDataBuilder *pBuilder = pCommitter->dWriter.pBuilder; - if (pBuilder->suid || pBuilder->uid) { - if (!TABLE_SAME_SCHEMA(pBuilder->suid, pBuilder->uid, id.suid, id.uid)) { - code = tsdbCommitSttBlk(pCommitter->dWriter.pWriter, pBuilder, pCommitter->dWriter.aSttBlk); - TSDB_CHECK_CODE(code, lino, _exit); - - tDiskDataBuilderClear(pBuilder); - } - } - - if (!pBuilder->suid && !pBuilder->uid) { - ASSERT(pCommitter->skmTable.suid == id.suid); - ASSERT(pCommitter->skmTable.uid == id.uid); - code = tDiskDataBuilderInit(pBuilder, pCommitter->skmTable.pTSchema, &id, pCommitter->cmprAlg, 0); - TSDB_CHECK_CODE(code, lino, _exit); - } -#else - SBlockData *pBData = &pCommitter->dWriter.bDatal; - if (pBData->suid || pBData->uid) { - if (!TABLE_SAME_SCHEMA(pBData->suid, pBData->uid, id.suid, id.uid)) { - code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, pBData, pCommitter->dWriter.aSttBlk, pCommitter->cmprAlg); - TSDB_CHECK_CODE(code, lino, _exit); - - tBlockDataReset(pBData); - } - } - - if (!pBData->suid && !pBData->uid) { - ASSERT(pCommitter->skmTable.suid == id.suid); - ASSERT(pCommitter->skmTable.uid == id.uid); - TABLEID tid = {.suid = id.suid, .uid = id.suid ? 0 : id.uid}; - code = tBlockDataInit(pBData, &tid, pCommitter->skmTable.pTSchema, NULL, 0); - TSDB_CHECK_CODE(code, lino, _exit); - } -#endif - -_exit: - if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, - tstrerror(code)); - } - return code; -} - -static int32_t tsdbAppendLastBlock(SCommitter *pCommitter) { - int32_t code = 0; - int32_t lino = 0; - - SBlockData *pBData = &pCommitter->dWriter.bData; - TABLEID id = {.suid = pBData->suid, .uid = pBData->uid}; - - code = tsdbInitSttBlockBuilderIfNeed(pCommitter, id); - TSDB_CHECK_CODE(code, lino, _exit); - - for (int32_t iRow = 0; iRow < pBData->nRow; iRow++) { - TSDBROW row = tsdbRowFromBlockData(pBData, iRow); - -#if USE_STREAM_COMPRESSION - code = tDiskDataAddRow(pCommitter->dWriter.pBuilder, &row, NULL, &id); - TSDB_CHECK_CODE(code, lino, _exit); - - if (pCommitter->dWriter.pBuilder->nRow >= pCommitter->maxRow) { - code = tsdbCommitSttBlk(pCommitter->dWriter.pWriter, pCommitter->dWriter.pBuilder, pCommitter->dWriter.aSttBlk); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbInitSttBlockBuilderIfNeed(pCommitter, id); - TSDB_CHECK_CODE(code, lino, _exit); - } -#else - code = tBlockDataAppendRow(&pCommitter->dWriter.bDatal, &row, NULL, id.uid); - TSDB_CHECK_CODE(code, lino, _exit); - - if (pCommitter->dWriter.bDatal.nRow >= pCommitter->maxRow) { - code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.bDatal, pCommitter->dWriter.aSttBlk, - pCommitter->cmprAlg); - TSDB_CHECK_CODE(code, lino, _exit); - } -#endif - } - -_exit: - if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, - tstrerror(code)); - } - return code; -} - -static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) { - int32_t code = 0; - int32_t lino = 0; - - SRowInfo *pRowInfo = tsdbGetCommitRow(pCommitter); - if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) { - pRowInfo = NULL; - } - - if (pRowInfo == NULL) goto _exit; - - if (pCommitter->toLastOnly) { - code = tsdbInitSttBlockBuilderIfNeed(pCommitter, id); - TSDB_CHECK_CODE(code, lino, _exit); - - while (pRowInfo) { - STSchema *pTSchema = NULL; - if (pRowInfo->row.type == TSDBROW_ROW_FMT) { - code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row)); - TSDB_CHECK_CODE(code, lino, _exit); - pTSchema = pCommitter->skmRow.pTSchema; - } - -#if USE_STREAM_COMPRESSION - code = tDiskDataAddRow(pCommitter->dWriter.pBuilder, &pRowInfo->row, pTSchema, &id); -#else - code = tBlockDataAppendRow(&pCommitter->dWriter.bDatal, &pRowInfo->row, pTSchema, id.uid); -#endif - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbNextCommitRow(pCommitter); - TSDB_CHECK_CODE(code, lino, _exit); - - pRowInfo = tsdbGetCommitRow(pCommitter); - if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) { - pRowInfo = NULL; - } - -#if USE_STREAM_COMPRESSION - if (pCommitter->dWriter.pBuilder->nRow >= pCommitter->maxRow) { - code = tsdbCommitSttBlk(pCommitter->dWriter.pWriter, pCommitter->dWriter.pBuilder, pCommitter->dWriter.aSttBlk); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbInitSttBlockBuilderIfNeed(pCommitter, id); - TSDB_CHECK_CODE(code, lino, _exit); - } -#else - if (pCommitter->dWriter.bDatal.nRow >= pCommitter->maxRow) { - code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.bDatal, pCommitter->dWriter.aSttBlk, - pCommitter->cmprAlg); - TSDB_CHECK_CODE(code, lino, _exit); - } -#endif - } - } else { - SBlockData *pBData = &pCommitter->dWriter.bData; - ASSERT(pBData->nRow == 0); - - while (pRowInfo) { - STSchema *pTSchema = NULL; - if (pRowInfo->row.type == TSDBROW_ROW_FMT) { - code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row)); - TSDB_CHECK_CODE(code, lino, _exit); - pTSchema = pCommitter->skmRow.pTSchema; - } - - code = tBlockDataAppendRow(pBData, &pRowInfo->row, pTSchema, id.uid); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbNextCommitRow(pCommitter); - TSDB_CHECK_CODE(code, lino, _exit); - - pRowInfo = tsdbGetCommitRow(pCommitter); - if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) { - pRowInfo = NULL; - } - - if (pBData->nRow >= pCommitter->maxRow) { - code = - tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBData, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); - TSDB_CHECK_CODE(code, lino, _exit); - } - } - - if (pBData->nRow) { - if (pBData->nRow > pCommitter->minRow) { - code = - tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBData, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); - TSDB_CHECK_CODE(code, lino, _exit); - } else { - code = tsdbAppendLastBlock(pCommitter); - TSDB_CHECK_CODE(code, lino, _exit); - } - } - } - -_exit: - if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, - tstrerror(code)); - } - return code; -} - -static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter) { - int32_t code = 0; - int32_t lino = 0; - - SRowInfo *pRowInfo; - TABLEID id = {0}; - while ((pRowInfo = tsdbGetCommitRow(pCommitter)) != NULL) { - ASSERT(pRowInfo->suid != id.suid || pRowInfo->uid != id.uid); - id.suid = pRowInfo->suid; - id.uid = pRowInfo->uid; - - code = tsdbMoveCommitData(pCommitter, id); - TSDB_CHECK_CODE(code, lino, _exit); - - // start - tMapDataReset(&pCommitter->dWriter.mBlock); - - // impl - code = tsdbUpdateTableSchema(pCommitter->pTsdb->pVnode->pMeta, id.suid, id.uid, &pCommitter->skmTable); - TSDB_CHECK_CODE(code, lino, _exit); - code = tBlockDataInit(&pCommitter->dReader.bData, &id, pCommitter->skmTable.pTSchema, NULL, 0); - TSDB_CHECK_CODE(code, lino, _exit); - code = tBlockDataInit(&pCommitter->dWriter.bData, &id, pCommitter->skmTable.pTSchema, NULL, 0); - TSDB_CHECK_CODE(code, lino, _exit); - - /* merge with data in .data file */ - code = tsdbMergeTableData(pCommitter, id); - TSDB_CHECK_CODE(code, lino, _exit); - - /* handle remain table data */ - code = tsdbCommitTableData(pCommitter, id); - TSDB_CHECK_CODE(code, lino, _exit); - - // end - if (pCommitter->dWriter.mBlock.nItem > 0) { - SBlockIdx blockIdx = {.suid = id.suid, .uid = id.uid}; - code = tsdbWriteDataBlk(pCommitter->dWriter.pWriter, &pCommitter->dWriter.mBlock, &blockIdx); - TSDB_CHECK_CODE(code, lino, _exit); - - if (taosArrayPush(pCommitter->dWriter.aBlockIdx, &blockIdx) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - } - } - - id.suid = INT64_MAX; - id.uid = INT64_MAX; - code = tsdbMoveCommitData(pCommitter, id); - TSDB_CHECK_CODE(code, lino, _exit); - -#if USE_STREAM_COMPRESSION - code = tsdbCommitSttBlk(pCommitter->dWriter.pWriter, pCommitter->dWriter.pBuilder, pCommitter->dWriter.aSttBlk); -#else - code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.bDatal, pCommitter->dWriter.aSttBlk, - pCommitter->cmprAlg); -#endif - TSDB_CHECK_CODE(code, lino, _exit); - -_exit: - if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, - tstrerror(code)); - } - return code; -} - -int32_t tsdbFinishCommit(STsdb *pTsdb) { - int32_t code = 0; - int32_t lino = 0; - SMemTable *pMemTable = pTsdb->imem; - - // lock - taosThreadMutexLock(&pTsdb->mutex); - - code = tsdbFSCommit(pTsdb); - if (code) { - taosThreadMutexUnlock(&pTsdb->mutex); - TSDB_CHECK_CODE(code, lino, _exit); - } - - pTsdb->imem = NULL; - - // unlock - taosThreadMutexUnlock(&pTsdb->mutex); - if (pMemTable) { - tsdbUnrefMemTable(pMemTable, NULL, true); - } - -_exit: - if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); - } else { - tsdbInfo("vgId:%d, tsdb finish commit", TD_VID(pTsdb->pVnode)); - } - return code; -} - -int32_t tsdbRollbackCommit(STsdb *pTsdb) { - int32_t code = 0; - int32_t lino = 0; - - code = tsdbFSRollback(pTsdb); - TSDB_CHECK_CODE(code, lino, _exit); - -_exit: - if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); - } else { - tsdbInfo("vgId:%d, tsdb rollback commit", TD_VID(pTsdb->pVnode)); - } - return code; -} diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 732f46467e..a796f5121c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -702,7 +702,7 @@ _exit: } // EXPOSED APIS ==================================================================================== -int32_t tsdbFSCommit(STsdb *pTsdb) { +static int32_t tsdbFSCommit(STsdb *pTsdb) { int32_t code = 0; int32_t lino = 0; STsdbFS fs = {0}; @@ -738,7 +738,7 @@ _exit: return code; } -int32_t tsdbFSRollback(STsdb *pTsdb) { +static int32_t tsdbFSRollback(STsdb *pTsdb) { int32_t code = 0; int32_t lino = 0; @@ -833,312 +833,3 @@ int32_t tsdbFSClose(STsdb *pTsdb) { return code; } - -int32_t tsdbFSCopy(STsdb *pTsdb, STsdbFS *pFS) { - int32_t code = 0; - int32_t lino = 0; - - pFS->pDelFile = NULL; - if (pFS->aDFileSet) { - taosArrayClear(pFS->aDFileSet); - } else { - pFS->aDFileSet = taosArrayInit(taosArrayGetSize(pTsdb->fs.aDFileSet), sizeof(SDFileSet)); - if (pFS->aDFileSet == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - } - - if (pTsdb->fs.pDelFile) { - pFS->pDelFile = (SDelFile *)taosMemoryMalloc(sizeof(SDelFile)); - if (pFS->pDelFile == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - - *pFS->pDelFile = *pTsdb->fs.pDelFile; - } - - for (int32_t iSet = 0; iSet < taosArrayGetSize(pTsdb->fs.aDFileSet); iSet++) { - SDFileSet *pSet = (SDFileSet *)taosArrayGet(pTsdb->fs.aDFileSet, iSet); - SDFileSet fSet = {.diskId = pSet->diskId, .fid = pSet->fid}; - - // head - fSet.pHeadF = (SHeadFile *)taosMemoryMalloc(sizeof(SHeadFile)); - if (fSet.pHeadF == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - *fSet.pHeadF = *pSet->pHeadF; - - // data - fSet.pDataF = (SDataFile *)taosMemoryMalloc(sizeof(SDataFile)); - if (fSet.pDataF == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - *fSet.pDataF = *pSet->pDataF; - - // sma - fSet.pSmaF = (SSmaFile *)taosMemoryMalloc(sizeof(SSmaFile)); - if (fSet.pSmaF == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - *fSet.pSmaF = *pSet->pSmaF; - - // stt - for (fSet.nSttF = 0; fSet.nSttF < pSet->nSttF; fSet.nSttF++) { - fSet.aSttF[fSet.nSttF] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile)); - if (fSet.aSttF[fSet.nSttF] == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - *fSet.aSttF[fSet.nSttF] = *pSet->aSttF[fSet.nSttF]; - } - - if (taosArrayPush(pFS->aDFileSet, &fSet) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } - } - -_exit: - if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); - } - return code; -} - -int32_t tsdbFSUpsertDelFile(STsdbFS *pFS, SDelFile *pDelFile) { - int32_t code = 0; - - if (pDelFile) { - if (pFS->pDelFile == NULL) { - pFS->pDelFile = (SDelFile *)taosMemoryMalloc(sizeof(SDelFile)); - if (pFS->pDelFile == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - } - *pFS->pDelFile = *pDelFile; - } else { - if (pFS->pDelFile) { - taosMemoryFree(pFS->pDelFile); - pFS->pDelFile = NULL; - } - } - -_exit: - return code; -} - -int32_t tsdbFSUpsertFSet(STsdbFS *pFS, SDFileSet *pSet) { - int32_t code = 0; - int32_t idx = taosArraySearchIdx(pFS->aDFileSet, pSet, tDFileSetCmprFn, TD_GE); - - if (idx < 0) { - idx = taosArrayGetSize(pFS->aDFileSet); - } else { - SDFileSet *pDFileSet = (SDFileSet *)taosArrayGet(pFS->aDFileSet, idx); - int32_t c = tDFileSetCmprFn(pSet, pDFileSet); - if (c == 0) { - *pDFileSet->pHeadF = *pSet->pHeadF; - *pDFileSet->pDataF = *pSet->pDataF; - *pDFileSet->pSmaF = *pSet->pSmaF; - // stt - if (pSet->nSttF > pDFileSet->nSttF) { - ASSERT(pSet->nSttF == pDFileSet->nSttF + 1); - - pDFileSet->aSttF[pDFileSet->nSttF] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile)); - if (pDFileSet->aSttF[pDFileSet->nSttF] == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - *pDFileSet->aSttF[pDFileSet->nSttF] = *pSet->aSttF[pSet->nSttF - 1]; - pDFileSet->nSttF++; - } else if (pSet->nSttF < pDFileSet->nSttF) { - ASSERT(pSet->nSttF == 1); - for (int32_t iStt = 1; iStt < pDFileSet->nSttF; iStt++) { - taosMemoryFree(pDFileSet->aSttF[iStt]); - } - - *pDFileSet->aSttF[0] = *pSet->aSttF[0]; - pDFileSet->nSttF = 1; - } else { - for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) { - *pDFileSet->aSttF[iStt] = *pSet->aSttF[iStt]; - } - } - - pDFileSet->diskId = pSet->diskId; - goto _exit; - } - } - - ASSERT(pSet->nSttF == 1); - SDFileSet fSet = {.diskId = pSet->diskId, .fid = pSet->fid, .nSttF = 1}; - - // head - fSet.pHeadF = (SHeadFile *)taosMemoryMalloc(sizeof(SHeadFile)); - if (fSet.pHeadF == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - *fSet.pHeadF = *pSet->pHeadF; - - // data - fSet.pDataF = (SDataFile *)taosMemoryMalloc(sizeof(SDataFile)); - if (fSet.pDataF == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - *fSet.pDataF = *pSet->pDataF; - - // sma - fSet.pSmaF = (SSmaFile *)taosMemoryMalloc(sizeof(SSmaFile)); - if (fSet.pSmaF == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - *fSet.pSmaF = *pSet->pSmaF; - - // stt - fSet.aSttF[0] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile)); - if (fSet.aSttF[0] == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - *fSet.aSttF[0] = *pSet->aSttF[0]; - - if (taosArrayInsert(pFS->aDFileSet, idx, &fSet) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - -_exit: - return code; -} - -int32_t tsdbFSPrepareCommit(STsdb *pTsdb, STsdbFS *pFSNew) { - int32_t code = 0; - int32_t lino = 0; - char tfname[TSDB_FILENAME_LEN]; - - tsdbGetCurrentFName(pTsdb, NULL, tfname); - - // gnrt CURRENT.t - code = tsdbSaveFSToFile(pFSNew, tfname); - TSDB_CHECK_CODE(code, lino, _exit); - -_exit: - if (code) { - tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); - } - return code; -} - -int32_t tsdbFSRef(STsdb *pTsdb, STsdbFS *pFS) { - int32_t code = 0; - int32_t nRef; - - pFS->aDFileSet = taosArrayInit(taosArrayGetSize(pTsdb->fs.aDFileSet), sizeof(SDFileSet)); - if (pFS->aDFileSet == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - - pFS->pDelFile = pTsdb->fs.pDelFile; - if (pFS->pDelFile) { - nRef = atomic_fetch_add_32(&pFS->pDelFile->nRef, 1); - ASSERT(nRef > 0); - } - - SDFileSet fSet; - for (int32_t iSet = 0; iSet < taosArrayGetSize(pTsdb->fs.aDFileSet); iSet++) { - SDFileSet *pSet = (SDFileSet *)taosArrayGet(pTsdb->fs.aDFileSet, iSet); - fSet = *pSet; - - nRef = atomic_fetch_add_32(&pSet->pHeadF->nRef, 1); - ASSERT(nRef > 0); - - nRef = atomic_fetch_add_32(&pSet->pDataF->nRef, 1); - ASSERT(nRef > 0); - - nRef = atomic_fetch_add_32(&pSet->pSmaF->nRef, 1); - ASSERT(nRef > 0); - - for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) { - nRef = atomic_fetch_add_32(&pSet->aSttF[iStt]->nRef, 1); - ASSERT(nRef > 0); - } - - if (taosArrayPush(pFS->aDFileSet, &fSet) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - } - -_exit: - return code; -} - -void tsdbFSUnref(STsdb *pTsdb, STsdbFS *pFS) { - int32_t nRef; - char fname[TSDB_FILENAME_LEN]; - - if (pFS->pDelFile) { - nRef = atomic_sub_fetch_32(&pFS->pDelFile->nRef, 1); - ASSERT(nRef >= 0); - if (nRef == 0) { - tsdbDelFileName(pTsdb, pFS->pDelFile, fname); - (void)taosRemoveFile(fname); - taosMemoryFree(pFS->pDelFile); - } - } - - for (int32_t iSet = 0; iSet < taosArrayGetSize(pFS->aDFileSet); iSet++) { - SDFileSet *pSet = (SDFileSet *)taosArrayGet(pFS->aDFileSet, iSet); - - // head - nRef = atomic_sub_fetch_32(&pSet->pHeadF->nRef, 1); - ASSERT(nRef >= 0); - if (nRef == 0) { - tsdbHeadFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pHeadF, fname); - (void)taosRemoveFile(fname); - taosMemoryFree(pSet->pHeadF); - } - - // data - nRef = atomic_sub_fetch_32(&pSet->pDataF->nRef, 1); - ASSERT(nRef >= 0); - if (nRef == 0) { - tsdbDataFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pDataF, fname); - (void)taosRemoveFile(fname); - taosMemoryFree(pSet->pDataF); - } - - // sma - nRef = atomic_sub_fetch_32(&pSet->pSmaF->nRef, 1); - ASSERT(nRef >= 0); - if (nRef == 0) { - tsdbSmaFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pSmaF, fname); - (void)taosRemoveFile(fname); - taosMemoryFree(pSet->pSmaF); - } - - // stt - for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) { - nRef = atomic_sub_fetch_32(&pSet->aSttF[iStt]->nRef, 1); - ASSERT(nRef >= 0); - if (nRef == 0) { - tsdbSttFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSttF[iStt], fname); - (void)taosRemoveFile(fname); - taosMemoryFree(pSet->aSttF[iStt]); - /* code */ - } - } - } - - taosArrayDestroy(pFS->aDFileSet); -} diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 5ad067c113..1f54f13592 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -22,8 +22,8 @@ #include "tsdbUtil2.h" #include "tsimplehash.h" -#define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) -#define getCurrentKeyInSttBlock(_r) ((_r)->currentKey) +#define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) +#define getCurrentKeyInSttBlock(_r) ((_r)->currentKey) typedef struct { bool overlapWithNeighborBlock; @@ -41,7 +41,7 @@ static int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, i static TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pReader); static int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pScanInfo, STsdbReader* pReader); static int32_t doMergeRowsInSttBlock(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo, int64_t ts, - SRowMerger* pMerger, SVersionRange* pVerRange, const char* id); + SRowMerger* pMerger, SVersionRange* pVerRange, const char* id); static int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDelList, STsdbReader* pReader); static int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, SRow* pTSRow, STableBlockScanInfo* pScanInfo); @@ -67,7 +67,7 @@ static SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond static int32_t doBuildDataBlock(STsdbReader* pReader); static TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader); static bool hasDataInFileBlock(const SBlockData* pBlockData, const SFileBlockDumpInfo* pDumpInfo); -static bool hasDataInSttBlock(STableBlockScanInfo *pInfo); +static bool hasDataInSttBlock(STableBlockScanInfo* pInfo); static void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter); static int32_t getInitialDelIndex(const SArray* pDelSkyline, int32_t order); static void resetTableListIndex(SReaderStatus* pStatus); @@ -1138,7 +1138,7 @@ static bool getNeighborBlockOfSameTable(SDataBlockIter* pBlockIter, SFileDataBlo return false; } - int32_t step = asc ? 1 : -1; + int32_t step = asc ? 1 : -1; STableDataBlockIdx* pTableDataBlockIdx = taosArrayGet(pTableBlockScanInfo->pBlockIdxList, pBlockInfo->tbBlockIdx + step); SFileDataBlockInfo* p = taosArrayGet(pBlockIter->blockList, pTableDataBlockIdx->globalIndex); @@ -1316,17 +1316,17 @@ static int32_t buildDataBlockFromBuf(STsdbReader* pReader, STableBlockScanInfo* return TSDB_CODE_SUCCESS; } - int64_t st = taosGetTimestampUs(); + int64_t st = taosGetTimestampUs(); SSDataBlock* pBlock = pReader->resBlockInfo.pResBlock; - int32_t code = buildDataBlockFromBufImpl(pBlockScanInfo, endKey, pReader->resBlockInfo.capacity, pReader); + int32_t code = buildDataBlockFromBufImpl(pBlockScanInfo, endKey, pReader->resBlockInfo.capacity, pReader); double el = (taosGetTimestampUs() - st) / 1000.0; updateComposedBlockInfo(pReader, el, pBlockScanInfo); tsdbDebug("%p build data block from cache completed, elapsed time:%.2f ms, numOfRows:%" PRId64 ", brange:%" PRId64 " - %" PRId64 ", uid:%" PRIu64 ", %s", - pReader, el, pBlock->info.rows, pBlock->info.window.skey, pBlock->info.window.ekey, - pBlockScanInfo->uid, pReader->idStr); + pReader, el, pBlock->info.rows, pBlock->info.window.skey, pBlock->info.window.ekey, pBlockScanInfo->uid, + pReader->idStr); pReader->cost.buildmemBlock += el; return code; @@ -1390,13 +1390,9 @@ static bool nextRowFromSttBlocks(SSttBlockReader* pSttBlockReader, STableBlockSc } } -static void doPinSttBlock(SSttBlockReader* pSttBlockReader) { - tMergeTreePinSttBlock(&pSttBlockReader->mergeTree); -} +static void doPinSttBlock(SSttBlockReader* pSttBlockReader) { tMergeTreePinSttBlock(&pSttBlockReader->mergeTree); } -static void doUnpinSttBlock(SSttBlockReader* pSttBlockReader) { - tMergeTreeUnpinSttBlock(&pSttBlockReader->mergeTree); -} +static void doUnpinSttBlock(SSttBlockReader* pSttBlockReader) { tMergeTreeUnpinSttBlock(&pSttBlockReader->mergeTree); } static bool tryCopyDistinctRowFromSttBlock(TSDBROW* fRow, SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo, int64_t ts, STsdbReader* pReader, @@ -1535,8 +1531,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (code != TSDB_CODE_SUCCESS) { return code; } - doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, tsLast, pMerger, &pReader->info.verRange, - pReader->idStr); + doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, tsLast, pMerger, &pReader->info.verRange, pReader->idStr); } if (minKey == k.ts) { @@ -1585,8 +1580,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (code != TSDB_CODE_SUCCESS) { return code; } - doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, tsLast, pMerger, &pReader->info.verRange, - pReader->idStr); + doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, tsLast, pMerger, &pReader->info.verRange, pReader->idStr); } if (minKey == key) { @@ -1648,7 +1642,7 @@ static int32_t doMergeFileBlockAndLastBlock(SSttBlockReader* pSttBlockReader, ST TSDBROW* pRow1 = tMergeTreeGetRow(&pSttBlockReader->mergeTree); tsdbRowMergerAdd(pMerger, pRow1, NULL); doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, tsLastBlock, pMerger, &pReader->info.verRange, - pReader->idStr); + pReader->idStr); code = tsdbRowMergerGetRow(pMerger, &pTSRow); if (code != TSDB_CODE_SUCCESS) { @@ -1671,7 +1665,7 @@ static int32_t doMergeFileBlockAndLastBlock(SSttBlockReader* pSttBlockReader, ST } doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, tsLastBlock, pMerger, &pReader->info.verRange, - pReader->idStr); + pReader->idStr); // merge with block data if ts == key if (tsLastBlock == pBlockData->aTSKEY[pDumpInfo->rowIndex]) { @@ -1740,7 +1734,7 @@ static int32_t mergeFileBlockAndSttBlock(STsdbReader* pReader, SSttBlockReader* // the following for key == tsLast // ASC: file block ------> stt block // DESC: stt block ------> file block - SRow* pTSRow = NULL; + SRow* pTSRow = NULL; if (ASCENDING_TRAVERSE(pReader->info.order)) { code = tsdbRowMergerAdd(pMerger, &fRow, NULL); if (code != TSDB_CODE_SUCCESS) { @@ -1889,8 +1883,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* return code; } - doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, tsLast, pMerger, &pReader->info.verRange, - pReader->idStr); + doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, tsLast, pMerger, &pReader->info.verRange, pReader->idStr); } if (minKey == ik.ts) { @@ -1948,8 +1941,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* return code; } - doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, tsLast, pMerger, &pReader->info.verRange, - pReader->idStr); + doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, tsLast, pMerger, &pReader->info.verRange, pReader->idStr); } if (minKey == key) { @@ -2120,7 +2112,7 @@ static bool initSttBlockReader(SSttBlockReader* pSttBlockReader, STableBlockScan }; SSttDataInfoForTable info = {.pTimeWindowList = taosArrayInit(4, sizeof(STimeWindow))}; - int32_t code = tMergeTreeOpen2(&pSttBlockReader->mergeTree, &conf, &info); + int32_t code = tMergeTreeOpen2(&pSttBlockReader->mergeTree, &conf, &info); if (code != TSDB_CODE_SUCCESS) { return false; } @@ -2138,7 +2130,7 @@ static bool initSttBlockReader(SSttBlockReader* pSttBlockReader, STableBlockScan pScanInfo->sttWindow.ekey = INT64_MIN; // calculate the time window for data in stt files - for(int32_t i = 0; i < taosArrayGetSize(info.pTimeWindowList); ++i) { + for (int32_t i = 0; i < taosArrayGetSize(info.pTimeWindowList); ++i) { STimeWindow* pWindow = taosArrayGet(info.pTimeWindowList, i); if (pScanInfo->sttWindow.skey > pWindow->skey) { pScanInfo->sttWindow.skey = pWindow->skey; @@ -2149,8 +2141,9 @@ static bool initSttBlockReader(SSttBlockReader* pSttBlockReader, STableBlockScan } } - pScanInfo->sttKeyInfo.status = taosArrayGetSize(info.pTimeWindowList)? STT_FILE_HAS_DATA:STT_FILE_NO_DATA; - pScanInfo->sttKeyInfo.nextProcKey = ASCENDING_TRAVERSE(pReader->info.order)? pScanInfo->sttWindow.skey:pScanInfo->sttWindow.ekey; + pScanInfo->sttKeyInfo.status = taosArrayGetSize(info.pTimeWindowList) ? STT_FILE_HAS_DATA : STT_FILE_NO_DATA; + pScanInfo->sttKeyInfo.nextProcKey = + ASCENDING_TRAVERSE(pReader->info.order) ? pScanInfo->sttWindow.skey : pScanInfo->sttWindow.ekey; hasData = true; } else { hasData = nextRowFromSttBlocks(pSttBlockReader, pScanInfo, &pReader->info.verRange); @@ -2168,9 +2161,7 @@ static bool initSttBlockReader(SSttBlockReader* pSttBlockReader, STableBlockScan return hasData; } -static bool hasDataInSttBlock(STableBlockScanInfo *pInfo) { - return pInfo->sttKeyInfo.status == STT_FILE_HAS_DATA; -} +static bool hasDataInSttBlock(STableBlockScanInfo* pInfo) { return pInfo->sttKeyInfo.status == STT_FILE_HAS_DATA; } bool hasDataInFileBlock(const SBlockData* pBlockData, const SFileBlockDumpInfo* pDumpInfo) { if ((pBlockData->nRow > 0) && (pBlockData->nRow != pDumpInfo->totalRows)) { @@ -2225,10 +2216,11 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc } } -int32_t mergeRowsInSttBlocks(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader) { - bool copied = false; - SRow* pTSRow = NULL; - int64_t tsLastBlock = getCurrentKeyInSttBlock(pSttBlockReader); +int32_t mergeRowsInSttBlocks(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pBlockScanInfo, + STsdbReader* pReader) { + bool copied = false; + SRow* pTSRow = NULL; + int64_t tsLastBlock = getCurrentKeyInSttBlock(pSttBlockReader); SRowMerger* pMerger = &pReader->status.merger; TSDBROW* pRow = tMergeTreeGetRow(&pSttBlockReader->mergeTree); @@ -2559,16 +2551,16 @@ static void prepareDurationForNextFileSet(STsdbReader* pReader) { pReader->status.bProcMemFirstFileset = false; } - int32_t fid = pReader->status.pCurrentFileset->fid; + int32_t fid = pReader->status.pCurrentFileset->fid; STimeWindow winFid = {0}; tsdbFidKeyRange(fid, pReader->pTsdb->keepCfg.days, pReader->pTsdb->keepCfg.precision, &winFid.skey, &winFid.ekey); if (ASCENDING_TRAVERSE(pReader->info.order)) { pReader->status.bProcMemPreFileset = !(pReader->status.memTableMaxKey < pReader->status.prevFilesetStartKey || - (winFid.skey-1) < pReader->status.memTableMinKey); + (winFid.skey - 1) < pReader->status.memTableMinKey); } else { - pReader->status.bProcMemPreFileset = !( pReader->status.memTableMaxKey < (winFid.ekey+1) || - pReader->status.prevFilesetEndKey < pReader->status.memTableMinKey); + pReader->status.bProcMemPreFileset = !(pReader->status.memTableMaxKey < (winFid.ekey + 1) || + pReader->status.prevFilesetEndKey < pReader->status.memTableMinKey); } if (pReader->status.bProcMemPreFileset) { @@ -2802,7 +2794,7 @@ static int32_t doLoadSttBlockSequentially(STsdbReader* pReader) { static bool notOverlapWithSttFiles(SFileDataBlockInfo* pBlockInfo, STableBlockScanInfo* pScanInfo, bool asc) { ASSERT(pScanInfo->sttKeyInfo.status != STT_FILE_READER_UNINIT); - if(pScanInfo->sttKeyInfo.status == STT_FILE_NO_DATA) { + if (pScanInfo->sttKeyInfo.status == STT_FILE_NO_DATA) { return true; } else { int64_t keyInStt = pScanInfo->sttKeyInfo.nextProcKey; @@ -2923,7 +2915,8 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { static int32_t buildBlockFromBufferSeqForPreFileset(STsdbReader* pReader, int64_t endKey) { SReaderStatus* pStatus = &pReader->status; - tsdbDebug("seq load data blocks from cache that preceeds fileset %d, %s", pReader->status.pCurrentFileset->fid, pReader->idStr); + tsdbDebug("seq load data blocks from cache that preceeds fileset %d, %s", pReader->status.pCurrentFileset->fid, + pReader->idStr); while (1) { if (pReader->code != TSDB_CODE_SUCCESS) { @@ -3847,7 +3840,6 @@ int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t e pBlockScanInfo->lastProcKey = row.pBlockData->aTSKEY[row.iRow]; } - // no data in buffer, return immediately if (!(pBlockScanInfo->iter.hasVal || pBlockScanInfo->iiter.hasVal)) { break; @@ -3945,7 +3937,7 @@ static int32_t doOpenReaderImpl(STsdbReader* pReader) { int32_t code = TSDB_CODE_SUCCESS; if (pStatus->fileIter.numOfFiles == 0) { pStatus->loadFromFile = false; -// } else if (READER_EXEC_DATA == pReader->info.readMode) { + // } else if (READER_EXEC_DATA == pReader->info.readMode) { // DO NOTHING } else { code = initForFirstBlockInFile(pReader, pBlockIter); @@ -4092,7 +4084,7 @@ int32_t tsdbReaderOpen2(void* pVnode, SQueryTableDataCond* pCond, void* pTableLi } pReader->flag = READER_STATUS_SUSPEND; - pReader->info.execMode = pCond->notLoadData? READER_EXEC_ROWS : READER_EXEC_DATA; + pReader->info.execMode = pCond->notLoadData ? READER_EXEC_ROWS : READER_EXEC_DATA; pReader->pIgnoreTables = pIgnoreTables; tsdbDebug("%p total numOfTable:%d, window:%" PRId64 " - %" PRId64 ", verRange:%" PRId64 " - %" PRId64 @@ -4157,7 +4149,7 @@ void tsdbReaderClose2(STsdbReader* pReader) { } SReadCostSummary* pCost = &pReader->cost; - SFilesetIter* pFilesetIter = &pReader->status.fileIter; + SFilesetIter* pFilesetIter = &pReader->status.fileIter; if (pFilesetIter->pSttBlockReader != NULL) { SSttBlockReader* pLReader = pFilesetIter->pSttBlockReader; tMergeTreeClose(&pLReader->mergeTree); @@ -4204,7 +4196,7 @@ int32_t tsdbReaderSuspend2(STsdbReader* pReader) { SReaderStatus* pStatus = &pReader->status; STableBlockScanInfo* pBlockScanInfo = NULL; - pReader->status.suspendInvoked = true; // record the suspend status + pReader->status.suspendInvoked = true; // record the suspend status if (pStatus->loadFromFile) { SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); @@ -4227,7 +4219,7 @@ int32_t tsdbReaderSuspend2(STsdbReader* pReader) { // resetDataBlockScanInfo excluding lastKey STableBlockScanInfo** p = NULL; - int32_t step = ASCENDING_TRAVERSE(pReader->info.order)? 1:-1; + int32_t step = ASCENDING_TRAVERSE(pReader->info.order) ? 1 : -1; int32_t iter = 0; while ((p = tSimpleHashIterate(pStatus->pTableMap, p, &iter)) != NULL) { @@ -4248,7 +4240,7 @@ int32_t tsdbReaderSuspend2(STsdbReader* pReader) { pReader->flag = READER_STATUS_SUSPEND; #if SUSPEND_RESUME_TEST - tsem_post(&pReader->resumeAfterSuspend); + tsem_post(&pReader->resumeAfterSuspend); #endif tsdbDebug("reader: %p suspended uid %" PRIu64 " in this query %s", pReader, pBlockScanInfo ? pBlockScanInfo->uid : 0, @@ -4331,13 +4323,13 @@ _err: } static int32_t buildFromPreFilesetBuffer(STsdbReader* pReader) { - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; SReaderStatus* pStatus = &pReader->status; SSDataBlock* pBlock = pReader->resBlockInfo.pResBlock; - int32_t fid = pReader->status.pCurrentFileset->fid; - STimeWindow win = {0}; + int32_t fid = pReader->status.pCurrentFileset->fid; + STimeWindow win = {0}; tsdbFidKeyRange(fid, pReader->pTsdb->keepCfg.days, pReader->pTsdb->keepCfg.precision, &win.skey, &win.ekey); int64_t endKey = (ASCENDING_TRAVERSE(pReader->info.order)) ? win.skey : win.ekey; @@ -4359,8 +4351,8 @@ static int32_t buildFromPreFilesetBuffer(STsdbReader* pReader) { static int32_t doTsdbNextDataBlockFilesetDelimited(STsdbReader* pReader) { SReaderStatus* pStatus = &pReader->status; - int32_t code = TSDB_CODE_SUCCESS; - SSDataBlock* pBlock = pReader->resBlockInfo.pResBlock; + int32_t code = TSDB_CODE_SUCCESS; + SSDataBlock* pBlock = pReader->resBlockInfo.pResBlock; if (pStatus->loadFromFile) { if (pStatus->bProcMemPreFileset) { @@ -4375,12 +4367,12 @@ static int32_t doTsdbNextDataBlockFilesetDelimited(STsdbReader* pReader) { return code; } - tsdbTrace("block from file rows: %"PRId64", will process pre-file set buffer: %d. %s", - pBlock->info.rows, pStatus->bProcMemFirstFileset, pReader->idStr); + tsdbTrace("block from file rows: %" PRId64 ", will process pre-file set buffer: %d. %s", pBlock->info.rows, + pStatus->bProcMemFirstFileset, pReader->idStr); if (pStatus->bProcMemPreFileset) { if (pBlock->info.rows > 0) { if (pReader->notifyFn) { - int32_t fid = pReader->status.pCurrentFileset->fid; + int32_t fid = pReader->status.pCurrentFileset->fid; STsdReaderNotifyInfo info = {0}; info.duration.filesetId = fid; pReader->notifyFn(TSD_READER_NOTIFY_NEXT_DURATION_BLOCK, &info, pReader->notifyParam); @@ -4404,8 +4396,8 @@ static int32_t doTsdbNextDataBlockFilesetDelimited(STsdbReader* pReader) { static int32_t doTsdbNextDataBlockFilesFirst(STsdbReader* pReader) { SReaderStatus* pStatus = &pReader->status; - int32_t code = TSDB_CODE_SUCCESS; - SSDataBlock* pBlock = pReader->resBlockInfo.pResBlock; + int32_t code = TSDB_CODE_SUCCESS; + SSDataBlock* pBlock = pReader->resBlockInfo.pResBlock; if (pStatus->loadFromFile) { code = buildBlockFromFiles(pReader); @@ -4940,7 +4932,7 @@ static void getMemTableTimeRange(STsdbReader* pReader, int64_t* pMaxKey, int64_t int64_t minKey = INT64_MAX; void* pHashIter = tSimpleHashIterate(pStatus->pTableMap, NULL, &iter); - while (pHashIter!= NULL) { + while (pHashIter != NULL) { STableBlockScanInfo* pBlockScanInfo = *(STableBlockScanInfo**)pHashIter; STbData* d = NULL; @@ -5144,7 +5136,6 @@ void tsdbUntakeReadSnap2(STsdbReader* pReader, STsdbReadSnap* pSnap, bool proact tsdbUnrefMemTable(pSnap->pIMem, pSnap->pINode, proactive); } - tsdbFSUnref(pTsdb, &pSnap->fs); if (pSnap->pNode) taosMemoryFree(pSnap->pNode); if (pSnap->pINode) taosMemoryFree(pSnap->pINode); @@ -5165,9 +5156,7 @@ void tsdbReaderSetId2(STsdbReader* pReader, const char* idstr) { void tsdbReaderSetCloseFlag(STsdbReader* pReader) { /*pReader->code = TSDB_CODE_TSC_QUERY_CANCELLED;*/ } -void tsdbSetFilesetDelimited(STsdbReader* pReader) { - pReader->bFilesetDelimited = true; -} +void tsdbSetFilesetDelimited(STsdbReader* pReader) { pReader->bFilesetDelimited = true; } void tsdbReaderSetNotifyCb(STsdbReader* pReader, TsdReaderNotifyCbFn notifyFn, void* param) { pReader->notifyFn = notifyFn; From f14c3b93c138420dea53a21297446417879d30a5 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 25 Dec 2023 18:22:36 +0800 Subject: [PATCH 66/94] refactor: do some internal refactor. --- source/libs/stream/src/streamTaskSm.c | 81 ++++++++++++++++----------- 1 file changed, 47 insertions(+), 34 deletions(-) diff --git a/source/libs/stream/src/streamTaskSm.c b/source/libs/stream/src/streamTaskSm.c index 68ae1cce36..51e6ad23fe 100644 --- a/source/libs/stream/src/streamTaskSm.c +++ b/source/libs/stream/src/streamTaskSm.c @@ -160,6 +160,45 @@ static STaskStateTrans* streamTaskFindTransform(ETaskStatus state, const EStream return NULL; } +static int32_t doHandleWaitingEvent(SStreamTaskSM* pSM, const char* pEventName, SStreamTask* pTask) { + int32_t code = TSDB_CODE_SUCCESS; + int64_t el = (taosGetTimestampMs() - pSM->startTs); + stDebug("s-task:%s handle event:%s completed, elapsed time:%" PRId64 "ms state:%s -> %s", pTask->id.idStr, + pEventName, el, pSM->prev.state.name, pSM->current.name); + + SAttachedEventInfo* pEvtInfo = taosArrayGet(pSM->pWaitingEventList, 0); + + // OK, let's handle the attached event, since the task has reached the required status now + if (pSM->current.state == pEvtInfo->status) { + stDebug("s-task:%s handle the event:%s in waiting list, state:%s", pTask->id.idStr, + GET_EVT_NAME(pEvtInfo->event), pSM->current.name); + + // remove it + taosArrayPop(pSM->pWaitingEventList); + + STaskStateTrans* pNextTrans = streamTaskFindTransform(pSM->current.state, pEvtInfo->event); + ASSERT(pSM->pActiveTrans == NULL && pNextTrans != NULL); + + pSM->pActiveTrans = pNextTrans; + pSM->startTs = taosGetTimestampMs(); + taosThreadMutexUnlock(&pTask->lock); + + code = pNextTrans->pAction(pSM->pTask); + if (pNextTrans->autoInvokeEndFn) { + return streamTaskOnHandleEventSuccess(pSM, pNextTrans->event); + } else { + return code; + } + } else { + taosThreadMutexUnlock(&pTask->lock); + stDebug("s-task:%s state:%s event:%s in waiting list, req state:%s not fulfilled, put it back", pTask->id.idStr, + pSM->current.name, GET_EVT_NAME(pEvtInfo->event), + StreamTaskStatusList[pEvtInfo->status].name); + } + + return code; +} + void streamTaskRestoreStatus(SStreamTask* pTask) { SStreamTaskSM* pSM = pTask->status.pSM; @@ -175,7 +214,13 @@ void streamTaskRestoreStatus(SStreamTask* pTask) { pSM->prev.evt = 0; pSM->startTs = taosGetTimestampMs(); - stDebug("s-task:%s restore status, %s -> %s", pTask->id.idStr, pSM->prev.state.name, pSM->current.name); + + if (taosArrayGetSize(pSM->pWaitingEventList) > 0) { + stDebug("s-task:%s restore status, %s -> %s, and then handle waiting event", pTask->id.idStr, pSM->prev.state.name, pSM->current.name); + doHandleWaitingEvent(pSM, "restore-pause/halt", pTask); + } else { + stDebug("s-task:%s restore status, %s -> %s", pTask->id.idStr, pSM->prev.state.name, pSM->current.name); + } taosThreadMutexUnlock(&pTask->lock); } @@ -343,39 +388,7 @@ int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent even pTrans->pSuccAction(pTask); if (taosArrayGetSize(pSM->pWaitingEventList) > 0) { - int64_t el = (taosGetTimestampMs() - pSM->startTs); - stDebug("s-task:%s handle event:%s completed, elapsed time:%" PRId64 "ms state:%s -> %s", pTask->id.idStr, - GET_EVT_NAME(pTrans->event), el, pSM->prev.state.name, pSM->current.name); - - SAttachedEventInfo* pEvtInfo = taosArrayGet(pSM->pWaitingEventList, 0); - - // OK, let's handle the attached event, since the task has reached the required status now - if (pSM->current.state == pEvtInfo->status) { - stDebug("s-task:%s handle the event:%s in waiting list, state:%s", pTask->id.idStr, - GET_EVT_NAME(pEvtInfo->event), pSM->current.name); - - // remove it - taosArrayPop(pSM->pWaitingEventList); - - STaskStateTrans* pNextTrans = streamTaskFindTransform(pSM->current.state, pEvtInfo->event); - ASSERT(pSM->pActiveTrans == NULL && pNextTrans != NULL); - - pSM->pActiveTrans = pNextTrans; - pSM->startTs = taosGetTimestampMs(); - taosThreadMutexUnlock(&pTask->lock); - - int32_t code = pNextTrans->pAction(pSM->pTask); - if (pNextTrans->autoInvokeEndFn) { - return streamTaskOnHandleEventSuccess(pSM, pNextTrans->event); - } else { - return code; - } - } else { - taosThreadMutexUnlock(&pTask->lock); - stDebug("s-task:%s state:%s event:%s in waiting list, req state:%s not fulfilled, put it back", pTask->id.idStr, - pSM->current.name, GET_EVT_NAME(pEvtInfo->event), - StreamTaskStatusList[pEvtInfo->status].name); - } + doHandleWaitingEvent(pSM, GET_EVT_NAME(pTrans->event), pTask); } else { taosThreadMutexUnlock(&pTask->lock); From ac3c76f60895023168689eb9adfca9eb02f402c2 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Mon, 25 Dec 2023 19:38:20 +0800 Subject: [PATCH 67/94] init dummy function --- .../executor/src/streamtimewindowoperator.c | 4 +++ tests/script/tsim/stream/session1.sim | 36 +++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index 7b68b65c17..c9490e2c55 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -1623,6 +1623,7 @@ void initDummyFunction(SqlFunctionCtx* pDummy, SqlFunctionCtx* pCtx, int32_t num pDummy[i].functionId = pCtx[i].functionId; pDummy[i].isNotNullFunc = pCtx[i].isNotNullFunc; pDummy[i].isPseudoFunc = pCtx[i].isPseudoFunc; + pDummy[i].fpSet.init = pCtx[i].fpSet.init; } } @@ -2774,6 +2775,9 @@ void streamSessionSemiReloadState(SOperatorInfo* pOperator) { for (int32_t i = 0; i < num; i++) { SResultWindowInfo winInfo = {0}; getSessionWindowInfoByKey(pAggSup, pSeKeyBuf + i, &winInfo); + if (!IS_VALID_SESSION_WIN(winInfo)) { + continue; + } compactSessionSemiWindow(pOperator, &winInfo); saveSessionOutputBuf(pAggSup, &winInfo); } diff --git a/tests/script/tsim/stream/session1.sim b/tests/script/tsim/stream/session1.sim index cf42159d84..aae17053b2 100644 --- a/tests/script/tsim/stream/session1.sim +++ b/tests/script/tsim/stream/session1.sim @@ -327,4 +327,40 @@ if $rows != 1 then goto loop17 endi +sql create database test2 vgroups 4; +sql use test2; +sql create stable st(ts timestamp,a int,b int,c int) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); +sql create stream streams4 trigger at_once ignore update 0 ignore expired 0 into streamt4 as select _wstart, count(*) c1, count(a) c2 from st session(ts, 2s) ; + +sql insert into t1 values(1648791255100,1,2,3); +sql insert into t1 values(1648791255300,1,2,3); + +sleep 1000 + +sql insert into t1 values(1648791253000,1,2,3) (1648791254000,1,2,3); + +$loop_count = 0 +loop18: +sleep 1000 +sql select * from streamt4; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 1 then + print =====rows=$rows + goto loop18 +endi + +if $data01 != 4 then + print =====data01=$data01 + goto loop18 +endi + +print =====over + system sh/exec.sh -n dnode1 -s stop -x SIGINT From 5f0b0b4338e69437e34bbe82ab6d524e8e3392fd Mon Sep 17 00:00:00 2001 From: root Date: Mon, 25 Dec 2023 19:56:56 +0800 Subject: [PATCH 68/94] case: mlevel_basic.py passed --- .../enterprise/multi-level/mlevel_basic.py | 10 ++++++---- tests/army/frame/caseBase.py | 8 ++++---- tests/army/frame/eos.py | 2 +- tests/army/frame/epath.py | 18 +++++++++--------- tests/army/frame/etool.py | 18 ++++++++++-------- tests/army/frame/sql.py | 2 +- 6 files changed, 31 insertions(+), 27 deletions(-) diff --git a/tests/army/enterprise/multi-level/mlevel_basic.py b/tests/army/enterprise/multi-level/mlevel_basic.py index a12b1b59eb..70bfb9b66e 100644 --- a/tests/army/enterprise/multi-level/mlevel_basic.py +++ b/tests/army/enterprise/multi-level/mlevel_basic.py @@ -16,12 +16,13 @@ import time import taos import frame +import frame.etool from frame.log import * from frame.cases import * from frame.sql import * from frame.caseBase import * - +from frame import * class TDTestCase(TBase): @@ -30,9 +31,10 @@ class TDTestCase(TBase): def insertData(self): tdLog.info(f"insert data.") # taosBenchmark run - json = frame.etool.curFile(__file__, "mlevel_basic.json") - frame.etool.runBenchmark(json=json) + json = etool.curFile(__file__, "mlevel_basic.json") + etool.runBenchmark(json=json) + tdSql.execute(f"use {self.db}") # set insert data information self.childtable_count = 2 self.insert_rows = 1000000 @@ -76,4 +78,4 @@ class TDTestCase(TBase): tdCases.addLinux(__file__, TDTestCase()) -tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/army/frame/caseBase.py b/tests/army/frame/caseBase.py index 7b371cd822..11c44b28e9 100644 --- a/tests/army/frame/caseBase.py +++ b/tests/army/frame/caseBase.py @@ -16,11 +16,11 @@ import os import time import datetime -from log import * -from sql import * +from frame.log import * +from frame.sql import * # test case base -class DbBase: +class TBase: # # frame call @@ -85,7 +85,7 @@ class DbBase: tdSql.checkAgg(sql, self.childtable_count) # check step - sql = f"select count(*) from (select diff(ts) as dif from {self.stb} group by tbname) where dif != {self.timestamp_step}" + sql = f"select count(*) from (select diff(ts) as dif from {self.stb} partition by tbname) where dif != {self.timestamp_step}" tdSql.checkAgg(sql, 0) # save agg result diff --git a/tests/army/frame/eos.py b/tests/army/frame/eos.py index 2d8559f3eb..ab4d2c1bd0 100644 --- a/tests/army/frame/eos.py +++ b/tests/army/frame/eos.py @@ -22,7 +22,7 @@ import datetime # if windows platform return True def isWin(): - return sys.platform.system().lower() == 'windows' + return sys.platform.lower() == 'windows' # wait util execute file finished def exe(file): diff --git a/tests/army/frame/epath.py b/tests/army/frame/epath.py index 644e448fb7..edff9c78a4 100644 --- a/tests/army/frame/epath.py +++ b/tests/army/frame/epath.py @@ -19,13 +19,13 @@ import os from frame.log import * # build/bin path -binPath = "" +binDir = "" -def binPath(self): - global binPath +def binPath(): + global binDir - if binPath != "": - return binPath + if binDir != "": + return binDir selfPath = os.path.dirname(os.path.realpath(__file__)) @@ -46,8 +46,8 @@ def binPath(self): else: tdLog.info(f"taosd found in {buildPath}") # return - binPath = buildPath + "/build/bin/" - return binPath + binDir = buildPath + "/build/bin/" + return binDir -def binFile(self, file): - return binPath() + file \ No newline at end of file +def binFile(filename): + return binPath() + filename diff --git a/tests/army/frame/etool.py b/tests/army/frame/etool.py index 5ba29ee1c3..f24871b3dd 100644 --- a/tests/army/frame/etool.py +++ b/tests/army/frame/etool.py @@ -19,22 +19,24 @@ import sys import os import time import datetime -import epath -import eos +import frame.epath +import frame.eos # run taosBenchmark with command or json file mode def runBenchmark(command = "", json = "") : # get taosBenchmark path - bmFile = epath.binFile("taosBenchmark") - if eos.isWin(): + bmFile = frame.epath.binFile("taosBenchmark") + if frame.eos.isWin(): bmFile += ".exe" # run if command != "": - eos.exe(bmFile + " " + command) + frame.eos.exe(bmFile + " " + command) if json != "": - eos.exe(bmFile + " -f " + json) + cmd = f"{bmFile} -f {json}" + print(cmd) + frame.eos.exe(cmd) # get current directory file name -def curFile(fullPath, file): - return os.path.dirname(fullPath) + "/" + file +def curFile(fullPath, filename): + return os.path.dirname(fullPath) + "/" + filename diff --git a/tests/army/frame/sql.py b/tests/army/frame/sql.py index b3ee1ed5f8..2e14f0c2f0 100644 --- a/tests/army/frame/sql.py +++ b/tests/army/frame/sql.py @@ -538,7 +538,7 @@ class TDSql: # check like select count(*) ... sql def checkAgg(self, sql, expectCnt): self.query(sql) - self.checkRowCol(0, 0, expectCnt) + self.checkData(0, 0, expectCnt) # get first value def getFirstValue(self, sql) : From 7b967ffdc63c397e684b820f7e30dc1671a5ec86 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Mon, 25 Dec 2023 20:22:18 +0800 Subject: [PATCH 69/94] enh: retry on fsm commit when rpc out of memory in syncFsmExecute --- source/libs/sync/src/syncPipeline.c | 37 ++++++++++++++++++----------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c index f2386797c1..28ee5ba841 100644 --- a/source/libs/sync/src/syncPipeline.c +++ b/source/libs/sync/src/syncPipeline.c @@ -558,22 +558,31 @@ int32_t syncFsmExecute(SSyncNode* pNode, SSyncFSM* pFsm, ESyncState role, SyncTe pEntry->term); } - SRpcMsg rpcMsg = {.code = applyCode}; - syncEntry2OriginalRpc(pEntry, &rpcMsg); + int32_t code = 0; + bool retry = false; + do { + SRpcMsg rpcMsg = {.code = applyCode}; + syncEntry2OriginalRpc(pEntry, &rpcMsg); - SFsmCbMeta cbMeta = {0}; - cbMeta.index = pEntry->index; - cbMeta.lastConfigIndex = syncNodeGetSnapshotConfigIndex(pNode, pEntry->index); - cbMeta.isWeak = pEntry->isWeak; - cbMeta.code = applyCode; - cbMeta.state = role; - cbMeta.seqNum = pEntry->seqNum; - cbMeta.term = pEntry->term; - cbMeta.currentTerm = term; - cbMeta.flag = -1; + SFsmCbMeta cbMeta = {0}; + cbMeta.index = pEntry->index; + cbMeta.lastConfigIndex = syncNodeGetSnapshotConfigIndex(pNode, pEntry->index); + cbMeta.isWeak = pEntry->isWeak; + cbMeta.code = applyCode; + cbMeta.state = role; + cbMeta.seqNum = pEntry->seqNum; + cbMeta.term = pEntry->term; + cbMeta.currentTerm = term; + cbMeta.flag = -1; - (void)syncRespMgrGetAndDel(pNode->pSyncRespMgr, cbMeta.seqNum, &rpcMsg.info); - int32_t code = pFsm->FpCommitCb(pFsm, &rpcMsg, &cbMeta); + (void)syncRespMgrGetAndDel(pNode->pSyncRespMgr, cbMeta.seqNum, &rpcMsg.info); + code = pFsm->FpCommitCb(pFsm, &rpcMsg, &cbMeta); + retry = (code != 0) && (terrno == TSDB_CODE_OUT_OF_RPC_MEMORY_QUEUE); + if (retry) { + taosMsleep(10); + sError("vgId:%d, retry on fsm commit since %s. index:%" PRId64, pNode->vgId, terrstr(), pEntry->index); + } + } while (retry); return code; } From 9071ff39736daa9ec3d61c76cddbffa189955246 Mon Sep 17 00:00:00 2001 From: root Date: Mon, 25 Dec 2023 21:20:59 +0800 Subject: [PATCH 70/94] fix: add compactDb flushDb --- .../enterprise/multi-level/mlevel_basic.json | 4 ++-- .../enterprise/multi-level/mlevel_basic.py | 19 +++++++------------ tests/army/frame/caseBase.py | 2 ++ tests/army/frame/eos.py | 3 ++- tests/army/frame/etool.py | 6 +++++- tests/army/frame/server/dnodes.py | 8 +++++++- tests/army/test.py | 2 ++ 7 files changed, 27 insertions(+), 17 deletions(-) diff --git a/tests/army/enterprise/multi-level/mlevel_basic.json b/tests/army/enterprise/multi-level/mlevel_basic.json index 7df5b1caa8..1c2b9274d2 100644 --- a/tests/army/enterprise/multi-level/mlevel_basic.json +++ b/tests/army/enterprise/multi-level/mlevel_basic.json @@ -24,12 +24,12 @@ { "name": "stb", "child_table_exists": "no", - "childtable_count": 2, + "childtable_count": 4, "insert_rows": 1000000, "childtable_prefix": "d", "insert_mode": "taosc", "timestamp_step": 1000, - "start_timestamp":"now-10d", + "start_timestamp":"now-12d", "columns": [ { "type": "bool", "name": "bc"}, { "type": "float", "name": "fc" }, diff --git a/tests/army/enterprise/multi-level/mlevel_basic.py b/tests/army/enterprise/multi-level/mlevel_basic.py index 70bfb9b66e..3bec2bfb72 100644 --- a/tests/army/enterprise/multi-level/mlevel_basic.py +++ b/tests/army/enterprise/multi-level/mlevel_basic.py @@ -36,23 +36,15 @@ class TDTestCase(TBase): tdSql.execute(f"use {self.db}") # set insert data information - self.childtable_count = 2 + self.childtable_count = 4 self.insert_rows = 1000000 self.timestamp_step = 1000 def doAction(self): - tdLog.info(f"trim database.") + tdLog.info(f"do action.") + self.flushDb() self.trimDb() - - def saveData(self): - tdLog.info(f"check correct.") - - def checkSaveCorrent(self): - tdLog.info(f"check correct.") - - def checkCorrect(self): - tdLog.info(f"check correct.") - + self.compactDb() # run def run(self): @@ -73,6 +65,9 @@ class TDTestCase(TBase): # check save agg result correct self.checkAggCorrect() + # check insert correct again + self.checkInsertCorrect() + tdLog.success(f"{__file__} successfully executed") diff --git a/tests/army/frame/caseBase.py b/tests/army/frame/caseBase.py index 11c44b28e9..441196f050 100644 --- a/tests/army/frame/caseBase.py +++ b/tests/army/frame/caseBase.py @@ -69,6 +69,8 @@ class TBase: def compactDb(self): tdSql.execute(f"compact database {self.db}") + def flushDb(self): + tdSql.execute(f"flush database {self.db}") # # check db correct diff --git a/tests/army/frame/eos.py b/tests/army/frame/eos.py index ab4d2c1bd0..dcb63ff3aa 100644 --- a/tests/army/frame/eos.py +++ b/tests/army/frame/eos.py @@ -19,10 +19,11 @@ import sys import os import time import datetime +import platform # if windows platform return True def isWin(): - return sys.platform.lower() == 'windows' + return platform.system().lower() == 'windows' # wait util execute file finished def exe(file): diff --git a/tests/army/frame/etool.py b/tests/army/frame/etool.py index f24871b3dd..35c390dc1a 100644 --- a/tests/army/frame/etool.py +++ b/tests/army/frame/etool.py @@ -21,6 +21,7 @@ import time import datetime import frame.epath import frame.eos +from frame.log import * # run taosBenchmark with command or json file mode def runBenchmark(command = "", json = "") : @@ -35,7 +36,10 @@ def runBenchmark(command = "", json = "") : if json != "": cmd = f"{bmFile} -f {json}" print(cmd) - frame.eos.exe(cmd) + status = frame.eos.exe(cmd) + if status !=0: + tdLog.exit(f"run failed {cmd} status={status}") + # get current directory file name def curFile(fullPath, filename): diff --git a/tests/army/frame/server/dnodes.py b/tests/army/frame/server/dnodes.py index 1e51fbc3be..0d40b665dd 100644 --- a/tests/army/frame/server/dnodes.py +++ b/tests/army/frame/server/dnodes.py @@ -911,4 +911,10 @@ class TDDnodes: def getAsan(self): return self.asan -tdDnodes = TDDnodes() \ No newline at end of file + def setLevelDisk(self, level, disk): + for i in range(len(self.dnodes)): + self.dnodes[i].level = int(level) + self.dnodes[i].disk = int(disk) + + +tdDnodes = TDDnodes() diff --git a/tests/army/test.py b/tests/army/test.py index 83d065b24b..a3e28b772d 100644 --- a/tests/army/test.py +++ b/tests/army/test.py @@ -373,6 +373,7 @@ if __name__ == "__main__": tAdapter.stop(force_kill=True) if dnodeNums == 1 : + tdDnodes.setLevelDisk(level, disk) tdDnodes.deploy(1,updateCfgDict) tdDnodes.start(1) tdCases.logSql(logSql) @@ -542,6 +543,7 @@ if __name__ == "__main__": if dnodeNums == 1 : # dnode is one + tdDnodes.setLevelDisk(level, disk) tdDnodes.deploy(1,updateCfgDict) tdDnodes.start(1) tdCases.logSql(logSql) From 0e6ea7af712f4c918df6b1a323b825ba1169f494 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 26 Dec 2023 10:54:40 +0800 Subject: [PATCH 71/94] enh: check snapshot begin index for unexpected change in syncNodeOnSnapshotBegin --- source/libs/sync/src/syncSnapshot.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 2277b70c8f..10a8734617 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -806,6 +806,13 @@ static int32_t syncNodeOnSnapshotBegin(SSyncNode *pSyncNode, SyncSnapshotSend *p goto _SEND_REPLY; } + SyncIndex beginIndex = syncNodeGetSnapBeginIndex(pSyncNode); + if (pReceiver->snapshotParam.start != beginIndex) { + sRError(pReceiver, "snapshot begin index is changed unexpectedly. sver:%" PRId64 ", beginIndex:%" PRId64, + pReceiver->snapshotParam.start, beginIndex); + goto _SEND_REPLY; + } + code = 0; _SEND_REPLY: if (code != 0 && terrno != 0) { From 1aae0e52e38bb5b3c3e6950ee4926a71cda82ff4 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Tue, 26 Dec 2023 14:48:05 +0800 Subject: [PATCH 72/94] fix: like wildcard match --- source/util/src/tcompare.c | 13 ++- source/util/test/utilTests.cpp | 56 ++++++++++- tests/parallel_test/cases.task | 4 + tests/system-test/2-query/like.py | 148 ++++++++++++++++++++++++++++++ tests/system-test/win-test-file | 4 + 5 files changed, 221 insertions(+), 4 deletions(-) create mode 100644 tests/system-test/2-query/like.py diff --git a/source/util/src/tcompare.c b/source/util/src/tcompare.c index b6b71855f3..2baa21f5d9 100644 --- a/source/util/src/tcompare.c +++ b/source/util/src/tcompare.c @@ -1070,6 +1070,11 @@ int32_t patternMatch(const char *pattern, size_t psize, const char *str, size_t return TSDB_PATTERN_MATCH; /* "*" at the end of the pattern matches */ } + if (c == '\\' && (pattern[i] == '_' || pattern[i] == '%')) { + c = pattern[i]; + i++; + } + char rejectList[2] = {toupper(c), tolower(c)}; str += nMatchChar; @@ -1139,6 +1144,11 @@ int32_t wcsPatternMatch(const TdUcs4 *pattern, size_t psize, const TdUcs4 *str, return TSDB_PATTERN_MATCH; } + if (c == '\\' && (pattern[i] == '_' || pattern[i] == '%')) { + c = pattern[i]; + i++; + } + TdUcs4 rejectList[2] = {towupper(c), towlower(c)}; str += nMatchChar; @@ -1166,7 +1176,8 @@ int32_t wcsPatternMatch(const TdUcs4 *pattern, size_t psize, const TdUcs4 *str, c1 = str[j++]; nMatchChar++; - if (c == L'\\' && pattern[i] == L'_' && c1 == L'_') { + if (c == '\\' && pattern[i] == c1 && + (c1 == '_' || c1 == '%')) { i++; continue; } diff --git a/source/util/test/utilTests.cpp b/source/util/test/utilTests.cpp index 01a55ae710..6e32167f88 100644 --- a/source/util/test/utilTests.cpp +++ b/source/util/test/utilTests.cpp @@ -76,6 +76,16 @@ TEST(utilTest, wchar_pattern_match_test) { const TdWchar* str12 = L""; ret = wcsPatternMatch(reinterpret_cast(pattern12), 4, reinterpret_cast(str12), 0, &pInfo); ASSERT_EQ(ret, TSDB_PATTERN_NOMATCH); + + const TdWchar* pattern13 = L"%\\_6 "; + const TdWchar* str13 = L"6a6 "; + ret = wcsPatternMatch(reinterpret_cast(pattern13), 6, reinterpret_cast(str13), 4, &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH); + + const TdWchar* pattern14 = L"%\\%6 "; + const TdWchar* str14 = L"6a6 "; + ret = wcsPatternMatch(reinterpret_cast(pattern14), 6, reinterpret_cast(str14), 4, &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH); } TEST(utilTest, wchar_pattern_match_no_terminated) { @@ -126,14 +136,24 @@ TEST(utilTest, wchar_pattern_match_no_terminated) { ret = wcsPatternMatch(reinterpret_cast(pattern8), 8, reinterpret_cast(str8), 6, &pInfo); ASSERT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH); - const TdWchar* pattern9 = L"6\\_6 "; + const TdWchar* pattern9 = L"6\\_6 "; const TdWchar* str9 = L"6_6 "; - ret = wcsPatternMatch(reinterpret_cast(pattern9), 4, reinterpret_cast(str9), 3, &pInfo); + ret = wcsPatternMatch(reinterpret_cast(pattern9), 6, reinterpret_cast(str9), 4, &pInfo); ASSERT_EQ(ret, TSDB_PATTERN_MATCH); const TdWchar* pattern10 = L"% "; const TdWchar* str10 = L"6_6 "; - ret = wcsPatternMatch(reinterpret_cast(pattern10), 1, reinterpret_cast(str10), 3, &pInfo); + ret = wcsPatternMatch(reinterpret_cast(pattern10), 2, reinterpret_cast(str10), 4, &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_MATCH); + + const TdWchar* pattern11 = L"%\\_6 "; + const TdWchar* str11 = L"6_6 "; + ret = wcsPatternMatch(reinterpret_cast(pattern11), 6, reinterpret_cast(str11), 4, &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_MATCH); + + const TdWchar* pattern12 = L"%\\%6 "; + const TdWchar* str12 = L"6%6 "; + ret = wcsPatternMatch(reinterpret_cast(pattern12), 6, reinterpret_cast(str12), 4, &pInfo); ASSERT_EQ(ret, TSDB_PATTERN_MATCH); } @@ -209,6 +229,36 @@ TEST(utilTest, char_pattern_match_test) { const char* str13 = "a%c"; ret = patternMatch(pattern13, 5, str13, strlen(str13), &pInfo); ASSERT_EQ(ret, TSDB_PATTERN_MATCH); + + const char* pattern14 = "%a\\%c"; + const char* str14 = "a%c"; + ret = patternMatch(pattern14, strlen(pattern14), str14, strlen(str14), &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_MATCH); + + const char* pattern15 = "_a\\%c"; + const char* str15 = "ba%c"; + ret = patternMatch(pattern15, strlen(pattern15), str15, strlen(str15), &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_MATCH); + + const char* pattern16 = "_\\%c"; + const char* str16 = "a%c"; + ret = patternMatch(pattern16, strlen(pattern16), str16, strlen(str16), &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_MATCH); + + const char* pattern17 = "_\\%c"; + const char* str17 = "ba%c"; + ret = patternMatch(pattern17, strlen(pattern17), str17, strlen(str17), &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_NOMATCH); + + const char* pattern18 = "%\\%c"; + const char* str18 = "abc"; + ret = patternMatch(pattern18, strlen(pattern18), str18, strlen(str18), &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH); + + const char* pattern19 = "%\\_c"; + const char* str19 = "abc"; + ret = patternMatch(pattern19, strlen(pattern19), str19, strlen(str19), &pInfo); + ASSERT_EQ(ret, TSDB_PATTERN_NOWILDCARDMATCH); } TEST(utilTest, char_pattern_match_no_terminated) { diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index bcdd143cfc..0aab8de088 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -131,6 +131,10 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ts-4233.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ts-4233.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ts-4233.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/like.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/like.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/like.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/like.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 3-enterprise/restore/restoreDnode.py -N 5 -M 3 -i False ,,y,system-test,./pytest.sh python3 ./test.py -f 3-enterprise/restore/restoreVnode.py -N 5 -M 3 -i False diff --git a/tests/system-test/2-query/like.py b/tests/system-test/2-query/like.py new file mode 100644 index 0000000000..95e2daf6d3 --- /dev/null +++ b/tests/system-test/2-query/like.py @@ -0,0 +1,148 @@ +import taos +import sys +import datetime +import inspect + +from util.log import * +from util.sql import * +from util.cases import * +from util.common import tdCom + +class TDTestCase: + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def initDB(self): + tdSql.execute("drop database if exists db") + tdSql.execute("create database if not exists db") + + def stopTest(self): + tdSql.execute("drop database if exists db") + + def like_wildcard_test(self): + tdSql.execute("create table db.t1x (ts timestamp, c1 varchar(100))") + tdSql.execute("create table db.t_1x (ts timestamp, c1 varchar(100))") + + tdSql.query("select * from information_schema.ins_columns where table_name like '%1x'") + tdSql.checkRows(4) + + tdSql.query("select * from information_schema.ins_columns where table_name like '%\_1x'") + tdSql.checkRows(2) + + + tdSql.query("insert into db.t1x values(now, 'abc'), (now+1s, 'a%c'),(now+2s, 'a_c'),(now+3s, '_c'),(now+4s, '%c')") + + tdSql.query("select * from db.t1x") + tdSql.checkRows(5) + + tdSql.query("select * from db.t1x where c1 like '%_c'") + tdSql.checkRows(5) + + tdSql.query("select * from db.t1x where c1 like '%__c'") + tdSql.checkRows(3) + + tdSql.query("select * from db.t1x where c1 like '%\_c'") + tdSql.checkRows(2) + + tdSql.query("select * from db.t1x where c1 like '%\%c'") + tdSql.checkRows(2) + + tdSql.query("select * from db.t1x where c1 like '_\%c'") + tdSql.checkRows(1) + tdSql.checkData(0, 1, "a%c") + + tdSql.query("select * from db.t1x where c1 like '_\_c'") + tdSql.checkRows(1) + tdSql.checkData(0, 1, "a_c") + + tdSql.query("select * from db.t1x where c1 like '%%_c'") + tdSql.checkRows(5) + + tdSql.query("select * from db.t1x where c1 like '%_%c'") + tdSql.checkRows(5) + + tdSql.query("select * from db.t1x where c1 like '__%c'") + tdSql.checkRows(3) + + tdSql.query("select * from db.t1x where c1 not like '__%c'") + tdSql.checkRows(2) + + def like_cnc_wildcard_test(self): + tdSql.execute("create table db.t3x (ts timestamp, c1 varchar(100))") + + tdSql.execute("insert into db.t3x values(now, '我是中文'), (now+1s, '我是_中文'), (now+2s, '我是%中文'), (now+3s, '%中文'),(now+4s, '_中文')") + tdSql.query("select * from db.t3x") + tdSql.checkRows(5) + + tdSql.query("select * from db.t3x where c1 like '%中文'") + tdSql.checkRows(5) + + tdSql.query("select * from db.t3x where c1 like '%中_文'") + tdSql.checkRows(0) + + tdSql.query("select * from db.t3x where c1 like '%\%中文'") + tdSql.checkRows(2) + + tdSql.query("select * from db.t3x where c1 like '%\_中文'") + tdSql.checkRows(2) + + tdSql.query("select * from db.t3x where c1 like '_中文'") + tdSql.checkRows(2) + + tdSql.query("select * from db.t3x where c1 like '\_中文'") + tdSql.checkRows(1) + + def like_multi_wildcard_test(self): + tdSql.execute("create table db.t4x (ts timestamp, c1 varchar(100))") + + # 插入测试数据 + tdSql.execute("insert into db.t4x values(now, 'abc'), (now+1s, 'a%c'),(now+2s, 'a_c'),(now+3s, '_c'),(now+4s, '%c')") + tdSql.execute("insert into db.t4x values(now+5s, '%%%c'),(now+6s, '___c'),(now+7s, '%_%c'),(now+8s, '%\\c')") + + tdSql.query("select * from db.t4x where c1 like '%%%_'") + tdSql.checkRows(9) + + tdSql.query("select * from db.t4x where c1 like '\%\%\%_'") + tdSql.checkRows(1) + + tdSql.query("select * from db.t4x where c1 like '%\_%%'") + tdSql.checkRows(4) + + tdSql.query("select * from db.t4x where c1 like '_\%\%'") + tdSql.checkRows(0) + + tdSql.query("select * from db.t4x where c1 like '%abc%'") + tdSql.checkRows(1) + + tdSql.query("select * from db.t4x where c1 like '_%abc%'") + tdSql.checkRows(0) + + tdSql.query("select * from db.t4x where c1 like '\%%\%%'") + tdSql.checkRows(2) + + tdSql.query("select * from db.t4x where c1 like '\%\_%\%%'") + tdSql.checkRows(1) + + def run(self): + tdLog.printNoPrefix("==========start like_wildcard_test run ...............") + tdSql.prepare(replica = self.replicaVar) + + + self.initDB() + self.like_wildcard_test() + self.like_cnc_wildcard_test() + self.like_multi_wildcard_test() + tdLog.printNoPrefix("==========end like_wildcard_test run ...............") + + self.stopTest() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/win-test-file b/tests/system-test/win-test-file index aefdb1e824..0b4ceace93 100644 --- a/tests/system-test/win-test-file +++ b/tests/system-test/win-test-file @@ -99,6 +99,10 @@ python3 ./test.py -f 2-query/ts-4233.py python3 ./test.py -f 2-query/ts-4233.py -Q 2 python3 ./test.py -f 2-query/ts-4233.py -Q 3 python3 ./test.py -f 2-query/ts-4233.py -Q 4 +python3 ./test.py -f 2-query/like.py +python3 ./test.py -f 2-query/like.py -Q 2 +python3 ./test.py -f 2-query/like.py -Q 3 +python3 ./test.py -f 2-query/like.py -Q 4 python3 ./test.py -f 3-enterprise/restore/restoreDnode.py -N 5 -M 3 -i False python3 ./test.py -f 3-enterprise/restore/restoreVnode.py -N 5 -M 3 -i False python3 ./test.py -f 3-enterprise/restore/restoreMnode.py -N 5 -M 3 -i False From dc01625ea14aba9e2f303c58193a8a91153c4299 Mon Sep 17 00:00:00 2001 From: Yihao Deng Date: Tue, 26 Dec 2023 06:52:33 +0000 Subject: [PATCH 73/94] fix tag index double free --- source/dnode/vnode/src/meta/metaTable.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 976ee616f9..35a0adc4da 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -36,7 +36,6 @@ static int metaDeleteBtimeIdx(SMeta *pMeta, const SMetaEntry *pME); static int metaUpdateNcolIdx(SMeta *pMeta, const SMetaEntry *pME); static int metaDeleteNcolIdx(SMeta *pMeta, const SMetaEntry *pME); - static void metaGetEntryInfo(const SMetaEntry *pEntry, SMetaInfo *pInfo) { pInfo->uid = pEntry->uid; pInfo->version = pEntry->version; @@ -562,6 +561,7 @@ int metaAddIndexToSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { tdbTbUpsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, pMeta->txn); metaULock(pMeta); metaDestroyTagIdxKey(pTagIdxKey); + pTagIdxKey = NULL; } nStbEntry.version = version; @@ -692,6 +692,7 @@ int metaDropIndexFromSTable(SMeta *pMeta, int64_t version, SDropIndexReq *pReq) tdbTbDelete(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, pMeta->txn); metaULock(pMeta); metaDestroyTagIdxKey(pTagIdxKey); + pTagIdxKey = NULL; } // clear idx flag @@ -1076,7 +1077,7 @@ static int metaDeleteTtl(SMeta *pMeta, const SMetaEntry *pME) { return ttlMgrDeleteTtl(pMeta->pTtlMgr, &ctx); } -static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *pSuid, int8_t* pSysTbl) { +static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *pSuid, int8_t *pSysTbl) { void *pData = NULL; int nData = 0; int rc = 0; @@ -1146,6 +1147,7 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *p tdbTbDelete(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, pMeta->txn); } metaDestroyTagIdxKey(pTagIdxKey); + pTagIdxKey = NULL; } } tDecoderClear(&tdc); @@ -1865,6 +1867,7 @@ static int metaAddTagIndex(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTb } tdbTbUpsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, pMeta->txn); metaDestroyTagIdxKey(pTagIdxKey); + pTagIdxKey = NULL; } tdbTbcClose(pCtbIdxc); return 0; @@ -2122,7 +2125,7 @@ int metaUpdateChangeTimeWithLock(SMeta *pMeta, tb_uid_t uid, int64_t changeTimeM if (!tsTtlChangeOnWrite) return 0; metaWLock(pMeta); - int ret = metaUpdateChangeTime(pMeta, uid, changeTimeMs); + int ret = metaUpdateChangeTime(pMeta, uid, changeTimeMs); metaULock(pMeta); return ret; } @@ -2237,6 +2240,7 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { tdbTbUpsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, pMeta->txn); } metaDestroyTagIdxKey(pTagIdxKey); + pTagIdxKey = NULL; } } end: From 40aa5f697f6c31e9ac21c180e2e684c77483322a Mon Sep 17 00:00:00 2001 From: Yihao Deng Date: Tue, 26 Dec 2023 07:21:02 +0000 Subject: [PATCH 74/94] fix tag index double free --- source/dnode/vnode/src/meta/metaTable.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 35a0adc4da..ac2486eda1 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -2231,14 +2231,12 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { nTagData = tDataTypes[pTagColumn->type].bytes; } - if (pTagData != NULL) { - if (metaCreateTagIdxKey(pCtbEntry->ctbEntry.suid, pTagColumn->colId, pTagData, nTagData, pTagColumn->type, - pCtbEntry->uid, &pTagIdxKey, &nTagIdxKey) < 0) { - ret = -1; - goto end; - } - tdbTbUpsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, pMeta->txn); + if (metaCreateTagIdxKey(pCtbEntry->ctbEntry.suid, pTagColumn->colId, pTagData, nTagData, pTagColumn->type, + pCtbEntry->uid, &pTagIdxKey, &nTagIdxKey) < 0) { + ret = -1; + goto end; } + tdbTbUpsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, pMeta->txn); metaDestroyTagIdxKey(pTagIdxKey); pTagIdxKey = NULL; } From 2ba586513f9c62031cca9bb3152858f83a64bd45 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 26 Dec 2023 16:41:33 +0800 Subject: [PATCH 75/94] fix(tsdb/cache): erase entry from lru if deleted --- source/dnode/vnode/src/tsdb/tsdbCache.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index d7fea4956d..a39c74a679 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -1183,8 +1183,10 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE LRUHandle *h = taosLRUCacheLookup(pTsdb->lruCache, keys_list[i], klen); if (h) { SLastCol *pLastCol = (SLastCol *)taosLRUCacheValue(pTsdb->lruCache, h); - if (pLastCol->dirty && (pLastCol->ts <= eKey && pLastCol->ts >= sKey)) { + if (pLastCol->dirty) { pLastCol->dirty = 0; + } + if (pLastCol->ts <= eKey && pLastCol->ts >= sKey) { erase = true; } taosLRUCacheRelease(pTsdb->lruCache, h, erase); @@ -1197,8 +1199,10 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE h = taosLRUCacheLookup(pTsdb->lruCache, keys_list[num_keys + i], klen); if (h) { SLastCol *pLastCol = (SLastCol *)taosLRUCacheValue(pTsdb->lruCache, h); - if (pLastCol->dirty && (pLastCol->ts <= eKey && pLastCol->ts >= sKey)) { + if (pLastCol->dirty) { pLastCol->dirty = 0; + } + if (pLastCol->ts <= eKey && pLastCol->ts >= sKey) { erase = true; } taosLRUCacheRelease(pTsdb->lruCache, h, erase); From a46c921f688b1643fc4de8131fa11e21bd01822e Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Wed, 27 Dec 2023 11:29:03 +0800 Subject: [PATCH 76/94] check block rows --- source/libs/executor/src/streamtimewindowoperator.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index c9490e2c55..b9f3dc19d8 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -2343,10 +2343,6 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDa } int32_t code = pAPI->stateStore.streamStateGetByPos(pState, pPos, (void**)&pRow); - if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) { - ASSERT(pBlock->info.rows > 0); - break; - } if (code == -1) { // for history @@ -2363,6 +2359,11 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDa continue; } + if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) { + ASSERT(pBlock->info.rows > 0); + break; + } + pGroupResInfo->index += 1; for (int32_t j = 0; j < numOfExprs; ++j) { @@ -2383,6 +2384,7 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDa SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId); char* in = GET_ROWCELL_INTERBUF(pCtx[j].resultInfo); for (int32_t k = 0; k < pRow->numOfRows; ++k) { + qDebug("===stream===block rows:%d, rows:%d, exprs:%d, capacity:%d", pBlock->info.rows, i, j, pBlock->info.capacity); colDataSetVal(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes); } } From a97c2e0e165b32ddc02392041ee843aff76172c3 Mon Sep 17 00:00:00 2001 From: charles Date: Wed, 27 Dec 2023 13:27:38 +0800 Subject: [PATCH 77/94] add test case for jira TS-4382 --- tests/parallel_test/cases.task | 1 + tests/system-test/2-query/test_ts4382.py | 75 ++++++++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 tests/system-test/2-query/test_ts4382.py diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index e35bc60e20..e7c26b9833 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -518,6 +518,7 @@ e ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/json_tag.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQueryInterval.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/systable_func.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/test_ts4382.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity_1.py diff --git a/tests/system-test/2-query/test_ts4382.py b/tests/system-test/2-query/test_ts4382.py new file mode 100644 index 0000000000..50ec67ed23 --- /dev/null +++ b/tests/system-test/2-query/test_ts4382.py @@ -0,0 +1,75 @@ +import random +import string +from util.log import * +from util.cases import * +from util.sql import * +from util.sqlset import * +from util import constant +from util.common import * + + +class TDTestCase: + """Verify the jira TS-4382 + """ + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + self.dbname = 'db' + self.stbname = 'st' + self.ctbname_list = ["ct1", "ct2"] + self.tag_value_list = ['{"instance":"100"}', '{"instance":"200"}'] + + def prepareData(self): + # db + tdSql.execute("create database {};".format(self.dbname)) + tdSql.execute("use {};".format(self.dbname)) + tdLog.debug("Create database %s" % self.dbname) + + # super table + tdSql.execute("create table {} (ts timestamp, col1 int) tags (t1 json);".format(self.stbname)) + tdLog.debug("Create super table %s" % self.stbname) + + # child table + for i in range(len(self.ctbname_list)): + tdSql.execute("create table {} using {} tags('{}');".format(self.ctbname_list[i], self.stbname, self.tag_value_list[i])) + tdLog.debug("Create child table %s" % self.ctbname_list) + + # insert data + tdSql.execute("insert into {} values(now, 1)(now+1s, 2)".format(self.ctbname_list[0])) + tdSql.execute("insert into {} values(now, null)(now+1s, null)".format(self.ctbname_list[1])) + + def run(self): + self.prepareData() + sql_list = [ + # super table query with correct tag name of json type + { + "sql": "select to_char(ts, 'yyyy-mm-dd hh24:mi:ss') as time, irate(col1) from st group by to_char(ts, 'yyyy-mm-dd hh24:mi:ss'), t1->'instance' order by time;", + "result_check": "0.0" + }, + # child table query with incorrect tag name of json type + { + "sql": "select to_char(ts, 'yyyy-mm-dd hh24:mi:ss') as time, irate(col1) from ct1 group by to_char(ts, 'yyyy-mm-dd hh24:mi:ss'), t1->'name' order by time;", + "result_check": "None" + }, + # child table query with null value + { + "sql": "select ts, avg(col1) from ct2 group by ts, t1->'name' order by ts;", + "result_check": "None" + } + ] + for sql_dic in sql_list: + tdSql.query(sql_dic["sql"]) + tdLog.debug("execute sql: %s" % sql_dic["sql"]) + for item in [row[1] for row in tdSql.queryResult]: + if sql_dic["result_check"] in str(item): + tdLog.debug("Check query result of '{}' successfully".format(sql_dic["sql"])) + break + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) + From 382569591fd797e9a89208857124114be63b11d3 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Wed, 27 Dec 2023 14:01:38 +0800 Subject: [PATCH 78/94] log --- source/libs/executor/src/streamtimewindowoperator.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index b9f3dc19d8..024a44d107 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -2384,7 +2384,6 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDa SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId); char* in = GET_ROWCELL_INTERBUF(pCtx[j].resultInfo); for (int32_t k = 0; k < pRow->numOfRows; ++k) { - qDebug("===stream===block rows:%d, rows:%d, exprs:%d, capacity:%d", pBlock->info.rows, i, j, pBlock->info.capacity); colDataSetVal(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes); } } From 7eb0e42bb9906fe0c11df5d28ddae2af193b0fb2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 26 Dec 2023 19:02:52 +0800 Subject: [PATCH 79/94] fix(stream): handle error when checkpoint is interrupted by nodeUpdate. --- source/dnode/vnode/src/tq/tq.c | 4 ++-- source/libs/stream/src/streamDispatch.c | 2 +- source/libs/stream/src/streamExec.c | 25 ++++++++++++++++++++++--- 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index c8cc4f9d54..33f937a909 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1294,8 +1294,8 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) if (status == TASK_STATUS__CK) { ASSERT(pTask->chkInfo.checkpointingId == req.checkpointId); tqWarn("s-task:%s recv checkpoint-source msg again checkpointId:%" PRId64 - " already received, ignore this msg and continue process checkpoint", - pTask->id.idStr, pTask->chkInfo.checkpointingId); + " transId:%d already received, ignore this msg and continue process checkpoint", + pTask->id.idStr, pTask->chkInfo.checkpointingId, req.transId); taosThreadMutexUnlock(&pTask->lock); streamMetaReleaseTask(pMeta, pTask); diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index bfa269b0d5..00a8940b6a 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -746,7 +746,7 @@ int32_t streamTaskSendCheckpointSourceRsp(SStreamTask* pTask) { taosArrayClear(pTask->pReadyMsgList); stDebug("s-task:%s level:%d source checkpoint completed msg sent to mnode", pTask->id.idStr, pTask->info.taskLevel); } else { - stDebug("s-task:%s level:%d already send rsp to mnode", pTask->id.idStr, pTask->info.taskLevel); + stDebug("s-task:%s level:%d already send rsp checkpoint success to mnode", pTask->id.idStr, pTask->info.taskLevel); } taosThreadMutexUnlock(&pTask->lock); diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 25f32195be..5822b18bc3 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -625,10 +625,29 @@ int32_t doStreamExecTask(SStreamTask* pTask) { // todo other thread may change the status // do nothing after sync executor state to storage backend, untill the vnode-level checkpoint is completed. if (type == STREAM_INPUT__CHECKPOINT) { + + // todo add lock char* p = NULL; - streamTaskGetStatus(pTask, &p); - stDebug("s-task:%s checkpoint block received, set status:%s", pTask->id.idStr, p); - streamTaskBuildCheckpoint(pTask); + ETaskStatus s = streamTaskGetStatus(pTask, &p); + if (s == TASK_STATUS__CK) { + stDebug("s-task:%s checkpoint block received, set status:%s", pTask->id.idStr, p); + streamTaskBuildCheckpoint(pTask); + } else { + // todo refactor + int32_t code = 0; + if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { + code = streamTaskSendCheckpointSourceRsp(pTask); + } else { + code = streamTaskSendCheckpointReadyMsg(pTask); + } + + if (code != TSDB_CODE_SUCCESS) { + // todo: let's retry send rsp to upstream/mnode + stError("s-task:%s failed to send checkpoint rsp to upstream, checkpointId:%" PRId64 ", code:%s", pTask->id.idStr, + 0, tstrerror(code)); + } + } + return 0; } } From 646f24659e94336b67906c4f7181e0d76b2e7fd6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 26 Dec 2023 19:16:49 +0800 Subject: [PATCH 80/94] fix(stream): fix syntax error. --- source/libs/stream/src/streamExec.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 5822b18bc3..5b4f603f87 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -643,7 +643,7 @@ int32_t doStreamExecTask(SStreamTask* pTask) { if (code != TSDB_CODE_SUCCESS) { // todo: let's retry send rsp to upstream/mnode - stError("s-task:%s failed to send checkpoint rsp to upstream, checkpointId:%" PRId64 ", code:%s", pTask->id.idStr, + stError("s-task:%s failed to send checkpoint rsp to upstream, checkpointId:%d, code:%s", pTask->id.idStr, 0, tstrerror(code)); } } From 1659164926c8c862819be06dfced8af8142c27d6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 27 Dec 2023 23:00:18 +0800 Subject: [PATCH 81/94] fix(stream): remove invalid checkpoint id check. --- source/dnode/vnode/src/tq/tq.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 33f937a909..58f1fe954a 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1292,7 +1292,6 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) // check if the checkpoint msg already sent or not. if (status == TASK_STATUS__CK) { - ASSERT(pTask->chkInfo.checkpointingId == req.checkpointId); tqWarn("s-task:%s recv checkpoint-source msg again checkpointId:%" PRId64 " transId:%d already received, ignore this msg and continue process checkpoint", pTask->id.idStr, pTask->chkInfo.checkpointingId, req.transId); From 659c0a831e118455826f3720d100cf61140747ae Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 28 Dec 2023 10:25:37 +0800 Subject: [PATCH 82/94] fix: change the result schema --- source/libs/parser/src/parTranslater.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 16e0332ee9..3ab038be9d 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -8769,7 +8769,7 @@ static int32_t extractShowVariablesResultSchema(int32_t* numOfCols, SSchema** pS (*pSchema)[0].type = TSDB_DATA_TYPE_BINARY; (*pSchema)[0].bytes = TSDB_CONFIG_OPTION_LEN; - strcpy((*pSchema)[0].name, "result"); + strcpy((*pSchema)[0].name, "name"); (*pSchema)[1].type = TSDB_DATA_TYPE_BINARY; (*pSchema)[1].bytes = TSDB_CONFIG_VALUE_LEN; @@ -8791,7 +8791,7 @@ static int32_t extractCompactDbResultSchema(int32_t* numOfCols, SSchema** pSchem (*pSchema)[0].type = TSDB_DATA_TYPE_BINARY; (*pSchema)[0].bytes = COMPACT_DB_RESULT_FIELD1_LEN; - strcpy((*pSchema)[0].name, "name"); + strcpy((*pSchema)[0].name, "result"); (*pSchema)[1].type = TSDB_DATA_TYPE_INT; (*pSchema)[1].bytes = tDataTypes[TSDB_DATA_TYPE_INT].bytes; From 631eb9a72c6a727aaaee2489e7edcd11512076af Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 28 Dec 2023 10:55:50 +0800 Subject: [PATCH 83/94] fix: add test case --- tests/parallel_test/cases.task | 1 + tests/system-test/2-query/compact-col.py | 71 ++++++++++++++++++++++++ 2 files changed, 72 insertions(+) create mode 100644 tests/system-test/2-query/compact-col.py diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index bcdd143cfc..4bc226d93f 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -29,6 +29,7 @@ ,,n,system-test,python3 ./test.py -f 8-stream/snode_restart_with_checkpoint.py -N 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tbname_vgroup.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/compact-col.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stbJoin.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stbJoin.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stbJoin.py -Q 3 diff --git a/tests/system-test/2-query/compact-col.py b/tests/system-test/2-query/compact-col.py new file mode 100644 index 0000000000..0a4614d710 --- /dev/null +++ b/tests/system-test/2-query/compact-col.py @@ -0,0 +1,71 @@ +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import tdDnodes +from math import inf + +class TDTestCase: + def caseDescription(self): + ''' + case1: [TD-] + ''' + return + + def init(self, conn, logSql, replicaVer=1): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), True) + self.conn = conn + + def restartTaosd(self, index=1, dbname="db"): + tdDnodes.stop(index) + tdDnodes.startWithoutSleep(index) + tdSql.execute(f"use tbname_vgroup") + + def run(self): + print("running {}".format(__file__)) + tdSql.execute("drop database if exists tbname_vgroup") + tdSql.execute("create database if not exists tbname_vgroup") + tdSql.execute('use tbname_vgroup') + tdSql.execute('drop database if exists dbvg') + tdSql.execute('create database dbvg vgroups 8;') + + tdSql.execute('use dbvg;') + + tdSql.execute('create table st(ts timestamp, f int) tags (t int);') + + tdSql.execute("insert into ct1 using st tags(1) values('2021-04-19 00:00:01', 1)") + + tdSql.execute("insert into ct2 using st tags(2) values('2021-04-19 00:00:02', 2)") + + tdSql.execute("insert into ct3 using st tags(3) values('2021-04-19 00:00:03', 3)") + + tdSql.execute("insert into ct4 using st tags(4) values('2021-04-19 00:00:04', 4)") + + + tdSql.execute('create table st2(ts timestamp, f int) tags (t int);') + + tdSql.execute("insert into ct21 using st2 tags(1) values('2021-04-19 00:00:01', 1)") + + tdSql.execute("insert into ct22 using st2 tags(2) values('2021-04-19 00:00:02', 2)") + + tdSql.execute("insert into ct23 using st2 tags(3) values('2021-04-19 00:00:03', 3)") + + tdSql.execute("insert into ct24 using st2 tags(4) values('2021-04-19 00:00:04', 4)") + + + col_names = tdSql.getColNameList("compact database tbname_vgroup") + if col_names[0] != "result": + raise Exception("first column name of compact result shall be result") + + col_names = tdSql.getColNameList("show variables") + if col_names[0] != "name": + raise Exception("first column name of compact result shall be name") + + tdSql.execute('drop database tbname_vgroup') + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From c124bcf52fca3a9528ba93d51d7563564a083525 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Wed, 27 Dec 2023 11:29:03 +0800 Subject: [PATCH 84/94] check block rows --- source/libs/executor/src/streamtimewindowoperator.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index c9490e2c55..b9f3dc19d8 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -2343,10 +2343,6 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDa } int32_t code = pAPI->stateStore.streamStateGetByPos(pState, pPos, (void**)&pRow); - if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) { - ASSERT(pBlock->info.rows > 0); - break; - } if (code == -1) { // for history @@ -2363,6 +2359,11 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDa continue; } + if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) { + ASSERT(pBlock->info.rows > 0); + break; + } + pGroupResInfo->index += 1; for (int32_t j = 0; j < numOfExprs; ++j) { @@ -2383,6 +2384,7 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDa SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId); char* in = GET_ROWCELL_INTERBUF(pCtx[j].resultInfo); for (int32_t k = 0; k < pRow->numOfRows; ++k) { + qDebug("===stream===block rows:%d, rows:%d, exprs:%d, capacity:%d", pBlock->info.rows, i, j, pBlock->info.capacity); colDataSetVal(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes); } } From f1304680c590473a3c9c24eaaa582be9119b60f4 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Wed, 27 Dec 2023 14:01:38 +0800 Subject: [PATCH 85/94] log --- source/libs/executor/src/streamtimewindowoperator.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index b9f3dc19d8..024a44d107 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -2384,7 +2384,6 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, void* pState, SSDa SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId); char* in = GET_ROWCELL_INTERBUF(pCtx[j].resultInfo); for (int32_t k = 0; k < pRow->numOfRows; ++k) { - qDebug("===stream===block rows:%d, rows:%d, exprs:%d, capacity:%d", pBlock->info.rows, i, j, pBlock->info.capacity); colDataSetVal(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes); } } From 448b7c939f122dd1f25d11a4ae59a4cef5b79ed0 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Thu, 28 Dec 2023 11:09:38 +0800 Subject: [PATCH 86/94] reset datablock window info --- source/libs/executor/src/scanoperator.c | 1 + .../executor/src/streamtimewindowoperator.c | 27 ++++++++++--------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index ef2a99d1d1..b38b9bca49 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2316,6 +2316,7 @@ FETCH_NEXT_BLOCK: doCheckUpdate(pInfo, pBlockInfo->window.ekey, pInfo->pRes); doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL); + blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex); int64_t numOfUpdateRes = pInfo->pUpdateDataRes->info.rows; qDebug("%s %" PRId64 " rows in datablock, update res:%" PRId64, id, pBlockInfo->rows, numOfUpdateRes); diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index 024a44d107..c749c3814d 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -801,10 +801,24 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDat SRowBuffPos* pResPos = NULL; SResultRow* pResult = NULL; int32_t forwardRows = 0; + int32_t endRowId = pSDataBlock->info.rows - 1; SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); tsCols = (int64_t*)pColDataInfo->pData; + if (pSDataBlock->info.window.skey != tsCols[0] || pSDataBlock->info.window.ekey != tsCols[endRowId]) { + qError("table uid %" PRIu64 " data block timestamp range may not be calculated! minKey %" PRId64 + ",maxKey %" PRId64, + pSDataBlock->info.id.uid, pSDataBlock->info.window.skey, pSDataBlock->info.window.ekey); + blockDataUpdateTsWindow(pSDataBlock, pInfo->primaryTsIndex); + + // timestamp of the data is incorrect + if (pSDataBlock->info.window.skey <= 0 || pSDataBlock->info.window.ekey <= 0) { + qError("table uid %" PRIu64 " data block timestamp is out of range! minKey %" PRId64 ",maxKey %" PRId64, + pSDataBlock->info.id.uid, pSDataBlock->info.window.skey, pSDataBlock->info.window.ekey); + } + } + int32_t startPos = 0; TSKEY ts = getStartTsKey(&pSDataBlock->info.window, tsCols); STimeWindow nextWin = {0}; @@ -902,19 +916,6 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDat pInfo->delKey = key; } int32_t prevEndPos = (forwardRows - 1) * step + startPos; - if (pSDataBlock->info.window.skey <= 0 || pSDataBlock->info.window.ekey <= 0) { - qError("table uid %" PRIu64 " data block timestamp range may not be calculated! minKey %" PRId64 - ",maxKey %" PRId64, - pSDataBlock->info.id.uid, pSDataBlock->info.window.skey, pSDataBlock->info.window.ekey); - blockDataUpdateTsWindow(pSDataBlock, 0); - - // timestamp of the data is incorrect - if (pSDataBlock->info.window.skey <= 0 || pSDataBlock->info.window.ekey <= 0) { - qError("table uid %" PRIu64 " data block timestamp is out of range! minKey %" PRId64 ",maxKey %" PRId64, - pSDataBlock->info.id.uid, pSDataBlock->info.window.skey, pSDataBlock->info.window.ekey); - } - } - if (IS_FINAL_INTERVAL_OP(pOperator)) { startPos = getNextQualifiedFinalWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos); } else { From 9c997e34ea84f3c80eee0b1e353a903f74dc7154 Mon Sep 17 00:00:00 2001 From: charles Date: Thu, 28 Dec 2023 13:32:13 +0800 Subject: [PATCH 87/94] update test case to replace 'gdb' with 'local variables' --- .../test_hot_refresh_configurations.py | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/tests/system-test/0-others/test_hot_refresh_configurations.py b/tests/system-test/0-others/test_hot_refresh_configurations.py index 59222281ae..09e634974b 100644 --- a/tests/system-test/0-others/test_hot_refresh_configurations.py +++ b/tests/system-test/0-others/test_hot_refresh_configurations.py @@ -159,20 +159,21 @@ class TDTestCase: ] } - def get_param_value_with_gdb(self, config_name, process_name): - res = subprocess.Popen("gdb -q -nx -p `pidof {}` --batch -ex 'set height 0' -ex 'p {}'".format(process_name, config_name), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - r_lines = res.stdout.read() - if r_lines: - for line in r_lines.decode().split("\n"): - if "$1 = " in line: - tdLog.debug("gdb result: {}".format(line)) - return line.split(" = ")[1] + def get_param_value(self, config_name): + tdSql.query("show local variables;") + for row in tdSql.queryResult: + if config_name == row[0]: + tdLog.debug("Found variable '{}'".format(row[0])) + return row[1] def cli_check(self, name, values, except_values=False): if not except_values: for v in values: tdLog.debug("Set {} to {}".format(name, v)) tdSql.execute(f'alter local "{name} {v}";') + value = self.get_param_value(name) + tdLog.debug("Get {} value: {}".format(name, value)) + assert(v == int(value)) else: for v in values: tdLog.debug("Set {} to {}".format(name, v)) @@ -190,9 +191,7 @@ class TDTestCase: for v in values: dnode = random.choice(p_list) tdSql.execute(f'alter {dnode} "{name} {v}";') - if platform.system() == "Linux" and platform.machine() == "aarch64": - continue - value = self.get_param_value_with_gdb(alias, "taosd") + value = self.get_param_value(alias) if value: tdLog.debug(f"value: {value}") assert(value == str(bool(v)).lower() if is_bool else str(v)) From 51cd8e9fb77822374cff9b0583874638151d8828 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Thu, 28 Dec 2023 15:34:13 +0800 Subject: [PATCH 88/94] reset unclose window info --- source/libs/executor/inc/executorInt.h | 1 + source/libs/executor/src/executil.c | 7 ++- source/libs/executor/src/scanoperator.c | 1 - .../executor/src/streameventwindowoperator.c | 14 +++--- .../executor/src/streamtimewindowoperator.c | 44 ++++++++++--------- 5 files changed, 37 insertions(+), 30 deletions(-) diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index e3e504cdbc..e98777544b 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -845,6 +845,7 @@ void resetWinRange(STimeWindow* winRange); bool checkExpiredData(SStateStore* pAPI, SUpdateInfo* pUpdateInfo, STimeWindowAggSupp* pTwSup, uint64_t tableId, TSKEY ts); int64_t getDeleteMark(SWindowPhysiNode* pWinPhyNode, int64_t interval); void resetUnCloseSessionWinInfo(SSHashObj* winMap); +void setStreamOperatorCompleted(struct SOperatorInfo* pOperator); int32_t encodeSSessionKey(void** buf, SSessionKey* key); void* decodeSSessionKey(void* buf, SSessionKey* key); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 377de99fc0..825d716926 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -2244,8 +2244,11 @@ void printDataBlock(SSDataBlock* pBlock, const char* flag, const char* taskIdStr } void printSpecDataBlock(SSDataBlock* pBlock, const char* flag, const char* opStr, const char* taskIdStr) { - if (!pBlock || pBlock->info.rows == 0) { - qDebug("%s===stream===%s: Block is Null or Empty", taskIdStr, flag); + if (!pBlock) { + qDebug("%s===stream===%s: Block is Null", taskIdStr, flag); + return; + } else if (pBlock->info.rows == 0) { + qDebug("%s===stream===%s: Block is Empty. type:%d", taskIdStr, flag, pBlock->info.type); return; } if (qDebugFlag & DEBUG_DEBUG) { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index ef2a99d1d1..32407dcf71 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1997,7 +1997,6 @@ void streamScanOperatorSaveCheckpoint(SStreamScanInfo* pInfo) { int32_t len = streamScanOperatorEncode(pInfo, &pBuf); pInfo->stateStore.streamStateSaveInfo(pInfo->pState, STREAM_SCAN_OP_CHECKPOINT_NAME, strlen(STREAM_SCAN_OP_CHECKPOINT_NAME), pBuf, len); taosMemoryFree(pBuf); - pInfo->stateStore.streamStateCommit(pInfo->pState); } // other properties are recovered from the execution plan diff --git a/source/libs/executor/src/streameventwindowoperator.c b/source/libs/executor/src/streameventwindowoperator.c index 6adab74344..3868edf645 100644 --- a/source/libs/executor/src/streameventwindowoperator.c +++ b/source/libs/executor/src/streameventwindowoperator.c @@ -480,18 +480,18 @@ static SSDataBlock* doStreamEventAgg(SOperatorInfo* pOperator) { return resBlock; } + if (pInfo->recvGetAll) { + pInfo->recvGetAll = false; + resetUnCloseSessionWinInfo(pInfo->streamAggSup.pResultRows); + } + if (pInfo->reCkBlock) { pInfo->reCkBlock = false; printDataBlock(pInfo->pCheckpointRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo)); return pInfo->pCheckpointRes; } - if (pInfo->recvGetAll) { - pInfo->recvGetAll = false; - resetUnCloseSessionWinInfo(pInfo->streamAggSup.pResultRows); - } - - setOperatorCompleted(pOperator); + setStreamOperatorCompleted(pOperator); return NULL; } @@ -561,7 +561,7 @@ static SSDataBlock* doStreamEventAgg(SOperatorInfo* pOperator) { if (resBlock != NULL) { return resBlock; } - setOperatorCompleted(pOperator); + setStreamOperatorCompleted(pOperator); return NULL; } diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index c9490e2c55..a96d0ef633 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -1223,7 +1223,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { return pInfo->pCheckpointRes; } - setOperatorCompleted(pOperator); + setStreamOperatorCompleted(pOperator); if (!IS_FINAL_INTERVAL_OP(pOperator)) { clearFunctionContext(&pOperator->exprSupp); // semi interval operator clear disk buffer @@ -2609,18 +2609,18 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { return opRes; } + if (pInfo->recvGetAll) { + pInfo->recvGetAll = false; + resetUnCloseSessionWinInfo(pInfo->streamAggSup.pResultRows); + } + if (pInfo->reCkBlock) { pInfo->reCkBlock = false; printDataBlock(pInfo->pCheckpointRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo)); return pInfo->pCheckpointRes; } - if (pInfo->recvGetAll) { - pInfo->recvGetAll = false; - resetUnCloseSessionWinInfo(pInfo->streamAggSup.pResultRows); - } - - setOperatorCompleted(pOperator); + setStreamOperatorCompleted(pOperator); return NULL; } @@ -2718,7 +2718,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { return opRes; } - setOperatorCompleted(pOperator); + setStreamOperatorCompleted(pOperator); return NULL; } @@ -3001,7 +3001,7 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { clearFunctionContext(&pOperator->exprSupp); // semi session operator clear disk buffer clearStreamSessionOperator(pInfo); - setOperatorCompleted(pOperator); + setStreamOperatorCompleted(pOperator); pInfo->clearState = false; return NULL; } @@ -3074,7 +3074,7 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { clearFunctionContext(&pOperator->exprSupp); // semi session operator clear disk buffer clearStreamSessionOperator(pInfo); - setOperatorCompleted(pOperator); + setStreamOperatorCompleted(pOperator); return NULL; } @@ -3551,18 +3551,18 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { return resBlock; } + if (pInfo->recvGetAll) { + pInfo->recvGetAll = false; + resetUnCloseSessionWinInfo(pInfo->streamAggSup.pResultRows); + } + if (pInfo->reCkBlock) { pInfo->reCkBlock = false; printDataBlock(pInfo->pCheckpointRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo)); return pInfo->pCheckpointRes; } - if (pInfo->recvGetAll) { - pInfo->recvGetAll = false; - resetUnCloseSessionWinInfo(pInfo->streamAggSup.pResultRows); - } - - setOperatorCompleted(pOperator); + setStreamOperatorCompleted(pOperator); return NULL; } @@ -3628,7 +3628,7 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { if (resBlock != NULL) { return resBlock; } - setOperatorCompleted(pOperator); + setStreamOperatorCompleted(pOperator); return NULL; } @@ -3864,11 +3864,11 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { if (pInfo->reCkBlock) { pInfo->reCkBlock = false; - // printDataBlock(pInfo->pCheckpointRes, "single interval ck"); + printDataBlock(pInfo->pCheckpointRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo)); return pInfo->pCheckpointRes; } - setOperatorCompleted(pOperator); + setStreamOperatorCompleted(pOperator); return NULL; } @@ -3900,7 +3900,6 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { doDeleteWindows(pOperator, &pInfo->interval, pBlock, pInfo->pDelWins, pInfo->pUpdatedMap); continue; } else if (pBlock->info.type == STREAM_GET_ALL) { - qDebug("===stream===%s recv|block type STREAM_GET_ALL", getStreamOpName(pOperator->operatorType)); pInfo->recvGetAll = true; getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pInfo->pUpdatedMap); continue; @@ -4083,3 +4082,8 @@ _error: pTaskInfo->code = code; return NULL; } + +void setStreamOperatorCompleted(SOperatorInfo* pOperator) { + setOperatorCompleted(pOperator); + qDebug("stask:%s %s status: %d. set completed", GET_TASKID(pOperator->pTaskInfo), getStreamOpName(pOperator->operatorType), pOperator->status); +} From 68cb86602eff35725e39592c054ab22954ee9cb6 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Fri, 29 Dec 2023 11:37:34 +0800 Subject: [PATCH 89/94] fix: reuse cid with fs eid updated in tsdbFSetRAWWriteFileDataBegin --- source/dnode/vnode/src/tsdb/tsdbFS2.c | 6 ++++++ source/dnode/vnode/src/tsdb/tsdbFS2.h | 1 + source/dnode/vnode/src/tsdb/tsdbFSetRAW.c | 6 ++++-- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index b2ccc3e859..1d2fd60df2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -805,6 +805,12 @@ int64_t tsdbFSAllocEid(STFileSystem *fs) { return cid; } +void tsdbFSUpdateEid(STFileSystem *fs, int64_t cid) { + taosThreadMutexLock(&fs->tsdb->mutex); + fs->neid = TMAX(fs->neid, cid); + taosThreadMutexUnlock(&fs->tsdb->mutex); +} + int32_t tsdbFSEditBegin(STFileSystem *fs, const TFileOpArray *opArray, EFEditT etype) { int32_t code = 0; int32_t lino; diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.h b/source/dnode/vnode/src/tsdb/tsdbFS2.h index 714bf5bf16..7099ce8b26 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.h +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.h @@ -52,6 +52,7 @@ int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ev int32_t tsdbFSDestroyRefRangedSnapshot(TFileSetRangeArray **fsrArr); // txn int64_t tsdbFSAllocEid(STFileSystem *fs); +void tsdbFSUpdateEid(STFileSystem *fs, int64_t cid); int32_t tsdbFSEditBegin(STFileSystem *fs, const TFileOpArray *opArray, EFEditT etype); int32_t tsdbFSEditCommit(STFileSystem *fs); int32_t tsdbFSEditAbort(STFileSystem *fs); diff --git a/source/dnode/vnode/src/tsdb/tsdbFSetRAW.c b/source/dnode/vnode/src/tsdb/tsdbFSetRAW.c index 03c12502d5..ddb8e7e27e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSetRAW.c +++ b/source/dnode/vnode/src/tsdb/tsdbFSetRAW.c @@ -14,6 +14,7 @@ */ #include "tsdbFSetRAW.h" +#include "tsdbFS2.h" // SFSetRAWWriter ================================================== typedef struct SFSetRAWWriter { @@ -76,7 +77,7 @@ static int32_t tsdbFSetRAWWriteFileDataBegin(SFSetRAWWriter *writer, STsdbDataRA .szPage = writer->config->szPage, .fid = bHdr->file.fid, .did = writer->config->did, - .cid = writer->config->cid, + .cid = bHdr->file.cid, .level = writer->config->level, .file = @@ -84,7 +85,7 @@ static int32_t tsdbFSetRAWWriteFileDataBegin(SFSetRAWWriter *writer, STsdbDataRA .type = bHdr->file.type, .fid = bHdr->file.fid, .did = writer->config->did, - .cid = writer->config->cid, + .cid = bHdr->file.cid, .size = bHdr->file.size, .minVer = bHdr->file.minVer, .maxVer = bHdr->file.maxVer, @@ -94,6 +95,7 @@ static int32_t tsdbFSetRAWWriteFileDataBegin(SFSetRAWWriter *writer, STsdbDataRA }, }; + tsdbFSUpdateEid(config.tsdb->pFS, config.cid); writer->ctx->offset = 0; writer->ctx->file = config.file; From 7abeb5a27a3cd6077d0e4e0c1c659c00e41d3b26 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 29 Dec 2023 17:44:10 +0800 Subject: [PATCH 90/94] fix(stream): update the stream error code. --- source/dnode/mnode/impl/src/mndStream.c | 2 +- tests/system-test/8-stream/max_delay_session.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 25f51b85df..57f3ac174c 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -792,7 +792,7 @@ static int32_t checkForNumOfStreams(SMnode *pMnode, SStreamObj *pStreamObj) { / if (pStream->targetStbUid == pStreamObj->targetStbUid) { mError("Cannot write the same stable as other stream:%s", pStream->name); sdbCancelFetch(pMnode->pSdb, pIter); - terrno = TSDB_CODE_MND_TOO_MANY_STREAMS; + terrno = TSDB_CODE_MND_INVALID_TARGET_TABLE; return terrno; } } diff --git a/tests/system-test/8-stream/max_delay_session.py b/tests/system-test/8-stream/max_delay_session.py index 46c4c5801d..7e9d365fc8 100644 --- a/tests/system-test/8-stream/max_delay_session.py +++ b/tests/system-test/8-stream/max_delay_session.py @@ -27,6 +27,8 @@ class TDTestCase: self.tb_stream_des_table = f'{self.tb_name}{self.tdCom.des_table_suffix}' self.tdCom.date_time = self.tdCom.dataDict["start_ts"] + time.sleep(1) + if watermark is not None: watermark_value = f'{self.tdCom.dataDict["watermark"]}s' else: From 1e8244f3c692cf181fa1a29d1c4d4b3a7493fa52 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 30 Dec 2023 10:31:15 +0800 Subject: [PATCH 91/94] feat: add s3 minio ci case --- source/common/src/tglobal.c | 2 +- tests/army/enterprise/s3/s3_basic.json | 58 ++++++++++++++ tests/army/enterprise/s3/s3_basic.py | 105 +++++++++++++++++++++++++ tests/army/frame/caseBase.py | 4 + tests/army/frame/srvCtl.py | 5 ++ 5 files changed, 173 insertions(+), 1 deletion(-) create mode 100644 tests/army/enterprise/s3/s3_basic.json create mode 100644 tests/army/enterprise/s3/s3_basic.py diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index fdac3882c3..d6d1e1810d 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -789,7 +789,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "s3PageCacheSize", tsS3PageCacheSize, 4, 1024 * 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "s3UploadDelaySec", tsS3UploadDelaySec, 60 * 10, 60 * 60 * 24 * 30, CFG_SCOPE_SERVER, + if (cfgAddInt32(pCfg, "s3UploadDelaySec", tsS3UploadDelaySec, 60 * 1, 60 * 60 * 24 * 30, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; diff --git a/tests/army/enterprise/s3/s3_basic.json b/tests/army/enterprise/s3/s3_basic.json new file mode 100644 index 0000000000..1c2b9274d2 --- /dev/null +++ b/tests/army/enterprise/s3/s3_basic.json @@ -0,0 +1,58 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "connection_pool_size": 8, + "num_of_records_per_req": 2000, + "thread_count": 2, + "create_table_thread_count": 1, + "confirm_parameter_prompt": "no", + "databases": [ + { + "dbinfo": { + "name": "db", + "drop": "yes", + "vgroups": 2, + "replica": 1, + "duration":"1d", + "keep": "3d,6d,30d" + }, + "super_tables": [ + { + "name": "stb", + "child_table_exists": "no", + "childtable_count": 4, + "insert_rows": 1000000, + "childtable_prefix": "d", + "insert_mode": "taosc", + "timestamp_step": 1000, + "start_timestamp":"now-12d", + "columns": [ + { "type": "bool", "name": "bc"}, + { "type": "float", "name": "fc" }, + { "type": "double", "name": "dc"}, + { "type": "tinyint", "name": "ti", "values":["1"]}, + { "type": "smallint", "name": "si" }, + { "type": "int", "name": "ic" }, + { "type": "bigint", "name": "bi" }, + { "type": "utinyint", "name": "uti"}, + { "type": "usmallint", "name": "usi"}, + { "type": "uint", "name": "ui" }, + { "type": "ubigint", "name": "ubi"}, + { "type": "binary", "name": "bin", "len": 32}, + { "type": "nchar", "name": "nch", "len": 64} + ], + "tags": [ + {"type": "tinyint", "name": "groupid","max": 10,"min": 1}, + {"name": "location","type": "binary", "len": 16, "values": + ["San Francisco", "Los Angles", "San Diego", "San Jose", "Palo Alto", "Campbell", "Mountain View","Sunnyvale", "Santa Clara", "Cupertino"] + } + ] + } + ] + } + ] +} diff --git a/tests/army/enterprise/s3/s3_basic.py b/tests/army/enterprise/s3/s3_basic.py new file mode 100644 index 0000000000..12c0720112 --- /dev/null +++ b/tests/army/enterprise/s3/s3_basic.py @@ -0,0 +1,105 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import time + +import taos +import frame +import frame.etool + +from frame.log import * +from frame.cases import * +from frame.sql import * +from frame.caseBase import * +from frame.srvCtl import * +from frame import * + +# +# 192.168.1.52 MINIO S3 API KEY: MQCEIoaPGUs1mhXgpUAu:XTgpN2dEMInnYgqN4gj3G5zgb39ROtsisKKy0GFa +# + +''' +s3EndPoint http://192.168.1.52:9000 +s3AccessKey MQCEIoaPGUs1mhXgpUAu:XTgpN2dEMInnYgqN4gj3G5zgb39ROtsisKKy0GFa +s3BucketName ci-bucket +s3UploadDelaySec 60 +''' + + +class TDTestCase(TBase): + updatecfgDict = { + 's3EndPoint': 'http://192.168.1.52:9000', + 's3AccessKey': 'MQCEIoaPGUs1mhXgpUAu:XTgpN2dEMInnYgqN4gj3G5zgb39ROtsisKKy0GFa', + 's3BucketName': 'ci-bucket', + 's3UploadDelaySec':'60' + } + + def insertData(self): + tdLog.info(f"insert data.") + # taosBenchmark run + json = etool.curFile(__file__, "mlevel_basic.json") + etool.runBenchmark(json=json) + + tdSql.execute(f"use {self.db}") + # set insert data information + self.childtable_count = 4 + self.insert_rows = 1000000 + self.timestamp_step = 1000 + + def doAction(self): + tdLog.info(f"do action.") + self.flushDb() + self.compactDb() + + # sleep 70s + time.sleep(70) + self.trimDb() + + loop = 0 + while len(sc.dnodeDataFiles()) > 0 and loop < 10: + time.sleep(10) + self.trimDb() + loop += 1 + + # run + def run(self): + tdLog.debug(f"start to excute {__file__}") + + # insert data + self.insertData() + + # check insert data correct + self.checkInsertCorrect() + + # save + self.snapshotAgg() + + # do action + self.doAction() + + # check save agg result correct + self.checkAggCorrect() + + # check insert correct again + self.checkInsertCorrect() + + # drop database and free s3 file + self.dropDb() + + tdLog.success(f"{__file__} successfully executed") + + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/army/frame/caseBase.py b/tests/army/frame/caseBase.py index 441196f050..6009111d43 100644 --- a/tests/army/frame/caseBase.py +++ b/tests/army/frame/caseBase.py @@ -72,6 +72,10 @@ class TBase: def flushDb(self): tdSql.execute(f"flush database {self.db}") + def dropDb(self): + tdSql.execute(f"drop database {self.db}") + + # # check db correct # diff --git a/tests/army/frame/srvCtl.py b/tests/army/frame/srvCtl.py index e7ef08cde9..99f503d4b6 100644 --- a/tests/army/frame/srvCtl.py +++ b/tests/army/frame/srvCtl.py @@ -24,4 +24,9 @@ class srvCtl: self.mLevel = 0 self.mLevelDisk = 0 + # return dnode data files list + def dnodeDataFiles(self, idx): + files = [] + return files + sc = srvCtl() \ No newline at end of file From 64164f5684289af8a36ec52a2d41845359536551 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 30 Dec 2023 14:39:39 +0800 Subject: [PATCH 92/94] fix: run s3_basic.py ok --- tests/army/enterprise/s3/s3_basic.json | 2 +- tests/army/enterprise/s3/s3_basic.py | 16 ++++++++++++---- tests/army/frame/caseBase.py | 16 ++++++++-------- tests/army/frame/eos.py | 21 +++++++++++++++++++++ tests/army/frame/epath.py | 3 +++ tests/army/frame/srvCtl.py | 12 ++++++++++++ 6 files changed, 57 insertions(+), 13 deletions(-) diff --git a/tests/army/enterprise/s3/s3_basic.json b/tests/army/enterprise/s3/s3_basic.json index 1c2b9274d2..169da290bf 100644 --- a/tests/army/enterprise/s3/s3_basic.json +++ b/tests/army/enterprise/s3/s3_basic.json @@ -29,7 +29,7 @@ "childtable_prefix": "d", "insert_mode": "taosc", "timestamp_step": 1000, - "start_timestamp":"now-12d", + "start_timestamp":"now-15d", "columns": [ { "type": "bool", "name": "bc"}, { "type": "float", "name": "fc" }, diff --git a/tests/army/enterprise/s3/s3_basic.py b/tests/army/enterprise/s3/s3_basic.py index 12c0720112..84c91209b3 100644 --- a/tests/army/enterprise/s3/s3_basic.py +++ b/tests/army/enterprise/s3/s3_basic.py @@ -17,6 +17,7 @@ import time import taos import frame import frame.etool +import frame.eos from frame.log import * from frame.cases import * @@ -24,6 +25,7 @@ from frame.sql import * from frame.caseBase import * from frame.srvCtl import * from frame import * +from frame.eos import * # # 192.168.1.52 MINIO S3 API KEY: MQCEIoaPGUs1mhXgpUAu:XTgpN2dEMInnYgqN4gj3G5zgb39ROtsisKKy0GFa @@ -42,13 +44,16 @@ class TDTestCase(TBase): 's3EndPoint': 'http://192.168.1.52:9000', 's3AccessKey': 'MQCEIoaPGUs1mhXgpUAu:XTgpN2dEMInnYgqN4gj3G5zgb39ROtsisKKy0GFa', 's3BucketName': 'ci-bucket', + 's3BlockSize': '10240', + 's3BlockCacheSize': '320', + 's3PageCacheSize': '10240', 's3UploadDelaySec':'60' } def insertData(self): tdLog.info(f"insert data.") # taosBenchmark run - json = etool.curFile(__file__, "mlevel_basic.json") + json = etool.curFile(__file__, "s3_basic.json") etool.runBenchmark(json=json) tdSql.execute(f"use {self.db}") @@ -63,12 +68,15 @@ class TDTestCase(TBase): self.compactDb() # sleep 70s + tdLog.info(f"wait 70s ...") time.sleep(70) self.trimDb() - + + rootPath = sc.clusterRootPath() + cmd = f"ls {rootPath}/dnode1/data02/vnode/vnode*/tsdb/*.data" loop = 0 - while len(sc.dnodeDataFiles()) > 0 and loop < 10: - time.sleep(10) + while len(eos.runRetList(cmd)) > 0 and loop < 40: + time.sleep(5) self.trimDb() loop += 1 diff --git a/tests/army/frame/caseBase.py b/tests/army/frame/caseBase.py index 6009111d43..ad6e744fe6 100644 --- a/tests/army/frame/caseBase.py +++ b/tests/army/frame/caseBase.py @@ -63,17 +63,17 @@ class TBase: # db action # - def trimDb(self): - tdSql.execute(f"trim database {self.db}") + def trimDb(self, show = False): + tdSql.execute(f"trim database {self.db}", show = show) - def compactDb(self): - tdSql.execute(f"compact database {self.db}") + def compactDb(self, show = False): + tdSql.execute(f"compact database {self.db}", show = show) - def flushDb(self): - tdSql.execute(f"flush database {self.db}") + def flushDb(self, show = False): + tdSql.execute(f"flush database {self.db}", show = show) - def dropDb(self): - tdSql.execute(f"drop database {self.db}") + def dropDb(self, show = False): + tdSql.execute(f"drop database {self.db}", show = show) # diff --git a/tests/army/frame/eos.py b/tests/army/frame/eos.py index dcb63ff3aa..b24be74393 100644 --- a/tests/army/frame/eos.py +++ b/tests/army/frame/eos.py @@ -20,11 +20,16 @@ import os import time import datetime import platform +import subprocess # if windows platform return True def isWin(): return platform.system().lower() == 'windows' +# +# execute programe +# + # wait util execute file finished def exe(file): return os.system(file) @@ -34,3 +39,19 @@ def exeNoWait(file): print("exe no wait") +# run return output and error +def run(command): + process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + process.wait(3) + + output = process.stdout.read().decode(encoding="gbk") + error = process.stderr.read().decode(encoding="gbk") + + return output, error + + +# return list after run +def runRetList(command): + lines = [] + output,error = run(command) + return output.splitlines() diff --git a/tests/army/frame/epath.py b/tests/army/frame/epath.py index edff9c78a4..c7796befbc 100644 --- a/tests/army/frame/epath.py +++ b/tests/army/frame/epath.py @@ -51,3 +51,6 @@ def binPath(): def binFile(filename): return binPath() + filename + + + diff --git a/tests/army/frame/srvCtl.py b/tests/army/frame/srvCtl.py index 99f503d4b6..75f0d79513 100644 --- a/tests/army/frame/srvCtl.py +++ b/tests/army/frame/srvCtl.py @@ -16,6 +16,8 @@ import os import time import datetime +from frame.server.dnodes import * + class srvCtl: def __init__(self): # record server information @@ -24,9 +26,19 @@ class srvCtl: self.mLevel = 0 self.mLevelDisk = 0 + # + # about path + # + + # get cluster root path like /root/TDinternal/sim/ + def clusterRootPath(self): + return tdDnodes.path + # return dnode data files list def dnodeDataFiles(self, idx): files = [] return files + + sc = srvCtl() \ No newline at end of file From 26023864f14ab5a6b6926bca68a6a43c2cb12a8a Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 30 Dec 2023 14:48:36 +0800 Subject: [PATCH 93/94] fix: add show path information --- tests/army/enterprise/s3/s3_basic.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/army/enterprise/s3/s3_basic.py b/tests/army/enterprise/s3/s3_basic.py index 84c91209b3..85beb195c3 100644 --- a/tests/army/enterprise/s3/s3_basic.py +++ b/tests/army/enterprise/s3/s3_basic.py @@ -70,14 +70,15 @@ class TDTestCase(TBase): # sleep 70s tdLog.info(f"wait 70s ...") time.sleep(70) - self.trimDb() + self.trimDb(True) rootPath = sc.clusterRootPath() cmd = f"ls {rootPath}/dnode1/data02/vnode/vnode*/tsdb/*.data" + tdLog.info(cmd) loop = 0 while len(eos.runRetList(cmd)) > 0 and loop < 40: time.sleep(5) - self.trimDb() + self.trimDb(True) loop += 1 # run From 1600f0654d6c3f46aae1242199aa61f7af223a2a Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Sat, 30 Dec 2023 15:16:29 +0800 Subject: [PATCH 94/94] fix: s3 for minio finished --- tests/army/enterprise/s3/s3_basic.json | 2 +- tests/army/enterprise/s3/s3_basic.py | 6 +++--- tests/army/frame/srvCtl.py | 2 +- tests/parallel_test/cases.task | 11 +++++++++-- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/tests/army/enterprise/s3/s3_basic.json b/tests/army/enterprise/s3/s3_basic.json index 169da290bf..e56a18e757 100644 --- a/tests/army/enterprise/s3/s3_basic.json +++ b/tests/army/enterprise/s3/s3_basic.json @@ -29,7 +29,7 @@ "childtable_prefix": "d", "insert_mode": "taosc", "timestamp_step": 1000, - "start_timestamp":"now-15d", + "start_timestamp":"now-13d", "columns": [ { "type": "bool", "name": "bc"}, { "type": "float", "name": "fc" }, diff --git a/tests/army/enterprise/s3/s3_basic.py b/tests/army/enterprise/s3/s3_basic.py index 85beb195c3..389b8a52c7 100644 --- a/tests/army/enterprise/s3/s3_basic.py +++ b/tests/army/enterprise/s3/s3_basic.py @@ -68,12 +68,12 @@ class TDTestCase(TBase): self.compactDb() # sleep 70s - tdLog.info(f"wait 70s ...") - time.sleep(70) + tdLog.info(f"wait 65s ...") + time.sleep(65) self.trimDb(True) rootPath = sc.clusterRootPath() - cmd = f"ls {rootPath}/dnode1/data02/vnode/vnode*/tsdb/*.data" + cmd = f"ls {rootPath}/dnode1/data20/vnode/vnode*/tsdb/*.data" tdLog.info(cmd) loop = 0 while len(eos.runRetList(cmd)) > 0 and loop < 40: diff --git a/tests/army/frame/srvCtl.py b/tests/army/frame/srvCtl.py index 75f0d79513..155c83094d 100644 --- a/tests/army/frame/srvCtl.py +++ b/tests/army/frame/srvCtl.py @@ -32,7 +32,7 @@ class srvCtl: # get cluster root path like /root/TDinternal/sim/ def clusterRootPath(self): - return tdDnodes.path + return tdDnodes.getDnodesRootDir() # return dnode data files list def dnodeDataFiles(self, idx): diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index b5cde02d25..e7395a26f1 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -5,10 +5,17 @@ #unit-test ,,y,unit-test,bash test.sh -#army-test +# +# army-test +# ,,y,army,./pytest.sh python3 ./test.py -f enterprise/multi-level/mlevel_basic.py -N 3 -L 3 -D 2 +,,y,army,./pytest.sh python3 ./test.py -f enterprise/s3/s3_basic.py -L 3 -D 1 -#system test + + +# +# system test +# ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/stream_basic.py ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/scalar_function.py ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/at_once_interval.py