From 8a05bad89952a8a1706115bf2c1c3af2a47126eb Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 10 Nov 2023 14:14:22 +0800 Subject: [PATCH 01/29] fix:snode error --- include/libs/stream/tstream.h | 1 - source/dnode/mgmt/mgmt_snode/src/smHandle.c | 2 + source/dnode/snode/src/snode.c | 41 +++++++++++++++++++-- source/libs/stream/src/streamDispatch.c | 6 +-- source/libs/stream/src/streamMeta.c | 7 +--- 5 files changed, 44 insertions(+), 13 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index eab3ecf04e..6ca6b176d0 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -828,7 +828,6 @@ int32_t streamMetaCommit(SStreamMeta* pMeta); int32_t streamMetaLoadAllTasks(SStreamMeta* pMeta); void streamMetaNotifyClose(SStreamMeta* pMeta); void streamMetaStartHb(SStreamMeta* pMeta); -void streamMetaInitForSnode(SStreamMeta* pMeta); bool streamMetaTaskInTimer(SStreamMeta* pMeta); int32_t streamMetaUpdateTaskDownstreamStatus(SStreamTask* pTask, int64_t startTs, int64_t endTs, bool succ); void streamMetaRLock(SStreamMeta* pMeta); diff --git a/source/dnode/mgmt/mgmt_snode/src/smHandle.c b/source/dnode/mgmt/mgmt_snode/src/smHandle.c index b29c5c1eb4..509a7a6b7f 100644 --- a/source/dnode/mgmt/mgmt_snode/src/smHandle.c +++ b/source/dnode/mgmt/mgmt_snode/src/smHandle.c @@ -88,6 +88,8 @@ SArray *smGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_SCAN_HISTORY_FINISH, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_SCAN_HISTORY_FINISH_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_CHECK_POINT_SOURCE, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECKPOINT_READY, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; + code = 0; _OVER: diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 6f5b370826..74f792cc76 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -117,7 +117,7 @@ SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) { } pSnode->msgCb = pOption->msgCb; - pSnode->pMeta = streamMetaOpen(path, pSnode, (FTaskExpand *)sndExpandTask, SNODE_HANDLE, -1); + pSnode->pMeta = streamMetaOpen(path, pSnode, (FTaskExpand *)sndExpandTask, SNODE_HANDLE, taosGetTimestampMs()); if (pSnode->pMeta == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto FAIL; @@ -126,8 +126,6 @@ SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) { stopRsync(); startRsync(); - // todo fix it: send msg to mnode to rollback to an existed checkpoint - streamMetaInitForSnode(pSnode->pMeta); return pSnode; FAIL: @@ -270,6 +268,9 @@ int32_t sndProcessTaskDispatchRsp(SSnode *pSnode, SRpcMsg *pMsg) { pRsp->upstreamTaskId = htonl(pRsp->upstreamTaskId); pRsp->streamId = htobe64(pRsp->streamId); + pRsp->downstreamTaskId = htonl(pRsp->downstreamTaskId); + pRsp->downstreamNodeId = htonl(pRsp->downstreamNodeId); + pRsp->stage = htobe64(pRsp->stage); pRsp->msgId = htonl(pRsp->msgId); SStreamTask *pTask = streamMetaAcquireTask(pSnode->pMeta, pRsp->streamId, pRsp->upstreamTaskId); @@ -335,6 +336,38 @@ int32_t sndProcessTaskRecoverFinishRsp(SSnode *pSnode, SRpcMsg *pMsg) { return 0; } +// downstream task has complete the stream task checkpoint procedure, let's start the handle the rsp by execute task +int32_t sndProcessTaskCheckpointReadyMsg(SSnode *pSnode, SRpcMsg* pMsg) { + SStreamMeta* pMeta = pSnode->pMeta; + char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + int32_t len = pMsg->contLen - sizeof(SMsgHead); + int32_t code = 0; + + SStreamCheckpointReadyMsg req = {0}; + + SDecoder decoder; + tDecoderInit(&decoder, (uint8_t*)msg, len); + if (tDecodeStreamCheckpointReadyMsg(&decoder, &req) < 0) { + code = TSDB_CODE_MSG_DECODE_ERROR; + tDecoderClear(&decoder); + return code; + } + tDecoderClear(&decoder); + + SStreamTask* pTask = streamMetaAcquireTask(pMeta, req.streamId, req.upstreamTaskId); + if (pTask == NULL) { + qError("vgId:%d failed to find s-task:0x%x, it may have been destroyed already", pMeta->vgId, req.downstreamTaskId); + return code; + } + + qDebug("snode vgId:%d s-task:%s received the checkpoint ready msg from task:0x%x (vgId:%d), handle it", pMeta->vgId, + pTask->id.idStr, req.downstreamTaskId, req.downstreamNodeId); + + streamProcessCheckpointReadyMsg(pTask); + streamMetaReleaseTask(pMeta, pTask); + return code; +} + int32_t sndProcessStreamTaskCheckReq(SSnode *pSnode, SRpcMsg *pMsg) { char *msgStr = pMsg->pCont; char *msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead)); @@ -450,6 +483,8 @@ int32_t sndProcessStreamMsg(SSnode *pSnode, SRpcMsg *pMsg) { return sndProcessStreamTaskCheckReq(pSnode, pMsg); case TDMT_VND_STREAM_TASK_CHECK_RSP: return sndProcessStreamTaskCheckRsp(pSnode, pMsg); + case TDMT_STREAM_TASK_CHECKPOINT_READY: + return sndProcessTaskCheckpointReadyMsg(pSnode, pMsg); default: ASSERT(0); } diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 5665e7a917..15b4bd634a 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -371,7 +371,7 @@ static int32_t doBuildDispatchMsg(SStreamTask* pTask, const SStreamDataBlock* pD pTask->msgInfo.pData = pReqs; } - stDebug("s-task:%s build dispatch msg success, msgId:%d", pTask->id.idStr, pTask->execInfo.dispatch); + stDebug("s-task:%s build dispatch msg success, msgId:%d, stage:%" PRId64, pTask->id.idStr, pTask->execInfo.dispatch, pTask->pMeta->stage); return code; } @@ -926,8 +926,8 @@ int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamTaskId, initRpcMsg(&info.msg, TDMT_STREAM_TASK_CHECKPOINT_READY, buf, tlen + sizeof(SMsgHead)); info.msg.info.noResp = 1; // refactor later. - stDebug("s-task:%s (level:%d) prepare checkpoint ready msg to upstream s-task:0x%" PRIx64 ":0x%x (vgId:%d) idx:%d", - pTask->id.idStr, pTask->info.taskLevel, req.streamId, req.upstreamTaskId, req.upstreamNodeId, index); + stDebug("s-task:%s (level:%d) prepare checkpoint ready msg to upstream s-task:0x%" PRIx64 ":0x%x (vgId:%d) idx:%d, vgId:%d", + pTask->id.idStr, pTask->info.taskLevel, req.streamId, req.upstreamTaskId, req.upstreamNodeId, index, req.upstreamNodeId); if (pTask->pReadyMsgList == NULL) { pTask->pReadyMsgList = taosArrayInit(4, sizeof(SStreamChkptReadyInfo)); diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 17cd9fac57..4539afb1b3 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -175,6 +175,7 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF pMeta->ahandle = ahandle; pMeta->expandFunc = expandFunc; pMeta->stage = stage; + pMeta->role = (vgId == SNODE_HANDLE) ? NODE_ROLE_LEADER : NODE_ROLE_UNINIT; // send heartbeat every 5sec. pMeta->rid = taosAddRef(streamMetaId, pMeta); @@ -205,7 +206,6 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF } pMeta->streamBackendRid = taosAddRef(streamBackendId, pMeta->streamBackend); - pMeta->role = NODE_ROLE_UNINIT; code = streamBackendLoadCheckpointInfo(pMeta); taosInitRWLatch(&pMeta->lock); @@ -1093,11 +1093,6 @@ void streamMetaStartHb(SStreamMeta* pMeta) { metaHbToMnode(pRid, NULL); } -void streamMetaInitForSnode(SStreamMeta* pMeta) { - pMeta->stage = 0; - pMeta->role = NODE_ROLE_LEADER; -} - void streamMetaResetStartInfo(STaskStartInfo* pStartInfo) { taosHashClear(pStartInfo->pReadyTaskSet); taosHashClear(pStartInfo->pFailedTaskSet); From 6a44bd54ac040414cb7ec7e4edea5b719052c2e4 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 13 Nov 2023 09:12:15 +0800 Subject: [PATCH 02/29] fix:load task in snode --- source/dnode/snode/src/snode.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 74f792cc76..5fd185ba4d 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -123,6 +123,10 @@ SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) { goto FAIL; } + if (streamMetaLoadAllTasks(pSnode->pMeta) < 0) { + goto FAIL; + } + stopRsync(); startRsync(); From 1cbe568be7cdec1802b3714dc7bf27ecfe156cab Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 13 Nov 2023 18:39:42 +0800 Subject: [PATCH 03/29] fix:add logic when snode restart --- include/libs/stream/tstream.h | 2 +- source/common/src/systable.c | 2 +- source/dnode/mnode/impl/src/mndStream.c | 31 ++++++++++++++++++++++--- source/libs/stream/src/streamMeta.c | 6 ++--- 4 files changed, 33 insertions(+), 8 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 6ca6b176d0..5de0c3180b 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -656,7 +656,7 @@ int32_t tDecodeStreamCheckpointReadyMsg(SDecoder* pDecoder, SStreamCheckpointRea typedef struct STaskStatusEntry { STaskId id; int32_t status; - int32_t stage; + int64_t stage; int32_t nodeId; int64_t verStart; // start version in WAL, only valid for source task int64_t verEnd; // end version in WAL, only valid for source task diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 5f44d3e7fc..481c671d0f 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -165,7 +165,7 @@ static const SSysDbTableSchema streamTaskSchema[] = { {.name = "node_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, {.name = "level", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, {.name = "status", .bytes = 15 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, - {.name = "stage", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, + {.name = "stage", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = false}, {.name = "in_queue", .bytes = 20, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, // {.name = "out_queue", .bytes = 20, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, {.name = "info", .bytes = 25, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index fd0c349dd2..221518f4c3 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -2053,6 +2053,9 @@ static SVgroupChangeInfo mndFindChangedNodeInfo(SMnode *pMnode, const SArray *pP epsetAssign(&updateInfo.newEp, &pCurrent->epset); taosArrayPush(info.pUpdateNodeList, &updateInfo); + + } + if(pCurrent->nodeId != SNODE_HANDLE){ SVgObj *pVgroup = mndAcquireVgroup(pMnode, pCurrent->nodeId); taosHashPut(info.pDBMap, pVgroup->dbName, strlen(pVgroup->dbName), NULL, 0); mndReleaseVgroup(pMnode, pVgroup); @@ -2107,6 +2110,24 @@ static SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool* allReady) { sdbRelease(pSdb, pVgroup); } + SSnodeObj *pObj = NULL; + while (1) { + pIter = sdbFetch(pSdb, SDB_SNODE, pIter, (void **)&pObj); + if (pIter == NULL) { + break; + } + + SNodeEntry entry = {0}; + addEpIntoEpSet(&entry.epset, pObj->pDnode->fqdn, pObj->pDnode->port); + entry.nodeId = SNODE_HANDLE; + + char buf[256] = {0}; + EPSET_TO_STR(&entry.epset, buf); + mDebug("take snode snapshot, nodeId:%d %s", entry.nodeId, buf); + taosArrayPush(pVgroupListSnapshot, &entry); + sdbRelease(pSdb, pObj); + } + return pVgroupListSnapshot; } @@ -2284,6 +2305,8 @@ int32_t removeExpirednodeEntryAndTask(SArray *pNodeSnapshot) { STaskId* pId = taosArrayGet(execInfo.pTaskList, i); STaskStatusEntry* pEntry = taosHashGet(execInfo.pTaskMap, pId, sizeof(*pId)); + if(pEntry->nodeId == SNODE_HANDLE) continue; + bool existed = taskNodeExists(pNodeSnapshot, pEntry->nodeId); if (!existed) { taosArrayPush(pRemovedTasks, pId); @@ -2629,13 +2652,13 @@ int32_t setNodeEpsetExpiredFlag(const SArray* pNodeList) { return TSDB_CODE_SUCCESS; } -static void updateStageInfo(STaskStatusEntry* pTaskEntry, int32_t stage) { +static void updateStageInfo(STaskStatusEntry* pTaskEntry, int64_t stage) { int32_t numOfNodes = taosArrayGetSize(execInfo.pNodeEntryList); for(int32_t j = 0; j < numOfNodes; ++j) { SNodeEntry* pNodeEntry = taosArrayGet(execInfo.pNodeEntryList, j); if (pNodeEntry->nodeId == pTaskEntry->nodeId) { - mInfo("vgId:%d stage updated from %d to %d, nodeUpdate trigger by s-task:0x%" PRIx64, pTaskEntry->nodeId, + mInfo("vgId:%d stage updated from %"PRId64 " to %"PRId64 ", nodeUpdate trigger by s-task:0x%" PRIx64, pTaskEntry->nodeId, pTaskEntry->stage, stage, pTaskEntry->id.taskId); pNodeEntry->stageUpdated = true; @@ -2672,6 +2695,7 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { setNodeEpsetExpiredFlag(req.pUpdateNodes); + bool snodeChanged = false; for (int32_t i = 0; i < req.numOfTasks; ++i) { STaskStatusEntry *p = taosArrayGet(req.pTaskStatus, i); STaskStatusEntry *pTaskEntry = taosHashGet(execInfo.pTaskMap, &p->id, sizeof(p->id)); @@ -2682,6 +2706,7 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { if (pTaskEntry->stage != p->stage && pTaskEntry->stage != -1) { updateStageInfo(pTaskEntry, p->stage); + if(pTaskEntry->nodeId == SNODE_HANDLE) snodeChanged = true; } else { streamTaskStatusCopy(pTaskEntry, p); if (p->activeCheckpointId != 0) { @@ -2710,7 +2735,7 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { SArray* p = mndTakeVgroupSnapshot(pMnode, &allReady); taosArrayDestroy(p); - if (allReady) { + if (allReady || snodeChanged) { // if the execInfo.activeCheckpoint == 0, the checkpoint is restoring from wal mInfo("checkpointId:%" PRId64 " failed, issue task-reset trans to reset all tasks status", execInfo.activeCheckpoint); diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 4539afb1b3..0a3c44441c 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -770,7 +770,7 @@ int32_t tEncodeStreamHbMsg(SEncoder* pEncoder, const SStreamHbMsg* pReq) { if (tEncodeI64(pEncoder, ps->id.streamId) < 0) return -1; if (tEncodeI32(pEncoder, ps->id.taskId) < 0) return -1; if (tEncodeI32(pEncoder, ps->status) < 0) return -1; - if (tEncodeI32(pEncoder, ps->stage) < 0) return -1; + if (tEncodeI64(pEncoder, ps->stage) < 0) return -1; if (tEncodeI32(pEncoder, ps->nodeId) < 0) return -1; if (tEncodeDouble(pEncoder, ps->inputQUsed) < 0) return -1; if (tEncodeDouble(pEncoder, ps->inputRate) < 0) return -1; @@ -808,7 +808,7 @@ int32_t tDecodeStreamHbMsg(SDecoder* pDecoder, SStreamHbMsg* pReq) { if (tDecodeI64(pDecoder, &entry.id.streamId) < 0) return -1; if (tDecodeI32(pDecoder, &taskId) < 0) return -1; if (tDecodeI32(pDecoder, &entry.status) < 0) return -1; - if (tDecodeI32(pDecoder, &entry.stage) < 0) return -1; + if (tDecodeI64(pDecoder, &entry.stage) < 0) return -1; if (tDecodeI32(pDecoder, &entry.nodeId) < 0) return -1; if (tDecodeDouble(pDecoder, &entry.inputQUsed) < 0) return -1; if (tDecodeDouble(pDecoder, &entry.inputRate) < 0) return -1; @@ -893,7 +893,7 @@ void metaHbToMnode(void* param, void* tmrId) { SStreamHbMsg hbMsg = {0}; SEpSet epset = {0}; bool hasMnodeEpset = false; - int32_t stage = 0; + int64_t stage = 0; streamMetaRLock(pMeta); From 8a9ded764d54da97548a645103edb8f8e1f2313f Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 15 Nov 2023 11:50:14 +0800 Subject: [PATCH 04/29] fix:add finish history msg to snode --- include/util/tlog.h | 1 + source/common/src/tglobal.c | 4 + source/dnode/snode/src/snode.c | 90 +++++++++++++---- source/util/src/tlog.c | 1 + tests/system-test/8-stream/snodeRestart.py | 107 +++++++++++++++++++++ tests/system-test/output.txt | 0 6 files changed, 183 insertions(+), 20 deletions(-) create mode 100644 tests/system-test/8-stream/snodeRestart.py delete mode 100644 tests/system-test/output.txt diff --git a/include/util/tlog.h b/include/util/tlog.h index a6d146a79e..6d393bfefb 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -66,6 +66,7 @@ extern int32_t udfDebugFlag; extern int32_t smaDebugFlag; extern int32_t idxDebugFlag; extern int32_t tdbDebugFlag; +extern int32_t sndDebugFlag; int32_t taosInitLog(const char *logName, int32_t maxFiles); void taosCloseLog(); diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 4a1ba9e391..47988b0de9 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -430,6 +430,7 @@ static int32_t taosAddServerLogCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "tdbDebugFlag", tdbDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; if (cfgAddInt32(pCfg, "metaDebugFlag", metaDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; if (cfgAddInt32(pCfg, "stDebugFlag", stDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "sndDebugFlag", sndDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; return 0; } @@ -946,6 +947,7 @@ static void taosSetServerLogCfg(SConfig *pCfg) { tdbDebugFlag = cfgGetItem(pCfg, "tdbDebugFlag")->i32; metaDebugFlag = cfgGetItem(pCfg, "metaDebugFlag")->i32; stDebugFlag = cfgGetItem(pCfg, "stDebugFlag")->i32; + sndDebugFlag = cfgGetItem(pCfg, "sndDebugFlag")->i32; } static int32_t taosSetSlowLogScope(char *pScope) { @@ -1668,6 +1670,7 @@ void taosCfgDynamicOptions(const char *option, const char *value) { {"metaDebugFlag", &metaDebugFlag}, {"jniDebugFlag", &jniDebugFlag}, {"stDebugFlag", &stDebugFlag}, + {"sndDebugFlag", &sndDebugFlag}, {"audit", &tsEnableAudit}, {"asynclog", &tsAsyncLog}, @@ -1786,6 +1789,7 @@ void taosSetAllDebugFlag(int32_t flag, bool rewrite) { taosSetDebugFlag(&tdbDebugFlag, "tdbDebugFlag", flag, rewrite); taosSetDebugFlag(&metaDebugFlag, "metaDebugFlag", flag, rewrite); taosSetDebugFlag(&stDebugFlag, "stDebugFlag", flag, rewrite); + taosSetDebugFlag(&sndDebugFlag, "sndDebugFlag", flag, rewrite); uInfo("all debug flag are set to %d", flag); } diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 5fd185ba4d..718f4e851b 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -19,6 +19,27 @@ #include "tstream.h" #include "tuuid.h" +#define sndError(...) \ + do { \ + if (sndDebugFlag & DEBUG_ERROR) { \ + taosPrintLog("SND ERROR ", DEBUG_ERROR, sndDebugFlag, __VA_ARGS__); \ + } \ + } while (0) + +#define sndInfo(...) \ + do { \ + if (sndDebugFlag & DEBUG_INFO) { \ + taosPrintLog("SND INFO ", DEBUG_INFO, sndDebugFlag, __VA_ARGS__); \ + } \ + } while (0) + +#define sndDebug(...) \ + do { \ + if (sndDebugFlag & DEBUG_DEBUG) { \ + taosPrintLog("SND ", DEBUG_DEBUG, sndDebugFlag, __VA_ARGS__); \ + } \ + } while (0) + void sndEnqueueStreamDispatch(SSnode *pSnode, SRpcMsg *pMsg) { char *msgStr = pMsg->pCont; char *msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead)); @@ -65,10 +86,10 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProcessVer pTask->pState = streamStateOpen(pSnode->path, pTask, false, -1, -1); if (pTask->pState == NULL) { - qError("s-task:%s failed to open state for task", pTask->id.idStr); + sndError("s-task:%s failed to open state for task", pTask->id.idStr); return -1; } else { - qDebug("s-task:%s state:%p", pTask->id.idStr, pTask->pState); + sndDebug("s-task:%s state:%p", pTask->id.idStr, pTask->pState); } int32_t numOfChildEp = taosArrayGetSize(pTask->upstreamInfo.pList); @@ -85,7 +106,7 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProcessVer // checkpoint ver is the kept version, handled data should be the next version. if (pTask->chkInfo.checkpointId != 0) { pTask->chkInfo.nextProcessVer = pTask->chkInfo.checkpointVer + 1; - qInfo("s-task:%s restore from the checkpointId:%" PRId64 " ver:%" PRId64 " nextProcessVer:%" PRId64, pTask->id.idStr, + sndInfo("s-task:%s restore from the checkpointId:%" PRId64 " ver:%" PRId64 " nextProcessVer:%" PRId64, pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer); } else { if (pTask->chkInfo.nextProcessVer == -1) { @@ -96,7 +117,7 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProcessVer char* p = NULL; streamTaskGetStatus(pTask, &p); - qInfo("snode:%d expand stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64 + sndInfo("snode:%d expand stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64 " nextProcessVer:%" PRId64 " child id:%d, level:%d, status:%s fill-history:%d, trigger:%" PRId64 " ms", SNODE_HANDLE, pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer, pTask->info.selfChildId, pTask->info.taskLevel, p, pTask->info.fillHistory, pTask->info.triggerParam); @@ -186,24 +207,23 @@ int32_t sndProcessTaskDeployReq(SSnode *pSnode, char *msg, int32_t msgLen) { char* p = NULL; streamTaskGetStatus(pTask, &p); - qDebug("snode:%d s-task:%s is deployed on snode and add into meta, status:%s, numOfTasks:%d", SNODE_HANDLE, + sndDebug("snode:%d s-task:%s is deployed on snode and add into meta, status:%s, numOfTasks:%d", SNODE_HANDLE, pTask->id.idStr, p, numOfTasks); EStreamTaskEvent event = (HAS_RELATED_FILLHISTORY_TASK(pTask)) ? TASK_EVENT_INIT_STREAM_SCANHIST : TASK_EVENT_INIT; streamTaskHandleEvent(pTask->status.pSM, event); - streamTaskCheckDownstream(pTask); return 0; } int32_t sndProcessTaskDropReq(SSnode *pSnode, char *msg, int32_t msgLen) { SVDropStreamTaskReq *pReq = (SVDropStreamTaskReq *)msg; - qDebug("snode:%d receive msg to drop stream task:0x%x", pSnode->pMeta->vgId, pReq->taskId); + sndDebug("snode:%d receive msg to drop stream task:0x%x", pSnode->pMeta->vgId, pReq->taskId); streamMetaUnregisterTask(pSnode->pMeta, pReq->streamId, pReq->taskId); // commit the update streamMetaWLock(pSnode->pMeta); int32_t numOfTasks = streamMetaGetNumOfTasks(pSnode->pMeta); - qDebug("vgId:%d task:0x%x dropped, remain tasks:%d", pSnode->pMeta->vgId, pReq->taskId, numOfTasks); + sndDebug("vgId:%d task:0x%x dropped, remain tasks:%d", pSnode->pMeta->vgId, pReq->taskId, numOfTasks); if (streamMetaCommit(pSnode->pMeta) < 0) { // persist to disk @@ -308,7 +328,7 @@ int32_t sndProcessWriteMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg *pRsp) { return 0; } -int32_t sndProcessStreamTaskScanHistoryFinishReq(SSnode *pSnode, SRpcMsg *pMsg) { +int32_t sndProcessTaskScanHistoryFinishReq(SSnode *pSnode, SRpcMsg *pMsg) { char *msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); @@ -335,8 +355,38 @@ int32_t sndProcessStreamTaskScanHistoryFinishReq(SSnode *pSnode, SRpcMsg *pMsg) return 0; } -int32_t sndProcessTaskRecoverFinishRsp(SSnode *pSnode, SRpcMsg *pMsg) { - // +int32_t sndProcessTaskScanHistoryFinishRsp(SSnode *pSnode, SRpcMsg *pMsg) { + char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); + + // deserialize + SStreamCompleteHistoryMsg req = {0}; + + SDecoder decoder; + tDecoderInit(&decoder, (uint8_t*)msg, msgLen); + tDecodeCompleteHistoryDataMsg(&decoder, &req); + tDecoderClear(&decoder); + + SStreamTask* pTask = streamMetaAcquireTask(pSnode->pMeta, req.streamId, req.upstreamTaskId); + if (pTask == NULL) { + sndError("vgId:%d process scan history finish rsp, failed to find task:0x%x, it may be destroyed", + pSnode->pMeta->vgId, req.upstreamTaskId); + return -1; + } + + int32_t remain = atomic_sub_fetch_32(&pTask->notReadyTasks, 1); + if (remain > 0) { + sndDebug("s-task:%s scan-history finish rsp received from downstream task:0x%x, unfinished remain:%d", + pTask->id.idStr, req.downstreamId, remain); + } else { + sndDebug( + "s-task:%s scan-history finish rsp received from downstream task:0x%x, all downstream tasks rsp scan-history " + "completed msg", + pTask->id.idStr, req.downstreamId); + streamProcessScanHistoryFinishRsp(pTask); + } + + streamMetaReleaseTask(pSnode->pMeta, pTask); return 0; } @@ -360,11 +410,11 @@ int32_t sndProcessTaskCheckpointReadyMsg(SSnode *pSnode, SRpcMsg* pMsg) { SStreamTask* pTask = streamMetaAcquireTask(pMeta, req.streamId, req.upstreamTaskId); if (pTask == NULL) { - qError("vgId:%d failed to find s-task:0x%x, it may have been destroyed already", pMeta->vgId, req.downstreamTaskId); + sndError("vgId:%d failed to find s-task:0x%x, it may have been destroyed already", pMeta->vgId, req.downstreamTaskId); return code; } - qDebug("snode vgId:%d s-task:%s received the checkpoint ready msg from task:0x%x (vgId:%d), handle it", pMeta->vgId, + sndDebug("snode vgId:%d s-task:%s received the checkpoint ready msg from task:0x%x (vgId:%d), handle it", pMeta->vgId, pTask->id.idStr, req.downstreamTaskId, req.downstreamNodeId); streamProcessCheckpointReadyMsg(pTask); @@ -403,11 +453,11 @@ int32_t sndProcessStreamTaskCheckReq(SSnode *pSnode, SRpcMsg *pMsg) { streamMetaReleaseTask(pSnode->pMeta, pTask); char* p = NULL; streamTaskGetStatus(pTask, &p); - qDebug("s-task:%s status:%s, recv task check req(reqId:0x%" PRIx64 ") task:0x%x (vgId:%d), ready:%d", + sndDebug("s-task:%s status:%s, recv task check req(reqId:0x%" PRIx64 ") task:0x%x (vgId:%d), ready:%d", pTask->id.idStr, p, rsp.reqId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status); } else { rsp.status = TASK_DOWNSTREAM_NOT_READY; - qDebug("recv task check(taskId:0x%x not built yet) req(reqId:0x%" PRIx64 ") from task:0x%x (vgId:%d), rsp status %d", + sndDebug("recv task check(taskId:0x%x not built yet) req(reqId:0x%" PRIx64 ") from task:0x%x (vgId:%d), rsp status %d", taskId, rsp.reqId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status); } @@ -417,7 +467,7 @@ int32_t sndProcessStreamTaskCheckReq(SSnode *pSnode, SRpcMsg *pMsg) { tEncodeSize(tEncodeStreamTaskCheckRsp, &rsp, len, code); if (code < 0) { - qError("vgId:%d failed to encode task check rsp, task:0x%x", pSnode->pMeta->vgId, taskId); + sndError("vgId:%d failed to encode task check rsp, task:0x%x", pSnode->pMeta->vgId, taskId); return -1; } @@ -452,12 +502,12 @@ int32_t sndProcessStreamTaskCheckRsp(SSnode* pSnode, SRpcMsg* pMsg) { } tDecoderClear(&decoder); - qDebug("tq task:0x%x (vgId:%d) recv check rsp(reqId:0x%" PRIx64 ") from 0x%x (vgId:%d) status %d", + sndDebug("tq task:0x%x (vgId:%d) recv check rsp(reqId:0x%" PRIx64 ") from 0x%x (vgId:%d) status %d", rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, rsp.status); SStreamTask* pTask = streamMetaAcquireTask(pSnode->pMeta, rsp.streamId, rsp.upstreamTaskId); if (pTask == NULL) { - qError("tq failed to locate the stream task:0x%x (vgId:%d), it may have been destroyed", rsp.upstreamTaskId, + sndError("tq failed to locate the stream task:0x%x (vgId:%d), it may have been destroyed", rsp.upstreamTaskId, pSnode->pMeta->vgId); return -1; } @@ -480,9 +530,9 @@ int32_t sndProcessStreamMsg(SSnode *pSnode, SRpcMsg *pMsg) { case TDMT_STREAM_RETRIEVE_RSP: return sndProcessTaskRetrieveRsp(pSnode, pMsg); case TDMT_VND_STREAM_SCAN_HISTORY_FINISH: - return sndProcessStreamTaskScanHistoryFinishReq(pSnode, pMsg); + return sndProcessTaskScanHistoryFinishReq(pSnode, pMsg); case TDMT_VND_STREAM_SCAN_HISTORY_FINISH_RSP: - return sndProcessTaskRecoverFinishRsp(pSnode, pMsg); + return sndProcessTaskScanHistoryFinishRsp(pSnode, pMsg); case TDMT_VND_STREAM_TASK_CHECK: return sndProcessStreamTaskCheckReq(pSnode, pMsg); case TDMT_VND_STREAM_TASK_CHECK_RSP: diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index aa6719f604..46b25dee6a 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -110,6 +110,7 @@ int32_t metaDebugFlag = 131; int32_t udfDebugFlag = 131; int32_t smaDebugFlag = 131; int32_t idxDebugFlag = 131; +int32_t sndDebugFlag = 131; int64_t dbgEmptyW = 0; int64_t dbgWN = 0; diff --git a/tests/system-test/8-stream/snodeRestart.py b/tests/system-test/8-stream/snodeRestart.py new file mode 100644 index 0000000000..65ca090df9 --- /dev/null +++ b/tests/system-test/8-stream/snodeRestart.py @@ -0,0 +1,107 @@ + +import taos +import sys +import time +import socket +import os +import threading +import math + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +from util.cluster import * + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + + def case1(self): + tdLog.debug("case1 start") + + os.system("nohup taosBenchmark -y -B 1 -t 4 -S 1000 -n 1000 -i 1000 -v 2 > /dev/null 2>&1 &") + time.sleep(1) + tdSql.query("use test") + tdSql.query("create snode on dnode 4") + tdSql.query("create stream if not exists s1 trigger at_once ignore expired 0 ignore update 0 fill_history 1 into st1 as select sum(voltage) from meters partition by groupid interval(4s)") + tdLog.debug("create stream use snode and insert data") + time.sleep(10) + + tdDnodes = cluster.dnodes + tdDnodes[3].stoptaosd() + time.sleep(2) + tdDnodes[3].starttaosd() + tdLog.debug("snode restart ok") + + time.sleep(500) + os.system("kill -9 `pgrep taosBenchmark`") + + tdSql.query("select sum(voltage) from meters partition by groupid interval(4s)") + # tdSql.checkRows(1) + # + # tdSql.checkData(0, 0, '2016-01-01 08:00:07.000') + # tdSql.checkData(0, 1, 2000) + + tdSql.query("drop snode on dnode 4") + tdSql.query("drop stream if exists s1") + tdSql.query("drop database test") + + tdLog.debug("case1 end") + + def case2(self): + tdLog.debug("case2 start") + + updatecfgDict = {'checkpointInterval': 5} + print("===================: ", updatecfgDict) + + tdDnodes = cluster.dnodes + tdDnodes[0].stoptaosd() + tdDnodes[1].stoptaosd() + tdDnodes[2].stoptaosd() + tdDnodes[3].stoptaosd() + time.sleep(2) + tdDnodes[0].starttaosd() + tdDnodes[1].starttaosd() + tdDnodes[2].starttaosd() + tdDnodes[3].starttaosd() + tdLog.debug("taosd restart ok") + + os.system("taosBenchmark -y -B 1 -t 4 -S 1000 -n 1000 -i 1000 -v 2 &" ) + time.sleep(1) + tdSql.query("use test") + tdSql.query("create snode on dnode 4") + tdSql.query("create stream if not exists s1 trigger at_once ignore expired 0 ignore update 0 fill_history 1 into st1 as select sum(voltage) from meters partition by groupid interval(4s)") + tdLog.debug("create stream use snode and insert data") + time.sleep(10) + + tdDnodes[3].stoptaosd() + time.sleep(2) + tdDnodes[3].starttaosd() + tdLog.debug("snode restart ok") + + time.sleep(5) + os.system("kill -9 `pgrep taosBenchmark`") + + tdSql.query("select sum(voltage) from meters partition by groupid interval(4s)") + # tdSql.checkRows(1) + # + # tdSql.checkData(0, 0, '2016-01-01 08:00:07.000') + # tdSql.checkData(0, 1, 2000) + + tdLog.debug("case2 end") + + def run(self): + self.case1() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/output.txt b/tests/system-test/output.txt deleted file mode 100644 index e69de29bb2..0000000000 From 188a4ce6a50ce30d32961e2350680d388aa11a1f Mon Sep 17 00:00:00 2001 From: kailixu Date: Wed, 15 Nov 2023 16:33:05 +0800 Subject: [PATCH 05/29] enh: the latest active code take effect --- include/common/tgrant.h | 5 ++ source/dnode/mnode/impl/inc/mndCluster.h | 2 + source/dnode/mnode/impl/inc/mndDef.h | 2 + source/dnode/mnode/impl/src/mndCluster.c | 61 +++++++++++++++++++++++- 4 files changed, 69 insertions(+), 1 deletion(-) diff --git a/include/common/tgrant.h b/include/common/tgrant.h index f06fca8014..b18eaf8a6f 100644 --- a/include/common/tgrant.h +++ b/include/common/tgrant.h @@ -50,6 +50,11 @@ typedef enum { TSDB_GRANT_TABLE, } EGrantType; +typedef struct { + int64_t grantedTime; + int64_t connGrantedTime; +} SGrantedInfo; + int32_t grantCheck(EGrantType grant); int32_t grantAlterActiveCode(int32_t did, const char* old, const char* newer, char* out, int8_t type); diff --git a/source/dnode/mnode/impl/inc/mndCluster.h b/source/dnode/mnode/impl/inc/mndCluster.h index e33ffdb372..2b868b567a 100644 --- a/source/dnode/mnode/impl/inc/mndCluster.h +++ b/source/dnode/mnode/impl/inc/mndCluster.h @@ -27,7 +27,9 @@ void mndCleanupCluster(SMnode *pMnode); int32_t mndGetClusterName(SMnode *pMnode, char *clusterName, int32_t len); int64_t mndGetClusterId(SMnode *pMnode); int64_t mndGetClusterCreateTime(SMnode *pMnode); +int32_t mndGetClusterGrantedInfo(SMnode *pMnode, SGrantedInfo *pInfo); int64_t mndGetClusterUpTime(SMnode *pMnode); +int32_t mndProcessGrantedTime(SMnode *pMnode, int64_t grantedTime); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index efa99db74b..fc9086eebf 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -192,6 +192,8 @@ typedef struct { int64_t createdTime; int64_t updateTime; int32_t upTime; + int64_t grantedTime; + int64_t connGrantedTime; } SClusterObj; typedef struct { diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index 4c799e1e1e..20e25d918c 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -19,7 +19,7 @@ #include "mndTrans.h" #define CLUSTER_VER_NUMBE 1 -#define CLUSTER_RESERVE_SIZE 60 +#define CLUSTER_RESERVE_SIZE 44 int64_t tsExpireTime = 0; static SSdbRaw *mndClusterActionEncode(SClusterObj *pCluster); @@ -112,6 +112,19 @@ int64_t mndGetClusterCreateTime(SMnode *pMnode) { return createTime; } +int32_t mndGetClusterGrantedInfo(SMnode *pMnode, SGrantedInfo *pInfo) { + void *pIter = NULL; + SClusterObj *pCluster = mndAcquireCluster(pMnode, &pIter); + if (pCluster != NULL) { + pInfo->grantedTime = pCluster->grantedTime; + pInfo->connGrantedTime = pCluster->connGrantedTime; + mndReleaseCluster(pMnode, pCluster, pIter); + return 0; + } + + return -1; +} + static int32_t mndGetClusterUpTimeImp(SClusterObj *pCluster) { #if 0 int32_t upTime = taosGetTimestampSec() - pCluster->updateTime / 1000; @@ -146,6 +159,8 @@ static SSdbRaw *mndClusterActionEncode(SClusterObj *pCluster) { SDB_SET_INT64(pRaw, dataPos, pCluster->updateTime, _OVER) SDB_SET_BINARY(pRaw, dataPos, pCluster->name, TSDB_CLUSTER_ID_LEN, _OVER) SDB_SET_INT32(pRaw, dataPos, pCluster->upTime, _OVER) + SDB_SET_INT64(pRaw, dataPos, pCluster->grantedTime, _OVER) + SDB_SET_INT64(pRaw, dataPos, pCluster->connGrantedTime, _OVER) SDB_SET_RESERVE(pRaw, dataPos, CLUSTER_RESERVE_SIZE, _OVER) terrno = 0; @@ -186,6 +201,8 @@ static SSdbRow *mndClusterActionDecode(SSdbRaw *pRaw) { SDB_GET_INT64(pRaw, dataPos, &pCluster->updateTime, _OVER) SDB_GET_BINARY(pRaw, dataPos, pCluster->name, TSDB_CLUSTER_ID_LEN, _OVER) SDB_GET_INT32(pRaw, dataPos, &pCluster->upTime, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pCluster->grantedTime, _OVER); + SDB_GET_INT64(pRaw, dataPos, &pCluster->connGrantedTime, _OVER); SDB_GET_RESERVE(pRaw, dataPos, CLUSTER_RESERVE_SIZE, _OVER) terrno = 0; @@ -218,6 +235,7 @@ static int32_t mndClusterActionUpdate(SSdb *pSdb, SClusterObj *pOld, SClusterObj mTrace("cluster:%" PRId64 ", perform update action, old row:%p new row:%p, uptime from %d to %d", pOld->id, pOld, pNew, pOld->upTime, pNew->upTime); pOld->upTime = pNew->upTime; + pOld->grantedTime = pNew->grantedTime; pOld->updateTime = taosGetTimestampMs(); return 0; } @@ -359,3 +377,44 @@ static int32_t mndProcessUptimeTimer(SRpcMsg *pReq) { mndTransDrop(pTrans); return 0; } + +int32_t mndProcessGrantedTime(SMnode *pMnode, int64_t grantedTime) { + SClusterObj clusterObj = {0}; + void *pIter = NULL; + SClusterObj *pCluster = mndAcquireCluster(pMnode, &pIter); + if (pCluster != NULL) { + if (pCluster->grantedTime >= grantedTime) { + mndReleaseCluster(pMnode, pCluster, pIter); + return 0; + } + memcpy(&clusterObj, pCluster, sizeof(SClusterObj)); + clusterObj.grantedTime = grantedTime; + mndReleaseCluster(pMnode, pCluster, pIter); + } + + if (clusterObj.id <= 0) { + mError("can't get cluster info while update uptime"); + return -1; + } + + mInfo("update cluster granted time to %" PRIi64, clusterObj.grantedTime); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, NULL, "granted-time"); + if (pTrans == NULL) return -1; + + SSdbRaw *pCommitRaw = mndClusterActionEncode(&clusterObj); + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); + + if (mndTransPrepare(pMnode, pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + + mndTransDrop(pTrans); + return 0; +} \ No newline at end of file From 45782f278b6e557ecf2ea4c9d08d3b3af473ce01 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 15 Nov 2023 18:13:35 +0800 Subject: [PATCH 06/29] fix:send update msg to vnodes if snode restart --- include/dnode/vnode/stream.h | 18 + source/dnode/mgmt/mgmt_snode/src/smHandle.c | 2 +- source/dnode/snode/CMakeLists.txt | 1 + source/dnode/snode/src/snode.c | 407 +++++++++++++++++++- source/dnode/vnode/CMakeLists.txt | 5 + source/dnode/vnode/src/inc/tq.h | 3 - source/dnode/vnode/src/tq/tq.c | 7 +- source/dnode/vnode/src/tq/tqStreamTask.c | 1 + tests/system-test/8-stream/snodeRestart.py | 2 +- 9 files changed, 415 insertions(+), 31 deletions(-) create mode 100644 include/dnode/vnode/stream.h diff --git a/include/dnode/vnode/stream.h b/include/dnode/vnode/stream.h new file mode 100644 index 0000000000..6d86847542 --- /dev/null +++ b/include/dnode/vnode/stream.h @@ -0,0 +1,18 @@ +// +// Created by mingming wanng on 2023/11/15. +// + +#ifndef TDENGINE_STREAM_H +#define TDENGINE_STREAM_H + +#define STREAM_EXEC_EXTRACT_DATA_IN_WAL_ID (-1) +#define STREAM_EXEC_START_ALL_TASKS_ID (-2) +#define STREAM_EXEC_RESTART_ALL_TASKS_ID (-3) + +typedef struct STaskUpdateEntry { + int64_t streamId; + int32_t taskId; + int32_t transId; +} STaskUpdateEntry; + +#endif // TDENGINE_STREAM_H diff --git a/source/dnode/mgmt/mgmt_snode/src/smHandle.c b/source/dnode/mgmt/mgmt_snode/src/smHandle.c index 509a7a6b7f..cd81b9873f 100644 --- a/source/dnode/mgmt/mgmt_snode/src/smHandle.c +++ b/source/dnode/mgmt/mgmt_snode/src/smHandle.c @@ -73,6 +73,7 @@ SArray *smGetMsgHandles() { SArray *pArray = taosArrayInit(4, sizeof(SMgmtHandle)); if (pArray == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_UPDATE, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DEPLOY, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DROP, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RUN, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; @@ -87,7 +88,6 @@ SArray *smGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_CHECK_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_SCAN_HISTORY_FINISH, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_SCAN_HISTORY_FINISH_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_CHECK_POINT_SOURCE, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECKPOINT_READY, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; diff --git a/source/dnode/snode/CMakeLists.txt b/source/dnode/snode/CMakeLists.txt index ebfe80ecab..2da1f9adac 100644 --- a/source/dnode/snode/CMakeLists.txt +++ b/source/dnode/snode/CMakeLists.txt @@ -3,6 +3,7 @@ add_library(snode STATIC ${SNODE_SRC}) target_include_directories( snode PUBLIC "${TD_SOURCE_DIR}/include/dnode/snode" + PUBLIC "${TD_SOURCE_DIR}/include/dnode/vnode" private "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) target_link_libraries( diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 718f4e851b..25400220b8 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -18,6 +18,7 @@ #include "sndInt.h" #include "tstream.h" #include "tuuid.h" +#include "stream.h" #define sndError(...) \ do { \ @@ -84,7 +85,16 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProcessVer streamTaskOpenAllUpstreamInput(pTask); - pTask->pState = streamStateOpen(pSnode->path, pTask, false, -1, -1); + SStreamTask* pSateTask = pTask; + SStreamTask task = {0}; + if (pTask->info.fillHistory) { + task.id.streamId = pTask->streamTaskId.streamId; + task.id.taskId = pTask->streamTaskId.taskId; + task.pMeta = pTask->pMeta; + pSateTask = &task; + } + + pTask->pState = streamStateOpen(pSnode->path, pSateTask, false, -1, -1); if (pTask->pState == NULL) { sndError("s-task:%s failed to open state for task", pTask->id.idStr); return -1; @@ -92,12 +102,20 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProcessVer sndDebug("s-task:%s state:%p", pTask->id.idStr, pTask->pState); } - int32_t numOfChildEp = taosArrayGetSize(pTask->upstreamInfo.pList); - SReadHandle handle = { .vnode = NULL, .numOfVgroups = numOfChildEp, .pStateBackend = pTask->pState, .fillHistory = pTask->info.fillHistory }; + int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTask->upstreamInfo.pList); + SReadHandle handle = { + .checkpointId = pTask->chkInfo.checkpointId, + .vnode = NULL, + .numOfVgroups = numOfVgroups, + .pStateBackend = pTask->pState, + .fillHistory = pTask->info.fillHistory, + .winRange = pTask->dataRange.window, + }; initStreamStateAPI(&handle.api); - pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle, 0, pTask->id.taskId); + pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle, SNODE_HANDLE, pTask->id.taskId); ASSERT(pTask->exec.pExecutor); + qSetTaskId(pTask->exec.pExecutor, pTask->id.taskId, pTask->id.streamId); streamTaskResetUpstreamStageInfo(pTask); streamSetupScheduleTrigger(pTask); @@ -169,6 +187,167 @@ void sndClose(SSnode *pSnode) { int32_t sndGetLoad(SSnode *pSnode, SSnodeLoad *pLoad) { return 0; } +int32_t sndStartStreamTasks(SSnode* pSnode) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t vgId = SNODE_HANDLE; + SStreamMeta* pMeta = pSnode->pMeta; + + int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); + sndDebug("vgId:%d start to check all %d stream task(s) downstream status", vgId, numOfTasks); + if (numOfTasks == 0) { + return TSDB_CODE_SUCCESS; + } + + SArray* pTaskList = NULL; + streamMetaWLock(pMeta); + pTaskList = taosArrayDup(pMeta->pTaskList, NULL); + taosHashClear(pMeta->startInfo.pReadyTaskSet); + taosHashClear(pMeta->startInfo.pFailedTaskSet); + pMeta->startInfo.startTs = taosGetTimestampMs(); + streamMetaWUnLock(pMeta); + + // broadcast the check downstream tasks msg + for (int32_t i = 0; i < numOfTasks; ++i) { + SStreamTaskId* pTaskId = taosArrayGet(pTaskList, i); + SStreamTask* pTask = streamMetaAcquireTask(pMeta, pTaskId->streamId, pTaskId->taskId); + if (pTask == NULL) { + continue; + } + + // fill-history task can only be launched by related stream tasks. + if (pTask->info.fillHistory == 1) { + streamMetaReleaseTask(pMeta, pTask); + continue; + } + + if (pTask->status.downstreamReady == 1) { + if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { + sndDebug("s-task:%s downstream ready, no need to check downstream, check only related fill-history task", + pTask->id.idStr); + streamLaunchFillHistoryTask(pTask); + } + + streamMetaUpdateTaskDownstreamStatus(pTask, pTask->execInfo.init, pTask->execInfo.start, true); + streamMetaReleaseTask(pMeta, pTask); + continue; + } + + EStreamTaskEvent event = (HAS_RELATED_FILLHISTORY_TASK(pTask)) ? TASK_EVENT_INIT_STREAM_SCANHIST : TASK_EVENT_INIT; + int32_t ret = streamTaskHandleEvent(pTask->status.pSM, event); + if (ret != TSDB_CODE_SUCCESS) { + code = ret; + } + + streamMetaReleaseTask(pMeta, pTask); + } + + taosArrayDestroy(pTaskList); + return code; +} + +int32_t sndResetStreamTaskStatus(SSnode* pSnode) { + SStreamMeta* pMeta = pSnode->pMeta; + int32_t vgId = pMeta->vgId; + int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); + + sndDebug("vgId:%d reset all %d stream task(s) status to be uninit", vgId, numOfTasks); + if (numOfTasks == 0) { + return TSDB_CODE_SUCCESS; + } + + for (int32_t i = 0; i < numOfTasks; ++i) { + SStreamTaskId* pTaskId = taosArrayGet(pMeta->pTaskList, i); + + STaskId id = {.streamId = pTaskId->streamId, .taskId = pTaskId->taskId}; + SStreamTask** pTask = taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); + streamTaskResetStatus(*pTask); + } + + return 0; +} + +int32_t sndRestartStreamTasks(SSnode* pSnode) { + SStreamMeta* pMeta = pSnode->pMeta; + int32_t vgId = pMeta->vgId; + int32_t code = 0; + int64_t st = taosGetTimestampMs(); + + while(1) { + int32_t startVal = atomic_val_compare_exchange_32(&pMeta->startInfo.taskStarting, 0, 1); + if (startVal == 0) { + break; + } + + sndDebug("vgId:%d in start stream tasks procedure, wait for 500ms and recheck", vgId); + taosMsleep(500); + } + + terrno = 0; + sndInfo("vgId:%d tasks are all updated and stopped, restart all tasks, triggered by transId:%d", vgId, + pMeta->updateInfo.transId); + + while (streamMetaTaskInTimer(pMeta)) { + sndDebug("vgId:%d some tasks in timer, wait for 100ms and recheck", pMeta->vgId); + taosMsleep(100); + } + + streamMetaWLock(pMeta); + + code = streamMetaReopen(pMeta); + if (code != TSDB_CODE_SUCCESS) { + sndError("vgId:%d failed to reopen stream meta", vgId); + streamMetaWUnLock(pMeta); + code = terrno; + return code; + } + + int64_t el = taosGetTimestampMs() - st; + + sndInfo("vgId:%d close&reload state elapsed time:%.3fms", vgId, el/1000.); + + code = streamMetaLoadAllTasks(pSnode->pMeta); + if (code != TSDB_CODE_SUCCESS) { + sndError("vgId:%d failed to load stream tasks, code:%s", vgId, tstrerror(terrno)); + streamMetaWUnLock(pMeta); + code = terrno; + return code; + } + sndInfo("vgId:%d restart all stream tasks after all tasks being updated", vgId); + sndResetStreamTaskStatus(pSnode); + sndStartStreamTasks(pSnode); + + streamMetaWUnLock(pMeta); + code = terrno; + return code; +} + +int32_t sndStartStreamTaskAsync(SSnode* pSnode, bool restart) { + SStreamMeta* pMeta = pSnode->pMeta; + int32_t vgId = pMeta->vgId; + + int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); + if (numOfTasks == 0) { + sndDebug("vgId:%d no stream tasks existed to run", vgId); + return 0; + } + + SStreamTaskRunReq* pRunReq = rpcMallocCont(sizeof(SStreamTaskRunReq)); + if (pRunReq == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + sndError("vgId:%d failed to create msg to start wal scanning to launch stream tasks, code:%s", vgId, terrstr()); + return -1; + } + + sndDebug("vgId:%d start all %d stream task(s) async", vgId, numOfTasks); + pRunReq->head.vgId = vgId; + pRunReq->streamId = 0; + pRunReq->taskId = restart? STREAM_EXEC_RESTART_ALL_TASKS_ID:STREAM_EXEC_START_ALL_TASKS_ID; + + SRpcMsg msg = {.msgType = TDMT_STREAM_TASK_RUN, .pCont = pRunReq, .contLen = sizeof(SStreamTaskRunReq)}; + tmsgPutToQueue(&pSnode->msgCb, STREAM_QUEUE, &msg); + return 0; +} + int32_t sndProcessTaskDeployReq(SSnode *pSnode, char *msg, int32_t msgLen) { int32_t code; @@ -235,6 +414,16 @@ int32_t sndProcessTaskDropReq(SSnode *pSnode, char *msg, int32_t msgLen) { int32_t sndProcessTaskRunReq(SSnode *pSnode, SRpcMsg *pMsg) { SStreamTaskRunReq *pReq = pMsg->pCont; + int32_t taskId = pReq->taskId; + + if (taskId == STREAM_EXEC_START_ALL_TASKS_ID) { + sndStartStreamTasks(pSnode); + return 0; + } else if (taskId == STREAM_EXEC_RESTART_ALL_TASKS_ID) { + sndRestartStreamTasks(pSnode); + return 0; + } + SStreamTask *pTask = streamMetaAcquireTask(pSnode->pMeta, pReq->streamId, pReq->taskId); if (pTask) { streamExecTask(pTask); @@ -312,22 +501,6 @@ int32_t sndProcessTaskRetrieveRsp(SSnode *pSnode, SRpcMsg *pMsg) { return 0; } -int32_t sndProcessWriteMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg *pRsp) { - switch (pMsg->msgType) { - case TDMT_STREAM_TASK_DEPLOY: { - void *pReq = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); - int32_t len = pMsg->contLen - sizeof(SMsgHead); - return sndProcessTaskDeployReq(pSnode, pReq, len); - } - - case TDMT_STREAM_TASK_DROP: - return sndProcessTaskDropReq(pSnode, pMsg->pCont, pMsg->contLen); - default: - ASSERT(0); - } - return 0; -} - int32_t sndProcessTaskScanHistoryFinishReq(SSnode *pSnode, SRpcMsg *pMsg) { char *msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); @@ -517,6 +690,181 @@ int32_t sndProcessStreamTaskCheckRsp(SSnode* pSnode, SRpcMsg* pMsg) { return code; } +int32_t sndProcessTaskUpdateReq(SSnode* pSnode, SRpcMsg* pMsg) { + SStreamMeta* pMeta = pSnode->pMeta; + int32_t vgId = SNODE_HANDLE; + char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + int32_t len = pMsg->contLen - sizeof(SMsgHead); + SRpcMsg rsp = {.info = pMsg->info, .code = TSDB_CODE_SUCCESS}; + + SStreamTaskNodeUpdateMsg req = {0}; + + SDecoder decoder; + tDecoderInit(&decoder, (uint8_t*)msg, len); + if (tDecodeStreamTaskUpdateMsg(&decoder, &req) < 0) { + rsp.code = TSDB_CODE_MSG_DECODE_ERROR; + sndError("vgId:%d failed to decode task update msg, code:%s", vgId, tstrerror(rsp.code)); + tDecoderClear(&decoder); + return rsp.code; + } + + tDecoderClear(&decoder); + + // update the nodeEpset when it exists + streamMetaWLock(pMeta); + + // the task epset may be updated again and again, when replaying the WAL, the task may be in stop status. + STaskId id = {.streamId = req.streamId, .taskId = req.taskId}; + SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); + if (ppTask == NULL || *ppTask == NULL) { + sndError("vgId:%d failed to acquire task:0x%x when handling update, it may have been dropped already", pMeta->vgId, + req.taskId); + rsp.code = TSDB_CODE_SUCCESS; + streamMetaWUnLock(pMeta); + + taosArrayDestroy(req.pNodeList); + return rsp.code; + } + + SStreamTask* pTask = *ppTask; + + if (pMeta->updateInfo.transId != req.transId) { + pMeta->updateInfo.transId = req.transId; + sndInfo("s-task:%s receive new trans to update nodeEp msg from mnode, transId:%d", pTask->id.idStr, req.transId); + // info needs to be kept till the new trans to update the nodeEp arrived. + taosHashClear(pMeta->updateInfo.pTasks); + } else { + sndDebug("s-task:%s recv trans to update nodeEp from mnode, transId:%d", pTask->id.idStr, req.transId); + } + + STaskUpdateEntry entry = {.streamId = req.streamId, .taskId = req.taskId, .transId = req.transId}; + void* exist = taosHashGet(pMeta->updateInfo.pTasks, &entry, sizeof(STaskUpdateEntry)); + if (exist != NULL) { + sndDebug("s-task:%s (vgId:%d) already update in trans:%d, discard the nodeEp update msg", pTask->id.idStr, vgId, + req.transId); + rsp.code = TSDB_CODE_SUCCESS; + streamMetaWUnLock(pMeta); + taosArrayDestroy(req.pNodeList); + return rsp.code; + } + + streamMetaWUnLock(pMeta); + + // the following two functions should not be executed within the scope of meta lock to avoid deadlock + streamTaskUpdateEpsetInfo(pTask, req.pNodeList); + streamTaskResetStatus(pTask); + + // continue after lock the meta again + streamMetaWLock(pMeta); + + SStreamTask** ppHTask = NULL; + if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { + ppHTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &pTask->hTaskInfo.id, sizeof(pTask->hTaskInfo.id)); + if (ppHTask == NULL || *ppHTask == NULL) { + sndError("vgId:%d failed to acquire fill-history task:0x%x when handling update, it may have been dropped already", + pMeta->vgId, req.taskId); + CLEAR_RELATED_FILLHISTORY_TASK(pTask); + } else { + sndDebug("s-task:%s fill-history task update nodeEp along with stream task", (*ppHTask)->id.idStr); + streamTaskUpdateEpsetInfo(*ppHTask, req.pNodeList); + } + } + + { + streamMetaSaveTask(pMeta, pTask); + if (ppHTask != NULL) { + streamMetaSaveTask(pMeta, *ppHTask); + } + + if (streamMetaCommit(pMeta) < 0) { + // persist to disk + } + } + + streamTaskStop(pTask); + + // keep the already handled info + taosHashPut(pMeta->updateInfo.pTasks, &entry, sizeof(entry), NULL, 0); + + if (ppHTask != NULL) { + streamTaskStop(*ppHTask); + sndDebug("s-task:%s task nodeEp update completed, streamTask and related fill-history task closed", pTask->id.idStr); + taosHashPut(pMeta->updateInfo.pTasks, &(*ppHTask)->id, sizeof(pTask->id), NULL, 0); + } else { + sndDebug("s-task:%s task nodeEp update completed, streamTask closed", pTask->id.idStr); + } + + rsp.code = 0; + + // possibly only handle the stream task. + int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta); + int32_t updateTasks = taosHashGetSize(pMeta->updateInfo.pTasks); + + pMeta->startInfo.tasksWillRestart = 1; + + if (updateTasks < numOfTasks) { + sndDebug("vgId:%d closed tasks:%d, unclosed:%d, all tasks will be started when nodeEp update completed", vgId, + updateTasks, (numOfTasks - updateTasks)); + streamMetaWUnLock(pMeta); + } else { + sndDebug("vgId:%d all %d task(s) nodeEp updated and closed", vgId, numOfTasks); +#if 1 + sndStartStreamTaskAsync(pSnode, true); + streamMetaWUnLock(pMeta); +#else + streamMetaWUnLock(pMeta); + + // For debug purpose. + // the following procedure consume many CPU resource, result in the re-election of leader + // with high probability. So we employ it as a test case for the stream processing framework, with + // checkpoint/restart/nodeUpdate etc. + while(1) { + int32_t startVal = atomic_val_compare_exchange_32(&pMeta->startInfo.taskStarting, 0, 1); + if (startVal == 0) { + break; + } + + tqDebug("vgId:%d in start stream tasks procedure, wait for 500ms and recheck", vgId); + taosMsleep(500); + } + + while (streamMetaTaskInTimer(pMeta)) { + tqDebug("vgId:%d some tasks in timer, wait for 100ms and recheck", pMeta->vgId); + taosMsleep(100); + } + + streamMetaWLock(pMeta); + + int32_t code = streamMetaReopen(pMeta); + if (code != 0) { + tqError("vgId:%d failed to reopen stream meta", vgId); + streamMetaWUnLock(pMeta); + taosArrayDestroy(req.pNodeList); + return -1; + } + + if (streamMetaLoadAllTasks(pTq->pStreamMeta) < 0) { + tqError("vgId:%d failed to load stream tasks", vgId); + streamMetaWUnLock(pMeta); + taosArrayDestroy(req.pNodeList); + return -1; + } + + if (vnodeIsRoleLeader(pTq->pVnode) && !tsDisableStream) { + tqInfo("vgId:%d start all stream tasks after all being updated", vgId); + tqResetStreamTaskStatus(pTq); + tqStartStreamTaskAsync(pTq, false); + } else { + tqInfo("vgId:%d, follower node not start stream tasks", vgId); + } + streamMetaWUnLock(pMeta); +#endif + } + + taosArrayDestroy(req.pNodeList); + return rsp.code; +} + int32_t sndProcessStreamMsg(SSnode *pSnode, SRpcMsg *pMsg) { switch (pMsg->msgType) { case TDMT_STREAM_TASK_RUN: @@ -544,3 +892,22 @@ int32_t sndProcessStreamMsg(SSnode *pSnode, SRpcMsg *pMsg) { } return 0; } + +int32_t sndProcessWriteMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg *pRsp) { + switch (pMsg->msgType) { + case TDMT_STREAM_TASK_DEPLOY: { + void *pReq = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + int32_t len = pMsg->contLen - sizeof(SMsgHead); + return sndProcessTaskDeployReq(pSnode, pReq, len); + } + + case TDMT_STREAM_TASK_DROP: + return sndProcessTaskDropReq(pSnode, pMsg->pCont, pMsg->contLen); + case TDMT_VND_STREAM_TASK_UPDATE: + sndProcessTaskUpdateReq(pSnode, pMsg); + break; + default: + ASSERT(0); + } + return 0; +} \ No newline at end of file diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index dc43da7fe7..635c15aa41 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -138,6 +138,11 @@ else() endif() endif() +target_include_directories( + vnode + PUBLIC "${TD_SOURCE_DIR}/include/dnode/vnode" +) + target_link_libraries( vnode PUBLIC os diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index fdd449bf36..b3f8317add 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -43,9 +43,6 @@ extern "C" { typedef struct STqOffsetStore STqOffsetStore; -#define STREAM_EXEC_EXTRACT_DATA_IN_WAL_ID (-1) -#define STREAM_EXEC_START_ALL_TASKS_ID (-2) -#define STREAM_EXEC_RESTART_ALL_TASKS_ID (-3) #define IS_OFFSET_RESET_TYPE(_t) ((_t) < 0) // tqExec diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 3ae0eb1ddf..521686d023 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -15,17 +15,12 @@ #include "tq.h" #include "vnd.h" +#include "stream.h" typedef struct { int8_t inited; } STqMgmt; -typedef struct STaskUpdateEntry { - int64_t streamId; - int32_t taskId; - int32_t transId; -} STaskUpdateEntry; - static STqMgmt tqMgmt = {0}; // 0: not init diff --git a/source/dnode/vnode/src/tq/tqStreamTask.c b/source/dnode/vnode/src/tq/tqStreamTask.c index e578638e9d..9f00f9cc10 100644 --- a/source/dnode/vnode/src/tq/tqStreamTask.c +++ b/source/dnode/vnode/src/tq/tqStreamTask.c @@ -15,6 +15,7 @@ #include "tq.h" #include "vnd.h" +#include "stream.h" #define MAX_REPEAT_SCAN_THRESHOLD 3 #define SCAN_WAL_IDLE_DURATION 100 diff --git a/tests/system-test/8-stream/snodeRestart.py b/tests/system-test/8-stream/snodeRestart.py index 65ca090df9..6adf874ecd 100644 --- a/tests/system-test/8-stream/snodeRestart.py +++ b/tests/system-test/8-stream/snodeRestart.py @@ -27,7 +27,7 @@ class TDTestCase: time.sleep(1) tdSql.query("use test") tdSql.query("create snode on dnode 4") - tdSql.query("create stream if not exists s1 trigger at_once ignore expired 0 ignore update 0 fill_history 1 into st1 as select sum(voltage) from meters partition by groupid interval(4s)") + tdSql.query("create stream if not exists s1 trigger at_once ignore expired 0 ignore update 0 fill_history 1 into st1 as select _wstart, sum(voltage) from meters partition by groupid interval(4s)") tdLog.debug("create stream use snode and insert data") time.sleep(10) From 50126e44252f0671dbe480ddc6a13494f2ce67d4 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 16 Nov 2023 19:52:05 +0800 Subject: [PATCH 07/29] enh: support spread active codes --- include/util/taoserror.h | 1 + source/dnode/mnode/impl/inc/mndCluster.h | 2 +- source/dnode/mnode/impl/src/mndCluster.c | 8 +++++--- source/dnode/mnode/impl/src/mndDnode.c | 7 +++++-- source/util/src/terror.c | 1 + 5 files changed, 13 insertions(+), 6 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index a3ee294338..ce8db162b6 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -557,6 +557,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_GRANT_GEN_IVLD_KEY TAOS_DEF_ERROR_CODE(0, 0x0812) #define TSDB_CODE_GRANT_GEN_APP_LIMIT TAOS_DEF_ERROR_CODE(0, 0x0813) #define TSDB_CODE_GRANT_GEN_ENC_IVLD_KLEN TAOS_DEF_ERROR_CODE(0, 0x0814) +#define TSDB_CODE_GRANT_PAR_IVLD_DIST TAOS_DEF_ERROR_CODE(0, 0x0815) // sync // #define TSDB_CODE_SYN_INVALID_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0900) // 2.x diff --git a/source/dnode/mnode/impl/inc/mndCluster.h b/source/dnode/mnode/impl/inc/mndCluster.h index 2b868b567a..81cfce6530 100644 --- a/source/dnode/mnode/impl/inc/mndCluster.h +++ b/source/dnode/mnode/impl/inc/mndCluster.h @@ -29,7 +29,7 @@ int64_t mndGetClusterId(SMnode *pMnode); int64_t mndGetClusterCreateTime(SMnode *pMnode); int32_t mndGetClusterGrantedInfo(SMnode *pMnode, SGrantedInfo *pInfo); int64_t mndGetClusterUpTime(SMnode *pMnode); -int32_t mndProcessGrantedTime(SMnode *pMnode, int64_t grantedTime); +int32_t mndProcessGrantedInfo(SMnode *pMnode, SGrantedInfo *pInfo); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index 20e25d918c..bcef419c31 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -236,6 +236,7 @@ static int32_t mndClusterActionUpdate(SSdb *pSdb, SClusterObj *pOld, SClusterObj pNew, pOld->upTime, pNew->upTime); pOld->upTime = pNew->upTime; pOld->grantedTime = pNew->grantedTime; + pOld->connGrantedTime = pNew->connGrantedTime; pOld->updateTime = taosGetTimestampMs(); return 0; } @@ -378,17 +379,18 @@ static int32_t mndProcessUptimeTimer(SRpcMsg *pReq) { return 0; } -int32_t mndProcessGrantedTime(SMnode *pMnode, int64_t grantedTime) { +int32_t mndProcessGrantedInfo(SMnode *pMnode, SGrantedInfo *pInfo) { SClusterObj clusterObj = {0}; void *pIter = NULL; SClusterObj *pCluster = mndAcquireCluster(pMnode, &pIter); if (pCluster != NULL) { - if (pCluster->grantedTime >= grantedTime) { + if (pCluster->grantedTime >= pInfo->grantedTime && pCluster->connGrantedTime > pInfo->connGrantedTime) { mndReleaseCluster(pMnode, pCluster, pIter); return 0; } memcpy(&clusterObj, pCluster, sizeof(SClusterObj)); - clusterObj.grantedTime = grantedTime; + if (pCluster->grantedTime < pInfo->grantedTime) clusterObj.grantedTime = pInfo->grantedTime; + if (pCluster->connGrantedTime < pInfo->connGrantedTime) clusterObj.connGrantedTime = pInfo->connGrantedTime; mndReleaseCluster(pMnode, pCluster, pIter); } diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index e224aceec2..d46f431ade 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -1284,6 +1284,10 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { strcpy(dcfgReq.config, "supportvnodes"); snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag); } else if (strncasecmp(cfgReq.config, "activeCode", 10) == 0 || strncasecmp(cfgReq.config, "cActiveCode", 11) == 0) { + if (cfgReq.dnodeId != -1) { + terrno = TSDB_CODE_INVALID_CFG; + goto _err_out; + } int8_t opt = strncasecmp(cfgReq.config, "a", 1) == 0 ? DND_ACTIVE_CODE : DND_CONN_ACTIVE_CODE; int8_t index = opt == DND_ACTIVE_CODE ? 10 : 11; if (' ' != cfgReq.config[index] && 0 != cfgReq.config[index]) { @@ -1304,9 +1308,8 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { strcpy(dcfgReq.config, opt == DND_ACTIVE_CODE ? "activeCode" : "cActiveCode"); snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%s", cfgReq.value); - if (mndConfigDnode(pMnode, pReq, &cfgReq, opt) != 0) { + if ((terrno = mndConfigDnode(pMnode, pReq, &cfgReq, opt)) != 0) { mError("dnode:%d, failed to config activeCode since %s", cfgReq.dnodeId, terrstr()); - terrno = TSDB_CODE_INVALID_CFG; goto _err_out; } tFreeSMCfgDnodeReq(&cfgReq); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 21022f2016..f310db53ef 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -444,6 +444,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_PAR_DEC_IVLD_KLEN, "Invalid klen to decod TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_GEN_IVLD_KEY, "Invalid key to gen active code") TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_GEN_APP_LIMIT, "Limited app num to gen active code") TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_GEN_ENC_IVLD_KLEN, "Invalid klen to encode active code") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_PAR_IVLD_DIST, "Invalid dist to parse active code") // sync TAOS_DEFINE_ERROR(TSDB_CODE_SYN_TIMEOUT, "Sync timeout") From 9e12ad408e92eab11a0a10fee096b8a3f7b7673b Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 16 Nov 2023 20:01:54 +0800 Subject: [PATCH 08/29] enh: support spread active codes --- source/dnode/mnode/impl/src/mndCluster.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index bcef419c31..b6d36763cc 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -384,7 +384,7 @@ int32_t mndProcessGrantedInfo(SMnode *pMnode, SGrantedInfo *pInfo) { void *pIter = NULL; SClusterObj *pCluster = mndAcquireCluster(pMnode, &pIter); if (pCluster != NULL) { - if (pCluster->grantedTime >= pInfo->grantedTime && pCluster->connGrantedTime > pInfo->connGrantedTime) { + if (pCluster->grantedTime >= pInfo->grantedTime && pCluster->connGrantedTime >= pInfo->connGrantedTime) { mndReleaseCluster(pMnode, pCluster, pIter); return 0; } @@ -395,12 +395,12 @@ int32_t mndProcessGrantedInfo(SMnode *pMnode, SGrantedInfo *pInfo) { } if (clusterObj.id <= 0) { - mError("can't get cluster info while update uptime"); + mError("can't get cluster info while update granted info"); return -1; } - mInfo("update cluster granted time to %" PRIi64, clusterObj.grantedTime); - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, NULL, "granted-time"); + mInfo("update cluster granted info to %" PRIi64 ",%" PRIi64, clusterObj.grantedTime, clusterObj.connGrantedTime); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, NULL, "granted-info"); if (pTrans == NULL) return -1; SSdbRaw *pCommitRaw = mndClusterActionEncode(&clusterObj); From 63a17295a797b6bca40a53061ce681a98bbecac4 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 16 Nov 2023 20:09:56 +0800 Subject: [PATCH 09/29] enh: support spread active codes --- source/dnode/mnode/impl/inc/mndCluster.h | 3 ++- source/dnode/mnode/impl/src/mndCluster.c | 3 +-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndCluster.h b/source/dnode/mnode/impl/inc/mndCluster.h index 81cfce6530..93496b9330 100644 --- a/source/dnode/mnode/impl/inc/mndCluster.h +++ b/source/dnode/mnode/impl/inc/mndCluster.h @@ -28,8 +28,9 @@ int32_t mndGetClusterName(SMnode *pMnode, char *clusterName, int32_t len); int64_t mndGetClusterId(SMnode *pMnode); int64_t mndGetClusterCreateTime(SMnode *pMnode); int32_t mndGetClusterGrantedInfo(SMnode *pMnode, SGrantedInfo *pInfo); +int32_t mndSetClusterGrantedInfo(SMnode *pMnode, SGrantedInfo *pInfo); int64_t mndGetClusterUpTime(SMnode *pMnode); -int32_t mndProcessGrantedInfo(SMnode *pMnode, SGrantedInfo *pInfo); + #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index b6d36763cc..26c678b513 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -379,7 +379,7 @@ static int32_t mndProcessUptimeTimer(SRpcMsg *pReq) { return 0; } -int32_t mndProcessGrantedInfo(SMnode *pMnode, SGrantedInfo *pInfo) { +int32_t mndSetClusterGrantedInfo(SMnode *pMnode, SGrantedInfo *pInfo) { SClusterObj clusterObj = {0}; void *pIter = NULL; SClusterObj *pCluster = mndAcquireCluster(pMnode, &pIter); @@ -399,7 +399,6 @@ int32_t mndProcessGrantedInfo(SMnode *pMnode, SGrantedInfo *pInfo) { return -1; } - mInfo("update cluster granted info to %" PRIi64 ",%" PRIi64, clusterObj.grantedTime, clusterObj.connGrantedTime); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, NULL, "granted-info"); if (pTrans == NULL) return -1; From b64e506292f7034b0500a77d8ab0836190a2e02d Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Thu, 16 Nov 2023 20:15:41 +0800 Subject: [PATCH 10/29] Update mndCluster.h --- source/dnode/mnode/impl/inc/mndCluster.h | 1 - 1 file changed, 1 deletion(-) diff --git a/source/dnode/mnode/impl/inc/mndCluster.h b/source/dnode/mnode/impl/inc/mndCluster.h index 93496b9330..2b59d9dbf5 100644 --- a/source/dnode/mnode/impl/inc/mndCluster.h +++ b/source/dnode/mnode/impl/inc/mndCluster.h @@ -31,7 +31,6 @@ int32_t mndGetClusterGrantedInfo(SMnode *pMnode, SGrantedInfo *pInfo); int32_t mndSetClusterGrantedInfo(SMnode *pMnode, SGrantedInfo *pInfo); int64_t mndGetClusterUpTime(SMnode *pMnode); - #ifdef __cplusplus } #endif From a5de1b032bd99c99001e05811c97da3c3081963e Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 16 Nov 2023 21:19:58 +0800 Subject: [PATCH 11/29] fix:snode upport stream task --- include/dnode/snode/snode.h | 1 + include/libs/stream/tstream.h | 4 +- source/dnode/mgmt/mgmt_snode/src/smInt.c | 5 + source/dnode/snode/src/snode.c | 108 ++++++++++--------- source/dnode/vnode/src/tq/tq.c | 4 +- source/libs/stream/src/streamStart.c | 13 +-- tests/system-test/7-tmq/tmqVnodeReplicate.py | 1 - tests/system-test/8-stream/snodeRestart.py | 45 +++++--- tests/system-test/test.py | 2 +- 9 files changed, 108 insertions(+), 75 deletions(-) diff --git a/include/dnode/snode/snode.h b/include/dnode/snode/snode.h index e8c64b07c4..c3dfd3a611 100644 --- a/include/dnode/snode/snode.h +++ b/include/dnode/snode/snode.h @@ -45,6 +45,7 @@ typedef struct { */ SSnode *sndOpen(const char *path, const SSnodeOpt *pOption); +int32_t sndInit(SSnode * pSnode); /** * @brief Stop Snode in Dnode. * diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 3ce2de476e..cb13a50ff4 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -592,7 +592,7 @@ typedef struct { int32_t downstreamNodeId; int32_t downstreamTaskId; int32_t childId; - int32_t oldStage; + int64_t oldStage; int8_t status; } SStreamTaskCheckRsp; @@ -760,7 +760,7 @@ void initRpcMsg(SRpcMsg* pMsg, int32_t msgType, void* pCont, int32_t contLen) // recover and fill history void streamTaskCheckDownstream(SStreamTask* pTask); -int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_t vgId, int64_t stage); +int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_t vgId, int64_t stage, int64_t* oldStage); int32_t streamTaskUpdateEpsetInfo(SStreamTask* pTask, SArray* pNodeList); void streamTaskResetUpstreamStageInfo(SStreamTask* pTask); bool streamTaskAllUpstreamClosed(SStreamTask* pTask); diff --git a/source/dnode/mgmt/mgmt_snode/src/smInt.c b/source/dnode/mgmt/mgmt_snode/src/smInt.c index 47c2993014..56744e4654 100644 --- a/source/dnode/mgmt/mgmt_snode/src/smInt.c +++ b/source/dnode/mgmt/mgmt_snode/src/smInt.c @@ -76,9 +76,14 @@ int32_t smOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { return 0; } +static int32_t smStartSnodes(SSnodeMgmt *pMgmt) { + return sndInit(pMgmt->pSnode); +} + SMgmtFunc smGetMgmtFunc() { SMgmtFunc mgmtFunc = {0}; mgmtFunc.openFp = smOpen; + mgmtFunc.startFp = (NodeStartFp)smStartSnodes; mgmtFunc.closeFp = (NodeCloseFp)smClose; mgmtFunc.createFp = (NodeCreateFp)smProcessCreateReq; mgmtFunc.dropFp = (NodeDropFp)smProcessDropReq; diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 25400220b8..4863a1f7b7 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -143,50 +143,6 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProcessVer return 0; } -SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) { - SSnode *pSnode = taosMemoryCalloc(1, sizeof(SSnode)); - if (pSnode == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - pSnode->path = taosStrdup(path); - if (pSnode->path == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto FAIL; - } - - pSnode->msgCb = pOption->msgCb; - pSnode->pMeta = streamMetaOpen(path, pSnode, (FTaskExpand *)sndExpandTask, SNODE_HANDLE, taosGetTimestampMs()); - if (pSnode->pMeta == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto FAIL; - } - - if (streamMetaLoadAllTasks(pSnode->pMeta) < 0) { - goto FAIL; - } - - stopRsync(); - startRsync(); - - return pSnode; - -FAIL: - taosMemoryFree(pSnode->path); - taosMemoryFree(pSnode); - return NULL; -} - -void sndClose(SSnode *pSnode) { - streamMetaNotifyClose(pSnode->pMeta); - streamMetaCommit(pSnode->pMeta); - streamMetaClose(pSnode->pMeta); - taosMemoryFree(pSnode->path); - taosMemoryFree(pSnode); -} - -int32_t sndGetLoad(SSnode *pSnode, SSnodeLoad *pLoad) { return 0; } - int32_t sndStartStreamTasks(SSnode* pSnode) { int32_t code = TSDB_CODE_SUCCESS; int32_t vgId = SNODE_HANDLE; @@ -223,7 +179,7 @@ int32_t sndStartStreamTasks(SSnode* pSnode) { if (pTask->status.downstreamReady == 1) { if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { sndDebug("s-task:%s downstream ready, no need to check downstream, check only related fill-history task", - pTask->id.idStr); + pTask->id.idStr); streamLaunchFillHistoryTask(pTask); } @@ -284,7 +240,7 @@ int32_t sndRestartStreamTasks(SSnode* pSnode) { terrno = 0; sndInfo("vgId:%d tasks are all updated and stopped, restart all tasks, triggered by transId:%d", vgId, - pMeta->updateInfo.transId); + pMeta->updateInfo.transId); while (streamMetaTaskInTimer(pMeta)) { sndDebug("vgId:%d some tasks in timer, wait for 100ms and recheck", pMeta->vgId); @@ -301,11 +257,12 @@ int32_t sndRestartStreamTasks(SSnode* pSnode) { return code; } + streamMetaInitBackend(pMeta); int64_t el = taosGetTimestampMs() - st; - sndInfo("vgId:%d close&reload state elapsed time:%.3fms", vgId, el/1000.); + sndInfo("vgId:%d close&reload state elapsed time:%.3fs", vgId, el/1000.); - code = streamMetaLoadAllTasks(pSnode->pMeta); + code = streamMetaLoadAllTasks(pMeta); if (code != TSDB_CODE_SUCCESS) { sndError("vgId:%d failed to load stream tasks, code:%s", vgId, tstrerror(terrno)); streamMetaWUnLock(pMeta); @@ -314,13 +271,64 @@ int32_t sndRestartStreamTasks(SSnode* pSnode) { } sndInfo("vgId:%d restart all stream tasks after all tasks being updated", vgId); sndResetStreamTaskStatus(pSnode); - sndStartStreamTasks(pSnode); streamMetaWUnLock(pMeta); + sndStartStreamTasks(pSnode); + code = terrno; return code; } +SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) { + SSnode *pSnode = taosMemoryCalloc(1, sizeof(SSnode)); + if (pSnode == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + pSnode->path = taosStrdup(path); + if (pSnode->path == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto FAIL; + } + + pSnode->msgCb = pOption->msgCb; + pSnode->pMeta = streamMetaOpen(path, pSnode, (FTaskExpand *)sndExpandTask, SNODE_HANDLE, taosGetTimestampMs()); + if (pSnode->pMeta == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto FAIL; + } + + if (streamMetaLoadAllTasks(pSnode->pMeta) < 0) { + goto FAIL; + } + + stopRsync(); + startRsync(); + + return pSnode; + +FAIL: + taosMemoryFree(pSnode->path); + taosMemoryFree(pSnode); + return NULL; +} + +int32_t sndInit(SSnode * pSnode) { + sndResetStreamTaskStatus(pSnode); + sndStartStreamTasks(pSnode); + return 0; +} + +void sndClose(SSnode *pSnode) { + streamMetaNotifyClose(pSnode->pMeta); + streamMetaCommit(pSnode->pMeta); + streamMetaClose(pSnode->pMeta); + taosMemoryFree(pSnode->path); + taosMemoryFree(pSnode); +} + +int32_t sndGetLoad(SSnode *pSnode, SSnodeLoad *pLoad) { return 0; } + int32_t sndStartStreamTaskAsync(SSnode* pSnode, bool restart) { SStreamMeta* pMeta = pSnode->pMeta; int32_t vgId = pMeta->vgId; @@ -622,7 +630,7 @@ int32_t sndProcessStreamTaskCheckReq(SSnode *pSnode, SRpcMsg *pMsg) { SStreamTask *pTask = streamMetaAcquireTask(pSnode->pMeta, req.streamId, taskId); if (pTask != NULL) { - rsp.status = streamTaskCheckStatus(pTask, req.upstreamTaskId, req.upstreamNodeId, req.stage); + rsp.status = streamTaskCheckStatus(pTask, req.upstreamTaskId, req.upstreamNodeId, req.stage, &rsp.oldStage); streamMetaReleaseTask(pSnode->pMeta, pTask); char* p = NULL; streamTaskGetStatus(pTask, &p); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 8ee0edcb31..6ba5529049 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -923,12 +923,12 @@ int32_t tqProcessTaskCheckReq(STQ* pTq, SRpcMsg* pMsg) { } else { SStreamTask* pTask = streamMetaAcquireTask(pMeta, req.streamId, taskId); if (pTask != NULL) { - rsp.status = streamTaskCheckStatus(pTask, req.upstreamTaskId, req.upstreamNodeId, req.stage); + rsp.status = streamTaskCheckStatus(pTask, req.upstreamTaskId, req.upstreamNodeId, req.stage, &rsp.oldStage); streamMetaReleaseTask(pMeta, pTask); char* p = NULL; streamTaskGetStatus(pTask, &p); - tqDebug("s-task:%s status:%s, stage:%d recv task check req(reqId:0x%" PRIx64 ") task:0x%x (vgId:%d), check_status:%d", + tqDebug("s-task:%s status:%s, stage:%"PRId64" recv task check req(reqId:0x%" PRIx64 ") task:0x%x (vgId:%d), check_status:%d", pTask->id.idStr, p, rsp.oldStage, rsp.reqId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status); } else { rsp.status = TASK_DOWNSTREAM_NOT_READY; diff --git a/source/libs/stream/src/streamStart.c b/source/libs/stream/src/streamStart.c index 97eb7b79a2..a1c0fa2040 100644 --- a/source/libs/stream/src/streamStart.c +++ b/source/libs/stream/src/streamStart.c @@ -290,10 +290,11 @@ static void recheckDownstreamTasks(void* param, void* tmrId) { stDebug("s-task:%s complete send check in timer, ref:%d", pTask->id.idStr, ref); } -int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_t vgId, int64_t stage) { +int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_t vgId, int64_t stage, int64_t* oldStage) { SStreamChildEpInfo* pInfo = streamTaskGetUpstreamTaskEpInfo(pTask, upstreamTaskId); ASSERT(pInfo != NULL); + *oldStage = pInfo->stage; const char* id = pTask->id.idStr; if (stage == -1) { stDebug("s-task:%s receive check msg from upstream task:0x%x(vgId:%d), invalid stageId:%" PRId64 ", not ready", id, @@ -459,9 +460,9 @@ int32_t streamProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRs if (pRsp->status == TASK_UPSTREAM_NEW_STAGE || pRsp->status == TASK_DOWNSTREAM_NOT_LEADER) { if (pRsp->status == TASK_UPSTREAM_NEW_STAGE) { stError( - "s-task:%s vgId:%d self vnode-transfer/leader-change/restart detected, old stage:%d, current stage:%d, " + "s-task:%s vgId:%d self vnode-transfer/leader-change/restart detected, old stage:%"PRId64", current stage:%"PRId64", " "not check wait for downstream task nodeUpdate, and all tasks restart", - id, pRsp->upstreamNodeId, pRsp->oldStage, (int32_t)pTask->pMeta->stage); + id, pRsp->upstreamNodeId, pRsp->oldStage, pTask->pMeta->stage); } else { stError( "s-task:%s downstream taskId:0x%x (vgId:%d) not leader, self dispatch epset needs to be updated, not check " @@ -476,7 +477,7 @@ int32_t streamProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRs STaskRecheckInfo* pInfo = createRecheckInfo(pTask, pRsp); int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1); - stDebug("s-task:%s downstream taskId:0x%x (vgId:%d) not ready, stage:%d, retry in 100ms, ref:%d ", id, + stDebug("s-task:%s downstream taskId:0x%x (vgId:%d) not ready, stage:%"PRId64", retry in 100ms, ref:%d ", id, pRsp->downstreamTaskId, pRsp->downstreamNodeId, pRsp->oldStage, ref); pInfo->checkTimer = taosTmrStart(recheckDownstreamTasks, CHECK_DOWNSTREAM_INTERVAL, pInfo, streamEnv.timer); } @@ -926,7 +927,7 @@ int32_t tEncodeStreamTaskCheckRsp(SEncoder* pEncoder, const SStreamTaskCheckRsp* if (tEncodeI32(pEncoder, pRsp->downstreamNodeId) < 0) return -1; if (tEncodeI32(pEncoder, pRsp->downstreamTaskId) < 0) return -1; if (tEncodeI32(pEncoder, pRsp->childId) < 0) return -1; - if (tEncodeI32(pEncoder, pRsp->oldStage) < 0) return -1; + if (tEncodeI64(pEncoder, pRsp->oldStage) < 0) return -1; if (tEncodeI8(pEncoder, pRsp->status) < 0) return -1; tEndEncode(pEncoder); return pEncoder->pos; @@ -941,7 +942,7 @@ int32_t tDecodeStreamTaskCheckRsp(SDecoder* pDecoder, SStreamTaskCheckRsp* pRsp) if (tDecodeI32(pDecoder, &pRsp->downstreamNodeId) < 0) return -1; if (tDecodeI32(pDecoder, &pRsp->downstreamTaskId) < 0) return -1; if (tDecodeI32(pDecoder, &pRsp->childId) < 0) return -1; - if (tDecodeI32(pDecoder, &pRsp->oldStage) < 0) return -1; + if (tDecodeI64(pDecoder, &pRsp->oldStage) < 0) return -1; if (tDecodeI8(pDecoder, &pRsp->status) < 0) return -1; tEndDecode(pDecoder); return 0; diff --git a/tests/system-test/7-tmq/tmqVnodeReplicate.py b/tests/system-test/7-tmq/tmqVnodeReplicate.py index fd8ece02e0..afb3319491 100644 --- a/tests/system-test/7-tmq/tmqVnodeReplicate.py +++ b/tests/system-test/7-tmq/tmqVnodeReplicate.py @@ -105,7 +105,6 @@ class TDTestCase: topicNameList = ['topic1'] # expectRowsList = [] - tmqCom.initConsumerTable("cdb", self.replicaVar) tdLog.info("create topics from stb with filter") queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName']) diff --git a/tests/system-test/8-stream/snodeRestart.py b/tests/system-test/8-stream/snodeRestart.py index 6adf874ecd..d2b9062c6e 100644 --- a/tests/system-test/8-stream/snodeRestart.py +++ b/tests/system-test/8-stream/snodeRestart.py @@ -15,40 +15,59 @@ from util.common import * from util.cluster import * class TDTestCase: + updatecfgDict = {'checkpointInterval': 1100} + print("===================: ", updatecfgDict) + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) def case1(self): - tdLog.debug("case1 start") + tdLog.debug("========case1 start========") os.system("nohup taosBenchmark -y -B 1 -t 4 -S 1000 -n 1000 -i 1000 -v 2 > /dev/null 2>&1 &") - time.sleep(1) + time.sleep(4) tdSql.query("use test") tdSql.query("create snode on dnode 4") - tdSql.query("create stream if not exists s1 trigger at_once ignore expired 0 ignore update 0 fill_history 1 into st1 as select _wstart, sum(voltage) from meters partition by groupid interval(4s)") - tdLog.debug("create stream use snode and insert data") - time.sleep(10) + tdSql.query("create stream if not exists s1 trigger at_once ignore expired 0 ignore update 0 fill_history 1 into st1 as select _wstart,sum(voltage),groupid from meters partition by groupid interval(2s)") + tdLog.debug("========create stream useing snode and insert data ok========") + time.sleep(4) tdDnodes = cluster.dnodes tdDnodes[3].stoptaosd() time.sleep(2) tdDnodes[3].starttaosd() - tdLog.debug("snode restart ok") + tdLog.debug("========snode restart ok========") - time.sleep(500) + time.sleep(30) os.system("kill -9 `pgrep taosBenchmark`") + tdLog.debug("========stop insert ok========") + time.sleep(2) - tdSql.query("select sum(voltage) from meters partition by groupid interval(4s)") - # tdSql.checkRows(1) - # + tdSql.query("select _wstart,sum(voltage),groupid from meters partition by groupid interval(2s) order by groupid,_wstart") + rowCnt = tdSql.getRows() + results = [] + for i in range(rowCnt): + results.append(tdSql.getData(i,1)) + + tdSql.query("select * from st1 order by groupid,_wstart") + tdSql.checkRows(rowCnt) + for i in range(rowCnt): + data1 = tdSql.getData(i,1) + data2 = results[i] + if data1 != data2: + tdLog.info("num: %d, act data: %d, expect data: %d"%(i, data1, data2)) + tdLog.exit("check data error!") # tdSql.checkData(0, 0, '2016-01-01 08:00:07.000') # tdSql.checkData(0, 1, 2000) - tdSql.query("drop snode on dnode 4") - tdSql.query("drop stream if exists s1") - tdSql.query("drop database test") + # tdLog.debug("========sleep 500s========") + # time.sleep(500) + # + # tdSql.query("drop snode on dnode 4") + # tdSql.query("drop stream if exists s1") + # tdSql.query("drop database test") tdLog.debug("case1 end") diff --git a/tests/system-test/test.py b/tests/system-test/test.py index 81f98fea22..795132b14e 100644 --- a/tests/system-test/test.py +++ b/tests/system-test/test.py @@ -582,7 +582,7 @@ if __name__ == "__main__": tdDnodes.setAsan(asan) tdDnodes.stopAll() for dnode in tdDnodes.dnodes: - tdDnodes.deploy(dnode.index,{}) + tdDnodes.deploy(dnode.index,updateCfgDict) for dnode in tdDnodes.dnodes: tdDnodes.starttaosd(dnode.index) tdCases.logSql(logSql) From da233a452441b71f751784f479b4a891825bd175 Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 17 Nov 2023 08:38:13 +0800 Subject: [PATCH 12/29] enh: adjust test case --- tests/system-test/0-others/information_schema.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/system-test/0-others/information_schema.py b/tests/system-test/0-others/information_schema.py index ac952d383a..a4c18d2938 100644 --- a/tests/system-test/0-others/information_schema.py +++ b/tests/system-test/0-others/information_schema.py @@ -247,7 +247,10 @@ class TDTestCase: tdSql.error('alter all dnodes "activeCode" "' + self.str510 + '"') tdSql.query(f'select * from information_schema.ins_dnodes') tdSql.checkEqual(tdSql.queryResult[0][8],"") - tdSql.execute('alter dnode 1 "activeCode" ""') + tdSql.error('alter dnode 1 "activeCode" ""') + tdSql.error('alter dnode 1 "activeCode"') + tdSql.Execute('alter all dnodes "activeCode" ""') + tdSql.Execute('alter all dnodes "activeCode"') tdSql.query(f'select active_code,c_active_code from information_schema.ins_dnodes') tdSql.checkEqual(tdSql.queryResult[0][0],"") tdSql.checkEqual(tdSql.queryResult[0][1],'') @@ -259,6 +262,10 @@ class TDTestCase: tdSql.error('alter all dnodes "cActiveCode" "' + self.str257 + '"') tdSql.error('alter all dnodes "cActiveCode" "' + self.str254 + '"') tdSql.error('alter dnode 1 "cActiveCode" "' + self.str510 + '"') + tdSql.error('alter dnode 1 "cActiveCode" ""') + tdSql.error('alter dnode 1 "cActiveCode"') + tdSql.Execute('alter all dnodes "cActiveCode" ""') + tdSql.Execute('alter all dnodes "cActiveCode"') tdSql.query(f'select active_code,c_active_code from information_schema.ins_dnodes') tdSql.checkEqual(tdSql.queryResult[0][0],"") tdSql.checkEqual(tdSql.queryResult[0][1],"") From 14a2be8ed2b770ac23ad10690dd1cadc14d1fcbe Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 17 Nov 2023 08:41:47 +0800 Subject: [PATCH 13/29] enh: adjust test case --- source/dnode/mnode/impl/inc/mndCluster.h | 1 - tests/system-test/0-others/information_schema.py | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndCluster.h b/source/dnode/mnode/impl/inc/mndCluster.h index 93496b9330..2b59d9dbf5 100644 --- a/source/dnode/mnode/impl/inc/mndCluster.h +++ b/source/dnode/mnode/impl/inc/mndCluster.h @@ -31,7 +31,6 @@ int32_t mndGetClusterGrantedInfo(SMnode *pMnode, SGrantedInfo *pInfo); int32_t mndSetClusterGrantedInfo(SMnode *pMnode, SGrantedInfo *pInfo); int64_t mndGetClusterUpTime(SMnode *pMnode); - #ifdef __cplusplus } #endif diff --git a/tests/system-test/0-others/information_schema.py b/tests/system-test/0-others/information_schema.py index a4c18d2938..27a3c73fb7 100644 --- a/tests/system-test/0-others/information_schema.py +++ b/tests/system-test/0-others/information_schema.py @@ -249,8 +249,8 @@ class TDTestCase: tdSql.checkEqual(tdSql.queryResult[0][8],"") tdSql.error('alter dnode 1 "activeCode" ""') tdSql.error('alter dnode 1 "activeCode"') - tdSql.Execute('alter all dnodes "activeCode" ""') - tdSql.Execute('alter all dnodes "activeCode"') + tdSql.execute('alter all dnodes "activeCode" ""') + tdSql.execute('alter all dnodes "activeCode"') tdSql.query(f'select active_code,c_active_code from information_schema.ins_dnodes') tdSql.checkEqual(tdSql.queryResult[0][0],"") tdSql.checkEqual(tdSql.queryResult[0][1],'') @@ -264,8 +264,8 @@ class TDTestCase: tdSql.error('alter dnode 1 "cActiveCode" "' + self.str510 + '"') tdSql.error('alter dnode 1 "cActiveCode" ""') tdSql.error('alter dnode 1 "cActiveCode"') - tdSql.Execute('alter all dnodes "cActiveCode" ""') - tdSql.Execute('alter all dnodes "cActiveCode"') + tdSql.execute('alter all dnodes "cActiveCode" ""') + tdSql.execute('alter all dnodes "cActiveCode"') tdSql.query(f'select active_code,c_active_code from information_schema.ins_dnodes') tdSql.checkEqual(tdSql.queryResult[0][0],"") tdSql.checkEqual(tdSql.queryResult[0][1],"") From cb9f784c33bca916ed5dbc5f21ed2c8b61995d94 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 17 Nov 2023 09:34:27 +0800 Subject: [PATCH 14/29] fix:add test cases --- tests/parallel_test/cases.task | 3 + .../{snodeRestart.py => snode_restart.py} | 48 ------------ .../8-stream/snode_restart_with_checkpoint.py | 78 +++++++++++++++++++ tests/system-test/8-stream/vnode_restart.py | 77 ++++++++++++++++++ 4 files changed, 158 insertions(+), 48 deletions(-) rename tests/system-test/8-stream/{snodeRestart.py => snode_restart.py} (58%) create mode 100644 tests/system-test/8-stream/snode_restart_with_checkpoint.py create mode 100644 tests/system-test/8-stream/vnode_restart.py diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 92eaec52b5..40eb926efb 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -20,6 +20,9 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/window_close_session_ext.py ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/partition_interval.py ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/pause_resume_test.py +#,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/vnode_restart.py +#,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/snode_restart.py +,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/snode_restart_with_checkpoint.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tbname_vgroup.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stbJoin.py diff --git a/tests/system-test/8-stream/snodeRestart.py b/tests/system-test/8-stream/snode_restart.py similarity index 58% rename from tests/system-test/8-stream/snodeRestart.py rename to tests/system-test/8-stream/snode_restart.py index d2b9062c6e..3657163ab0 100644 --- a/tests/system-test/8-stream/snodeRestart.py +++ b/tests/system-test/8-stream/snode_restart.py @@ -59,60 +59,12 @@ class TDTestCase: if data1 != data2: tdLog.info("num: %d, act data: %d, expect data: %d"%(i, data1, data2)) tdLog.exit("check data error!") - # tdSql.checkData(0, 0, '2016-01-01 08:00:07.000') - # tdSql.checkData(0, 1, 2000) # tdLog.debug("========sleep 500s========") # time.sleep(500) - # - # tdSql.query("drop snode on dnode 4") - # tdSql.query("drop stream if exists s1") - # tdSql.query("drop database test") tdLog.debug("case1 end") - def case2(self): - tdLog.debug("case2 start") - - updatecfgDict = {'checkpointInterval': 5} - print("===================: ", updatecfgDict) - - tdDnodes = cluster.dnodes - tdDnodes[0].stoptaosd() - tdDnodes[1].stoptaosd() - tdDnodes[2].stoptaosd() - tdDnodes[3].stoptaosd() - time.sleep(2) - tdDnodes[0].starttaosd() - tdDnodes[1].starttaosd() - tdDnodes[2].starttaosd() - tdDnodes[3].starttaosd() - tdLog.debug("taosd restart ok") - - os.system("taosBenchmark -y -B 1 -t 4 -S 1000 -n 1000 -i 1000 -v 2 &" ) - time.sleep(1) - tdSql.query("use test") - tdSql.query("create snode on dnode 4") - tdSql.query("create stream if not exists s1 trigger at_once ignore expired 0 ignore update 0 fill_history 1 into st1 as select sum(voltage) from meters partition by groupid interval(4s)") - tdLog.debug("create stream use snode and insert data") - time.sleep(10) - - tdDnodes[3].stoptaosd() - time.sleep(2) - tdDnodes[3].starttaosd() - tdLog.debug("snode restart ok") - - time.sleep(5) - os.system("kill -9 `pgrep taosBenchmark`") - - tdSql.query("select sum(voltage) from meters partition by groupid interval(4s)") - # tdSql.checkRows(1) - # - # tdSql.checkData(0, 0, '2016-01-01 08:00:07.000') - # tdSql.checkData(0, 1, 2000) - - tdLog.debug("case2 end") - def run(self): self.case1() diff --git a/tests/system-test/8-stream/snode_restart_with_checkpoint.py b/tests/system-test/8-stream/snode_restart_with_checkpoint.py new file mode 100644 index 0000000000..9567bbe439 --- /dev/null +++ b/tests/system-test/8-stream/snode_restart_with_checkpoint.py @@ -0,0 +1,78 @@ + +import taos +import sys +import time +import socket +import os +import threading +import math + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +from util.cluster import * + +class TDTestCase: + # updatecfgDict = {'checkpointInterval': 5} + # print("===================: ", updatecfgDict) + + def init(self, conn, logSql, replicaVar=1): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + + def case1(self): + tdLog.debug("========case1 start========") + + os.system("nohup taosBenchmark -y -B 1 -t 4 -S 1000 -n 1000 -i 1000 -v 2 > /dev/null 2>&1 &") + time.sleep(4) + tdSql.query("use test") + tdSql.query("create snode on dnode 4") + tdSql.query("create stream if not exists s1 trigger at_once ignore expired 0 ignore update 0 fill_history 1 into st1 as select _wstart,sum(voltage),groupid from meters partition by groupid interval(2s)") + tdLog.debug("========create stream useing snode and insert data ok========") + time.sleep(60) + + tdDnodes = cluster.dnodes + tdDnodes[3].stoptaosd() + time.sleep(2) + tdDnodes[3].starttaosd() + tdLog.debug("========snode restart ok========") + + time.sleep(30) + os.system("kill -9 `pgrep taosBenchmark`") + tdLog.debug("========stop insert ok========") + time.sleep(2) + + tdSql.query("select _wstart,sum(voltage),groupid from meters partition by groupid interval(2s) order by groupid,_wstart") + rowCnt = tdSql.getRows() + results = [] + for i in range(rowCnt): + results.append(tdSql.getData(i,1)) + + tdSql.query("select * from st1 order by groupid,_wstart") + tdSql.checkRows(rowCnt) + for i in range(rowCnt): + data1 = tdSql.getData(i,1) + data2 = results[i] + if data1 != data2: + tdLog.info("num: %d, act data: %d, expect data: %d"%(i, data1, data2)) + tdLog.exit("check data error!") + + # tdLog.debug("========sleep 500s========") + # time.sleep(500) + + tdLog.debug("case1 end") + + def run(self): + self.case1() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/8-stream/vnode_restart.py b/tests/system-test/8-stream/vnode_restart.py new file mode 100644 index 0000000000..a53432b77a --- /dev/null +++ b/tests/system-test/8-stream/vnode_restart.py @@ -0,0 +1,77 @@ + +import taos +import sys +import time +import socket +import os +import threading +import math + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +from util.cluster import * + +class TDTestCase: + updatecfgDict = {'checkpointInterval': 1100} + print("===================: ", updatecfgDict) + + def init(self, conn, logSql, replicaVar=1): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + + def case1(self): + tdLog.debug("========case1 start========") + + os.system("nohup taosBenchmark -y -B 1 -t 4 -S 1000 -n 1000 -i 1000 -v 2 > /dev/null 2>&1 &") + time.sleep(4) + tdSql.query("use test") + tdSql.query("create stream if not exists s1 trigger at_once ignore expired 0 ignore update 0 fill_history 1 into st1 as select _wstart,sum(voltage),groupid from meters partition by groupid interval(2s)") + tdLog.debug("========create stream useing snode and insert data ok========") + time.sleep(4) + + tdDnodes = cluster.dnodes + tdDnodes[2].stoptaosd() + time.sleep(2) + tdDnodes[2].starttaosd() + tdLog.debug("========vnode restart ok========") + + time.sleep(30) + os.system("kill -9 `pgrep taosBenchmark`") + tdLog.debug("========stop insert ok========") + time.sleep(2) + + tdSql.query("select _wstart,sum(voltage),groupid from meters partition by groupid interval(2s) order by groupid,_wstart") + rowCnt = tdSql.getRows() + results = [] + for i in range(rowCnt): + results.append(tdSql.getData(i,1)) + + tdSql.query("select * from st1 order by groupid,_wstart") + tdSql.checkRows(rowCnt) + for i in range(rowCnt): + data1 = tdSql.getData(i,1) + data2 = results[i] + if data1 != data2: + tdLog.info("num: %d, act data: %d, expect data: %d"%(i, data1, data2)) + tdLog.exit("check data error!") + + # tdLog.debug("========sleep 500s========") + # time.sleep(500) + + tdLog.debug("case1 end") + + def run(self): + self.case1() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From 88973c7464b2685c3d87be05c1d42ed410c4468a Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Fri, 17 Nov 2023 15:32:14 +0800 Subject: [PATCH 15/29] feat: skip tables out of limit --- source/libs/executor/inc/executorInt.h | 3 ++ source/libs/executor/src/scanoperator.c | 55 +++++++++++++++++++------ 2 files changed, 45 insertions(+), 13 deletions(-) diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index 331ce44366..41ffc26cd2 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -293,6 +293,9 @@ typedef struct STableMergeScanInfo { int32_t readIdx; SSDataBlock* pResBlock; SSampleExecInfo sample; // sample execution info + SSHashObj* mTableNumRows; // uid->num of table rows + SHashObj* mSkipTables; + int64_t mergeLimit; SSortExecInfo sortExecInfo; } STableMergeScanInfo; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index c47e14ad0d..4b2b7889d2 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -3255,6 +3255,24 @@ static SSDataBlock* getBlockForTableMergeScan(void* param) { pBlock->info.id.groupId = tableListGetTableGroupId(pInfo->base.pTableListInfo, pBlock->info.id.uid); + if (pInfo->mergeLimit != -1) { + int64_t nRows = 0; + void* pNum = tSimpleHashGet(pInfo->mTableNumRows, &pBlock->info.id.uid, sizeof(pBlock->info.id.uid)); + if (pNum == NULL) { + nRows = pBlock->info.rows; + } else { + nRows += *(int64_t*)pNum + pBlock->info.rows; + } + tSimpleHashPut(pInfo->mTableNumRows, &pBlock->info.id.uid, sizeof(pBlock->info.id.uid), &nRows, sizeof(nRows)); + if (nRows >= pInfo->mergeLimit) { + if (pInfo->mSkipTables == NULL) { + pInfo->mSkipTables = taosHashInit(pInfo->tableEndIndex - pInfo->tableStartIndex + 1, + taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_NO_LOCK); + } + int bSkip = 1; + taosHashPut(pInfo->mSkipTables, &pBlock->info.id.uid, sizeof(pBlock->info.id.uid), &bSkip, sizeof(bSkip)); + } + } pOperator->resultInfo.totalRows += pBlock->info.rows; pInfo->base.readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0; @@ -3314,22 +3332,20 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) { int32_t tableStartIdx = pInfo->tableStartIndex; int32_t tableEndIdx = pInfo->tableEndIndex; - bool hasLimit = pInfo->limitInfo.limit.limit != -1 || pInfo->limitInfo.limit.offset != -1; - int64_t mergeLimit = -1; - if (hasLimit) { - mergeLimit = pInfo->limitInfo.limit.limit + pInfo->limitInfo.limit.offset; - } + tSimpleHashClear(pInfo->mTableNumRows); + size_t szRow = blockDataGetRowSize(pInfo->pResBlock); - if (hasLimit) { - pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_SINGLESOURCE_SORT, -1, -1, - NULL, pTaskInfo->id.str, mergeLimit, szRow+8, tsPQSortMemThreshold * 1024* 1024); - } else { +// if (pInfo->mergeLimit != -1) { +// pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_SINGLESOURCE_SORT, -1, -1, +// NULL, pTaskInfo->id.str, pInfo->mergeLimit, szRow+8, tsPQSortMemThreshold * 1024* 1024); +// } else + { pInfo->sortBufSize = 2048 * pInfo->bufPageSize; int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_BLOCK_TS_MERGE, pInfo->bufPageSize, numOfBufPage, pInfo->pSortInputBlock, pTaskInfo->id.str, 0, 0, 0); - tsortSetMergeLimit(pInfo->pSortHandle, mergeLimit); + tsortSetMergeLimit(pInfo->pSortHandle, pInfo->mergeLimit); tsortSetAbortCheckFn(pInfo->pSortHandle, isTaskKilled, pOperator->pTaskInfo); } @@ -3341,7 +3357,8 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) { STableMergeScanSortSourceParam *param = taosMemoryCalloc(1, sizeof(STableMergeScanSortSourceParam)); param->pOperator = pOperator; STableKeyInfo* startKeyInfo = tableListGetInfo(pInfo->base.pTableListInfo, tableStartIdx); - pAPI->tsdReader.tsdReaderOpen(pHandle->vnode, &pInfo->base.cond, startKeyInfo, numOfTable, pInfo->pReaderBlock, (void**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo), false, NULL); + pAPI->tsdReader.tsdReaderOpen(pHandle->vnode, &pInfo->base.cond, startKeyInfo, numOfTable, pInfo->pReaderBlock, + (void**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo), false, &pInfo->mSkipTables); SSortSource* ps = taosMemoryCalloc(1, sizeof(SSortSource)); ps->param = param; @@ -3383,6 +3400,8 @@ int32_t stopGroupTableMergeScan(SOperatorInfo* pOperator) { pInfo->pSortHandle = NULL; resetLimitInfoForNextGroup(&pInfo->limitInfo); + taosHashCleanup(pInfo->mSkipTables); + pInfo->pSortHandle = NULL; return TSDB_CODE_SUCCESS; } @@ -3491,7 +3510,10 @@ void destroyTableMergeScanOperatorInfo(void* param) { taosArrayDestroy(pTableScanInfo->sortSourceParams); tsortDestroySortHandle(pTableScanInfo->pSortHandle); pTableScanInfo->pSortHandle = NULL; - + tSimpleHashCleanup(pTableScanInfo->mTableNumRows); + pTableScanInfo->mTableNumRows = NULL; + taosHashCleanup(pTableScanInfo->mSkipTables); + pTableScanInfo->pSortHandle = NULL; destroyTableScanBase(&pTableScanInfo->base, &pTableScanInfo->base.readerAPI); pTableScanInfo->pResBlock = blockDataDestroy(pTableScanInfo->pResBlock); @@ -3581,7 +3603,14 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN pInfo->pSortInfo = generateSortByTsInfo(pInfo->base.matchInfo.pList, pInfo->base.cond.order); pInfo->pSortInputBlock = createOneDataBlock(pInfo->pResBlock, false); initLimitInfo(pTableScanNode->scan.node.pLimit, pTableScanNode->scan.node.pSlimit, &pInfo->limitInfo); - + pInfo->mTableNumRows = tSimpleHashInit(1024, + taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT)); + pInfo->mergeLimit = -1; + bool hasLimit = pInfo->limitInfo.limit.limit != -1 || pInfo->limitInfo.limit.offset != -1; + if (hasLimit) { + pInfo->mergeLimit = pInfo->limitInfo.limit.limit + pInfo->limitInfo.limit.offset; + pInfo->mSkipTables = NULL; + } pInfo->pReaderBlock = createOneDataBlock(pInfo->pResBlock, false); int32_t rowSize = pInfo->pResBlock->info.rowSize; From 3b62d555ab64e9ddefa2891d3bb2c756e0c65584 Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 17 Nov 2023 15:50:08 +0800 Subject: [PATCH 16/29] enh: code optimization for active code --- include/common/tgrant.h | 2 ++ source/dnode/mnode/impl/src/mndDnode.c | 13 +++++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/include/common/tgrant.h b/include/common/tgrant.h index b18eaf8a6f..a5f3ab2e3f 100644 --- a/include/common/tgrant.h +++ b/include/common/tgrant.h @@ -31,6 +31,8 @@ extern "C" { #endif #define GRANT_HEART_BEAT_MIN 2 +#define GRANT_ACTIVE_CODE "activeCode" +#define GRANT_C_ACTIVE_CODE "cActiveCode" typedef enum { TSDB_GRANT_ALL, diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index d46f431ade..b0bffcc83e 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -790,7 +790,9 @@ static int32_t mndConfigDnode(SMnode *pMnode, SRpcMsg *pReq, SMCfgDnodeReq *pCfg if (cfgAll) { // alter all dnodes: if (!failRecord) failRecord = taosArrayInit(1, sizeof(int32_t)); if (failRecord) taosArrayPush(failRecord, &pDnode->id); - if (0 == cfgAllErr) cfgAllErr = terrno; // output 1st terrno. + if (0 == cfgAllErr || cfgAllErr == TSDB_CODE_GRANT_PAR_IVLD_ACTIVE) { + cfgAllErr = terrno; // output 1st or more specific error + } } } else { terrno = 0; // no action for dup active code @@ -806,7 +808,9 @@ static int32_t mndConfigDnode(SMnode *pMnode, SRpcMsg *pReq, SMCfgDnodeReq *pCfg if (cfgAll) { if (!failRecord) failRecord = taosArrayInit(1, sizeof(int32_t)); if (failRecord) taosArrayPush(failRecord, &pDnode->id); - if (0 == cfgAllErr) cfgAllErr = terrno; + if (0 == cfgAllErr || cfgAllErr == TSDB_CODE_GRANT_PAR_IVLD_ACTIVE) { + cfgAllErr = terrno; // output 1st or more specific error + } } } else { terrno = 0; @@ -1283,7 +1287,8 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { strcpy(dcfgReq.config, "supportvnodes"); snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag); - } else if (strncasecmp(cfgReq.config, "activeCode", 10) == 0 || strncasecmp(cfgReq.config, "cActiveCode", 11) == 0) { + } else if (strncasecmp(cfgReq.config, GRANT_ACTIVE_CODE, 10) == 0 || + strncasecmp(cfgReq.config, GRANT_C_ACTIVE_CODE, 11) == 0) { if (cfgReq.dnodeId != -1) { terrno = TSDB_CODE_INVALID_CFG; goto _err_out; @@ -1305,7 +1310,7 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { goto _err_out; } - strcpy(dcfgReq.config, opt == DND_ACTIVE_CODE ? "activeCode" : "cActiveCode"); + strcpy(dcfgReq.config, opt == DND_ACTIVE_CODE ? GRANT_ACTIVE_CODE : GRANT_C_ACTIVE_CODE); snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%s", cfgReq.value); if ((terrno = mndConfigDnode(pMnode, pReq, &cfgReq, opt)) != 0) { From e7593b4bcfb2a3a1608c034afbe7164fb717d441 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 17 Nov 2023 15:52:42 +0800 Subject: [PATCH 17/29] fix:testcase error --- tests/parallel_test/cases.task | 6 +++--- tests/system-test/2-query/db.py | 2 +- tests/system-test/7-tmq/tmqVnodeReplicate.py | 18 +++++++++--------- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 40eb926efb..7822e9614a 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -20,9 +20,9 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/window_close_session_ext.py ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/partition_interval.py ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/pause_resume_test.py -#,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/vnode_restart.py -#,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/snode_restart.py -,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/snode_restart_with_checkpoint.py +#,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/vnode_restart.py -N 4 +#,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/snode_restart.py -N 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/snode_restart_with_checkpoint.py -N 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tbname_vgroup.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stbJoin.py diff --git a/tests/system-test/2-query/db.py b/tests/system-test/2-query/db.py index 6870c59a0d..0246626e40 100644 --- a/tests/system-test/2-query/db.py +++ b/tests/system-test/2-query/db.py @@ -55,7 +55,7 @@ class TDTestCase: tdSql.checkData(0, 2, 0) tdSql.query("show dnode 1 variables like '%debugFlag'") - tdSql.checkRows(22) + tdSql.checkRows(23) tdSql.query("show dnode 1 variables like '____debugFlag'") tdSql.checkRows(2) diff --git a/tests/system-test/7-tmq/tmqVnodeReplicate.py b/tests/system-test/7-tmq/tmqVnodeReplicate.py index afb3319491..0ee11781ed 100644 --- a/tests/system-test/7-tmq/tmqVnodeReplicate.py +++ b/tests/system-test/7-tmq/tmqVnodeReplicate.py @@ -132,14 +132,15 @@ class TDTestCase: tmqCom.getStartConsumeNotifyFromTmqsim() tmqCom.getStartCommitNotifyFromTmqsim() - tdSql.query("select * from information_schema.ins_vnodes") - # tdLog.debug(tdSql.queryResult) - tdDnodes = cluster.dnodes - for result in tdSql.queryResult: - if result[2] == 'dbt' and result[3] == 'leader': - tdLog.debug("leader is %d"%(result[0] - 1)) - tdDnodes[result[0] - 1].stoptaosd() - break + tdSql.query("balance vgroup leader") + # tdSql.query("select * from information_schema.ins_vnodes") + # # tdLog.debug(tdSql.queryResult) + # tdDnodes = cluster.dnodes + # for result in tdSql.queryResult: + # if result[2] == 'dbt' and result[3] == 'leader': + # tdLog.debug("leader is %d"%(result[0] - 1)) + # tdDnodes[result[0] - 1].stoptaosd() + # break pInsertThread.join() expectRows = 1 @@ -158,7 +159,6 @@ class TDTestCase: tdLog.printNoPrefix("======== test case 1 end ...... ") def run(self): - tdSql.prepare() self.prepareTestEnv() self.tmqCase1() From f969579f7f4d2c409baf81121cdcc1321aae3918 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Chappyguoxy=E2=80=9D?= <“happy_guoxy@163.com”> Date: Fri, 17 Nov 2023 16:18:58 +0800 Subject: [PATCH 18/29] fix: reset wrong variable --- source/libs/executor/src/scanoperator.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 79bc38fa76..6acd8589fb 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -3403,7 +3403,7 @@ int32_t stopGroupTableMergeScan(SOperatorInfo* pOperator) { resetLimitInfoForNextGroup(&pInfo->limitInfo); taosHashCleanup(pInfo->mSkipTables); - pInfo->pSortHandle = NULL; + pInfo->mSkipTables = NULL; return TSDB_CODE_SUCCESS; } @@ -3515,7 +3515,7 @@ void destroyTableMergeScanOperatorInfo(void* param) { tSimpleHashCleanup(pTableScanInfo->mTableNumRows); pTableScanInfo->mTableNumRows = NULL; taosHashCleanup(pTableScanInfo->mSkipTables); - pTableScanInfo->pSortHandle = NULL; + pTableScanInfo->mSkipTables = NULL; destroyTableScanBase(&pTableScanInfo->base, &pTableScanInfo->base.readerAPI); pTableScanInfo->pResBlock = blockDataDestroy(pTableScanInfo->pResBlock); From 3241f9bda54678a2623c50fabc1377f620a261e4 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 17 Nov 2023 17:26:52 +0800 Subject: [PATCH 19/29] fix:memory leak --- source/dnode/snode/src/snode.c | 7 +++++++ source/dnode/vnode/src/tq/tq.c | 2 ++ source/libs/stream/src/stream.c | 1 - source/libs/stream/src/streamBackendRocksdb.c | 1 + tests/parallel_test/cases.task | 6 +++--- 5 files changed, 13 insertions(+), 4 deletions(-) diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 4863a1f7b7..007243192c 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -62,10 +62,14 @@ void sndEnqueueStreamDispatch(SSnode *pSnode, SRpcMsg *pMsg) { if (pTask) { SRpcMsg rsp = { .info = pMsg->info, .code = 0 }; streamProcessDispatchMsg(pTask, &req, &rsp); + tDeleteStreamDispatchReq(&req); streamMetaReleaseTask(pSnode->pMeta, pTask); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); return; + } else { + tDeleteStreamDispatchReq(&req); + return; } FAIL: @@ -450,14 +454,17 @@ int32_t sndProcessTaskDispatchReq(SSnode *pSnode, SRpcMsg *pMsg, bool exec) { SDecoder decoder; tDecoderInit(&decoder, (uint8_t *)msgBody, msgLen); tDecodeStreamDispatchReq(&decoder, &req); + tDecoderClear(&decoder); SStreamTask *pTask = streamMetaAcquireTask(pSnode->pMeta, req.streamId, req.taskId); if (pTask) { SRpcMsg rsp = {.info = pMsg->info, .code = 0}; streamProcessDispatchMsg(pTask, &req, &rsp); + tDeleteStreamDispatchReq(&req); streamMetaReleaseTask(pSnode->pMeta, pTask); return 0; } else { + tDeleteStreamDispatchReq(&req); return -1; } } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 6ba5529049..db045bdd2f 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1362,6 +1362,7 @@ int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg, bool exec) { if (pTask) { SRpcMsg rsp = {.info = pMsg->info, .code = 0}; streamProcessDispatchMsg(pTask, &req, &rsp); + tDeleteStreamDispatchReq(&req); streamMetaReleaseTask(pTq->pStreamMeta, pTask); return 0; } else { @@ -1600,6 +1601,7 @@ int32_t vnodeEnqueueStreamMsg(SVnode* pVnode, SRpcMsg* pMsg) { if (pTask != NULL) { SRpcMsg rsp = {.info = pMsg->info, .code = 0}; streamProcessDispatchMsg(pTask, &req, &rsp); + tDeleteStreamDispatchReq(&req); streamMetaReleaseTask(pTq->pStreamMeta, pTask); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 34b4677235..ab7951bb92 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -283,7 +283,6 @@ int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, S tmsgSendRsp(pRsp); } - tDeleteStreamDispatchReq(pReq); streamSchedExec(pTask); return 0; diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index c23483fffb..9699386fd4 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -519,6 +519,7 @@ void* streamBackendInit(const char* streamPath, int64_t chkpId, int32_t vgId) { if (err != NULL) { stError("failed to open rocksdb, path:%s, reason:%s", backendPath, err); taosMemoryFreeClear(err); + rocksdb_list_column_families_destroy(cfs, nCf); goto _EXIT; } } else { diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 7822e9614a..ca1cc704ae 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -20,9 +20,9 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/window_close_session_ext.py ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/partition_interval.py ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/pause_resume_test.py -#,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/vnode_restart.py -N 4 -#,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/snode_restart.py -N 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/snode_restart_with_checkpoint.py -N 4 +#,,n,system-test,python3 ./test.py -f 8-stream/vnode_restart.py -N 4 +#,,n,system-test,python3 ./test.py -f 8-stream/snode_restart.py -N 4 +,,n,system-test,python3 ./test.py -f 8-stream/snode_restart_with_checkpoint.py -N 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tbname_vgroup.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stbJoin.py From 96112fcd25bf969efbf5ecf48955fd730e21a95c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 20 Nov 2023 15:24:01 +0800 Subject: [PATCH 20/29] reverse --- source/libs/tfs/src/tfsTier.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/tfs/src/tfsTier.c b/source/libs/tfs/src/tfsTier.c index d4f228a537..911fdc52b7 100644 --- a/source/libs/tfs/src/tfsTier.c +++ b/source/libs/tfs/src/tfsTier.c @@ -112,7 +112,7 @@ int32_t tfsAllocDiskOnTier(STfsTier *pTier) { int32_t retId = -1; int64_t avail = 0; for (int32_t id = 0; id < TFS_MAX_DISKS_PER_TIER; ++id) { -#if 0 // round-robin +#if 1 // round-robin int32_t diskId = (pTier->nextid + id) % pTier->ndisk; STfsDisk *pDisk = pTier->disks[diskId]; From 2d7c4890da36ed62db39693c3250f2f9ef6f9ed8 Mon Sep 17 00:00:00 2001 From: dmchen Date: Mon, 20 Nov 2023 08:43:12 +0000 Subject: [PATCH 21/29] TD-27416 --- source/dnode/mnode/impl/src/mndStream.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index ee990956ff..02d401d924 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -1394,7 +1394,7 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { tNameFromString(&name, dropReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); // reuse this function for stream - auditRecord(pReq, pMnode->clusterId, "dropStream", name.dbname, name.tname, dropReq.sql, dropReq.sqlLen); + auditRecord(pReq, pMnode->clusterId, "dropStream", "", name.dbname, dropReq.sql, dropReq.sqlLen); sdbRelease(pMnode->pSdb, pStream); mndTransDrop(pTrans); From 905dbaf3b4a783151beb16b1058f2cddd2c4d302 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 20 Nov 2023 18:05:48 +0800 Subject: [PATCH 22/29] fix: update dmodule lib --- deps/arm/dm_static/libdmodule.a | Bin 628362 -> 628282 bytes deps/darwin/arm/dm_static/libdmodule.a | Bin 23792 -> 23792 bytes deps/darwin/x64/dm_static/libdmodule.a | Bin 23576 -> 23576 bytes deps/x86/dm_static/libdmodule.a | Bin 615154 -> 609650 bytes 4 files changed, 0 insertions(+), 0 deletions(-) diff --git a/deps/arm/dm_static/libdmodule.a b/deps/arm/dm_static/libdmodule.a index dd2afa9037ff005aff13b38bb04fe31418c420bd..fff594360670a1f9d9d931e148d3dc3395864cdd 100644 GIT binary patch literal 628282 zcmeEvd3;pG()KxX<_r_|Eeiz5gb)Z2NJ5ZBqD)w1RghifO4t+>*-;QA0TfY0h@#@2 zAS$ll4x$%uzwY~TuM6(D@AraN<*TZvdeV~&$bGNx_xk&U)93e_Ic3nyQ)W3+&KNOu z&VuQ4rye(J%2bj#@7P&W7R;FHkR(j!$(?730S9ZfTi34AT(>Sg@_Tk;wxo7x|DneO zT$dWfkLx_(R7ob4@QrXxhzb{Veh%ypux^M*E&63XOe9J9+JtqKuS-}@`TB$nly6Ab zNcqNuO_Xm+n5=vXVXE?Jgw2$1PMEHI3&NJlw<63?K0=tOd~3oi<=YTuE8mu|o$@(^ z?UnC9*irdh!cNL}Cd^YlpRhprE`){3cO^VR`EG>WmG42=Q~4tadnsQ;c$D%-6ZTd< zN?5FXAHu%M_ap4D{4s?*@HXXdC%i-XI|;Wazm@PV z~ZhzOVcTgdZyZ5#h(me?s`F@_PwCQ~qh1@GIrNCj3VEZwbFs{(Hh7 zl>d?NC*}W1_%G%65$;$1XTpCg{|n)-%Kt`qKzS!Xah-oDwg(73<%5L2@*%=n%7+P) zl&?)#NBO#h^^~tq*g*M)gpHJMOxQ&Eri97Lrx2zppGMeB`R0V_%C{hFseCKK4CNz) znaZ~&%u>D$VYc#Z3EL^3L)c#V4ul<*&n4`nd}qQu7ol<)ehf%J(7ct9(Dg{>mRiI6(PL zi*UE{j}Shp{9}ZVEB^%Hlgd9u_&4RBCVWQuX9=HE{&~U|lz)-%CFNfxd`0IQuC?uXCIzh4<+G%HAC2%U-T%5Zzyq;q0$? zl=QL)R}i)%%%Jp+gjs}ngzX3mo&7u8Iom5b65pNDE+ySwl-7~bdPVnV(9l(Mr~1QD zL8oqXdv98z6CG!ixoY^f$(oQp^l5`@GUCoOfrvvNI-VbZRdP{pR*Q@SIyK>Tp?SJ^PW$t;-~vHujG0XDoKyr)g}zi%K7y z^}jBDYV$5{!t1{-emHKCYHfEJ=kEE8p99`mE zP3^9JpZiU=a=(o~XkBcN^@Dsw{XV_UZwGizkN9=*quTEs=r{GTvWc^QW2w`#q6znz zetBK-nxeHN{Ul7(&xULl>u^6+$MloZc)Vr&OrG?W^%L}!=l#y!&i49TC)pu=?H%9O zjiu3^(qHx|_qVssdn(6ec-~Y}8u#~$Urm1>*Z$ru{iSkMF=duR_LaT7Gu7F?^MHHH z&I1AZ_2@T9zdrqj=(iUAhUqtnerwZj9r~?%OEkM|gyU?TTR~;HPVvEHw>%Iodzo!R z8pRt*W0~hp3C#~~Ycln5FTBnQDUH?y&yT3(OWt4Du3cmo z=aam@vJkN&oor{`q-VBp8k3zpDTZ%%mxM=}blMZjL0UPf`Gvb)5+I<&7M)&w=WfnpJ#wAXHo*(dhwuPk-Q9*WTi>QUUSe~X+| zCVS-{(D{n@D(cI6ht6shya!M^%e-!$IfB-Ww7sh*r}Np6&tw(f&^qbJ<@Ef*`#H(> znWwjP29&*=^GRRIS!ck?Z{UBE?#daAbgljEnF841@cJM-D)VT4^d(#S(%!WsqY2HK z4#F?Y7#VS#!IgQu&Piv0!!bzCX~xzvwzW`gCFDKXwzUiE51@HV@qqQ{EK2bOdb}@3 z>Aa9x(yU8tZIQ0Cq}E7o%jiiTxDDz9*TrYsz5hNS?HW6xzhj4-3%HJI?NfhNS5lp^ z_SH7KO&ZG)JbpuR)Ngb)7r*iOeF)h!#PS>S#&6xpZ*;b=7^Z%sGqkZ+{Kn^QKIe0K zBEJn&zfr#UP4yD`jdaCt)Rw7F`oL|l-zY!kH?BwLn)r=%4q}J+jq}-WT+W_rRA+ zWU-gi6Zvfj<@0=_eDRy=CG;EVI=>8fh1+6%u21?vy!egs$#37ueB*rEZ>+<9({lE_ z7r${n`%Uw?eVK1mXUuP!XSZqmHY8d7M&}#x+c2_mB>%F_>Nn<%--eOiSjs<1{WiAB zUhx~rVlSsB^4m$2&witP@tf);^c(4l->5CtkNJ&w@f+pG{KomV-&lwJrsZn*jq}-W znjiBU)fw}f=84}}-;Qr|UgbFJ(AiJ?Hi2xM%)e~2`i*(xw+W5CtkNJ&w@f+pG{KomV-&lwJrsZn*jq}-WnjiBU z)fw}f=84}}-}2iKj-k3=ReVbA=(7)>P3uw|XQWrCg7l_RFr--H3~TD4XCKL1;=c;WJlU%K1^EX4kML$K!l+soNrLFta}ze9>JFDfZb z_A`#(yeF_vHSX8&Dd)3KHQ)Hub$0cXI})66tHzP_EuVhT!^}meT-xRRfa6~;s*__N zKW|a2-{k&>^VufNH#X5+>1pXW&_ZD>E_{zvN}9o+xWxZ3rVRIkf^4puwudaK|6$TK#@ed_;rQGTX8 ziT5*iWF=vy<1~KXbBEBq)nGZBkRID>;$)r9>f?y|m}5D|$|2+OhK_i zd_0os;BhBR=wr@jA8US`k9E(F`IzbyAJaL)_DLn31!6wt7{xx;`otI9hV5gmhtCso z$5ySrn2$|el<)BQj{B9M-h@7;J>sx@OnSDDJ7zhEsZsSY$9;~g6q6BK$;T8!ck=n; zRB1!vt@wCyO&@oqxWVyQe60DlkDV6LOE?}^Xgrj;O!abnt)z2DoS#XD{CrihL-$Uk zLv~BsvDnJ(*nZY}xNa{kdP(*A$j|Y0@!5*~%zaZ^W&E_>C#v0L%3P*BCFbWm>i;jr zukblK(}`X}bNLyX%TLR>hxFK9GnYH2sh>;K&$Fm6^XPY0bbrNJ+%NL;V)FAyvSqRQ z`7Ek~=Q3gSxh#I>e4fjiU&GI(RBtKOMduvxGu1189$C%Lq!aTq>4=}TPVon~WBXa_ z;kw1o)#{7++0;e(;%Dw#f_fABdER0Ane^iPTu=R6rhZ;QJ~*F#SCF4p6SAMzlAp(t zEo;@!tEmq5GhsqMb3Xf7^K1C|QmXe-%gY&Mt%vIt zKUb?S=4VqE<%^%WZwcy6=;!kf)6b+A=Vv}s=pMY2_G4Zfe2(~!&l6e)?VH=Few-(6 zKG=EUH91c-Y`pyC1C#Tg`E{`)=ZYUjy0=uM@>;X*TL*;6UV4_N{dZ@c$#dw;5M5N! z#9dU;F;Kqyz_OR=*`7YCfjiO4tN@=WSA`(D!f zn&T<87a(1ZGh83PYvOYY+fW<#kYYpiHs0g1q8I+IgziAgta_;)>L=GnbH-`enChBS z*38^@M)x;`Dx<-YuWLC-wT~4xP){ z=G_>Bnth4g&&t}By6GM^X8(s|Ki}&ZyZ2;O`J9x{2R=t;x#beiyVIQf_kk*sf$ljxkxx{AynKbuV&%}r_zNgo@#j&lTFW&ob zeX$riU?87KYCMzuFtYfTUl&Jj(HOcK_o9Er^H~FDe`Ou2uL0(2RBfk^*2=mu>xcJ3 zvPa_NO{Dv==iM5!m)5K6EPomA1fBK#K9O}3l|5J)-ba32{0L%w{rFhV@9WXFtn2-i z-8IhVIqXZZiDKHKidURP6}P!HWBd)`E75!80Ig?|J=!M%^y|_u)vhrX|z|H}8v(mwj$T~d5?<}U*-<-3yzecU5TaoFrd5`R3moMJW2J8Clz z^O?tn`Pe+`o_Lqcpmhcf}`PSicm z_>9*3cbNy)0pueie6yp?;6%a4ZnjtU zXG&-LdCxrPIg0GhX8XxkI`(r8rQ4i!C&%8yA5xd!_n0|Wfw?E|lMYos#ZftDoAX~8 z$0hBP#1WcfvR@r)-yt3D3&#b*xN}`BCfoat_Q%-B^EkHtdmo>8pQ#hwFEIe?UiX=* z&%f;4>GX_mJLcc-9LsjvYebK9X>CS(M)>nDw68AX{g~Gg)x&#CiSm4pd9Zp*s7yrl z%pO<9{-bzWxfAc#u=n6|D-HjHy=N!w6>;%=i&Op>;zd;)JB&T4xPKULW?FF*2u$ z>GK;E-tL}!_NI5WIDhkbc8k4djXA`gHROMN&#GZR`C=mZ)b>T9zF(H0?^4&UvNCQItjrYMsny746l8fldz9s-{}=*E@)pTa9?RmGlF=&;{GzfhCciAd;O&M^}pNC?&?FS zpFZzn=8tWcynp?p&o*?fNHjm*`t|qc2gjJ|WqAkT`EhkH&6{myweRnNQa%(+nF z;v%{ar}M6p@ccvXaO7N5b&i)DJ~nq=dL}TFK5O|8Y7t>)Hc^sQe!Nz$CTl- zYKMJZA=@R6a-R-WuY=D89(KKaw`tp3OykMt1^$fp@5X|T(e38*ur!YHLvvPsqJ8Kj z$UjeC)eB5KHTu8(`tN^z^WM(VUw(L@H*1WZv|#4^1@yg^sWayf8yOioeddCNoIYae zl*qt|^PPTECrzI?Gcw?~(VR+pv*u16MuU3H+^JLhjO^#knmu*q$k8L|3I{(UhxVsj z%AY!A*4&Booso%?S>2gVJpW9YN#CFCJe|K%D?g`9oIZn^nm;Qtck09`k?bjvNsH!B zofkP}?yNH+XHJ|uFEWkO=S`nEd1@qkp40ERei1HMtz31@v*ym4d1_?xw5gL%pSOVa z#OnG9+L<(K%A%@v64h^M{9gTHPt0g)pY85HV#Kf!5fc9V`O^b`df-nF{ON%|J@BUo z{`A0~9{AG(e|q3g5B%wYKRxiL2mZhCK%Vw8dtUZDxf5ZA(qgja~vFoHu>me7XdRoH=!FT)hX= zn>=Fy6<}Tdsj{P*{u#3;&YwPM23aP$vHoedJ8ReYTIk5^dAX4(v*`9`=B)X2A0&To zEi!NZ#JTeq%tkX#Oi#1r>c~==KX>BHd68MCoHB3f{K)iqbfJWTd2}O_H)Hywywj%6 zJbn7id3k5entS@Z*%K#E%^THk`b_G`%!xDdsQ**v&YnAc-qgH#b0_D`p0VIm))l8q zO6*U)sX5jAea5^~Bhx0%oHBztVU0o!|A?8hXU#PuAY*Up{3CJ0c>23RQejxG0O|hm z&_98&&kAy_SnJKc#0J9jhYkpW0GF;Q`EL=4Bx>O{r*Vo)-}lgh;q2pMY}@lWGf-7) zxE;MnLIdHPO~eL+K8=8;v>zPH>{t+EbJ-9yc2QO88fJa2;(PiHv5u=+>#MXbXSP-t zrSD@Q98M=^1kjTt=4(@`^Jdad{3kx@FrV~Kt~QXR(!Z0rKvo!ezd&jT=~qZ;kbZ*{ zfph>;j<`L^aUteIbXDvDF`!};Vo=3_5PcPgLky`n4r18tM*Rz9l|ZcR7Ku0wVqF)V znGLa_it`~hcAu55E`*rk%J5{B(Z`6{5veYZu9vl3>)-IKgaK}me^cTqpX((&8 zqxR@s8`*x9jB8p_1$zf0QVB6dU%06hz*UkT5z8dqJv;ydp?ggAI>@)D(F~Z<0E(A{+LG)B<_YzLFP#R17Hx zQXkZkul;E>VuCTV(I{q3e;SR36sF0*JrUP78pCXDs@=(Hbmn`pk_*v1y4z?Gnm`8| zosB$nu+b9ap@WUqBM%*HbQL6Yu+i01#hNq|o5g^}>D(Eam>h~_y7qP&=Qr-#k;Eo! zSmXY)sUX;6s!|7VP(;aIreq#^)p#J+rJF+I9*xJG6{~9;k2dN$p(^#ns=CfMb#>Re zPCAxS8b=z>xGPr4%qn(PRmt6^WQLZUeF>Lr*yOc;nvzZ4sAAs?(Pku;GW0IoP^PTKrNl@(Q7368@e&l8wsJXLgF@2`l>!YZk;-u;_nUGq+DpFvCMh9} zCs%ADJ#9R>l3itln{2f^ZM7{>OLiO5o}D5GfN(fSyw7}FV#H3*No6C(c`5a4ok9~> z;(kiQTC7AafY`VeW@i{;lUg%KGmzB~Vrs2R>5X+DD*~~Zmc_o8lCEX3@1?Y;#S=A< zg?*1iwmFa$McEcwGxj~oMMK!=NPI^c!p=q_tisL~6X&O$HOGk+#l}YBSQ=X|OE)T# z@1vpdvUFo1d6MLB3m8LEXij-q-^OIN#LL=OO5+)*1DK{Qjq0}YVzio{DG9;8k14~wwQcWBQF1KOxE^7rakDd ztCDF#ki9V(`Y(nk4Dw`>S+#_GB(qAl$=qhM$&;Ma30#wjxTz(W1)NKNjB+9*1o+fRTumS=2YHhrDYXUnxg=$m`x^=L zkDVb$yI4rs?OxVYJZ*B>SFwZ;9tTkLD%p$kWwIAHw8>ta zFO$8vp-uKIUnYAG*}hEn;)XWai}PiQhqYP)BT~FLU#56*zD)7re3|0K`!dCg_hpJ$ zRpm<~8(*e)@xDy);(cjk<4dYveVI0W_GBj|RrU9&uhTqv7|1FM=(PP{7kO8vZCT1h zZw?&;(922juAUU{>Phjgo)quuNw%vedGFY+o)quuN%5|p?0qIPQC&S*E7EB@SvnKQ zTBx>vxP;6V+gGLJ2YH`??FGTOX;Ki3nlL868 zfh;{VbP3iI<&w17)2BEog+c5JYI|3$SC?E@$>lYsefeY{nJ<0WkkUKUj8-f95)H-m zB^rwBOEeVMmuM)iFVT?ImuRT1oM|+gMMH6YiH72)d~v83`T`^RgyQfEOrAjS0lLZ%iPri7^3fLh2kF7=sp2bX))p4up6@0L~ol zw3Kb8r~5Ot5)ShtL9)c+6!vHe$BL8|#8KWXDjE*+u8uNAZfD8}hr3D{u0BP`4YZ@B z3`ZH>$im?Pu`*G2l9ZVyWjMH`2;mVSMfN5tFLqapV!0@C*hvwhiae#D*vGwB6dx2t zo-wdO6rs}BeM3~<5fxsO(mAc*822a1{kP=SqESj=Le_AoMh%1AW|R~TXAn(xk6;}k z{N$gaCULZZcBwl+w2u|-lUZAcD)Q2nb02TWa_;k;RuS5uc`;D0P(_$=;A00FBkJ-nqAriVxg;Sc=;-frTNS0?M@G0^*<776ZhY^0xNjqcuOn#1uTPfz5hi@M~B&BdzHZHQ# zN$b#;s5~&H-S-*g>GNi@)`60$XUXPm_`Iu+kIlRA@ED=HE%bsQHwq(DVh^=)#!JuQ zOgdaRPJVjFos>(s%k&|A?N)$GlO`LnGlinS9xY0;I5_xIOrp%h` zq(_k#K`n!jm*XUzOf{pyIWG_!4!=gUwaeywL%%X#Q@A@REK>eY1QO6n(i^$2(NRT4 zEWN3_J53COn5tKx7_sy;m)&EIp6ThDi=$_HD?PxqK-o4bMj*CVi2M3{dkRE-EoM3LN z)1Owi!_hB5N`vMbS4%l{`-eM3!h=-$UTEe!N$kdOmt9NR==XY-*muX zanRMKA)0IGG$&~ol@Euxme%3|ZWv?9CCpVtrD$<4X^ztvhr@h7Pl1dk+A^t|uh75e zvg|la0c5Xoon}0(!{PTS2WjP61k&X)iCi}j4045%*y1x8OfJUvqTz5~sw7M1U&~&x zhrO0Z@xxvlF4eYKg!Ga`gXQeBR?Rk28SzG|<{VOKF+fU(#EX@JKnq9>F}qu}oJFOw zBds!Cj+KtYmCod9V^5{6THirpTdv8MsMfXRdGxixcAPL@=mMc<3O!5cLZORB!^ZRfPLVq``;eg|M`%dv&_w$Q;AI@&_6Ep)Pl zJiA-@woqUTU2LH%PZDF!5w_3`g3QNAcb5WTI9!V;UQg&vKxVV!DM`i#*Ae!m1g<7< zDS;~pTr5OIzYNngGpioeAgyF(pDL}KK{VN2K{SQkn8Hn@@M6Z0XSTy2$=;NiV+-wV zp#!fw?OJ9>TgV+xHJY4G)=*`3wuL-f$hU<8yS6U2P-qKXZQ%%8=w=JuZJ~!P^t6Q| zZK0Pf6xqU2ws5p9^tOelEfm{AA6w{a3;iHSj)&gifEmQ^M}T&)~%DM4jKHc;wi*pa!Ep)RtqR09F9_DtzE9)JeLNc$qTUS)5F7yYfk6kJ0d*q!)>A!VZL0`GK&Cemha*UgPaV$ZGBJ8%@LMgHtxUHBa+sJeLZ6j&L z&0sb=;$}987`@ianjv$uFi7EMvuDk7+%}LfWNvn{XyUBpwoMm_PoPIox9Mqsst$+e z#@cGP3vIPu9BZpXTw5LE+RBYrNnTUC!cgKr&VpTgBCO$ zA9Dt}S($oH%NgWyh8d=u!7Ad)J!goFQXmUA);UA<_=Hnj&ao~pcXNu%IZj2K;&O($ zW2hDLiX!KD6?6CmLd7^!&K)T%f$+2k$6l+-rJk_G@_>{x(!G}Y=8va@h4ia4SB?`o zquptgq;sNuMmY)6=Rtu3?&l=PTnsn?-px#(X&ZxxXZAoKa0%5P$bt(3UV!a0E(inz zeR0-+e)}#I3Zf(tX>2&4M{%Q59Ot<#Wpn(9SXCZlzp3~|fJr4)Y@QX`j!@RV!OGKQfx0lk{p#vyhW5%_Le={M(3ztN~%Nn@2Q7$Pvs{kzOQ{77t4} z%o8abW~WK?XNXqM16LbHXo71~Z{j?nf(I|%J4G*@URp`C^13C$N;Ahe6n zLZMxS9wD@w(C$Ke2<<8KNTI!i770B{=+Q!Z3&pXdSol6d`wHzRw7<|}gboloQ0O3` zgM|(eI#lSfLXQ(VOz3c-#|s@HbfnNxLPrZ7BXq3LaY9cJdZN&igpL>b7ojH$oglPC z=tQBDgiaPZMd(zarwBb&=rp0zg`Ot#bfGhZo*{Im&{;xf3!Ni$uF!cx=L=mR^h}{= z30)|3k8dX3O)gx=oX<{h2ACfZlU)Gy;tacLhl#)fY5D1w+nqx z=nkO|34K`TUxn@zx=ZM8p^pfCROn+u9~b(B&?kjHCG>AXpBDOz&}W4{C-ixtF9>~6 z=u1Lh7W#_NSB1VN^mUn&)Z$b~`@ZRLO0xlo`@Bl%84+sHj0m6VJKy5%BKwUsRKz%?1Ktn(yKx04?KvO_6 zAO(;LNCPwjGzX*uS^!!CS^+Wu5kMxOH6RPn29OPC3up((0kj8n0CWW80y+UY1M&d* zfC4}lKp~(j;0QoBKzBe7Ku^GtfL?$iz)^ss0lfiHKrx^Xpf8{wpg-UkzyQENz#zb2 zz!1Ptz_Ebi0K)*o0mlPI07e2v0Y(GH0LB8w0Zss%2sjBa9`F~y$$$xf62L^jB*0|A z6u?x#DS%S}(*V-}rvXj}%mADLmR>L05=0R18xD_3b+k$JKzq$oq#QX zt$@1#cLVML+zYr5a6jMyz&5~kz=MDtfQJAN1O5uw3D^bL4R{3bDBv-`)^uY0)9p!t)uhR{-tw#+7#}DnSw4SqQ6SLKYgk}R$o262sr7VH z(M_8=d!7^MESmHT?f7hUgo?>iW>D+R^IoOoBujvoX zJAPbi@A|lS&?olyG&c##do;JUHuk>HpSR7N68O+RrHiTYBk_Ff4&Y-ggNu&9Co0xO zji0L66yjbL@qQ}snTmKJ75H4mTqu8`VpoV?s#pZ^D;4`e{947K5Wi7z6vS^;9IqYw z&cBj7wmpzp@470g1?q)%_4KX-&F5(DAc*Z% zJQiYydKi=AA$F`cj4q=BS^Oab!dw+6LhPi<(;;?NaSp^h6&FFwug9+jysTx~hXR!^ zP`QiBJ;*;Ja(vbVrv6pq1-m%CMax<+L2)V6oAV+-wV zA=ef<*+ORsGK02s)#VsKFLLv)rxa6rzAe~y__*x%N$cu6VrqL7Z+D^1**-5r=vR%OhTPU%G$#$73 zcGXjD+fM0kb^BCXZJJ%d>9#P#7G~MPY`cOvb~|(JcIMe-=3B1>a~IgcnYOUVuKH}d zr;F{JrM6IN3uShhWp>VTTUcS+cCMXso}IJO7S6YYRkm<}EnH~Vw%QghvV}FaaIr0{ zwQE~v=ak#RCAM&>EnIF3SJ=V^Te#XDxod3UT3fiz7AkFFqrFaV;ExKKNxq5SaTwu7 zTe!*Y+|71vx574`-yW6qjGdx#Z(Bfusq1!oc(>SfZMEyV%NFjnh5Kybep`6Ju5Fv0 z^N{Veowfy!*g22d!eh4Zxb3JX?3^cU;VE1An=L$T3(wfXv$pWO-GvuyLtnIWUb4%) zYzwbIpm60lotg!E>Fbb+&i%#`FRvgv_a8tzvU!iz`53(3#7mCO1GqfBM1tl(NO-N$ zc~Jc#&P&6~mQLxDsK$=mP^4jAOU{!*d97OYuc|&T<4uuY^U>)kn)iy`!9V%2+~obdb3}mkb3VB3I6OH<<{j{Lm!4ZwaPo{fBZ0n zS(C#~0o$QJRTd5pA==tyFV7+lZvy$H7TzE6u%&PYY4)etgQ6YFr07a1%41m(E6Nt8 zut){=A888Y-bE?l@B>8gPVyyIW53`fB$v{;BkAT*CYRwT@tPQ8zma{3LCPEqcLtcZsB={(t_Oul|bOQn_^jAEa_%5T+? z^yIA8Il`msy4m@iTl3tM!kzW)yh?3&UeNI!dD&VOq_+IghgigES3n~di!q(s&!8H! z!v(wFF*O%Fa#4gebleIaeS$Tb6g*xps%;lMfhCEy3!dZ$*Fcso|AMFXlD-Uim-H+u zQlG}xp^sst5wB~guy`T~a?K6u3Lg;a2@MMMQBmQ*Jge~FixL)A)#GMRt~6gbW+SoT z@MfZz@DCH0!cALKv~cW3N@${vFX|_gA7ROh}WRhN36d zFXn`%M3dbJ(G(_>!d`>QNd=e>7kY&!_wS?5E}Vck4QCgY zK*B&5PP7NQa1tcU`NGMNFy{-WK#IteD4fcn#(br%@Dxbxk#{OzQv|X)K$^yrfr7ga zaXQ3ZDxStS8T6Gs6;J0)guV`>;tXD@^aaBZ%ASE;WUq=d*&_PxPe}VdYZQ4SdspF{ z)V`+cbFqDC*XKdP0xz83o?K;Sb>RXoM6((#oyiTG1zvd86{N`_9GMFTQ9JxKO^Ayg zmfUqJp8b@F8&q7((}2DMsp1ldn^io=ld5!^EL_@FMA`@`PSf~dOiPCoo79r*%MKSV z>lf?Xa*bZ9wxSc|pu~A65*rSe5XHP*KpXx8^B@(jLsY0PuAd59nr|~{PvN)RIK6%nU`5*xyT*ao|>WDE)YkZ zC{>je(%`fwJN;qgj&F#`Cv?7aX$I$YY)Y*|J$IrQsZu9&Ac*15>&)dkFKm#PKQdOJ zz}~R)xv%Ewc;lp2Kr--~Tt`R1iYW`n%) zilu1s2bTl4SH4ZsMs+;ms#-JtD0r1F(6 z&poevhssyEJT1NQ7gfG4W*3IwT^W1sY_}ETYxdc@$Gwl*?p{5o$68YM%6Cnm|1~>d zx!2r7m2v1XipsBYW3Af_xE+U>?yeG^W*fYGf%Gm=UP?2C`-u_jeF)FZ9crrT?J^%M z-@Ky~l|@}%Nx|}`OhU1IXD3*`XGdwT+4JW*ULQ?((P;E_7fK!bOhP|#XRv(VPOY?m z%)e;l7#W{n`MsvZu`cf+!SXkElwt(ViTR-0jxn|yzg zNK<@VxqP-m51%ww@0VT~OKf!SJI-co74>=)(uPpdr24UZ>-kp2bQ*fYT? zzBJL~diz3BE!sj^UisDuMVo}#JHhFbQ2RI4yw_-cOY0x*sXVxQ&Ku!4eJ3qA#qlb& z$LGLXm2l6iMuhX8ItQk2To&PtWnAyMQ|ZsmZG!IJbNX=yy}Ow5PjT8wU~KAbo;ZxU z)s3=fFqRf|5N6#{r+D6pwjOxz7H>Fdms1vRBSq&4Q?ayW0J~OlRxRy$Yl3RrR*2WYj2T7ziVmHCSl%O+Gj@sW%ui%HzfbDrA5yQvu7z;k+9y& z-mj{cRw3%$`#beMYwG=$>+PGs1n=^m>T-`c=6D6UR{1Ce&KT)?n5W4ghcgn_i#Xhfa`8|;jGi0ff2NRHujx+UTzo*M ze1_BWE8R0}#lVVsMRjO3@#Hx``2jli(WF=wVagxafvsmB!q$%CjIC$aW8OR=2hOfn zAYw@kqDKUWcduS~k54Fr_LI`Y=dEQpZ_!xEaf=*Y zvj&`HHkVdEZQYK2*m3CiPYZhueCORt4bp*FQrJ(!#7Ar74^9jdf2?s?huqIo7==r< z@DH)VrQ&21E_FE_g-fNd?6Yo}IF>5nWr2!L)Qb3I1S?LoDmu~SufL)2NiM&I(qdz@ z7@uuXY>ZWGjJuhZ>haSZcdRV55vXA-YS1c%n*PVC7;059)aAFQD0eK(hQR~17%wTb zJkTmOPeAa>92UU6>SoOUMJ>JHk_iB~it9${1egw~x{nX=4WH3Kz1dd4qbZdZYKr64KeA4*I)j?fO@*e1X|HhIK}}pr3uQWkn&i4{E|;PvQX_0w^Pqcl9qyOuM|uuf#edJZMXY~PO-ty1b7#xC_0K z<~c*1W0MQ329U+BwKca9ZBa*brXH|rjY_(;bPJ@~adl1*!@Jj^$=aC8^yumoNX0#E z75dD4Rb6rB^rH5)$J~CAuf>m2{W>+;K3Y3K{YXvGs>$sys?5AR@eQ{GCJAn?B zR1G(*<@#yI>u|^W*P-r>rnPoVO)IW|8wZ%2YVI9q(%J5aG(4!vV2S}%&C>s}4zIDf zLp5E8zO))?Ok#uRzoAua>Nw4$w{wZm z&mP5qvo^+mRF#%*_=TOJA8^*>8-7VFZ7nh_y165S+ZDNf!*A6z&sqDeYlJtoQO{ZP zt!o5hC6})Lv!Mf0Dim4A@S8P_TG$#f!lqcWYqmy=VC+oPLK|M{^pmT|I)?A6Y21g1 z5!T0=UGr_k2*%DtEzHkG3kwxl$M8$4(((--iCMBH-|)uDMD@?k)cO^re&x#)gU;Hr z#~mZh)>ebg8cH>iu`^K{n={bHRz=n^{5DM!FBxHmHq0X(F@mu)Q44jomWEafhHqMx z7BT#$SlXJ1;fVJqfaQ=Brk#!9Jtfo;9)@2$YtPM~P)@2&OSedAW8I91wY(>^F z{CrKL7G4Y(VOy-(H7^E?VC+oP!jDO4VV@%F82%ScqZY~nMwk|Bc1>Bp2*%DtEo`F$ z2LC*y$U27qwyM>9!yAhd)%8#ab?s7Q9m7ASY3vWj2={8EzzXLz_;J%CL^%)!+0&d~F&2KNwP)t&P$sl(|MQ zRwinJV>-&1`p}fwIke#z%r~41hOedgXS^b$DmuTh^1P+dPgrrhB2ug9o0`slLzRDK zDDF~3Y85R?2LG($tBN@(RXR4eNz=QgBDF|youaMFd|f`&(a$xCn-upcBDac;Y61RS z#ZmOU$v@*2Ayv^JykCn`Jje5#BZLk54JD;-;+r3`+uaAP>F0aV*O?vn!C$_J?}tS(!#bugx*O%*vpb zU0XT7oj9Ol38u{`GzF<>G_ds;xD`7-CdRwtmmmw6&u^nxcLg?iJ# z*wb!2W`Ne`V??~?+`pi`_9vr^MZMqL#yX2q8!2-v{(BHpOKHzYGHYzM z=gvW{_QrIo&0HN0vCDs#O@+~qD*&4Pop!(D;&U@#bxUCG$lqN&>1>Ri{1(s`T^-`inOu5YwX6o58R7Q=^FxT z!FsLMxxVayUP`$(=J}Vno_nd0YOQzu+A?+OUZ#aoE;n|O4}xCO6-L+J;Hv3uu2kKm ztEzNi7pR{pb+$I+?cXjD)w?60Dc+XA)mn!ctDtvx;2KS9bFJ}l)uV;c*@OMNl?dL3w7M=>V!+Z zMO9mx?sUA>G@N^za#|`aSr{*-nt8iXZFfg(nDg(9u|-=<-&3}_z8op&zxvyun^XE- zrYl|xdmjsWo$q#i+4zHA!98&{s3j@)y1Jn^DKyM|Q1r&sGV{FY{V}a9qt)6l55)4e z87_6Z=}y{%hHIXtO-6R8%Yt6kLnb%789efw`>;uR$^EPArNs3IIv<?NpC z8?9cZcH`U<^g6v_=0~$vV~Y8&xf-5YzOIQn1O;!HoPsws$Lso5yiT`tIM8eMcVu{N z{-GMQZI;FQ-TiH2Zpu5c{2uR`#I*Nf&fKHfUg&+J81#<)z@#4cq2Y#6?cVT@AbPR= zttAb-EN|__@hva`%-PP%`NZ|oKUF1i?Ox-QR6TZj6FxKZBh8#dHGEy@e(vgF()+9X zh24_(nfqm&Tp9R^4BQy_+Qh!QCE=c`gm)z2-B?1<8}W@+;f>N0pf~zklSC`cJK;MM zewy{Oy6I);yg*;?4}OnAo|!Mu^*3Q3+gIxcnqA(BKdM&zPL^rr*cOjY7!r_nSVn{aLkw-jtRw(L3eersS!=7;f6H z;JmEgGMo3+Al zv(Gi$obwDfcctOxoo~1Ws|*7rC(y}PrtP9X0 zFE`bvUt%h2`T))R)b?6qYK}5#msYD|eXKwmG9{bJTo&+~TdOzD-WHb!yoRgk&&Nxu z*RF$=>vf8g3T=t}bA>i!y4JL#nv}d=|C?QtlbUP%up!>qW*1Rs)9JTGCujt{mRFi; zy;hy|(KoFfxg_7REa+ui6;q2`ZJd>Pjp=)nE2&Ff*0nLEHrK_n3Q&=kdc8?%R$=sz zMf+BzN%3;J7}MHsjI*8^XyR)g;uoGs{Pyf@U#^$^Zo(G{mCw=mM++}cL4H*DK1_pr zo8vS0C;ig{e|q3g5B%wYKRxiL2mZhC0A6R=yFXHDrrSE;52k6S*W_!Kt*L_)TeD{C z|2I?Qgd~@rdfbwaH26o#nMn>;q9Jc%{X98J@tI0WgwA3)v{2|Gp>l5-S}gn$q2~x) zDzsE+ncVY)mI0OnRsiUK*`S}$d4QFG^8u>>7XU5Rtos%Ls|vt0st=i{H>7g1#AO6BoKNS(qAF%gtQCr7!n_c^aS8r zz>k0x@}kIJBM`a@($#?L0TqCa0#ySUXgy)Va3_?-p41bweDK5XE-5{P_7sX?K0)}C zg-#GUMd(za(}d0zI!EZ4LS^uM8E{{Q+LxjBF)%&`#;3n!NIyQj?%xgRKEVBeX8^AN zbVNRaM1RVZi+>IHC!h=-S_8NOPyyHsxJw|k9nynWJdTm$=9!p)@ydDQ~@@wyLh5c~|z6+2*i=(T^&@OB?&9%+$j_3LHbS$7wq+{4s@Vs5ckoAf?(nVAr&>JEVs*+p zA-|JYj36xG*3Xxe0-^G`dcRQku0oFxiYMZOt?yoAY+cA}Fb)V|R;oEc!>dSIL|67V&=34fJH{pd6L2J2;`_GK&fvC*1fjaTFT z<%qA>05$?P1MURe3wRK)3-AO0@3Q@uA-xH}m;QZ?=bt0-TY$#%Umz_<(AAChVo2)& zR{=E0-vmiF;kzN}?u-4_#}4a1jl35D8uH(U^Z{Tm;A?>HzPbUFdE&~IfQtZ^0ImVt z0@wn00I&nF8}OunOqEorFH>l1p;420~D z?EH}Ig(2)9;`|Weu|(exPEit5L&Z{3BBczQl=LH^n25tbE6KCW^)n@nXNi=W zDD-e-QZ2ydC%G^SzkU8XIPS!l! z;1E;O*Rr0amL}3EtX2EqF6%aAtxVBQxEt(+_3MIMm8fA0l1fExDve+=N}Q8aOdNY) zC3yIyqK!v*ACK^d9OXleYuw&shEN~cFT+UciR21M8vs`W^q6ujBz?-)L(3*4-UzrA zpf><|lG%bpJwP;v5%w-5Q+mkqRV7S!J!ztitHOwmpTf^1WCcX ztg&xM9uC7YLq$H=1C3ZE8rXQNN+8}!1eE-!*j6kQ_V*PY5&szB(Ic$1!IFYKz{9 ziid<#nHM{s$3%P$p!0YSq>ltFlQX3XxG_t3bOHltZDuU44vrITrD!Khc`4pT?y$bfJUb4AyokM(5;c_ z1|({Lx*5`D0qN7BcVj%B!X4H+)vLFq((h8C8lrV0);{aAfyU8z+vBa($O*&DEna*a z)wr+~UAPON;WYk+ZyWM7s>VNk=wnC1M+<$Xz~h&A2eHE55oJx)yz~)m+0Oiq;tyHP zdcA_(8J8(I|Jf%$tgT`x)lVp545ABGxvVo^)|Nj}bfySJ1hr35>hk!B6~9@K@c<3 z4uvv5MGx~2`wQkAI)=D`nKwgBM65uU>~s3Us%}V)SWQ?*ttAfu9I+H@1-4)p>?u0f zm!iUBKgKE@xa;>bg%@wdeclS;(i(Oti7C-R%G3<2*mf~t#3TN}rnC4+KgVSssxY<0 zYd*Y&X@P!N-IgAn6F+n4E(p1s)$NJf`b$!eb1E3xB-O z5kfI4n9^8;7$NJZ=VM#IXvy(Qw+M_}iPSSuC`NIz@L0w&qCOTgmI;>hbkV_*IbHY} zLa}_X$Y)8)Y@u_6V!_W7e!kEJLeCV6NM;3s1eZnUh-QK#MZ%bOu1vpbXRGs(s7I{$ zuz0A4zxX3o!e_wvvx}a)5E1df0tIyO$M4fWAa5T)@60y^q!Tv+ z^fl?Nkn|$`SxEl`><9b;H~?5K(NsDv+~L0)lro6C|7px^ES5JBiddE`yhONzzT-rc z!=BJe^fQD;gk}oO7TQ*5JE4ehh<=?U1yQYLJnJgC-GtVRVTfLK+=_~3u~5XSno$ao z2{8%L2yqAz2(brIXPnf7-NBAJIFgn~?j)gzE{H3LD2OEzMR-&~H_K3lHUn-0+z!B9 zSZFJxdjR(W@UkPcO(Yp#_|5j170X2f{yp>_@ki|reWd?=g&zLC0PkZ*_@C_p7?l&H zU3+Zo9buy6+9PzZ_MdijpKUKiVxrk9|rXSMeo+`K+K7MT+v6v_@|cm zCq%t|iw~v=pBMGPMjsn@qP<;!-GD~`@sE^GB2feM-ymt6eg=|;>*pXn4|oCaB0#VE z^cAND_E(Xof&EQLe+Ot_*GH~HJOuckpup#V`1t-c67@Okdq^7Qaq#nhLZZG>-4E$! zz^?$kY{bWBWHsUphd75di4z;neOQNxZ8(|PQ4Qy?Vrz{dp5eTOlNQccSW7rp;Y5YA z)MzP-^OGIP4ic+;oON)@!R*(q`eyVm;ru+*#MG;cyHPyh{EVNN@iS8ApiVxWb&sPm zoo9!dQt?~=yC|ody-tR`NYp+5RqVl^3CQ^A$HMRlF#lEw#J;}1K#=zJRkVII>3gXB z23StfpPmF(0xkqxEFjfqO17MJ;=X`^X@g*IZ<2=&>)0^=*x=qzEIdZ&K%tm;*hn$` zu-%<5xw7f`X9$n!G)wqH&7i}GaPfCwI-l;v;Az0pNTnf4W788*IgHq(F9YylL_hv8 zryqy71=B;%bn%xpakn%wwmAC8c5H=$Lyu_>@s<4m)<;q>3N`(YB@h3-_ziFXu-q4WR{+ihoCjDbAihW!E&Evb ze`m)3?>*pRBo6;LV;?UMeUSO19&Kx`NrZSjsxFcdIa}yrp-Y7R;SZ^Lx3n4ya}D5P zK>QtUIT9}cTnaeUldC?m{%<_19x8k#+yLXZJUxrW?{(YJqxdaPZ{#uV#I8(_I#d8FnG!C}y*iU7+N3s_fo(uZRG{Oe*-`^DBT!ZH{Io*h@ zsnUi#eOloq!l@!RWd@(vwe*{Ds>l&$%!eq>hhH&IEp@~!Jm_Xhfi^<1dn2yHQ8?Ve zIf&(m;&26`ICjqCqy`+>&?$)v(yBN#WbFkZEB=S9{WpYK3uP-s6xx(E8HH3RziXUL%PB*lEhHl zLJ!3)ZRstBsOFVIzSx9Aa~I*^-LAsptb<(>2WabY*vFK#uL$hX!1U}V_5Amf6cZCy z2N(lH_!6mqqR=T!rMEIw{RoSp)=cq!&Hltm$l4u4m}9aeLmfn4_DfujP`mSBPW`7z=Yzj-gwV8_CSmb1(X+mqRfVkP<$IS+_oX*3fh?^V#?Prn_Z3)&a z!WzZg=q8q9Egt?n`A;9Q#!D-*()<$PCkd4i*DL2~k|L|lm-Xh)lavKQ&l0*+sCrf1 z8Gm7L72K&8vDZPW0Bi(o0_dq)FATOI@h*TqiD6r+KKk}YX#^Gf`zybPY!yY&r_boBIul5k(vyAHkNh`+?vN7y~6 z>;r&4(CU-y7m_H(Tip{G9hJo)GTeu$J3%@(S!lxh{2S32rjZoyB+8?iTFqH8!b-)8 zjKp;H#q$q=SW30A!(X-1%kTQEg|FjqouLTJp-iSe<{1tv5*ejcq_z_c*olaU9TfW| zBK9cJkzV*?gvV(dSBY@QDUyCFQ>#ztY1QmY7A=wOulz`fnt55G4LcFJu_lTo9d_br z0v5uOnUXtOsKgMd-8!C$EHfT|?alfM6=FD6E?R_7x=3wZg?1B)I0<`-B;{zKQK5Z= z%8n_mp$`%Z{IQa!@k=AWPNzytAiVVPH8{n^A^rPQ{xi^g4)8KSmxUfzaZdH$N8W~@ zG<&5laHoLf2s!8B`&W{RfyW_effT{{#TXSID(^=dI{>=?j|243{Q@MlO)n&WLn7`1 zef{8}UNY!M4fV75`XPM%2F7)=Cj9spNB1C6Km4Izy3>z)>_(n``c}V>@e&gM4)_r8 zDIos!)2~R>Z{o#&co#1$e7vdf@AAZ1_X8dV=*M;SbGlC>@dbc>2k$LNdjKB-^kcJM zLHYskBj6{%KLP&|kdec((7jKuz%ej^huv}G93Qtn0s5PfB;aZ=@Vp56N5mZ*C$ItH;5401h^4! z6X0gRX230gTLHHLZU@`}xD&7iuoZ9@;BLS@fO`S=35Wv`1F);Xnb=cDh|N|+6zvlv z74h&S;jvjDHl8dg*f>grpD1*)&?!PMz#`BumRydhcC~=C&|0)|gcb`uT__$eLQNZ*8Nm30uSTaS#JmR5n5Ms{9U2Gz|at*d+kT)&&L3LTks>KYtcJhzt=<3 zcf*yCHUe${B)nt18F_lusPB#MKw`oVglTBF7b^DwG*IZL#Po|>@!!?e&y4*a?%q5w zwydldt?tu_=zaRG`n+E6dl&aPO*fS$IVY(MC!zbaRH`aD)v2V?Rh68iTioO%m7LI- zNL9`N4T6X`%b+q3ip=1M3W%sU04lRMfddYxsEFcl0f+bdeZOz-+LcPB19;ybFWqOa zz1CiP*WPP=>zluCwRwrzfQwc+Qt}|6cHHE)?bqwK+*|GUR8!2CJxKh;t(>#OOrEhh z9JOQ4j$-%jvGX}Q3LKN5HRa^a>6}wG_kn7sWlk=fmO0gMvzZeyr(RCBoL;$)vva#nubk=q zWxb29;diO#EU-IUy;qIzSMfm=A6D@@Dx7pFjJ4+v)%eFMM9+HuTs7yQWgzbPYc>9@ z3U{skd(}Rz;-6Ifi;B;v_?!yoyxsBZX{T&ec~+8VsP;@1&YXXjYRXvB0}xUroKB$@t5x=e4@2dEH6(3dchbmma_!HIs zOvT4lxVZ5N)&54sCsnw~-h~f|8$F-Vwa=;eHx*C4MRTO$87iKs;yYA)w~Fsm@oW_? zpuR}87pr)giXT$(BPw30;x#H>tKtnRepJPetN2M3Z&L9yD&DN(ttx(A#V@G%B^AG{ z;#XC?Q^mVg{DzA6sCb`>52*N%3O6siUD*xFAJMfxRPo0u{zS!}sra}Gw>i79*{#fO zUUs|kKd9$Fs`zIW|4YSZRs5?8w>dvea=+W0-Prt0HGYSR?^5wSD!xy}vsFA-#Sf@> zo(k6`yhyc|sCc=GAGX4H1hKgPjZk5Afr(g7$yxy~xt2=Yf}|bs40qU7F`k|5kysz; zUM8()!HzR_tlCkWfY?m&1DC8_T!9!*aRp*OlTzEN^<1-K-HzfAjLo+15exk_cFn%B zt23y$rT@ozif4Mbor`k{?EJJH&)D%+J3hjWx7qP_J9Zl*aaY|&N}4S3TN!(fxULa9 z?_?^+tzCRr2ZQO6oEmp=W{#(>mw%an`$@V2c*e4d00aeJmi>=Dn`us#Po+T8ySt0(96|F3hI zw=Yf3W#^n}p2U4?wvy{CITKn#Htsf#g7j_z&&lDm8$1d#2rBg@Ul432*1KUYtA9aj z*zOE$OuMakt#x6)q}-$xtTE}7mF}`#K8|ZG^1%NmgJIgK9jJ_@)zfZ3OJ;f@Py0R=2lISPcSu zSHZAmO_ZzO+HK@JzNzzT1P8+QdW!J8Jw?`P5AHJTZBG;}5zZ0K&{v9n>h}dN!tY&h zx;<;m-c}IRrJrN{ZmTIVhW12JLmM5de~tY@#~QV3dV9@?u)d9VkMOi4@f&u(YiC?7 z^OBpa@f9juAL9m;H>lB-;4UWqtr}fg{5#b?t-_^6*K@dV^t(6PFML#mi#&g-+Q(J6 ztmC3YdzwVg@#BKUOHbLu+H)tmnBnrptJKB)XgwQB21m+ z9J+A}3CQbNvU|nVB{9@xYhSgaaMQY-Z`e_sfZ*(S>ff~f2X?$_$7^=HZb!lB6Ycy- zc6`u|PqyPz?C7A|uEB8=pzCE^hxn6%%Qva`X%()Q`8m~&Q|O;Sh3|$$SJu2o&wQT> z*V%kXwck?Fu1<8@yW1K+qAsq;`2*ElXY&_UOF;0Ka4f9eW1}KC6^;<*NYa2L%qDp( zgIRlu!EUHI5u zgI;;DDSvKH2lV!SvprLK%-&zpjLH_&>)`Q$*2Ic1CW+60-9Bc=PSSFxwRaBbW2`-E z$4NU1A=n`I*dVvr6uO~>E`#h&443T5-OxpH*dJ#-SL~QXBb^9DQ}BTG+_K}g9VJ$D znTVclXrvngaka86fQMYlzD;xF+U0Lo&8;1-Wd3e7x~}SifIuA}}LU28Y1I7)Pl zbvtHs#kGU5>#rR_9%ESAvR&`}G4=ct6@RM2 zZK`b;wa1m=f2}S@ayg&UwZB*4s_{>&_D?EYL;f$SxuX1ERr@y;|E|LIrW=eMKduw@n;JiThu%djrzn8qU}p>+UkVx8?`uYr zb8~;|tE`8uu!Lyp2u`@{$R55{-O}YIuH$T5?&ol%p;#s^{!XvVT)lGO=#{n0lcDys zhlRDdY}mbW)aaF|t5=R1y>9Vxam?*e?!4t@DamN|h(EfSBhxE!t@mT9xuMF9O8;(+ zJ#Jfi>M6TH)}-EVS4~Ex-tV+p&-bcuqtdfflWnQ@`MNG8>Z|VkFlX~B)nyf;- z@;>WzYRugBObNf0c@F?X@3z zhKg@f@k|xpuHrjXe5Z=yjaCcRJ_y*TOp!V z5=sr%SZjhB^`5cEbz)gz0O3?|g5m-9t-pzk_AKF>)7GkI-D>A{zxC*#hSsWH)7Doa zuP}+Qz&Bd^<9M{*694UMSnt)Z+B59BzSD#(t0%rNasPjdpO>K7c0J&gp2SyJs}|Ll z9lFb?kX`>uyKBmh`jR<2KM9_{&w3`mHp!)*0NZya9BlKH5-#q>y_+!A(~XC3+Pm2t zcjf1gR||ZzJ>jW0S<%khKSPb}-28W{@i?ab`_#qh%8|VN|3)6?1X94O^n$LpdyQ(Y zzH_gt;{=6vZOM`HLc8wdD6t^{mj9^?shG~h(*+_)J<)?U*X`5lqL(wBE@e4O;IflT zJsun>ai{m~8oXao;R+qs!MM`XS^lGV{lsh9itjXH_9h0$dgturJ9xgtlMdgZwQsG7 zUR{1qTU#bw^phb2F2gVL9qd^h7}Il$1?-T-Qa!?=kFfUJ>?poU9NMF;^T zdh^5I-R-wEmyTK`+$G+=%l9SssObspw`~rS0b$yr%I7Sp1~>ZNlNjIRZb&Eo?bH0# zL~rr@oue(-PL8C0S@5i{7Yt9@cW6E5?K}z9lGQ||C8Z@4mOY(xd&!<)E3@Z5JAbSl zzsZiF9Uo`M6+2$G{z$sx*a#{m{4u@{O8i=8}|Gs+VM$te9(?hv12(; z_^m1&a*k8D9Z8~H;nq%@+ZArd5#{65Z^vmI{&!Ggr@yYDJJK-W5~!1V{J z9BFm9Tl!dJI?s}MWA=}_?>NQTXVm4h zDn6&8U7P0mvu_c<+^#?S4mG;lY5P_s?tSWUQ7)+aUIQLH@;u%Cd=)QH(eB^Z+h+V`mD9^ZW}(M53vL$vGu_}0v45$W8;L)mn)}86jcV>5`$^T@OZHQ$xwC9rIB3s5 z>YBUG{UW z%~N%`Q}4H{(S3V8f7f@Z(LH^?M>Qq&>T$nc?FQ20?!V7bm*=YZeic8U;s;ecPsNdr z!Y|Ram#T2L;g_rC-ornv+ACDF`x3uOjjvYG?pgdgHNIYj`x(1d%-xNDOxJ!|#amRo zT}8V~@vo`TU5Z_C=AOmBu50dK{9e`E!T9~Exr6b^(^&n!?mN=!_)m4sosa)gwZBr~ z&c}bPn)@LCt!nOy{CBFkp3hy8|4EJRk^C>JxnuHYRCA5szp3``D%?%^sXc-^743<# zlp3qYotD2{UEFv1J5_TR=I>U`J(<5(HFs!!mTKE&2VbxtH?~s^-ql&sVKo zbLpxCsh2Til0*PCKW%e;%8L+tco|Qc#Dd+s`xn-Z&UH}D&DT*7gYSBieFOk z4i&$w;#XAss)}Dz@lF-*Qt@sTzpmmpRQ#rj_o#TUiub8_zlsm2_@IgpsrW4wA6D_( zDt<@B@2dDc6~C|IBPu?s;ty2(p^86J@y9AYrs7Xj{HcmRQ}O32{!+zXsrZD7zgF={ z6@RDVQ!4&m#XqR{w2FUJ@lPuLS;c2md{)KhRQ#)oe^c@AD*i*oQ>FKQnu>2xadOT2 zak}(p>tWAP@dGM;P{s39JYU5NRJ>5di&VT=#miK@T*WI?yh_EZRlG*UYgN2X#p_kP zLB$(Y{HTf_Q}N>}enQ1hs`x1tZ&LBID&DN(Eh^rs;^$SoUBxe1VZwIZP?H2vDfAoG za=hXY%T)Ye^+U;!b;#6gN`-oVMOlEJ+n2g^pBOQfB3> zPr;57;J?w%FWb?+&Yqi4msmZaSV;V?e({Xmd#fE2@8==I$uCQ&yB{NIdyx(3l3iW4<9&9#-;SE(qz2}3*1lp#P5zplYarL{yqnfV-b~l6 z&y(#aJ&9{ZT)pn9=wFoF`%5ZZ5%6oO$s*n3ih$oxqbsdlAK*?l@6)ySt8lfz2UTm= z4t!XRCsz`sFnczBVn zwd)^VrpA}6aBYNq1$yKw(DN!?b8WRQ_Fho`0a?IBkooSzNzzoMPp#^{^xLE3Q0w>Me#_pQfT+CGxFmY!7!n zQZsU#O2l;`?Me_=d%Qru@Iot$&*|i9#JZia*Syt^kFaCTj%I)9eH_nwSL}G1M|)P1 z(X{6#I4SW^0lRi(>>g!T#k7g*k$tIP?Xn<=b2$lzA{&vtV~;qI1AC5h8ScmMGEqLa z`k$QYyH@^4!uc&iDM!-JXX=`3-d)N55;eY5g_E@(vYHKHl8+S!tKkvzchNqpXvcdJ zLp5vXV*AQ=F6Qrf`sDU6|5-IkF`1aR?W;K1Eneo7zf9@Nh6_^h=XQQWc7XU*9ix2&|ARD4FA zvw5zj@<`6cC%v7)d7*A_21k%!pLo@VLJYDS3*9aMJM1g0lU9yd*WHGk9D(EIh`a0u zI;9Co4Vo2!n1Dz!f>zh#>|TxSs-3UdQBr_b-4$!uv|~Ga>nU@2zjU;Gwu8l6h=J^(rXy-=+L$6fvMfx5&`uS-+%-7a_){c`r+IwmlC+A;1ju%stU@Gs79!G!A*MPi0#S2xuNQFB} zoxl+O3f<5S46jn-aWHXY#EKije_XdaOt@LxQC2&I90?jfuiK9U4VR+ZLE}i7+Kub& zGPT>zehwf4YVJ^2aP!4`bnSyGjx$j_|Euh0U!dZJ zDqf`G#VTH+;-xBHrsCx)en`cO|1W#WXGNv{L!|Ag54Xlwt9Xry*IHpPs9x>XS~t_S z%Q7c0l(q{FE-F0pl)Zo?2K&jR?n&?S_0}b@;~6{J5>75qk3vYY4Wb8$C23FBosY1m z+-Aqy?INeFd_{tbuxtO-?$wt((#{RF^qjSGdt>X@ z!B1H0S6VOe9UVPR?!A|IR+pz7{SGJpg|_+87s}@=NxSvUQobg68!0cl(ey`i4D9_h zmR-J}Ct$ap({}8fbB(=bzT5j5MA$DSVXK>JH3>8Jj-5N9rv4|U?&j2ozXtDr)91e8 z%TqVu4=;#+H=Vn>vVU%C^YXdJ*LNP++-aPQ%(3|Du(5M+?tJvQ>KJ`{XUEH%O@G+& zeaZRXCmIF3PUBp2 ze`Tk!x4ZxOe^q1mU>|^}sovh$Zl2?$hjxaa_~Kp*5V+ag-T&eq(p<-Eex9S;<*T@& zICJd)(Ftpf8|?se1l$64+n-_}-QLvs;FZSqp)QT>fsOi3eSh=tOYQHiw0GU#^Daas z53!gA);F)$*ZvcRjc^bOIKB{Qh@gG-p2mrhQ70Bz-P#0(`n~Zb~Obau8AI- zJG{a^t5@o)4>S(8Ro!jWHn({@Slvc*ZLPkJ^B!~~{>5o;Duqwp+|}t+u~fj>msnqG zd>Bq|G{XA+{?=xrsa|*v2;u<#?X)!po$Dh#I!Bv!BmP&y>U&q;hNu~Dd8+D$d z8yfN2I^U=_clVm%>h3=8#0aeHZ|v#P_DYklg&Qz%_PVHH=yn^`8;#AKb&iu&3p@NZ zJDu9ZMA&(AYd4yOmHqvd8))T>Tv=)Gp*A`DtHYU9uV^yRyxS=3tZlCF>+rObT5D`R(Pw09cV}Z@ZTH}^4K1dq(LA`!p5XQE zbYN?5Y((|w>F5=mes${-z(Z=E}Z< zyUhUJj6aXx2*cJY38w)?JL{XAb-T2E88K`F>QXM?l5dh|clTmv1!djcY3e`9y7C zXiN>Zj0ZN?fX%_^s;yDqs;}~G@hzhZNvQO7#R6?|?Ow$~fY7z;G^>I6BC$&JSL z%If~E&wH|J!(zbApTR)t$JbYPn!ehjQ}nb`@u=#ywz-d2#yLJWK1avB-Nxqiq1MP_ zC|9-;)y-`%t+)+9)4(X#qtDdKZ|+`=-#og?_q6knQ>0&K!^+y4PhI>VfM0wLc;oq| zeY#&zg;(@*yNTyuungnj=bEdWEucO!#W+I%8cx=@p7FKO>DRVSc~5#=yju!tj6bffQ@~o zHQ2=5Mw;+6okxrn9;>!+rMbJkxf;O~dJ*qNRRb}Ql&J2P6Z9l%!NP3V_vx|i_tdWf zR1(jy`oN)ER#uzBrFsI{r22J-N&HHyj=7B1Du5_DO|)WuNiny!3}>N6IccxHPc(-Q zN6+>K^stdL_=;dNqTcw4_KUc^y}`g~rG+pKTCd8<-rrq~u&z^2v>r6-gk7BB)3Ekd zAMhDzgzH-?8;<-ON)uhUqBl20#C1mmB^b5nl&|ct!_-#jndXc@|U1Ay|1`vLAy0%?3 zyaKD!>OIx+99p4AXs)i{C2>wlyRx-GT4%HzC8+pH0|Uyp#)qvsn!suOdVSUT2-HAv zAXli9rVxw?s`$S8&ehHR-JNZ~sgFG-aeZgSxE)+~CdTg49i*W3>(0n*?yefn#C+PR zKO;WJG{onahp0OyDmsmN;Nek+BzLyGZTt$J?`L=?o+UA|)y3H96&qI^eQX*7!k62# z!HHhanmN1Hktd)6S0iX`RdD5?+3?Z8V_Fqlxwf*&K}0jp641*kpK;(nr!hoVOofkU zyukPN;{l9LF*rbVK6`lj1JDW82`+6HP{B+YuZ+|6tz9EyU;uytBGkaCClw<2B+v2k zi=X1KaiHm0d0@ z_S+i(lS;aw%KA#g)$@n(N?Q%EKwH2~1M3Gnh5&uA;_JFMUORYyur`;qaKz`}mC*?o z@Hl*?&d~|&1JQus=dorbw%8+h&Ywah;(>UIQH${OA?E5@Vp-{YAi5C`-)&@#j^K&L zrRPoZ(dj0K7N@HRjnHApR&F&S@Ybyq&5&wO9Elz^D*kVh~mLzwxKsU z{umfo!L4Y9g1_m493S8x&+GfU*Zek@40)ZZ7_{n`5ISX zpMac!cj|ekZo22xm4|)Rsp8V9@tZH)G;`%;7oM^4h?(V6|0PJ>e9KL} zXL|<^cyhY8dCSecr#|wvcg@^#)2Y8$U%KVyQ!n_shrQH0okMxb4&P@jclF+XdT<#% ze`4bv^nBlce^|4l=S$+Ax12il-ul_I_nn!UDP6d4WANI+#vuN;>%8w~9!_D4UK}s+ zS8nRHW^1|m(z%0EjkBlz>)F$G<*weDN4)%=*Up{^7b4r6V7iJ#w@GG}&T)1`l!p5x^F4zFC+ws%4^ql(8 zQ zExo6n^OU>J;jy2*>0!a?!TSsn9?{$Ey>I=#GtYkZ%g)T)hi};!I(Ow9Ug6zO{+7GZ z_suuIYXcv3fHyus>sy|@#MXD-{H`}{+%~gu2LI!71XIzs_x{|uA-)mTL+_Ofrw^We z_uU5q)Vq2w@YPP=J^1WW>0suJ#;SK_=1gg3*
ddZvlk~14KXJ(eqv<=bIPoMkx z!Q_}d{q(`?h3L2KiBCUs;T*pAQxD$5?|s%OJc>^`wXrmO>PtCxryudQrRCGl{__h@ zzi{`3({H)!g=!%` z9L_DwrfSvvWSA@G3*juT6z3MRrD8s;lrM#o%e6u^sAbF5FkhGo3zbT_Vm;G?{dnZy z@YqxzlvFecRH<(`*qhqZ<7TVV;Z$il%+1c%&~rFd$QHx7@?3Vd5Da8ewPJ22oG(Klv6zA|JrP9JY-k?;d;T^M;nP3ns%2poaV!Ga3T-oPY zdT%J~-w*qOG|&15?%rc-@TkD6K=t4wR?hCOuY>g8pGMcm_N8)#axK7NDlrPRYA(EY zp-@>4bJ@9Ep;Rj5j~Ro^AjV*RDx53SCJPJ2U}!j%FH9~>hxvusd1w@>#V}i4p34=> zVRovVDW~fXsOMSsN1Yvjpw&j|71dk5xNFVG#=-H{S1=Q9AcktouH{+R) zdQ@MkRK`Rt%mw2qOhm4ftyU-brI@32{0-tSwXhf#^Q8jyN~PLxhR_!lmgd7S7|M*J zH8;Nim|2*sh1E1ho;tFA$V52BH$fbxq!glIb@yPW8N^+Js81?fs8tC#VRcz$sW3NP zyBNgns;4#^*ViCyH^Z&;YX26W-D<(0t@iZ?>8-v#zVK8PuQb0*MHt>SS2CCz9MbyW zY~h<{=E|1pwUe&%-0rzBU*f=;-iR>fkC>moO@XP5ej^i>*}1R%=6J;iY1}c9B0emV#`K|ABaxtGK_C zodpmN8E0ME-KbG{UfZ2KSWjb&`|+>Jm5PT#n>Kf_-4{G5xVpQ!7TjT`f%`#Nax;8` z$KDx0#=*jG23*_0h2S!jFQ$u6Ti5;|?%Q{#Bb+-VJ2jHM!^Vy6%ez~R=q^4DH$)E? z;>HbwXj=!OFqxwfoW2muEtE=pzqm_d2Yv=SO?^t>-x3T2L;b;--OJ4tXd%~v+tvaI z)2j~z>(C0;8{fEd1_&bl-r#QGPWz^Tpw<3p`%JtTEE(wUpT3C~>%2L67hgNL%WlRQ zkZbFOA9W!}zA=cFzNA0->UjOA^^Cpoqi+L#zpdMQZ=!$l=QsQDwI-w9zxi0a{gd$@ zg(?YvQ;@=<5#`0ymFDi{`p{VVI`QabYTLu1u>jRnVU~C)7#=dSQwU`kWIBLdvjHe6 zujlKR4>kZ#4ZvZxG7IEaoSrkHU{DawU#bumr?Aemb6kHh_c2!vff*O3FNXPIb*4I> z%@vNJyF-~mU#e1=Cc2s|mjKAECrx9y)d=EYaj}#Sv$z8xM4l1WBx9B>kw*ZLnJVJbfZ^vrO>$e1o55NYg&e0U+36sY!Fz(@Vu=%* z3=KNbO%IId%^W3x09B2M@NEyZ5y8%1ER&!Qk2+l}1LcDInX47EK&tqS1$-V>cMb$J zSPgEyHCOhpz^avnYUM$cm%OY@SLUtmWu;uZXmyl*DWk7dP^K_%6gE3I zT?mH2NdO081VhqBp}rcjsytV9sBy2sUmW3#e#30RVS_5{5{7b*4Cf zsTzO{%T}i6L*tw((`dL3;K1Klw18%*Uzq}fe5Gb4?nYwiS7%GH2s013E9c53`;6{0 zGmK{w03aIKxXj)g@g}wMJO+WQaGs(HuFqmAf%_ui9U6tofB#@70Qp_ttZ%I~ z&YIJ}{@K>GR1FM9t$NA0iNRsxIHu=kY5^GdO8SwtM}kYgK0Mw}O4pYvV>-Zq23UtE zQULU@1m-f}N(7u3*AjE@w7>ASRFzB+DDp+hQNSPpIL7%(af#exh!HR1R*d^paVa>n z1ay#Kk?}~R>Z45MUGE6uYcQ~GlVbqi zN?-%{7IWZ0fY3_8$OPUVOS?EXO@eak^~8i3mUfvFHv)458*~*pGh<_u4og5nwn!GI zE9HgxYG~}XW(K%~wy&#o$VX2|LdUEzn-M<_cwa1Yu)6--;KJF$g8k2M22gattUALE0ylgigfvH}&DhLtUF1(3Kd z*T+Tx8eXixc>*4Xu>6OZZgokI_hB-DWPmKdww9`3?qgzYE2Uh%q0g>z!|#CmyPI> zQ>al$Poo*!+aW+b*J_EulOyZsGF+GkK`EpU2=h3Qar{Quh+vVWxm;2HZn}cfrIbY9_`l z)Qb2qO0<;XDD43Cl%{~z!^;PoTkvKLuN-XDLl|Vhd=oGxi?u2^kaD&L)qsRR2*pO> zBH<)iX~(c7eiqmqP)_nYkX9;&XcNMVV>`$$F`C2s8G+_}0qu>0MR%-N?pQ)n$iI(iy+O~d=}CfYmW+su|{Gl#F;bJ!f0kCrqQCm zlg?^}WWmDn)D&5gh=3V?1!5b^)br&_8RFeE4?x-1F?;PRSvaX(QE|-Y4-*NCy`O>* zzfghDI}b**3V2P)EhuP@cROy!I9EMt4D}e48ue}G_X_j zzW6Bex*Rtu1*t(SmdTDq!uU2Y*mXPrlFsgpSSp_dld%L81`1SO?iA0B+ko(Ku~3=^ z06~8urZ=3D1WjxZ;sP~Z$-jjX830&RX?0980Q4@n0Y<k(=6B?I` z#u`!8m8M~5YxQLMRL@tlyjeN#%MPk7Dm?;$I#o7T9!RingMJPaus^J24M5XjP0dPG4 zqxh6`eAy)Tz7~T|Zfzl->ob)LG4S#%XeI_9yf3(ec_<*E5ekN=Saeml%RNJb8L^QI zbH%A5c-VWOe>EPr3fT~2ei14dFuca(gNaKoE%gOUOKDTi_6IAp)LDbAx_;;>;D%}q zP!<*zArSQk>u@Bc<4x2&4$NPlnzhEx(NG*LMU-Y%+gdh-w)=h3*g zxu-sQQ2Z2dYTy~z0m^l_mD+OvExAIq%EsMQxJg>0Bu*3U>nqUF;P;?H>kr^>r3x7C z3aYfhs-d=wYV(Fw7V_wNu%@nL{j>NS>5-t*u3%9YW)~O$IGeqv4CO(p z0mkm?7Aeu@TKXym9vmxhnhPlSO-?97#CEYeTZ4343&sGa zki{pV{x_DGBVh7iTT7~$uTzr@rvIvz6PKE){mgQ*$hBzCf;M}yG?RnI1OHK}3G`Je z&rM${S0Lq~0dF!zeQeFSV=NTtUdpn)*u%hYS$P+2P4iGM?sQY;!tr4Fe$YxtYeS|;xlKc z;pKDb%4V~^Sl>^4Nf|;1k!mD-cb6hCuzm2H+S@PHcOVTy?8*Y*M@I%nQ+oyTkKYF= zc_RG0Xd9>%fVh^)V$PZbW&B9hpei(3Cm0z^$PK@#Eim z0ERDNc(7roy^(xtQdgDA<*N(T`5;%Tl+us9@kqQ05f#{XaGd3-G)!FmbZF||u5Vu^ zdW0Sc)gY35d4fKD15&&0uG{7Qz`KJfY3{BxRyQ|W5E(b%S+TuII6BfP0LZ9!naY-` zW`1aKZGNRv09F9OiE&SHp*A%h3=YcfKoXt@CdifUE2V(~O+V`l`s&sbx2!Dzr4-b% zIq<-cOQ53Aznvjai_b0_JS7iU|ZVp-uwOj|B~Qslmzs z?!{m(-PqdcH&qA+6ctFVRz2I;3U}b17V3xmY}r=Afy$I*Pz;%=4j}o z>O1h`H4JcVwE@qp%Z=btt_r>2B!r;D79ZMq78B_B-UVp!92HLkT|&ex&cT+BVTNVT zEKoHGFPUMgK`Go$14?;|>DAKd?#el-H_2$tby42|RDlF> z?E@%KS%B2uhNrb`*s(@^cs$j-9KvY~=-EIe(mds$Z^M$nA}9soXj=^?roy$0;Y|CX zIg`a-##zN%HsGh;H0@nW`3%)jLis#8y3iuV)rCo*N?-*i$267Z%H^4&lqs;OU@n$% zHCK_p2JjpPai#!JOjCaY?~2*^67fZ{8;tr8-@ISY*6~T`x>mmm!=9CQs-U4uVj8Ys zjfdg(%Hv@QyIm_~a1ir#0G1wfW*nv@yES<5n43(d!W#3NAO;_E*Cx!7t)Ieg`CWr2 zx=G?&{8K~xsr@lX;Y9)idM9cyK%r5FI%E=k(3M&h4GBC%DF~)`a$T?mo0x^gfL16Ds~~`J6Ycd> z2A22gJJ1BSfCYLnBQv%#V@hK8Nk%iBgK?r@dL|cOXb0jY7id;?Q&?rl-N*$7)WA&F z(s6;)%wxHQ!SD!GV)F%P9YLW3xO+$@Q-{!;5Uzb$?88)plu6Ze>M;#917A9xpL~|4cAzKO5{Jh|2cdu& z8X3U@>}xJA%m>j)3X4DDMF{~=zfpXqh@8$XlA7WOVv1-68pLvNV}V}Lt^+maW=#YK z69EMjW`&5|DwsJSwXqRsKs0H?DVp?%UK2LGlIf;OAk5r3 z=$&$rM#fy$C^W2@$tg6x0<85E{DzL>SVji3ASBsZy4DYAKAo=hrI$eL+SNf#PA&>? z#z0)}w>TUQ5EOt!#j0aQ%V|cT$LP}lzb?|!KiTJ6*f8y0j_G? zq7)?b0fev*7^Hd|#^20nimrS#?3H#`)*4V(6G#o5V!LxGwZ02P7fhAm?9@3Rru?Qp zm6lHURz)LgNdmK!ai!jwuw4xPDKjHP!L9r!(tjJv~ zC}hBhBtTgr6SU*==Db+F7o0*jcS;z3DA{zU&TYiGMbS-V3R(=wWChNG^{WtoV+$d) zQE-f#Kc8X_!4BT63~|MZbp@lY{kS-09qpKWoE{g zszsxe&>NEvrVX1cHXs_WgEK&GgeT$Nfrjhd&?q#u)D&|qGqM4{(^81<>~8KLu zY*jF~Lt7+~POL1E*0j;IYGq5)v}mD7)z%pG!Qg6qj#%nEG#enOK!Es|p|c~9C?>{7 z#=xqNj1Qe3JU=**0nL~g9Usn&PfQGr4Gq@^Mn<7tLL?egv*GGx(2o*u{VXh7Bltek zy0t{_xB*x$7QGGeo0|OyABpyMAn8aHufYR={KVkc`Jqt=+Oh0Mhm?z&w!MEzDPoKs^T46PeM8%*4dV$ng02 z%<$;==HU-1D7;N_8KO{ zRis&1-QI&1i>~8qW`;CkSXiLC^xgnk2IB}WmZ{OK23TE7k^&Q_y_V0;q+<#ohZ(AXlRM$I z9mfX~!-5qKu=zSTjZSCaEo;CjprklUo`J}gPOOoY+pSY%T=59A@wgvC0tR7!H}V0s z+A-^4f=rQ_>OpwHV3jNIA+$INEJ$$n2$RL3c_R=(Fkhz@HLxn)4pY1kSL|Xi!1sjmYKCtp#pctHS*;n}@TW zq7D*<9hgX{tESPZSW7>lA4bx&8mPENLx7eGftmmgi*Zx>3E-=)nLl&-?Af#Q_M+dp zTBaz4-yV3V4+DO9oetIW^gM@8R{9M56ax5^C;IdUt-JfNK;Ya5%vTLV=%JnrmoW?4 z5)fD zN?|KCuN>gw1z0M80cMNf(sMv0P%h;ojf&}aTE5*F82NY4(0`_@ST#641nlW_-sh9E zcO*Pg%7@Nl&7vphXp41Vc!Bs1(i)U=n6q2NN>a3VHM;~3&VAA-dkW! zlMC%=<0y63q`-0yJ3*h6ahA|H;SbnN1QJRWlbuRHl=Ou$ibQ6f3an`87J+XWpaLpQ zCj#=wS624rJY_5Idjd;-2)F<(Tk-)rlkmipSJS z&Y2p>dbja?LlYUuFXI!Vql4$ihccr!L+yw9z)!5mQ^P@xfe`o?+C=@G0LwGfEq>!yL z@VFcSPXf_uVr*=Rv>x78Te~-q#t#BSG0Z?T8-+ABF*ckT1KT=|ZumGSo3;R5GiJU4 z5=xH6BO_QSre2K#!9X}+X&M&P<)IahVmIcYV}g!7hY$vkN5c@tZxc;K zkSr8I*v$bolgo(=M|AH@IxJ%?BSaIZl|oDk>DCH+Rpbth!OuhG`H#-yT)H(&+(yVd(8J$+{v~I%mh!6}H3&~AtOk+TycgJYv(BM`KP zG63?Ck)iXbkDtfF90TMJLA?OsD>E`hvq#MJiu*NAY7%0AjRj9qHOc&lp`^?Y4j+oj z2Lrg)Le27Eumr3HJOUZ8kev!h%eU$)>rg$-=OLe&*EZT(75jlI#ARspAw3wEgi&v6 z0U0?!=8S#lV>u(v+Bi>dFV@~fya*RAJOm-saz<6ewwskKRSDh^Y=MT|Fei-lhMa*! zalzbvYtY%i|IzR&vk%%*F=nI+KHh>LCtT8qLhp zog^n=4Dmk}VHUzeWSaf~@K<8UDG-}8+z?7apI_iv@B$_Pg-8Qn($S%MYi&}RCRQ@L zj1jrD5Nu3+XE#_2p(e&)K*<%wLHWA_VHwK`oI@AQ(UGA@)XQBYgootii2^L^R0xy? z`oFlcbx=?5UVc1M1ob1AntR@;fx+s24VEr23I}!k2rbTj-5a%N*&aRN(eWt&9ugqD zR(20CBNjAI#s$tixAQ^<$}wFG!QMw`6HExiXp`xNhX&7Q#6_6~8Co(Js)F;QX554g z4I@mwk6F_qrzL<5X3orfRVYjb#|I}$M3ZxK2)e<*f`=uSOKtgl1?oVV3CZry*GhDN zNQMGUcqoulF^nU|(nIRa%N0|DP$N3cMEr#7#th%ac$FvR$$N~j!P-_iCaz|1I0JDM z_yoACT%FG)`9M@XvVhi*nq>h9#$dYT^YL0GcvOMtRT;4+8?`xX1!qLyaeWRdZaH@e zfwhJRBS7#53kw!4CTmcIZ?7~Sh_!+MJzDHKRa&m3K@nhONRPu454=w5%y)K?(~9x0 zKyEF#cgngbIE6fX01vY>uxut763Sg*mHal7*^!7~9&9(=F%Yv9=LjH)ZR5sHuIP?X zB0Q?7mnF*s{apEy`w(=OZ=ellU?DZQ^utoLH`M0B-{mQo@D?7{5l{ph^Y>IFX;?+1T{L;c!&M*UzL=;)X18TA`Ks#~k0bu<5= zBc8_h(_2Q}j{a4B2YW{Sj{1t$*TR*EUyGl`sDLyO)?i5^^2bC#u?j0NHPhp8REeN{ z{~AHjB0kryAsZ%=2}Be{x(=J!{H4m#CVVJS3>rZQ+?Km^ufK*qc+y!v;XGs9 z_Ho`b)`93Viix2pIgavNgH!P_4oxOW{e4a*Vq z()wNSJ3zj`KXEn@s0ztaP&1Pd*CGrAT+Os3@D{JMLQ)ZpvELW5ct&B1jU0(>UNk=ay7tdjoN)ZvF#4DW2v|7ijI+Tc^*vz53)746 ze@Ch)8s6h`E>T+4BgKGZY2&U7bJTSKV5q$zy*54Lxaq8bu^D;;heE36Y;w&F5km$i zDG8>@24|@<3?!kzYBJ@-1erJ}?#(Eaw3!7U9~RRDOS*!aaB`=mbh#ri|J3_4kG(Sr zS(QqXf)Mgq8Hku&3HtFkbRWh;^1x81G3QmBj44v{Ku3C1k|r1Az~#_-XddHKtWCl1 z+P%nYFidvmb_R!Yh)D>;x!T3_`YO;)lE;huB4AG6y@v6WbPius^h_z^k1^gDB3Krq z2v`FLa-j&-Nq{Hrhfl4+g=y?8R+v6f=9t$;tLZRc%Oz*#bi=3ZA8NC2Jl!Jn?gj*1 z7dJwA8FoP@VH4+Nzeuye3@+H10Q?B6!5ws{?+@-koS6Bp+S_tqr7Dt>eo_b1;wp@t zGTuAk>jRLahacsj^guUfwoa;)o0!08c?c>83p_Xyp@_j$kadDdp3VU&Dxn4y5pbTv zIxrWv3Y>j1bQ6S$fVm7N=cfXASuvL#{5~x48z2%ydis;zc?>8}>Qdmw(WAmHsx66I~>7D&P>j76?>jD0L&Z5n9+U z!6BQ6BiVC9h!Hu1!JcWJku_p9<{2P`!{Cua{9>+C)*(NF%@G(S1i<#Q)a2;-G4L=> zao__Enm`k6tYL~O67G~z1UJ6M