From 9ba1a32dc8b35e3ce6259f282e6dcf693fe10089 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 31 Oct 2023 14:14:44 +0800 Subject: [PATCH 01/56] change NumOfTaskQueueThreads size --- source/common/src/tglobal.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 0a155f4ea1..fc95e9335c 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -486,9 +486,6 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { tsNumOfTaskQueueThreads = tsNumOfCores / 2; tsNumOfTaskQueueThreads = TMAX(tsNumOfTaskQueueThreads, 4); - if (tsNumOfTaskQueueThreads >= 10) { - tsNumOfTaskQueueThreads = 10; - } if (cfgAddInt32(pCfg, "numOfTaskQueueThreads", tsNumOfTaskQueueThreads, 4, 1024, CFG_SCOPE_CLIENT) != 0) return -1; return 0; From 13b6d3174e06125e1d8580cab0ea9c6257b2aa61 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 1 Nov 2023 10:35:25 +0800 Subject: [PATCH 02/56] change NumOfTaskQueueThreads size --- source/common/src/tglobal.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index fc95e9335c..4c3fb065b4 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -486,6 +486,10 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { tsNumOfTaskQueueThreads = tsNumOfCores / 2; tsNumOfTaskQueueThreads = TMAX(tsNumOfTaskQueueThreads, 4); + + if (tsNumOfTaskQueueThreads >= 50) { + tsNumOfTaskQueueThreads = 50; + } if (cfgAddInt32(pCfg, "numOfTaskQueueThreads", tsNumOfTaskQueueThreads, 4, 1024, CFG_SCOPE_CLIENT) != 0) return -1; return 0; @@ -1798,4 +1802,4 @@ void taosSetAllDebugFlag(int32_t flag, bool rewrite) { uInfo("all debug flag are set to %d", flag); } -int8_t taosGranted() { return atomic_load_8(&tsGrant); } +int8_t taosGranted() { return atomic_load_8(&tsGrant); } \ No newline at end of file From 8a05bad89952a8a1706115bf2c1c3af2a47126eb Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 10 Nov 2023 14:14:22 +0800 Subject: [PATCH 03/56] fix:snode error --- include/libs/stream/tstream.h | 1 - source/dnode/mgmt/mgmt_snode/src/smHandle.c | 2 + source/dnode/snode/src/snode.c | 41 +++++++++++++++++++-- source/libs/stream/src/streamDispatch.c | 6 +-- source/libs/stream/src/streamMeta.c | 7 +--- 5 files changed, 44 insertions(+), 13 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index eab3ecf04e..6ca6b176d0 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -828,7 +828,6 @@ int32_t streamMetaCommit(SStreamMeta* pMeta); int32_t streamMetaLoadAllTasks(SStreamMeta* pMeta); void streamMetaNotifyClose(SStreamMeta* pMeta); void streamMetaStartHb(SStreamMeta* pMeta); -void streamMetaInitForSnode(SStreamMeta* pMeta); bool streamMetaTaskInTimer(SStreamMeta* pMeta); int32_t streamMetaUpdateTaskDownstreamStatus(SStreamTask* pTask, int64_t startTs, int64_t endTs, bool succ); void streamMetaRLock(SStreamMeta* pMeta); diff --git a/source/dnode/mgmt/mgmt_snode/src/smHandle.c b/source/dnode/mgmt/mgmt_snode/src/smHandle.c index b29c5c1eb4..509a7a6b7f 100644 --- a/source/dnode/mgmt/mgmt_snode/src/smHandle.c +++ b/source/dnode/mgmt/mgmt_snode/src/smHandle.c @@ -88,6 +88,8 @@ SArray *smGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_SCAN_HISTORY_FINISH, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_SCAN_HISTORY_FINISH_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_CHECK_POINT_SOURCE, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECKPOINT_READY, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; + code = 0; _OVER: diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 6f5b370826..74f792cc76 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -117,7 +117,7 @@ SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) { } pSnode->msgCb = pOption->msgCb; - pSnode->pMeta = streamMetaOpen(path, pSnode, (FTaskExpand *)sndExpandTask, SNODE_HANDLE, -1); + pSnode->pMeta = streamMetaOpen(path, pSnode, (FTaskExpand *)sndExpandTask, SNODE_HANDLE, taosGetTimestampMs()); if (pSnode->pMeta == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto FAIL; @@ -126,8 +126,6 @@ SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) { stopRsync(); startRsync(); - // todo fix it: send msg to mnode to rollback to an existed checkpoint - streamMetaInitForSnode(pSnode->pMeta); return pSnode; FAIL: @@ -270,6 +268,9 @@ int32_t sndProcessTaskDispatchRsp(SSnode *pSnode, SRpcMsg *pMsg) { pRsp->upstreamTaskId = htonl(pRsp->upstreamTaskId); pRsp->streamId = htobe64(pRsp->streamId); + pRsp->downstreamTaskId = htonl(pRsp->downstreamTaskId); + pRsp->downstreamNodeId = htonl(pRsp->downstreamNodeId); + pRsp->stage = htobe64(pRsp->stage); pRsp->msgId = htonl(pRsp->msgId); SStreamTask *pTask = streamMetaAcquireTask(pSnode->pMeta, pRsp->streamId, pRsp->upstreamTaskId); @@ -335,6 +336,38 @@ int32_t sndProcessTaskRecoverFinishRsp(SSnode *pSnode, SRpcMsg *pMsg) { return 0; } +// downstream task has complete the stream task checkpoint procedure, let's start the handle the rsp by execute task +int32_t sndProcessTaskCheckpointReadyMsg(SSnode *pSnode, SRpcMsg* pMsg) { + SStreamMeta* pMeta = pSnode->pMeta; + char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + int32_t len = pMsg->contLen - sizeof(SMsgHead); + int32_t code = 0; + + SStreamCheckpointReadyMsg req = {0}; + + SDecoder decoder; + tDecoderInit(&decoder, (uint8_t*)msg, len); + if (tDecodeStreamCheckpointReadyMsg(&decoder, &req) < 0) { + code = TSDB_CODE_MSG_DECODE_ERROR; + tDecoderClear(&decoder); + return code; + } + tDecoderClear(&decoder); + + SStreamTask* pTask = streamMetaAcquireTask(pMeta, req.streamId, req.upstreamTaskId); + if (pTask == NULL) { + qError("vgId:%d failed to find s-task:0x%x, it may have been destroyed already", pMeta->vgId, req.downstreamTaskId); + return code; + } + + qDebug("snode vgId:%d s-task:%s received the checkpoint ready msg from task:0x%x (vgId:%d), handle it", pMeta->vgId, + pTask->id.idStr, req.downstreamTaskId, req.downstreamNodeId); + + streamProcessCheckpointReadyMsg(pTask); + streamMetaReleaseTask(pMeta, pTask); + return code; +} + int32_t sndProcessStreamTaskCheckReq(SSnode *pSnode, SRpcMsg *pMsg) { char *msgStr = pMsg->pCont; char *msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead)); @@ -450,6 +483,8 @@ int32_t sndProcessStreamMsg(SSnode *pSnode, SRpcMsg *pMsg) { return sndProcessStreamTaskCheckReq(pSnode, pMsg); case TDMT_VND_STREAM_TASK_CHECK_RSP: return sndProcessStreamTaskCheckRsp(pSnode, pMsg); + case TDMT_STREAM_TASK_CHECKPOINT_READY: + return sndProcessTaskCheckpointReadyMsg(pSnode, pMsg); default: ASSERT(0); } diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 5665e7a917..15b4bd634a 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -371,7 +371,7 @@ static int32_t doBuildDispatchMsg(SStreamTask* pTask, const SStreamDataBlock* pD pTask->msgInfo.pData = pReqs; } - stDebug("s-task:%s build dispatch msg success, msgId:%d", pTask->id.idStr, pTask->execInfo.dispatch); + stDebug("s-task:%s build dispatch msg success, msgId:%d, stage:%" PRId64, pTask->id.idStr, pTask->execInfo.dispatch, pTask->pMeta->stage); return code; } @@ -926,8 +926,8 @@ int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamTaskId, initRpcMsg(&info.msg, TDMT_STREAM_TASK_CHECKPOINT_READY, buf, tlen + sizeof(SMsgHead)); info.msg.info.noResp = 1; // refactor later. - stDebug("s-task:%s (level:%d) prepare checkpoint ready msg to upstream s-task:0x%" PRIx64 ":0x%x (vgId:%d) idx:%d", - pTask->id.idStr, pTask->info.taskLevel, req.streamId, req.upstreamTaskId, req.upstreamNodeId, index); + stDebug("s-task:%s (level:%d) prepare checkpoint ready msg to upstream s-task:0x%" PRIx64 ":0x%x (vgId:%d) idx:%d, vgId:%d", + pTask->id.idStr, pTask->info.taskLevel, req.streamId, req.upstreamTaskId, req.upstreamNodeId, index, req.upstreamNodeId); if (pTask->pReadyMsgList == NULL) { pTask->pReadyMsgList = taosArrayInit(4, sizeof(SStreamChkptReadyInfo)); diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 17cd9fac57..4539afb1b3 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -175,6 +175,7 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF pMeta->ahandle = ahandle; pMeta->expandFunc = expandFunc; pMeta->stage = stage; + pMeta->role = (vgId == SNODE_HANDLE) ? NODE_ROLE_LEADER : NODE_ROLE_UNINIT; // send heartbeat every 5sec. pMeta->rid = taosAddRef(streamMetaId, pMeta); @@ -205,7 +206,6 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF } pMeta->streamBackendRid = taosAddRef(streamBackendId, pMeta->streamBackend); - pMeta->role = NODE_ROLE_UNINIT; code = streamBackendLoadCheckpointInfo(pMeta); taosInitRWLatch(&pMeta->lock); @@ -1093,11 +1093,6 @@ void streamMetaStartHb(SStreamMeta* pMeta) { metaHbToMnode(pRid, NULL); } -void streamMetaInitForSnode(SStreamMeta* pMeta) { - pMeta->stage = 0; - pMeta->role = NODE_ROLE_LEADER; -} - void streamMetaResetStartInfo(STaskStartInfo* pStartInfo) { taosHashClear(pStartInfo->pReadyTaskSet); taosHashClear(pStartInfo->pFailedTaskSet); From 6a44bd54ac040414cb7ec7e4edea5b719052c2e4 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 13 Nov 2023 09:12:15 +0800 Subject: [PATCH 04/56] fix:load task in snode --- source/dnode/snode/src/snode.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 74f792cc76..5fd185ba4d 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -123,6 +123,10 @@ SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) { goto FAIL; } + if (streamMetaLoadAllTasks(pSnode->pMeta) < 0) { + goto FAIL; + } + stopRsync(); startRsync(); From 1cbe568be7cdec1802b3714dc7bf27ecfe156cab Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 13 Nov 2023 18:39:42 +0800 Subject: [PATCH 05/56] fix:add logic when snode restart --- include/libs/stream/tstream.h | 2 +- source/common/src/systable.c | 2 +- source/dnode/mnode/impl/src/mndStream.c | 31 ++++++++++++++++++++++--- source/libs/stream/src/streamMeta.c | 6 ++--- 4 files changed, 33 insertions(+), 8 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 6ca6b176d0..5de0c3180b 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -656,7 +656,7 @@ int32_t tDecodeStreamCheckpointReadyMsg(SDecoder* pDecoder, SStreamCheckpointRea typedef struct STaskStatusEntry { STaskId id; int32_t status; - int32_t stage; + int64_t stage; int32_t nodeId; int64_t verStart; // start version in WAL, only valid for source task int64_t verEnd; // end version in WAL, only valid for source task diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 5f44d3e7fc..481c671d0f 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -165,7 +165,7 @@ static const SSysDbTableSchema streamTaskSchema[] = { {.name = "node_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, {.name = "level", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, {.name = "status", .bytes = 15 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, - {.name = "stage", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, + {.name = "stage", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = false}, {.name = "in_queue", .bytes = 20, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, // {.name = "out_queue", .bytes = 20, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, {.name = "info", .bytes = 25, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index fd0c349dd2..221518f4c3 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -2053,6 +2053,9 @@ static SVgroupChangeInfo mndFindChangedNodeInfo(SMnode *pMnode, const SArray *pP epsetAssign(&updateInfo.newEp, &pCurrent->epset); taosArrayPush(info.pUpdateNodeList, &updateInfo); + + } + if(pCurrent->nodeId != SNODE_HANDLE){ SVgObj *pVgroup = mndAcquireVgroup(pMnode, pCurrent->nodeId); taosHashPut(info.pDBMap, pVgroup->dbName, strlen(pVgroup->dbName), NULL, 0); mndReleaseVgroup(pMnode, pVgroup); @@ -2107,6 +2110,24 @@ static SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool* allReady) { sdbRelease(pSdb, pVgroup); } + SSnodeObj *pObj = NULL; + while (1) { + pIter = sdbFetch(pSdb, SDB_SNODE, pIter, (void **)&pObj); + if (pIter == NULL) { + break; + } + + SNodeEntry entry = {0}; + addEpIntoEpSet(&entry.epset, pObj->pDnode->fqdn, pObj->pDnode->port); + entry.nodeId = SNODE_HANDLE; + + char buf[256] = {0}; + EPSET_TO_STR(&entry.epset, buf); + mDebug("take snode snapshot, nodeId:%d %s", entry.nodeId, buf); + taosArrayPush(pVgroupListSnapshot, &entry); + sdbRelease(pSdb, pObj); + } + return pVgroupListSnapshot; } @@ -2284,6 +2305,8 @@ int32_t removeExpirednodeEntryAndTask(SArray *pNodeSnapshot) { STaskId* pId = taosArrayGet(execInfo.pTaskList, i); STaskStatusEntry* pEntry = taosHashGet(execInfo.pTaskMap, pId, sizeof(*pId)); + if(pEntry->nodeId == SNODE_HANDLE) continue; + bool existed = taskNodeExists(pNodeSnapshot, pEntry->nodeId); if (!existed) { taosArrayPush(pRemovedTasks, pId); @@ -2629,13 +2652,13 @@ int32_t setNodeEpsetExpiredFlag(const SArray* pNodeList) { return TSDB_CODE_SUCCESS; } -static void updateStageInfo(STaskStatusEntry* pTaskEntry, int32_t stage) { +static void updateStageInfo(STaskStatusEntry* pTaskEntry, int64_t stage) { int32_t numOfNodes = taosArrayGetSize(execInfo.pNodeEntryList); for(int32_t j = 0; j < numOfNodes; ++j) { SNodeEntry* pNodeEntry = taosArrayGet(execInfo.pNodeEntryList, j); if (pNodeEntry->nodeId == pTaskEntry->nodeId) { - mInfo("vgId:%d stage updated from %d to %d, nodeUpdate trigger by s-task:0x%" PRIx64, pTaskEntry->nodeId, + mInfo("vgId:%d stage updated from %"PRId64 " to %"PRId64 ", nodeUpdate trigger by s-task:0x%" PRIx64, pTaskEntry->nodeId, pTaskEntry->stage, stage, pTaskEntry->id.taskId); pNodeEntry->stageUpdated = true; @@ -2672,6 +2695,7 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { setNodeEpsetExpiredFlag(req.pUpdateNodes); + bool snodeChanged = false; for (int32_t i = 0; i < req.numOfTasks; ++i) { STaskStatusEntry *p = taosArrayGet(req.pTaskStatus, i); STaskStatusEntry *pTaskEntry = taosHashGet(execInfo.pTaskMap, &p->id, sizeof(p->id)); @@ -2682,6 +2706,7 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { if (pTaskEntry->stage != p->stage && pTaskEntry->stage != -1) { updateStageInfo(pTaskEntry, p->stage); + if(pTaskEntry->nodeId == SNODE_HANDLE) snodeChanged = true; } else { streamTaskStatusCopy(pTaskEntry, p); if (p->activeCheckpointId != 0) { @@ -2710,7 +2735,7 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { SArray* p = mndTakeVgroupSnapshot(pMnode, &allReady); taosArrayDestroy(p); - if (allReady) { + if (allReady || snodeChanged) { // if the execInfo.activeCheckpoint == 0, the checkpoint is restoring from wal mInfo("checkpointId:%" PRId64 " failed, issue task-reset trans to reset all tasks status", execInfo.activeCheckpoint); diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 4539afb1b3..0a3c44441c 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -770,7 +770,7 @@ int32_t tEncodeStreamHbMsg(SEncoder* pEncoder, const SStreamHbMsg* pReq) { if (tEncodeI64(pEncoder, ps->id.streamId) < 0) return -1; if (tEncodeI32(pEncoder, ps->id.taskId) < 0) return -1; if (tEncodeI32(pEncoder, ps->status) < 0) return -1; - if (tEncodeI32(pEncoder, ps->stage) < 0) return -1; + if (tEncodeI64(pEncoder, ps->stage) < 0) return -1; if (tEncodeI32(pEncoder, ps->nodeId) < 0) return -1; if (tEncodeDouble(pEncoder, ps->inputQUsed) < 0) return -1; if (tEncodeDouble(pEncoder, ps->inputRate) < 0) return -1; @@ -808,7 +808,7 @@ int32_t tDecodeStreamHbMsg(SDecoder* pDecoder, SStreamHbMsg* pReq) { if (tDecodeI64(pDecoder, &entry.id.streamId) < 0) return -1; if (tDecodeI32(pDecoder, &taskId) < 0) return -1; if (tDecodeI32(pDecoder, &entry.status) < 0) return -1; - if (tDecodeI32(pDecoder, &entry.stage) < 0) return -1; + if (tDecodeI64(pDecoder, &entry.stage) < 0) return -1; if (tDecodeI32(pDecoder, &entry.nodeId) < 0) return -1; if (tDecodeDouble(pDecoder, &entry.inputQUsed) < 0) return -1; if (tDecodeDouble(pDecoder, &entry.inputRate) < 0) return -1; @@ -893,7 +893,7 @@ void metaHbToMnode(void* param, void* tmrId) { SStreamHbMsg hbMsg = {0}; SEpSet epset = {0}; bool hasMnodeEpset = false; - int32_t stage = 0; + int64_t stage = 0; streamMetaRLock(pMeta); From 8a9ded764d54da97548a645103edb8f8e1f2313f Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 15 Nov 2023 11:50:14 +0800 Subject: [PATCH 06/56] fix:add finish history msg to snode --- include/util/tlog.h | 1 + source/common/src/tglobal.c | 4 + source/dnode/snode/src/snode.c | 90 +++++++++++++---- source/util/src/tlog.c | 1 + tests/system-test/8-stream/snodeRestart.py | 107 +++++++++++++++++++++ tests/system-test/output.txt | 0 6 files changed, 183 insertions(+), 20 deletions(-) create mode 100644 tests/system-test/8-stream/snodeRestart.py delete mode 100644 tests/system-test/output.txt diff --git a/include/util/tlog.h b/include/util/tlog.h index a6d146a79e..6d393bfefb 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -66,6 +66,7 @@ extern int32_t udfDebugFlag; extern int32_t smaDebugFlag; extern int32_t idxDebugFlag; extern int32_t tdbDebugFlag; +extern int32_t sndDebugFlag; int32_t taosInitLog(const char *logName, int32_t maxFiles); void taosCloseLog(); diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 4a1ba9e391..47988b0de9 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -430,6 +430,7 @@ static int32_t taosAddServerLogCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "tdbDebugFlag", tdbDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; if (cfgAddInt32(pCfg, "metaDebugFlag", metaDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; if (cfgAddInt32(pCfg, "stDebugFlag", stDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "sndDebugFlag", sndDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; return 0; } @@ -946,6 +947,7 @@ static void taosSetServerLogCfg(SConfig *pCfg) { tdbDebugFlag = cfgGetItem(pCfg, "tdbDebugFlag")->i32; metaDebugFlag = cfgGetItem(pCfg, "metaDebugFlag")->i32; stDebugFlag = cfgGetItem(pCfg, "stDebugFlag")->i32; + sndDebugFlag = cfgGetItem(pCfg, "sndDebugFlag")->i32; } static int32_t taosSetSlowLogScope(char *pScope) { @@ -1668,6 +1670,7 @@ void taosCfgDynamicOptions(const char *option, const char *value) { {"metaDebugFlag", &metaDebugFlag}, {"jniDebugFlag", &jniDebugFlag}, {"stDebugFlag", &stDebugFlag}, + {"sndDebugFlag", &sndDebugFlag}, {"audit", &tsEnableAudit}, {"asynclog", &tsAsyncLog}, @@ -1786,6 +1789,7 @@ void taosSetAllDebugFlag(int32_t flag, bool rewrite) { taosSetDebugFlag(&tdbDebugFlag, "tdbDebugFlag", flag, rewrite); taosSetDebugFlag(&metaDebugFlag, "metaDebugFlag", flag, rewrite); taosSetDebugFlag(&stDebugFlag, "stDebugFlag", flag, rewrite); + taosSetDebugFlag(&sndDebugFlag, "sndDebugFlag", flag, rewrite); uInfo("all debug flag are set to %d", flag); } diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 5fd185ba4d..718f4e851b 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -19,6 +19,27 @@ #include "tstream.h" #include "tuuid.h" +#define sndError(...) \ + do { \ + if (sndDebugFlag & DEBUG_ERROR) { \ + taosPrintLog("SND ERROR ", DEBUG_ERROR, sndDebugFlag, __VA_ARGS__); \ + } \ + } while (0) + +#define sndInfo(...) \ + do { \ + if (sndDebugFlag & DEBUG_INFO) { \ + taosPrintLog("SND INFO ", DEBUG_INFO, sndDebugFlag, __VA_ARGS__); \ + } \ + } while (0) + +#define sndDebug(...) \ + do { \ + if (sndDebugFlag & DEBUG_DEBUG) { \ + taosPrintLog("SND ", DEBUG_DEBUG, sndDebugFlag, __VA_ARGS__); \ + } \ + } while (0) + void sndEnqueueStreamDispatch(SSnode *pSnode, SRpcMsg *pMsg) { char *msgStr = pMsg->pCont; char *msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead)); @@ -65,10 +86,10 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProcessVer pTask->pState = streamStateOpen(pSnode->path, pTask, false, -1, -1); if (pTask->pState == NULL) { - qError("s-task:%s failed to open state for task", pTask->id.idStr); + sndError("s-task:%s failed to open state for task", pTask->id.idStr); return -1; } else { - qDebug("s-task:%s state:%p", pTask->id.idStr, pTask->pState); + sndDebug("s-task:%s state:%p", pTask->id.idStr, pTask->pState); } int32_t numOfChildEp = taosArrayGetSize(pTask->upstreamInfo.pList); @@ -85,7 +106,7 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProcessVer // checkpoint ver is the kept version, handled data should be the next version. if (pTask->chkInfo.checkpointId != 0) { pTask->chkInfo.nextProcessVer = pTask->chkInfo.checkpointVer + 1; - qInfo("s-task:%s restore from the checkpointId:%" PRId64 " ver:%" PRId64 " nextProcessVer:%" PRId64, pTask->id.idStr, + sndInfo("s-task:%s restore from the checkpointId:%" PRId64 " ver:%" PRId64 " nextProcessVer:%" PRId64, pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer); } else { if (pTask->chkInfo.nextProcessVer == -1) { @@ -96,7 +117,7 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProcessVer char* p = NULL; streamTaskGetStatus(pTask, &p); - qInfo("snode:%d expand stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64 + sndInfo("snode:%d expand stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64 " nextProcessVer:%" PRId64 " child id:%d, level:%d, status:%s fill-history:%d, trigger:%" PRId64 " ms", SNODE_HANDLE, pTask->id.idStr, pChkInfo->checkpointId, pChkInfo->checkpointVer, pChkInfo->nextProcessVer, pTask->info.selfChildId, pTask->info.taskLevel, p, pTask->info.fillHistory, pTask->info.triggerParam); @@ -186,24 +207,23 @@ int32_t sndProcessTaskDeployReq(SSnode *pSnode, char *msg, int32_t msgLen) { char* p = NULL; streamTaskGetStatus(pTask, &p); - qDebug("snode:%d s-task:%s is deployed on snode and add into meta, status:%s, numOfTasks:%d", SNODE_HANDLE, + sndDebug("snode:%d s-task:%s is deployed on snode and add into meta, status:%s, numOfTasks:%d", SNODE_HANDLE, pTask->id.idStr, p, numOfTasks); EStreamTaskEvent event = (HAS_RELATED_FILLHISTORY_TASK(pTask)) ? TASK_EVENT_INIT_STREAM_SCANHIST : TASK_EVENT_INIT; streamTaskHandleEvent(pTask->status.pSM, event); - streamTaskCheckDownstream(pTask); return 0; } int32_t sndProcessTaskDropReq(SSnode *pSnode, char *msg, int32_t msgLen) { SVDropStreamTaskReq *pReq = (SVDropStreamTaskReq *)msg; - qDebug("snode:%d receive msg to drop stream task:0x%x", pSnode->pMeta->vgId, pReq->taskId); + sndDebug("snode:%d receive msg to drop stream task:0x%x", pSnode->pMeta->vgId, pReq->taskId); streamMetaUnregisterTask(pSnode->pMeta, pReq->streamId, pReq->taskId); // commit the update streamMetaWLock(pSnode->pMeta); int32_t numOfTasks = streamMetaGetNumOfTasks(pSnode->pMeta); - qDebug("vgId:%d task:0x%x dropped, remain tasks:%d", pSnode->pMeta->vgId, pReq->taskId, numOfTasks); + sndDebug("vgId:%d task:0x%x dropped, remain tasks:%d", pSnode->pMeta->vgId, pReq->taskId, numOfTasks); if (streamMetaCommit(pSnode->pMeta) < 0) { // persist to disk @@ -308,7 +328,7 @@ int32_t sndProcessWriteMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg *pRsp) { return 0; } -int32_t sndProcessStreamTaskScanHistoryFinishReq(SSnode *pSnode, SRpcMsg *pMsg) { +int32_t sndProcessTaskScanHistoryFinishReq(SSnode *pSnode, SRpcMsg *pMsg) { char *msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); @@ -335,8 +355,38 @@ int32_t sndProcessStreamTaskScanHistoryFinishReq(SSnode *pSnode, SRpcMsg *pMsg) return 0; } -int32_t sndProcessTaskRecoverFinishRsp(SSnode *pSnode, SRpcMsg *pMsg) { - // +int32_t sndProcessTaskScanHistoryFinishRsp(SSnode *pSnode, SRpcMsg *pMsg) { + char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); + + // deserialize + SStreamCompleteHistoryMsg req = {0}; + + SDecoder decoder; + tDecoderInit(&decoder, (uint8_t*)msg, msgLen); + tDecodeCompleteHistoryDataMsg(&decoder, &req); + tDecoderClear(&decoder); + + SStreamTask* pTask = streamMetaAcquireTask(pSnode->pMeta, req.streamId, req.upstreamTaskId); + if (pTask == NULL) { + sndError("vgId:%d process scan history finish rsp, failed to find task:0x%x, it may be destroyed", + pSnode->pMeta->vgId, req.upstreamTaskId); + return -1; + } + + int32_t remain = atomic_sub_fetch_32(&pTask->notReadyTasks, 1); + if (remain > 0) { + sndDebug("s-task:%s scan-history finish rsp received from downstream task:0x%x, unfinished remain:%d", + pTask->id.idStr, req.downstreamId, remain); + } else { + sndDebug( + "s-task:%s scan-history finish rsp received from downstream task:0x%x, all downstream tasks rsp scan-history " + "completed msg", + pTask->id.idStr, req.downstreamId); + streamProcessScanHistoryFinishRsp(pTask); + } + + streamMetaReleaseTask(pSnode->pMeta, pTask); return 0; } @@ -360,11 +410,11 @@ int32_t sndProcessTaskCheckpointReadyMsg(SSnode *pSnode, SRpcMsg* pMsg) { SStreamTask* pTask = streamMetaAcquireTask(pMeta, req.streamId, req.upstreamTaskId); if (pTask == NULL) { - qError("vgId:%d failed to find s-task:0x%x, it may have been destroyed already", pMeta->vgId, req.downstreamTaskId); + sndError("vgId:%d failed to find s-task:0x%x, it may have been destroyed already", pMeta->vgId, req.downstreamTaskId); return code; } - qDebug("snode vgId:%d s-task:%s received the checkpoint ready msg from task:0x%x (vgId:%d), handle it", pMeta->vgId, + sndDebug("snode vgId:%d s-task:%s received the checkpoint ready msg from task:0x%x (vgId:%d), handle it", pMeta->vgId, pTask->id.idStr, req.downstreamTaskId, req.downstreamNodeId); streamProcessCheckpointReadyMsg(pTask); @@ -403,11 +453,11 @@ int32_t sndProcessStreamTaskCheckReq(SSnode *pSnode, SRpcMsg *pMsg) { streamMetaReleaseTask(pSnode->pMeta, pTask); char* p = NULL; streamTaskGetStatus(pTask, &p); - qDebug("s-task:%s status:%s, recv task check req(reqId:0x%" PRIx64 ") task:0x%x (vgId:%d), ready:%d", + sndDebug("s-task:%s status:%s, recv task check req(reqId:0x%" PRIx64 ") task:0x%x (vgId:%d), ready:%d", pTask->id.idStr, p, rsp.reqId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status); } else { rsp.status = TASK_DOWNSTREAM_NOT_READY; - qDebug("recv task check(taskId:0x%x not built yet) req(reqId:0x%" PRIx64 ") from task:0x%x (vgId:%d), rsp status %d", + sndDebug("recv task check(taskId:0x%x not built yet) req(reqId:0x%" PRIx64 ") from task:0x%x (vgId:%d), rsp status %d", taskId, rsp.reqId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status); } @@ -417,7 +467,7 @@ int32_t sndProcessStreamTaskCheckReq(SSnode *pSnode, SRpcMsg *pMsg) { tEncodeSize(tEncodeStreamTaskCheckRsp, &rsp, len, code); if (code < 0) { - qError("vgId:%d failed to encode task check rsp, task:0x%x", pSnode->pMeta->vgId, taskId); + sndError("vgId:%d failed to encode task check rsp, task:0x%x", pSnode->pMeta->vgId, taskId); return -1; } @@ -452,12 +502,12 @@ int32_t sndProcessStreamTaskCheckRsp(SSnode* pSnode, SRpcMsg* pMsg) { } tDecoderClear(&decoder); - qDebug("tq task:0x%x (vgId:%d) recv check rsp(reqId:0x%" PRIx64 ") from 0x%x (vgId:%d) status %d", + sndDebug("tq task:0x%x (vgId:%d) recv check rsp(reqId:0x%" PRIx64 ") from 0x%x (vgId:%d) status %d", rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, rsp.status); SStreamTask* pTask = streamMetaAcquireTask(pSnode->pMeta, rsp.streamId, rsp.upstreamTaskId); if (pTask == NULL) { - qError("tq failed to locate the stream task:0x%x (vgId:%d), it may have been destroyed", rsp.upstreamTaskId, + sndError("tq failed to locate the stream task:0x%x (vgId:%d), it may have been destroyed", rsp.upstreamTaskId, pSnode->pMeta->vgId); return -1; } @@ -480,9 +530,9 @@ int32_t sndProcessStreamMsg(SSnode *pSnode, SRpcMsg *pMsg) { case TDMT_STREAM_RETRIEVE_RSP: return sndProcessTaskRetrieveRsp(pSnode, pMsg); case TDMT_VND_STREAM_SCAN_HISTORY_FINISH: - return sndProcessStreamTaskScanHistoryFinishReq(pSnode, pMsg); + return sndProcessTaskScanHistoryFinishReq(pSnode, pMsg); case TDMT_VND_STREAM_SCAN_HISTORY_FINISH_RSP: - return sndProcessTaskRecoverFinishRsp(pSnode, pMsg); + return sndProcessTaskScanHistoryFinishRsp(pSnode, pMsg); case TDMT_VND_STREAM_TASK_CHECK: return sndProcessStreamTaskCheckReq(pSnode, pMsg); case TDMT_VND_STREAM_TASK_CHECK_RSP: diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index aa6719f604..46b25dee6a 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -110,6 +110,7 @@ int32_t metaDebugFlag = 131; int32_t udfDebugFlag = 131; int32_t smaDebugFlag = 131; int32_t idxDebugFlag = 131; +int32_t sndDebugFlag = 131; int64_t dbgEmptyW = 0; int64_t dbgWN = 0; diff --git a/tests/system-test/8-stream/snodeRestart.py b/tests/system-test/8-stream/snodeRestart.py new file mode 100644 index 0000000000..65ca090df9 --- /dev/null +++ b/tests/system-test/8-stream/snodeRestart.py @@ -0,0 +1,107 @@ + +import taos +import sys +import time +import socket +import os +import threading +import math + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +from util.cluster import * + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + + def case1(self): + tdLog.debug("case1 start") + + os.system("nohup taosBenchmark -y -B 1 -t 4 -S 1000 -n 1000 -i 1000 -v 2 > /dev/null 2>&1 &") + time.sleep(1) + tdSql.query("use test") + tdSql.query("create snode on dnode 4") + tdSql.query("create stream if not exists s1 trigger at_once ignore expired 0 ignore update 0 fill_history 1 into st1 as select sum(voltage) from meters partition by groupid interval(4s)") + tdLog.debug("create stream use snode and insert data") + time.sleep(10) + + tdDnodes = cluster.dnodes + tdDnodes[3].stoptaosd() + time.sleep(2) + tdDnodes[3].starttaosd() + tdLog.debug("snode restart ok") + + time.sleep(500) + os.system("kill -9 `pgrep taosBenchmark`") + + tdSql.query("select sum(voltage) from meters partition by groupid interval(4s)") + # tdSql.checkRows(1) + # + # tdSql.checkData(0, 0, '2016-01-01 08:00:07.000') + # tdSql.checkData(0, 1, 2000) + + tdSql.query("drop snode on dnode 4") + tdSql.query("drop stream if exists s1") + tdSql.query("drop database test") + + tdLog.debug("case1 end") + + def case2(self): + tdLog.debug("case2 start") + + updatecfgDict = {'checkpointInterval': 5} + print("===================: ", updatecfgDict) + + tdDnodes = cluster.dnodes + tdDnodes[0].stoptaosd() + tdDnodes[1].stoptaosd() + tdDnodes[2].stoptaosd() + tdDnodes[3].stoptaosd() + time.sleep(2) + tdDnodes[0].starttaosd() + tdDnodes[1].starttaosd() + tdDnodes[2].starttaosd() + tdDnodes[3].starttaosd() + tdLog.debug("taosd restart ok") + + os.system("taosBenchmark -y -B 1 -t 4 -S 1000 -n 1000 -i 1000 -v 2 &" ) + time.sleep(1) + tdSql.query("use test") + tdSql.query("create snode on dnode 4") + tdSql.query("create stream if not exists s1 trigger at_once ignore expired 0 ignore update 0 fill_history 1 into st1 as select sum(voltage) from meters partition by groupid interval(4s)") + tdLog.debug("create stream use snode and insert data") + time.sleep(10) + + tdDnodes[3].stoptaosd() + time.sleep(2) + tdDnodes[3].starttaosd() + tdLog.debug("snode restart ok") + + time.sleep(5) + os.system("kill -9 `pgrep taosBenchmark`") + + tdSql.query("select sum(voltage) from meters partition by groupid interval(4s)") + # tdSql.checkRows(1) + # + # tdSql.checkData(0, 0, '2016-01-01 08:00:07.000') + # tdSql.checkData(0, 1, 2000) + + tdLog.debug("case2 end") + + def run(self): + self.case1() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/output.txt b/tests/system-test/output.txt deleted file mode 100644 index e69de29bb2..0000000000 From 188a4ce6a50ce30d32961e2350680d388aa11a1f Mon Sep 17 00:00:00 2001 From: kailixu Date: Wed, 15 Nov 2023 16:33:05 +0800 Subject: [PATCH 07/56] enh: the latest active code take effect --- include/common/tgrant.h | 5 ++ source/dnode/mnode/impl/inc/mndCluster.h | 2 + source/dnode/mnode/impl/inc/mndDef.h | 2 + source/dnode/mnode/impl/src/mndCluster.c | 61 +++++++++++++++++++++++- 4 files changed, 69 insertions(+), 1 deletion(-) diff --git a/include/common/tgrant.h b/include/common/tgrant.h index f06fca8014..b18eaf8a6f 100644 --- a/include/common/tgrant.h +++ b/include/common/tgrant.h @@ -50,6 +50,11 @@ typedef enum { TSDB_GRANT_TABLE, } EGrantType; +typedef struct { + int64_t grantedTime; + int64_t connGrantedTime; +} SGrantedInfo; + int32_t grantCheck(EGrantType grant); int32_t grantAlterActiveCode(int32_t did, const char* old, const char* newer, char* out, int8_t type); diff --git a/source/dnode/mnode/impl/inc/mndCluster.h b/source/dnode/mnode/impl/inc/mndCluster.h index e33ffdb372..2b868b567a 100644 --- a/source/dnode/mnode/impl/inc/mndCluster.h +++ b/source/dnode/mnode/impl/inc/mndCluster.h @@ -27,7 +27,9 @@ void mndCleanupCluster(SMnode *pMnode); int32_t mndGetClusterName(SMnode *pMnode, char *clusterName, int32_t len); int64_t mndGetClusterId(SMnode *pMnode); int64_t mndGetClusterCreateTime(SMnode *pMnode); +int32_t mndGetClusterGrantedInfo(SMnode *pMnode, SGrantedInfo *pInfo); int64_t mndGetClusterUpTime(SMnode *pMnode); +int32_t mndProcessGrantedTime(SMnode *pMnode, int64_t grantedTime); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index efa99db74b..fc9086eebf 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -192,6 +192,8 @@ typedef struct { int64_t createdTime; int64_t updateTime; int32_t upTime; + int64_t grantedTime; + int64_t connGrantedTime; } SClusterObj; typedef struct { diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index 4c799e1e1e..20e25d918c 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -19,7 +19,7 @@ #include "mndTrans.h" #define CLUSTER_VER_NUMBE 1 -#define CLUSTER_RESERVE_SIZE 60 +#define CLUSTER_RESERVE_SIZE 44 int64_t tsExpireTime = 0; static SSdbRaw *mndClusterActionEncode(SClusterObj *pCluster); @@ -112,6 +112,19 @@ int64_t mndGetClusterCreateTime(SMnode *pMnode) { return createTime; } +int32_t mndGetClusterGrantedInfo(SMnode *pMnode, SGrantedInfo *pInfo) { + void *pIter = NULL; + SClusterObj *pCluster = mndAcquireCluster(pMnode, &pIter); + if (pCluster != NULL) { + pInfo->grantedTime = pCluster->grantedTime; + pInfo->connGrantedTime = pCluster->connGrantedTime; + mndReleaseCluster(pMnode, pCluster, pIter); + return 0; + } + + return -1; +} + static int32_t mndGetClusterUpTimeImp(SClusterObj *pCluster) { #if 0 int32_t upTime = taosGetTimestampSec() - pCluster->updateTime / 1000; @@ -146,6 +159,8 @@ static SSdbRaw *mndClusterActionEncode(SClusterObj *pCluster) { SDB_SET_INT64(pRaw, dataPos, pCluster->updateTime, _OVER) SDB_SET_BINARY(pRaw, dataPos, pCluster->name, TSDB_CLUSTER_ID_LEN, _OVER) SDB_SET_INT32(pRaw, dataPos, pCluster->upTime, _OVER) + SDB_SET_INT64(pRaw, dataPos, pCluster->grantedTime, _OVER) + SDB_SET_INT64(pRaw, dataPos, pCluster->connGrantedTime, _OVER) SDB_SET_RESERVE(pRaw, dataPos, CLUSTER_RESERVE_SIZE, _OVER) terrno = 0; @@ -186,6 +201,8 @@ static SSdbRow *mndClusterActionDecode(SSdbRaw *pRaw) { SDB_GET_INT64(pRaw, dataPos, &pCluster->updateTime, _OVER) SDB_GET_BINARY(pRaw, dataPos, pCluster->name, TSDB_CLUSTER_ID_LEN, _OVER) SDB_GET_INT32(pRaw, dataPos, &pCluster->upTime, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pCluster->grantedTime, _OVER); + SDB_GET_INT64(pRaw, dataPos, &pCluster->connGrantedTime, _OVER); SDB_GET_RESERVE(pRaw, dataPos, CLUSTER_RESERVE_SIZE, _OVER) terrno = 0; @@ -218,6 +235,7 @@ static int32_t mndClusterActionUpdate(SSdb *pSdb, SClusterObj *pOld, SClusterObj mTrace("cluster:%" PRId64 ", perform update action, old row:%p new row:%p, uptime from %d to %d", pOld->id, pOld, pNew, pOld->upTime, pNew->upTime); pOld->upTime = pNew->upTime; + pOld->grantedTime = pNew->grantedTime; pOld->updateTime = taosGetTimestampMs(); return 0; } @@ -359,3 +377,44 @@ static int32_t mndProcessUptimeTimer(SRpcMsg *pReq) { mndTransDrop(pTrans); return 0; } + +int32_t mndProcessGrantedTime(SMnode *pMnode, int64_t grantedTime) { + SClusterObj clusterObj = {0}; + void *pIter = NULL; + SClusterObj *pCluster = mndAcquireCluster(pMnode, &pIter); + if (pCluster != NULL) { + if (pCluster->grantedTime >= grantedTime) { + mndReleaseCluster(pMnode, pCluster, pIter); + return 0; + } + memcpy(&clusterObj, pCluster, sizeof(SClusterObj)); + clusterObj.grantedTime = grantedTime; + mndReleaseCluster(pMnode, pCluster, pIter); + } + + if (clusterObj.id <= 0) { + mError("can't get cluster info while update uptime"); + return -1; + } + + mInfo("update cluster granted time to %" PRIi64, clusterObj.grantedTime); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, NULL, "granted-time"); + if (pTrans == NULL) return -1; + + SSdbRaw *pCommitRaw = mndClusterActionEncode(&clusterObj); + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); + + if (mndTransPrepare(pMnode, pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + + mndTransDrop(pTrans); + return 0; +} \ No newline at end of file From 45782f278b6e557ecf2ea4c9d08d3b3af473ce01 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 15 Nov 2023 18:13:35 +0800 Subject: [PATCH 08/56] fix:send update msg to vnodes if snode restart --- include/dnode/vnode/stream.h | 18 + source/dnode/mgmt/mgmt_snode/src/smHandle.c | 2 +- source/dnode/snode/CMakeLists.txt | 1 + source/dnode/snode/src/snode.c | 407 +++++++++++++++++++- source/dnode/vnode/CMakeLists.txt | 5 + source/dnode/vnode/src/inc/tq.h | 3 - source/dnode/vnode/src/tq/tq.c | 7 +- source/dnode/vnode/src/tq/tqStreamTask.c | 1 + tests/system-test/8-stream/snodeRestart.py | 2 +- 9 files changed, 415 insertions(+), 31 deletions(-) create mode 100644 include/dnode/vnode/stream.h diff --git a/include/dnode/vnode/stream.h b/include/dnode/vnode/stream.h new file mode 100644 index 0000000000..6d86847542 --- /dev/null +++ b/include/dnode/vnode/stream.h @@ -0,0 +1,18 @@ +// +// Created by mingming wanng on 2023/11/15. +// + +#ifndef TDENGINE_STREAM_H +#define TDENGINE_STREAM_H + +#define STREAM_EXEC_EXTRACT_DATA_IN_WAL_ID (-1) +#define STREAM_EXEC_START_ALL_TASKS_ID (-2) +#define STREAM_EXEC_RESTART_ALL_TASKS_ID (-3) + +typedef struct STaskUpdateEntry { + int64_t streamId; + int32_t taskId; + int32_t transId; +} STaskUpdateEntry; + +#endif // TDENGINE_STREAM_H diff --git a/source/dnode/mgmt/mgmt_snode/src/smHandle.c b/source/dnode/mgmt/mgmt_snode/src/smHandle.c index 509a7a6b7f..cd81b9873f 100644 --- a/source/dnode/mgmt/mgmt_snode/src/smHandle.c +++ b/source/dnode/mgmt/mgmt_snode/src/smHandle.c @@ -73,6 +73,7 @@ SArray *smGetMsgHandles() { SArray *pArray = taosArrayInit(4, sizeof(SMgmtHandle)); if (pArray == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_UPDATE, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DEPLOY, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DROP, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RUN, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; @@ -87,7 +88,6 @@ SArray *smGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_CHECK_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_SCAN_HISTORY_FINISH, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_SCAN_HISTORY_FINISH_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_CHECK_POINT_SOURCE, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECKPOINT_READY, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; diff --git a/source/dnode/snode/CMakeLists.txt b/source/dnode/snode/CMakeLists.txt index ebfe80ecab..2da1f9adac 100644 --- a/source/dnode/snode/CMakeLists.txt +++ b/source/dnode/snode/CMakeLists.txt @@ -3,6 +3,7 @@ add_library(snode STATIC ${SNODE_SRC}) target_include_directories( snode PUBLIC "${TD_SOURCE_DIR}/include/dnode/snode" + PUBLIC "${TD_SOURCE_DIR}/include/dnode/vnode" private "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) target_link_libraries( diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 718f4e851b..25400220b8 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -18,6 +18,7 @@ #include "sndInt.h" #include "tstream.h" #include "tuuid.h" +#include "stream.h" #define sndError(...) \ do { \ @@ -84,7 +85,16 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProcessVer streamTaskOpenAllUpstreamInput(pTask); - pTask->pState = streamStateOpen(pSnode->path, pTask, false, -1, -1); + SStreamTask* pSateTask = pTask; + SStreamTask task = {0}; + if (pTask->info.fillHistory) { + task.id.streamId = pTask->streamTaskId.streamId; + task.id.taskId = pTask->streamTaskId.taskId; + task.pMeta = pTask->pMeta; + pSateTask = &task; + } + + pTask->pState = streamStateOpen(pSnode->path, pSateTask, false, -1, -1); if (pTask->pState == NULL) { sndError("s-task:%s failed to open state for task", pTask->id.idStr); return -1; @@ -92,12 +102,20 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProcessVer sndDebug("s-task:%s state:%p", pTask->id.idStr, pTask->pState); } - int32_t numOfChildEp = taosArrayGetSize(pTask->upstreamInfo.pList); - SReadHandle handle = { .vnode = NULL, .numOfVgroups = numOfChildEp, .pStateBackend = pTask->pState, .fillHistory = pTask->info.fillHistory }; + int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTask->upstreamInfo.pList); + SReadHandle handle = { + .checkpointId = pTask->chkInfo.checkpointId, + .vnode = NULL, + .numOfVgroups = numOfVgroups, + .pStateBackend = pTask->pState, + .fillHistory = pTask->info.fillHistory, + .winRange = pTask->dataRange.window, + }; initStreamStateAPI(&handle.api); - pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle, 0, pTask->id.taskId); + pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle, SNODE_HANDLE, pTask->id.taskId); ASSERT(pTask->exec.pExecutor); + qSetTaskId(pTask->exec.pExecutor, pTask->id.taskId, pTask->id.streamId); streamTaskResetUpstreamStageInfo(pTask); streamSetupScheduleTrigger(pTask); @@ -169,6 +187,167 @@ void sndClose(SSnode *pSnode) { int32_t sndGetLoad(SSnode *pSnode, SSnodeLoad *pLoad) { return 0; } +int32_t sndStartStreamTasks(SSnode* pSnode) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t vgId = SNODE_HANDLE; + SStreamMeta* pMeta = pSnode->pMeta; + + int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); + sndDebug("vgId:%d start to check all %d stream task(s) downstream status", vgId, numOfTasks); + if (numOfTasks == 0) { + return TSDB_CODE_SUCCESS; + } + + SArray* pTaskList = NULL; + streamMetaWLock(pMeta); + pTaskList = taosArrayDup(pMeta->pTaskList, NULL); + taosHashClear(pMeta->startInfo.pReadyTaskSet); + taosHashClear(pMeta->startInfo.pFailedTaskSet); + pMeta->startInfo.startTs = taosGetTimestampMs(); + streamMetaWUnLock(pMeta); + + // broadcast the check downstream tasks msg + for (int32_t i = 0; i < numOfTasks; ++i) { + SStreamTaskId* pTaskId = taosArrayGet(pTaskList, i); + SStreamTask* pTask = streamMetaAcquireTask(pMeta, pTaskId->streamId, pTaskId->taskId); + if (pTask == NULL) { + continue; + } + + // fill-history task can only be launched by related stream tasks. + if (pTask->info.fillHistory == 1) { + streamMetaReleaseTask(pMeta, pTask); + continue; + } + + if (pTask->status.downstreamReady == 1) { + if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { + sndDebug("s-task:%s downstream ready, no need to check downstream, check only related fill-history task", + pTask->id.idStr); + streamLaunchFillHistoryTask(pTask); + } + + streamMetaUpdateTaskDownstreamStatus(pTask, pTask->execInfo.init, pTask->execInfo.start, true); + streamMetaReleaseTask(pMeta, pTask); + continue; + } + + EStreamTaskEvent event = (HAS_RELATED_FILLHISTORY_TASK(pTask)) ? TASK_EVENT_INIT_STREAM_SCANHIST : TASK_EVENT_INIT; + int32_t ret = streamTaskHandleEvent(pTask->status.pSM, event); + if (ret != TSDB_CODE_SUCCESS) { + code = ret; + } + + streamMetaReleaseTask(pMeta, pTask); + } + + taosArrayDestroy(pTaskList); + return code; +} + +int32_t sndResetStreamTaskStatus(SSnode* pSnode) { + SStreamMeta* pMeta = pSnode->pMeta; + int32_t vgId = pMeta->vgId; + int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); + + sndDebug("vgId:%d reset all %d stream task(s) status to be uninit", vgId, numOfTasks); + if (numOfTasks == 0) { + return TSDB_CODE_SUCCESS; + } + + for (int32_t i = 0; i < numOfTasks; ++i) { + SStreamTaskId* pTaskId = taosArrayGet(pMeta->pTaskList, i); + + STaskId id = {.streamId = pTaskId->streamId, .taskId = pTaskId->taskId}; + SStreamTask** pTask = taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); + streamTaskResetStatus(*pTask); + } + + return 0; +} + +int32_t sndRestartStreamTasks(SSnode* pSnode) { + SStreamMeta* pMeta = pSnode->pMeta; + int32_t vgId = pMeta->vgId; + int32_t code = 0; + int64_t st = taosGetTimestampMs(); + + while(1) { + int32_t startVal = atomic_val_compare_exchange_32(&pMeta->startInfo.taskStarting, 0, 1); + if (startVal == 0) { + break; + } + + sndDebug("vgId:%d in start stream tasks procedure, wait for 500ms and recheck", vgId); + taosMsleep(500); + } + + terrno = 0; + sndInfo("vgId:%d tasks are all updated and stopped, restart all tasks, triggered by transId:%d", vgId, + pMeta->updateInfo.transId); + + while (streamMetaTaskInTimer(pMeta)) { + sndDebug("vgId:%d some tasks in timer, wait for 100ms and recheck", pMeta->vgId); + taosMsleep(100); + } + + streamMetaWLock(pMeta); + + code = streamMetaReopen(pMeta); + if (code != TSDB_CODE_SUCCESS) { + sndError("vgId:%d failed to reopen stream meta", vgId); + streamMetaWUnLock(pMeta); + code = terrno; + return code; + } + + int64_t el = taosGetTimestampMs() - st; + + sndInfo("vgId:%d close&reload state elapsed time:%.3fms", vgId, el/1000.); + + code = streamMetaLoadAllTasks(pSnode->pMeta); + if (code != TSDB_CODE_SUCCESS) { + sndError("vgId:%d failed to load stream tasks, code:%s", vgId, tstrerror(terrno)); + streamMetaWUnLock(pMeta); + code = terrno; + return code; + } + sndInfo("vgId:%d restart all stream tasks after all tasks being updated", vgId); + sndResetStreamTaskStatus(pSnode); + sndStartStreamTasks(pSnode); + + streamMetaWUnLock(pMeta); + code = terrno; + return code; +} + +int32_t sndStartStreamTaskAsync(SSnode* pSnode, bool restart) { + SStreamMeta* pMeta = pSnode->pMeta; + int32_t vgId = pMeta->vgId; + + int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); + if (numOfTasks == 0) { + sndDebug("vgId:%d no stream tasks existed to run", vgId); + return 0; + } + + SStreamTaskRunReq* pRunReq = rpcMallocCont(sizeof(SStreamTaskRunReq)); + if (pRunReq == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + sndError("vgId:%d failed to create msg to start wal scanning to launch stream tasks, code:%s", vgId, terrstr()); + return -1; + } + + sndDebug("vgId:%d start all %d stream task(s) async", vgId, numOfTasks); + pRunReq->head.vgId = vgId; + pRunReq->streamId = 0; + pRunReq->taskId = restart? STREAM_EXEC_RESTART_ALL_TASKS_ID:STREAM_EXEC_START_ALL_TASKS_ID; + + SRpcMsg msg = {.msgType = TDMT_STREAM_TASK_RUN, .pCont = pRunReq, .contLen = sizeof(SStreamTaskRunReq)}; + tmsgPutToQueue(&pSnode->msgCb, STREAM_QUEUE, &msg); + return 0; +} + int32_t sndProcessTaskDeployReq(SSnode *pSnode, char *msg, int32_t msgLen) { int32_t code; @@ -235,6 +414,16 @@ int32_t sndProcessTaskDropReq(SSnode *pSnode, char *msg, int32_t msgLen) { int32_t sndProcessTaskRunReq(SSnode *pSnode, SRpcMsg *pMsg) { SStreamTaskRunReq *pReq = pMsg->pCont; + int32_t taskId = pReq->taskId; + + if (taskId == STREAM_EXEC_START_ALL_TASKS_ID) { + sndStartStreamTasks(pSnode); + return 0; + } else if (taskId == STREAM_EXEC_RESTART_ALL_TASKS_ID) { + sndRestartStreamTasks(pSnode); + return 0; + } + SStreamTask *pTask = streamMetaAcquireTask(pSnode->pMeta, pReq->streamId, pReq->taskId); if (pTask) { streamExecTask(pTask); @@ -312,22 +501,6 @@ int32_t sndProcessTaskRetrieveRsp(SSnode *pSnode, SRpcMsg *pMsg) { return 0; } -int32_t sndProcessWriteMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg *pRsp) { - switch (pMsg->msgType) { - case TDMT_STREAM_TASK_DEPLOY: { - void *pReq = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); - int32_t len = pMsg->contLen - sizeof(SMsgHead); - return sndProcessTaskDeployReq(pSnode, pReq, len); - } - - case TDMT_STREAM_TASK_DROP: - return sndProcessTaskDropReq(pSnode, pMsg->pCont, pMsg->contLen); - default: - ASSERT(0); - } - return 0; -} - int32_t sndProcessTaskScanHistoryFinishReq(SSnode *pSnode, SRpcMsg *pMsg) { char *msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); @@ -517,6 +690,181 @@ int32_t sndProcessStreamTaskCheckRsp(SSnode* pSnode, SRpcMsg* pMsg) { return code; } +int32_t sndProcessTaskUpdateReq(SSnode* pSnode, SRpcMsg* pMsg) { + SStreamMeta* pMeta = pSnode->pMeta; + int32_t vgId = SNODE_HANDLE; + char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + int32_t len = pMsg->contLen - sizeof(SMsgHead); + SRpcMsg rsp = {.info = pMsg->info, .code = TSDB_CODE_SUCCESS}; + + SStreamTaskNodeUpdateMsg req = {0}; + + SDecoder decoder; + tDecoderInit(&decoder, (uint8_t*)msg, len); + if (tDecodeStreamTaskUpdateMsg(&decoder, &req) < 0) { + rsp.code = TSDB_CODE_MSG_DECODE_ERROR; + sndError("vgId:%d failed to decode task update msg, code:%s", vgId, tstrerror(rsp.code)); + tDecoderClear(&decoder); + return rsp.code; + } + + tDecoderClear(&decoder); + + // update the nodeEpset when it exists + streamMetaWLock(pMeta); + + // the task epset may be updated again and again, when replaying the WAL, the task may be in stop status. + STaskId id = {.streamId = req.streamId, .taskId = req.taskId}; + SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); + if (ppTask == NULL || *ppTask == NULL) { + sndError("vgId:%d failed to acquire task:0x%x when handling update, it may have been dropped already", pMeta->vgId, + req.taskId); + rsp.code = TSDB_CODE_SUCCESS; + streamMetaWUnLock(pMeta); + + taosArrayDestroy(req.pNodeList); + return rsp.code; + } + + SStreamTask* pTask = *ppTask; + + if (pMeta->updateInfo.transId != req.transId) { + pMeta->updateInfo.transId = req.transId; + sndInfo("s-task:%s receive new trans to update nodeEp msg from mnode, transId:%d", pTask->id.idStr, req.transId); + // info needs to be kept till the new trans to update the nodeEp arrived. + taosHashClear(pMeta->updateInfo.pTasks); + } else { + sndDebug("s-task:%s recv trans to update nodeEp from mnode, transId:%d", pTask->id.idStr, req.transId); + } + + STaskUpdateEntry entry = {.streamId = req.streamId, .taskId = req.taskId, .transId = req.transId}; + void* exist = taosHashGet(pMeta->updateInfo.pTasks, &entry, sizeof(STaskUpdateEntry)); + if (exist != NULL) { + sndDebug("s-task:%s (vgId:%d) already update in trans:%d, discard the nodeEp update msg", pTask->id.idStr, vgId, + req.transId); + rsp.code = TSDB_CODE_SUCCESS; + streamMetaWUnLock(pMeta); + taosArrayDestroy(req.pNodeList); + return rsp.code; + } + + streamMetaWUnLock(pMeta); + + // the following two functions should not be executed within the scope of meta lock to avoid deadlock + streamTaskUpdateEpsetInfo(pTask, req.pNodeList); + streamTaskResetStatus(pTask); + + // continue after lock the meta again + streamMetaWLock(pMeta); + + SStreamTask** ppHTask = NULL; + if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { + ppHTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &pTask->hTaskInfo.id, sizeof(pTask->hTaskInfo.id)); + if (ppHTask == NULL || *ppHTask == NULL) { + sndError("vgId:%d failed to acquire fill-history task:0x%x when handling update, it may have been dropped already", + pMeta->vgId, req.taskId); + CLEAR_RELATED_FILLHISTORY_TASK(pTask); + } else { + sndDebug("s-task:%s fill-history task update nodeEp along with stream task", (*ppHTask)->id.idStr); + streamTaskUpdateEpsetInfo(*ppHTask, req.pNodeList); + } + } + + { + streamMetaSaveTask(pMeta, pTask); + if (ppHTask != NULL) { + streamMetaSaveTask(pMeta, *ppHTask); + } + + if (streamMetaCommit(pMeta) < 0) { + // persist to disk + } + } + + streamTaskStop(pTask); + + // keep the already handled info + taosHashPut(pMeta->updateInfo.pTasks, &entry, sizeof(entry), NULL, 0); + + if (ppHTask != NULL) { + streamTaskStop(*ppHTask); + sndDebug("s-task:%s task nodeEp update completed, streamTask and related fill-history task closed", pTask->id.idStr); + taosHashPut(pMeta->updateInfo.pTasks, &(*ppHTask)->id, sizeof(pTask->id), NULL, 0); + } else { + sndDebug("s-task:%s task nodeEp update completed, streamTask closed", pTask->id.idStr); + } + + rsp.code = 0; + + // possibly only handle the stream task. + int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta); + int32_t updateTasks = taosHashGetSize(pMeta->updateInfo.pTasks); + + pMeta->startInfo.tasksWillRestart = 1; + + if (updateTasks < numOfTasks) { + sndDebug("vgId:%d closed tasks:%d, unclosed:%d, all tasks will be started when nodeEp update completed", vgId, + updateTasks, (numOfTasks - updateTasks)); + streamMetaWUnLock(pMeta); + } else { + sndDebug("vgId:%d all %d task(s) nodeEp updated and closed", vgId, numOfTasks); +#if 1 + sndStartStreamTaskAsync(pSnode, true); + streamMetaWUnLock(pMeta); +#else + streamMetaWUnLock(pMeta); + + // For debug purpose. + // the following procedure consume many CPU resource, result in the re-election of leader + // with high probability. So we employ it as a test case for the stream processing framework, with + // checkpoint/restart/nodeUpdate etc. + while(1) { + int32_t startVal = atomic_val_compare_exchange_32(&pMeta->startInfo.taskStarting, 0, 1); + if (startVal == 0) { + break; + } + + tqDebug("vgId:%d in start stream tasks procedure, wait for 500ms and recheck", vgId); + taosMsleep(500); + } + + while (streamMetaTaskInTimer(pMeta)) { + tqDebug("vgId:%d some tasks in timer, wait for 100ms and recheck", pMeta->vgId); + taosMsleep(100); + } + + streamMetaWLock(pMeta); + + int32_t code = streamMetaReopen(pMeta); + if (code != 0) { + tqError("vgId:%d failed to reopen stream meta", vgId); + streamMetaWUnLock(pMeta); + taosArrayDestroy(req.pNodeList); + return -1; + } + + if (streamMetaLoadAllTasks(pTq->pStreamMeta) < 0) { + tqError("vgId:%d failed to load stream tasks", vgId); + streamMetaWUnLock(pMeta); + taosArrayDestroy(req.pNodeList); + return -1; + } + + if (vnodeIsRoleLeader(pTq->pVnode) && !tsDisableStream) { + tqInfo("vgId:%d start all stream tasks after all being updated", vgId); + tqResetStreamTaskStatus(pTq); + tqStartStreamTaskAsync(pTq, false); + } else { + tqInfo("vgId:%d, follower node not start stream tasks", vgId); + } + streamMetaWUnLock(pMeta); +#endif + } + + taosArrayDestroy(req.pNodeList); + return rsp.code; +} + int32_t sndProcessStreamMsg(SSnode *pSnode, SRpcMsg *pMsg) { switch (pMsg->msgType) { case TDMT_STREAM_TASK_RUN: @@ -544,3 +892,22 @@ int32_t sndProcessStreamMsg(SSnode *pSnode, SRpcMsg *pMsg) { } return 0; } + +int32_t sndProcessWriteMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg *pRsp) { + switch (pMsg->msgType) { + case TDMT_STREAM_TASK_DEPLOY: { + void *pReq = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + int32_t len = pMsg->contLen - sizeof(SMsgHead); + return sndProcessTaskDeployReq(pSnode, pReq, len); + } + + case TDMT_STREAM_TASK_DROP: + return sndProcessTaskDropReq(pSnode, pMsg->pCont, pMsg->contLen); + case TDMT_VND_STREAM_TASK_UPDATE: + sndProcessTaskUpdateReq(pSnode, pMsg); + break; + default: + ASSERT(0); + } + return 0; +} \ No newline at end of file diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index dc43da7fe7..635c15aa41 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -138,6 +138,11 @@ else() endif() endif() +target_include_directories( + vnode + PUBLIC "${TD_SOURCE_DIR}/include/dnode/vnode" +) + target_link_libraries( vnode PUBLIC os diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index fdd449bf36..b3f8317add 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -43,9 +43,6 @@ extern "C" { typedef struct STqOffsetStore STqOffsetStore; -#define STREAM_EXEC_EXTRACT_DATA_IN_WAL_ID (-1) -#define STREAM_EXEC_START_ALL_TASKS_ID (-2) -#define STREAM_EXEC_RESTART_ALL_TASKS_ID (-3) #define IS_OFFSET_RESET_TYPE(_t) ((_t) < 0) // tqExec diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 3ae0eb1ddf..521686d023 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -15,17 +15,12 @@ #include "tq.h" #include "vnd.h" +#include "stream.h" typedef struct { int8_t inited; } STqMgmt; -typedef struct STaskUpdateEntry { - int64_t streamId; - int32_t taskId; - int32_t transId; -} STaskUpdateEntry; - static STqMgmt tqMgmt = {0}; // 0: not init diff --git a/source/dnode/vnode/src/tq/tqStreamTask.c b/source/dnode/vnode/src/tq/tqStreamTask.c index e578638e9d..9f00f9cc10 100644 --- a/source/dnode/vnode/src/tq/tqStreamTask.c +++ b/source/dnode/vnode/src/tq/tqStreamTask.c @@ -15,6 +15,7 @@ #include "tq.h" #include "vnd.h" +#include "stream.h" #define MAX_REPEAT_SCAN_THRESHOLD 3 #define SCAN_WAL_IDLE_DURATION 100 diff --git a/tests/system-test/8-stream/snodeRestart.py b/tests/system-test/8-stream/snodeRestart.py index 65ca090df9..6adf874ecd 100644 --- a/tests/system-test/8-stream/snodeRestart.py +++ b/tests/system-test/8-stream/snodeRestart.py @@ -27,7 +27,7 @@ class TDTestCase: time.sleep(1) tdSql.query("use test") tdSql.query("create snode on dnode 4") - tdSql.query("create stream if not exists s1 trigger at_once ignore expired 0 ignore update 0 fill_history 1 into st1 as select sum(voltage) from meters partition by groupid interval(4s)") + tdSql.query("create stream if not exists s1 trigger at_once ignore expired 0 ignore update 0 fill_history 1 into st1 as select _wstart, sum(voltage) from meters partition by groupid interval(4s)") tdLog.debug("create stream use snode and insert data") time.sleep(10) From 54638feca47654268bae168b00a7932c19c0bd41 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Thu, 16 Nov 2023 17:40:28 +0800 Subject: [PATCH 09/56] docs: build 3.2.1.0 --- cmake/cmake.version | 2 +- docs/en/28-releases/01-tdengine.md | 4 ++++ docs/zh/28-releases/01-tdengine.md | 4 ++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/cmake/cmake.version b/cmake/cmake.version index b8871dbce3..e9cf31e75b 100644 --- a/cmake/cmake.version +++ b/cmake/cmake.version @@ -2,7 +2,7 @@ IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "3.2.1.0.alpha") + SET(TD_VER_NUMBER "3.2.2.0.alpha") ENDIF () IF (DEFINED VERCOMPATIBLE) diff --git a/docs/en/28-releases/01-tdengine.md b/docs/en/28-releases/01-tdengine.md index e43cd638f6..f846ede52f 100644 --- a/docs/en/28-releases/01-tdengine.md +++ b/docs/en/28-releases/01-tdengine.md @@ -10,6 +10,10 @@ For TDengine 2.x installation packages by version, please visit [here](https://t import Release from "/components/ReleaseV3"; +## 3.2.1.0 + + + ## 3.2.0.0 diff --git a/docs/zh/28-releases/01-tdengine.md b/docs/zh/28-releases/01-tdengine.md index 5ebf3af1eb..cfc4b92eee 100644 --- a/docs/zh/28-releases/01-tdengine.md +++ b/docs/zh/28-releases/01-tdengine.md @@ -10,6 +10,10 @@ TDengine 2.x 各版本安装包请访问[这里](https://www.taosdata.com/all-do import Release from "/components/ReleaseV3"; +## 3.2.1.0 + + + ## 3.2.0.0 From 50126e44252f0671dbe480ddc6a13494f2ce67d4 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 16 Nov 2023 19:52:05 +0800 Subject: [PATCH 10/56] enh: support spread active codes --- include/util/taoserror.h | 1 + source/dnode/mnode/impl/inc/mndCluster.h | 2 +- source/dnode/mnode/impl/src/mndCluster.c | 8 +++++--- source/dnode/mnode/impl/src/mndDnode.c | 7 +++++-- source/util/src/terror.c | 1 + 5 files changed, 13 insertions(+), 6 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index a3ee294338..ce8db162b6 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -557,6 +557,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_GRANT_GEN_IVLD_KEY TAOS_DEF_ERROR_CODE(0, 0x0812) #define TSDB_CODE_GRANT_GEN_APP_LIMIT TAOS_DEF_ERROR_CODE(0, 0x0813) #define TSDB_CODE_GRANT_GEN_ENC_IVLD_KLEN TAOS_DEF_ERROR_CODE(0, 0x0814) +#define TSDB_CODE_GRANT_PAR_IVLD_DIST TAOS_DEF_ERROR_CODE(0, 0x0815) // sync // #define TSDB_CODE_SYN_INVALID_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0900) // 2.x diff --git a/source/dnode/mnode/impl/inc/mndCluster.h b/source/dnode/mnode/impl/inc/mndCluster.h index 2b868b567a..81cfce6530 100644 --- a/source/dnode/mnode/impl/inc/mndCluster.h +++ b/source/dnode/mnode/impl/inc/mndCluster.h @@ -29,7 +29,7 @@ int64_t mndGetClusterId(SMnode *pMnode); int64_t mndGetClusterCreateTime(SMnode *pMnode); int32_t mndGetClusterGrantedInfo(SMnode *pMnode, SGrantedInfo *pInfo); int64_t mndGetClusterUpTime(SMnode *pMnode); -int32_t mndProcessGrantedTime(SMnode *pMnode, int64_t grantedTime); +int32_t mndProcessGrantedInfo(SMnode *pMnode, SGrantedInfo *pInfo); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index 20e25d918c..bcef419c31 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -236,6 +236,7 @@ static int32_t mndClusterActionUpdate(SSdb *pSdb, SClusterObj *pOld, SClusterObj pNew, pOld->upTime, pNew->upTime); pOld->upTime = pNew->upTime; pOld->grantedTime = pNew->grantedTime; + pOld->connGrantedTime = pNew->connGrantedTime; pOld->updateTime = taosGetTimestampMs(); return 0; } @@ -378,17 +379,18 @@ static int32_t mndProcessUptimeTimer(SRpcMsg *pReq) { return 0; } -int32_t mndProcessGrantedTime(SMnode *pMnode, int64_t grantedTime) { +int32_t mndProcessGrantedInfo(SMnode *pMnode, SGrantedInfo *pInfo) { SClusterObj clusterObj = {0}; void *pIter = NULL; SClusterObj *pCluster = mndAcquireCluster(pMnode, &pIter); if (pCluster != NULL) { - if (pCluster->grantedTime >= grantedTime) { + if (pCluster->grantedTime >= pInfo->grantedTime && pCluster->connGrantedTime > pInfo->connGrantedTime) { mndReleaseCluster(pMnode, pCluster, pIter); return 0; } memcpy(&clusterObj, pCluster, sizeof(SClusterObj)); - clusterObj.grantedTime = grantedTime; + if (pCluster->grantedTime < pInfo->grantedTime) clusterObj.grantedTime = pInfo->grantedTime; + if (pCluster->connGrantedTime < pInfo->connGrantedTime) clusterObj.connGrantedTime = pInfo->connGrantedTime; mndReleaseCluster(pMnode, pCluster, pIter); } diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index e224aceec2..d46f431ade 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -1284,6 +1284,10 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { strcpy(dcfgReq.config, "supportvnodes"); snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag); } else if (strncasecmp(cfgReq.config, "activeCode", 10) == 0 || strncasecmp(cfgReq.config, "cActiveCode", 11) == 0) { + if (cfgReq.dnodeId != -1) { + terrno = TSDB_CODE_INVALID_CFG; + goto _err_out; + } int8_t opt = strncasecmp(cfgReq.config, "a", 1) == 0 ? DND_ACTIVE_CODE : DND_CONN_ACTIVE_CODE; int8_t index = opt == DND_ACTIVE_CODE ? 10 : 11; if (' ' != cfgReq.config[index] && 0 != cfgReq.config[index]) { @@ -1304,9 +1308,8 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { strcpy(dcfgReq.config, opt == DND_ACTIVE_CODE ? "activeCode" : "cActiveCode"); snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%s", cfgReq.value); - if (mndConfigDnode(pMnode, pReq, &cfgReq, opt) != 0) { + if ((terrno = mndConfigDnode(pMnode, pReq, &cfgReq, opt)) != 0) { mError("dnode:%d, failed to config activeCode since %s", cfgReq.dnodeId, terrstr()); - terrno = TSDB_CODE_INVALID_CFG; goto _err_out; } tFreeSMCfgDnodeReq(&cfgReq); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 21022f2016..f310db53ef 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -444,6 +444,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_PAR_DEC_IVLD_KLEN, "Invalid klen to decod TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_GEN_IVLD_KEY, "Invalid key to gen active code") TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_GEN_APP_LIMIT, "Limited app num to gen active code") TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_GEN_ENC_IVLD_KLEN, "Invalid klen to encode active code") +TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_PAR_IVLD_DIST, "Invalid dist to parse active code") // sync TAOS_DEFINE_ERROR(TSDB_CODE_SYN_TIMEOUT, "Sync timeout") From 9e12ad408e92eab11a0a10fee096b8a3f7b7673b Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 16 Nov 2023 20:01:54 +0800 Subject: [PATCH 11/56] enh: support spread active codes --- source/dnode/mnode/impl/src/mndCluster.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index bcef419c31..b6d36763cc 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -384,7 +384,7 @@ int32_t mndProcessGrantedInfo(SMnode *pMnode, SGrantedInfo *pInfo) { void *pIter = NULL; SClusterObj *pCluster = mndAcquireCluster(pMnode, &pIter); if (pCluster != NULL) { - if (pCluster->grantedTime >= pInfo->grantedTime && pCluster->connGrantedTime > pInfo->connGrantedTime) { + if (pCluster->grantedTime >= pInfo->grantedTime && pCluster->connGrantedTime >= pInfo->connGrantedTime) { mndReleaseCluster(pMnode, pCluster, pIter); return 0; } @@ -395,12 +395,12 @@ int32_t mndProcessGrantedInfo(SMnode *pMnode, SGrantedInfo *pInfo) { } if (clusterObj.id <= 0) { - mError("can't get cluster info while update uptime"); + mError("can't get cluster info while update granted info"); return -1; } - mInfo("update cluster granted time to %" PRIi64, clusterObj.grantedTime); - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, NULL, "granted-time"); + mInfo("update cluster granted info to %" PRIi64 ",%" PRIi64, clusterObj.grantedTime, clusterObj.connGrantedTime); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, NULL, "granted-info"); if (pTrans == NULL) return -1; SSdbRaw *pCommitRaw = mndClusterActionEncode(&clusterObj); From 63a17295a797b6bca40a53061ce681a98bbecac4 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 16 Nov 2023 20:09:56 +0800 Subject: [PATCH 12/56] enh: support spread active codes --- source/dnode/mnode/impl/inc/mndCluster.h | 3 ++- source/dnode/mnode/impl/src/mndCluster.c | 3 +-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndCluster.h b/source/dnode/mnode/impl/inc/mndCluster.h index 81cfce6530..93496b9330 100644 --- a/source/dnode/mnode/impl/inc/mndCluster.h +++ b/source/dnode/mnode/impl/inc/mndCluster.h @@ -28,8 +28,9 @@ int32_t mndGetClusterName(SMnode *pMnode, char *clusterName, int32_t len); int64_t mndGetClusterId(SMnode *pMnode); int64_t mndGetClusterCreateTime(SMnode *pMnode); int32_t mndGetClusterGrantedInfo(SMnode *pMnode, SGrantedInfo *pInfo); +int32_t mndSetClusterGrantedInfo(SMnode *pMnode, SGrantedInfo *pInfo); int64_t mndGetClusterUpTime(SMnode *pMnode); -int32_t mndProcessGrantedInfo(SMnode *pMnode, SGrantedInfo *pInfo); + #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index b6d36763cc..26c678b513 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -379,7 +379,7 @@ static int32_t mndProcessUptimeTimer(SRpcMsg *pReq) { return 0; } -int32_t mndProcessGrantedInfo(SMnode *pMnode, SGrantedInfo *pInfo) { +int32_t mndSetClusterGrantedInfo(SMnode *pMnode, SGrantedInfo *pInfo) { SClusterObj clusterObj = {0}; void *pIter = NULL; SClusterObj *pCluster = mndAcquireCluster(pMnode, &pIter); @@ -399,7 +399,6 @@ int32_t mndProcessGrantedInfo(SMnode *pMnode, SGrantedInfo *pInfo) { return -1; } - mInfo("update cluster granted info to %" PRIi64 ",%" PRIi64, clusterObj.grantedTime, clusterObj.connGrantedTime); STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, NULL, "granted-info"); if (pTrans == NULL) return -1; From b64e506292f7034b0500a77d8ab0836190a2e02d Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Thu, 16 Nov 2023 20:15:41 +0800 Subject: [PATCH 13/56] Update mndCluster.h --- source/dnode/mnode/impl/inc/mndCluster.h | 1 - 1 file changed, 1 deletion(-) diff --git a/source/dnode/mnode/impl/inc/mndCluster.h b/source/dnode/mnode/impl/inc/mndCluster.h index 93496b9330..2b59d9dbf5 100644 --- a/source/dnode/mnode/impl/inc/mndCluster.h +++ b/source/dnode/mnode/impl/inc/mndCluster.h @@ -31,7 +31,6 @@ int32_t mndGetClusterGrantedInfo(SMnode *pMnode, SGrantedInfo *pInfo); int32_t mndSetClusterGrantedInfo(SMnode *pMnode, SGrantedInfo *pInfo); int64_t mndGetClusterUpTime(SMnode *pMnode); - #ifdef __cplusplus } #endif From a5de1b032bd99c99001e05811c97da3c3081963e Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 16 Nov 2023 21:19:58 +0800 Subject: [PATCH 14/56] fix:snode upport stream task --- include/dnode/snode/snode.h | 1 + include/libs/stream/tstream.h | 4 +- source/dnode/mgmt/mgmt_snode/src/smInt.c | 5 + source/dnode/snode/src/snode.c | 108 ++++++++++--------- source/dnode/vnode/src/tq/tq.c | 4 +- source/libs/stream/src/streamStart.c | 13 +-- tests/system-test/7-tmq/tmqVnodeReplicate.py | 1 - tests/system-test/8-stream/snodeRestart.py | 45 +++++--- tests/system-test/test.py | 2 +- 9 files changed, 108 insertions(+), 75 deletions(-) diff --git a/include/dnode/snode/snode.h b/include/dnode/snode/snode.h index e8c64b07c4..c3dfd3a611 100644 --- a/include/dnode/snode/snode.h +++ b/include/dnode/snode/snode.h @@ -45,6 +45,7 @@ typedef struct { */ SSnode *sndOpen(const char *path, const SSnodeOpt *pOption); +int32_t sndInit(SSnode * pSnode); /** * @brief Stop Snode in Dnode. * diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 3ce2de476e..cb13a50ff4 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -592,7 +592,7 @@ typedef struct { int32_t downstreamNodeId; int32_t downstreamTaskId; int32_t childId; - int32_t oldStage; + int64_t oldStage; int8_t status; } SStreamTaskCheckRsp; @@ -760,7 +760,7 @@ void initRpcMsg(SRpcMsg* pMsg, int32_t msgType, void* pCont, int32_t contLen) // recover and fill history void streamTaskCheckDownstream(SStreamTask* pTask); -int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_t vgId, int64_t stage); +int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_t vgId, int64_t stage, int64_t* oldStage); int32_t streamTaskUpdateEpsetInfo(SStreamTask* pTask, SArray* pNodeList); void streamTaskResetUpstreamStageInfo(SStreamTask* pTask); bool streamTaskAllUpstreamClosed(SStreamTask* pTask); diff --git a/source/dnode/mgmt/mgmt_snode/src/smInt.c b/source/dnode/mgmt/mgmt_snode/src/smInt.c index 47c2993014..56744e4654 100644 --- a/source/dnode/mgmt/mgmt_snode/src/smInt.c +++ b/source/dnode/mgmt/mgmt_snode/src/smInt.c @@ -76,9 +76,14 @@ int32_t smOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { return 0; } +static int32_t smStartSnodes(SSnodeMgmt *pMgmt) { + return sndInit(pMgmt->pSnode); +} + SMgmtFunc smGetMgmtFunc() { SMgmtFunc mgmtFunc = {0}; mgmtFunc.openFp = smOpen; + mgmtFunc.startFp = (NodeStartFp)smStartSnodes; mgmtFunc.closeFp = (NodeCloseFp)smClose; mgmtFunc.createFp = (NodeCreateFp)smProcessCreateReq; mgmtFunc.dropFp = (NodeDropFp)smProcessDropReq; diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 25400220b8..4863a1f7b7 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -143,50 +143,6 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProcessVer return 0; } -SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) { - SSnode *pSnode = taosMemoryCalloc(1, sizeof(SSnode)); - if (pSnode == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - pSnode->path = taosStrdup(path); - if (pSnode->path == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto FAIL; - } - - pSnode->msgCb = pOption->msgCb; - pSnode->pMeta = streamMetaOpen(path, pSnode, (FTaskExpand *)sndExpandTask, SNODE_HANDLE, taosGetTimestampMs()); - if (pSnode->pMeta == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto FAIL; - } - - if (streamMetaLoadAllTasks(pSnode->pMeta) < 0) { - goto FAIL; - } - - stopRsync(); - startRsync(); - - return pSnode; - -FAIL: - taosMemoryFree(pSnode->path); - taosMemoryFree(pSnode); - return NULL; -} - -void sndClose(SSnode *pSnode) { - streamMetaNotifyClose(pSnode->pMeta); - streamMetaCommit(pSnode->pMeta); - streamMetaClose(pSnode->pMeta); - taosMemoryFree(pSnode->path); - taosMemoryFree(pSnode); -} - -int32_t sndGetLoad(SSnode *pSnode, SSnodeLoad *pLoad) { return 0; } - int32_t sndStartStreamTasks(SSnode* pSnode) { int32_t code = TSDB_CODE_SUCCESS; int32_t vgId = SNODE_HANDLE; @@ -223,7 +179,7 @@ int32_t sndStartStreamTasks(SSnode* pSnode) { if (pTask->status.downstreamReady == 1) { if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { sndDebug("s-task:%s downstream ready, no need to check downstream, check only related fill-history task", - pTask->id.idStr); + pTask->id.idStr); streamLaunchFillHistoryTask(pTask); } @@ -284,7 +240,7 @@ int32_t sndRestartStreamTasks(SSnode* pSnode) { terrno = 0; sndInfo("vgId:%d tasks are all updated and stopped, restart all tasks, triggered by transId:%d", vgId, - pMeta->updateInfo.transId); + pMeta->updateInfo.transId); while (streamMetaTaskInTimer(pMeta)) { sndDebug("vgId:%d some tasks in timer, wait for 100ms and recheck", pMeta->vgId); @@ -301,11 +257,12 @@ int32_t sndRestartStreamTasks(SSnode* pSnode) { return code; } + streamMetaInitBackend(pMeta); int64_t el = taosGetTimestampMs() - st; - sndInfo("vgId:%d close&reload state elapsed time:%.3fms", vgId, el/1000.); + sndInfo("vgId:%d close&reload state elapsed time:%.3fs", vgId, el/1000.); - code = streamMetaLoadAllTasks(pSnode->pMeta); + code = streamMetaLoadAllTasks(pMeta); if (code != TSDB_CODE_SUCCESS) { sndError("vgId:%d failed to load stream tasks, code:%s", vgId, tstrerror(terrno)); streamMetaWUnLock(pMeta); @@ -314,13 +271,64 @@ int32_t sndRestartStreamTasks(SSnode* pSnode) { } sndInfo("vgId:%d restart all stream tasks after all tasks being updated", vgId); sndResetStreamTaskStatus(pSnode); - sndStartStreamTasks(pSnode); streamMetaWUnLock(pMeta); + sndStartStreamTasks(pSnode); + code = terrno; return code; } +SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) { + SSnode *pSnode = taosMemoryCalloc(1, sizeof(SSnode)); + if (pSnode == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + pSnode->path = taosStrdup(path); + if (pSnode->path == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto FAIL; + } + + pSnode->msgCb = pOption->msgCb; + pSnode->pMeta = streamMetaOpen(path, pSnode, (FTaskExpand *)sndExpandTask, SNODE_HANDLE, taosGetTimestampMs()); + if (pSnode->pMeta == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto FAIL; + } + + if (streamMetaLoadAllTasks(pSnode->pMeta) < 0) { + goto FAIL; + } + + stopRsync(); + startRsync(); + + return pSnode; + +FAIL: + taosMemoryFree(pSnode->path); + taosMemoryFree(pSnode); + return NULL; +} + +int32_t sndInit(SSnode * pSnode) { + sndResetStreamTaskStatus(pSnode); + sndStartStreamTasks(pSnode); + return 0; +} + +void sndClose(SSnode *pSnode) { + streamMetaNotifyClose(pSnode->pMeta); + streamMetaCommit(pSnode->pMeta); + streamMetaClose(pSnode->pMeta); + taosMemoryFree(pSnode->path); + taosMemoryFree(pSnode); +} + +int32_t sndGetLoad(SSnode *pSnode, SSnodeLoad *pLoad) { return 0; } + int32_t sndStartStreamTaskAsync(SSnode* pSnode, bool restart) { SStreamMeta* pMeta = pSnode->pMeta; int32_t vgId = pMeta->vgId; @@ -622,7 +630,7 @@ int32_t sndProcessStreamTaskCheckReq(SSnode *pSnode, SRpcMsg *pMsg) { SStreamTask *pTask = streamMetaAcquireTask(pSnode->pMeta, req.streamId, taskId); if (pTask != NULL) { - rsp.status = streamTaskCheckStatus(pTask, req.upstreamTaskId, req.upstreamNodeId, req.stage); + rsp.status = streamTaskCheckStatus(pTask, req.upstreamTaskId, req.upstreamNodeId, req.stage, &rsp.oldStage); streamMetaReleaseTask(pSnode->pMeta, pTask); char* p = NULL; streamTaskGetStatus(pTask, &p); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 8ee0edcb31..6ba5529049 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -923,12 +923,12 @@ int32_t tqProcessTaskCheckReq(STQ* pTq, SRpcMsg* pMsg) { } else { SStreamTask* pTask = streamMetaAcquireTask(pMeta, req.streamId, taskId); if (pTask != NULL) { - rsp.status = streamTaskCheckStatus(pTask, req.upstreamTaskId, req.upstreamNodeId, req.stage); + rsp.status = streamTaskCheckStatus(pTask, req.upstreamTaskId, req.upstreamNodeId, req.stage, &rsp.oldStage); streamMetaReleaseTask(pMeta, pTask); char* p = NULL; streamTaskGetStatus(pTask, &p); - tqDebug("s-task:%s status:%s, stage:%d recv task check req(reqId:0x%" PRIx64 ") task:0x%x (vgId:%d), check_status:%d", + tqDebug("s-task:%s status:%s, stage:%"PRId64" recv task check req(reqId:0x%" PRIx64 ") task:0x%x (vgId:%d), check_status:%d", pTask->id.idStr, p, rsp.oldStage, rsp.reqId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status); } else { rsp.status = TASK_DOWNSTREAM_NOT_READY; diff --git a/source/libs/stream/src/streamStart.c b/source/libs/stream/src/streamStart.c index 97eb7b79a2..a1c0fa2040 100644 --- a/source/libs/stream/src/streamStart.c +++ b/source/libs/stream/src/streamStart.c @@ -290,10 +290,11 @@ static void recheckDownstreamTasks(void* param, void* tmrId) { stDebug("s-task:%s complete send check in timer, ref:%d", pTask->id.idStr, ref); } -int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_t vgId, int64_t stage) { +int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_t vgId, int64_t stage, int64_t* oldStage) { SStreamChildEpInfo* pInfo = streamTaskGetUpstreamTaskEpInfo(pTask, upstreamTaskId); ASSERT(pInfo != NULL); + *oldStage = pInfo->stage; const char* id = pTask->id.idStr; if (stage == -1) { stDebug("s-task:%s receive check msg from upstream task:0x%x(vgId:%d), invalid stageId:%" PRId64 ", not ready", id, @@ -459,9 +460,9 @@ int32_t streamProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRs if (pRsp->status == TASK_UPSTREAM_NEW_STAGE || pRsp->status == TASK_DOWNSTREAM_NOT_LEADER) { if (pRsp->status == TASK_UPSTREAM_NEW_STAGE) { stError( - "s-task:%s vgId:%d self vnode-transfer/leader-change/restart detected, old stage:%d, current stage:%d, " + "s-task:%s vgId:%d self vnode-transfer/leader-change/restart detected, old stage:%"PRId64", current stage:%"PRId64", " "not check wait for downstream task nodeUpdate, and all tasks restart", - id, pRsp->upstreamNodeId, pRsp->oldStage, (int32_t)pTask->pMeta->stage); + id, pRsp->upstreamNodeId, pRsp->oldStage, pTask->pMeta->stage); } else { stError( "s-task:%s downstream taskId:0x%x (vgId:%d) not leader, self dispatch epset needs to be updated, not check " @@ -476,7 +477,7 @@ int32_t streamProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRs STaskRecheckInfo* pInfo = createRecheckInfo(pTask, pRsp); int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1); - stDebug("s-task:%s downstream taskId:0x%x (vgId:%d) not ready, stage:%d, retry in 100ms, ref:%d ", id, + stDebug("s-task:%s downstream taskId:0x%x (vgId:%d) not ready, stage:%"PRId64", retry in 100ms, ref:%d ", id, pRsp->downstreamTaskId, pRsp->downstreamNodeId, pRsp->oldStage, ref); pInfo->checkTimer = taosTmrStart(recheckDownstreamTasks, CHECK_DOWNSTREAM_INTERVAL, pInfo, streamEnv.timer); } @@ -926,7 +927,7 @@ int32_t tEncodeStreamTaskCheckRsp(SEncoder* pEncoder, const SStreamTaskCheckRsp* if (tEncodeI32(pEncoder, pRsp->downstreamNodeId) < 0) return -1; if (tEncodeI32(pEncoder, pRsp->downstreamTaskId) < 0) return -1; if (tEncodeI32(pEncoder, pRsp->childId) < 0) return -1; - if (tEncodeI32(pEncoder, pRsp->oldStage) < 0) return -1; + if (tEncodeI64(pEncoder, pRsp->oldStage) < 0) return -1; if (tEncodeI8(pEncoder, pRsp->status) < 0) return -1; tEndEncode(pEncoder); return pEncoder->pos; @@ -941,7 +942,7 @@ int32_t tDecodeStreamTaskCheckRsp(SDecoder* pDecoder, SStreamTaskCheckRsp* pRsp) if (tDecodeI32(pDecoder, &pRsp->downstreamNodeId) < 0) return -1; if (tDecodeI32(pDecoder, &pRsp->downstreamTaskId) < 0) return -1; if (tDecodeI32(pDecoder, &pRsp->childId) < 0) return -1; - if (tDecodeI32(pDecoder, &pRsp->oldStage) < 0) return -1; + if (tDecodeI64(pDecoder, &pRsp->oldStage) < 0) return -1; if (tDecodeI8(pDecoder, &pRsp->status) < 0) return -1; tEndDecode(pDecoder); return 0; diff --git a/tests/system-test/7-tmq/tmqVnodeReplicate.py b/tests/system-test/7-tmq/tmqVnodeReplicate.py index fd8ece02e0..afb3319491 100644 --- a/tests/system-test/7-tmq/tmqVnodeReplicate.py +++ b/tests/system-test/7-tmq/tmqVnodeReplicate.py @@ -105,7 +105,6 @@ class TDTestCase: topicNameList = ['topic1'] # expectRowsList = [] - tmqCom.initConsumerTable("cdb", self.replicaVar) tdLog.info("create topics from stb with filter") queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName']) diff --git a/tests/system-test/8-stream/snodeRestart.py b/tests/system-test/8-stream/snodeRestart.py index 6adf874ecd..d2b9062c6e 100644 --- a/tests/system-test/8-stream/snodeRestart.py +++ b/tests/system-test/8-stream/snodeRestart.py @@ -15,40 +15,59 @@ from util.common import * from util.cluster import * class TDTestCase: + updatecfgDict = {'checkpointInterval': 1100} + print("===================: ", updatecfgDict) + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) def case1(self): - tdLog.debug("case1 start") + tdLog.debug("========case1 start========") os.system("nohup taosBenchmark -y -B 1 -t 4 -S 1000 -n 1000 -i 1000 -v 2 > /dev/null 2>&1 &") - time.sleep(1) + time.sleep(4) tdSql.query("use test") tdSql.query("create snode on dnode 4") - tdSql.query("create stream if not exists s1 trigger at_once ignore expired 0 ignore update 0 fill_history 1 into st1 as select _wstart, sum(voltage) from meters partition by groupid interval(4s)") - tdLog.debug("create stream use snode and insert data") - time.sleep(10) + tdSql.query("create stream if not exists s1 trigger at_once ignore expired 0 ignore update 0 fill_history 1 into st1 as select _wstart,sum(voltage),groupid from meters partition by groupid interval(2s)") + tdLog.debug("========create stream useing snode and insert data ok========") + time.sleep(4) tdDnodes = cluster.dnodes tdDnodes[3].stoptaosd() time.sleep(2) tdDnodes[3].starttaosd() - tdLog.debug("snode restart ok") + tdLog.debug("========snode restart ok========") - time.sleep(500) + time.sleep(30) os.system("kill -9 `pgrep taosBenchmark`") + tdLog.debug("========stop insert ok========") + time.sleep(2) - tdSql.query("select sum(voltage) from meters partition by groupid interval(4s)") - # tdSql.checkRows(1) - # + tdSql.query("select _wstart,sum(voltage),groupid from meters partition by groupid interval(2s) order by groupid,_wstart") + rowCnt = tdSql.getRows() + results = [] + for i in range(rowCnt): + results.append(tdSql.getData(i,1)) + + tdSql.query("select * from st1 order by groupid,_wstart") + tdSql.checkRows(rowCnt) + for i in range(rowCnt): + data1 = tdSql.getData(i,1) + data2 = results[i] + if data1 != data2: + tdLog.info("num: %d, act data: %d, expect data: %d"%(i, data1, data2)) + tdLog.exit("check data error!") # tdSql.checkData(0, 0, '2016-01-01 08:00:07.000') # tdSql.checkData(0, 1, 2000) - tdSql.query("drop snode on dnode 4") - tdSql.query("drop stream if exists s1") - tdSql.query("drop database test") + # tdLog.debug("========sleep 500s========") + # time.sleep(500) + # + # tdSql.query("drop snode on dnode 4") + # tdSql.query("drop stream if exists s1") + # tdSql.query("drop database test") tdLog.debug("case1 end") diff --git a/tests/system-test/test.py b/tests/system-test/test.py index 81f98fea22..795132b14e 100644 --- a/tests/system-test/test.py +++ b/tests/system-test/test.py @@ -582,7 +582,7 @@ if __name__ == "__main__": tdDnodes.setAsan(asan) tdDnodes.stopAll() for dnode in tdDnodes.dnodes: - tdDnodes.deploy(dnode.index,{}) + tdDnodes.deploy(dnode.index,updateCfgDict) for dnode in tdDnodes.dnodes: tdDnodes.starttaosd(dnode.index) tdCases.logSql(logSql) From da233a452441b71f751784f479b4a891825bd175 Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 17 Nov 2023 08:38:13 +0800 Subject: [PATCH 15/56] enh: adjust test case --- tests/system-test/0-others/information_schema.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/system-test/0-others/information_schema.py b/tests/system-test/0-others/information_schema.py index ac952d383a..a4c18d2938 100644 --- a/tests/system-test/0-others/information_schema.py +++ b/tests/system-test/0-others/information_schema.py @@ -247,7 +247,10 @@ class TDTestCase: tdSql.error('alter all dnodes "activeCode" "' + self.str510 + '"') tdSql.query(f'select * from information_schema.ins_dnodes') tdSql.checkEqual(tdSql.queryResult[0][8],"") - tdSql.execute('alter dnode 1 "activeCode" ""') + tdSql.error('alter dnode 1 "activeCode" ""') + tdSql.error('alter dnode 1 "activeCode"') + tdSql.Execute('alter all dnodes "activeCode" ""') + tdSql.Execute('alter all dnodes "activeCode"') tdSql.query(f'select active_code,c_active_code from information_schema.ins_dnodes') tdSql.checkEqual(tdSql.queryResult[0][0],"") tdSql.checkEqual(tdSql.queryResult[0][1],'') @@ -259,6 +262,10 @@ class TDTestCase: tdSql.error('alter all dnodes "cActiveCode" "' + self.str257 + '"') tdSql.error('alter all dnodes "cActiveCode" "' + self.str254 + '"') tdSql.error('alter dnode 1 "cActiveCode" "' + self.str510 + '"') + tdSql.error('alter dnode 1 "cActiveCode" ""') + tdSql.error('alter dnode 1 "cActiveCode"') + tdSql.Execute('alter all dnodes "cActiveCode" ""') + tdSql.Execute('alter all dnodes "cActiveCode"') tdSql.query(f'select active_code,c_active_code from information_schema.ins_dnodes') tdSql.checkEqual(tdSql.queryResult[0][0],"") tdSql.checkEqual(tdSql.queryResult[0][1],"") From 14a2be8ed2b770ac23ad10690dd1cadc14d1fcbe Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 17 Nov 2023 08:41:47 +0800 Subject: [PATCH 16/56] enh: adjust test case --- source/dnode/mnode/impl/inc/mndCluster.h | 1 - tests/system-test/0-others/information_schema.py | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndCluster.h b/source/dnode/mnode/impl/inc/mndCluster.h index 93496b9330..2b59d9dbf5 100644 --- a/source/dnode/mnode/impl/inc/mndCluster.h +++ b/source/dnode/mnode/impl/inc/mndCluster.h @@ -31,7 +31,6 @@ int32_t mndGetClusterGrantedInfo(SMnode *pMnode, SGrantedInfo *pInfo); int32_t mndSetClusterGrantedInfo(SMnode *pMnode, SGrantedInfo *pInfo); int64_t mndGetClusterUpTime(SMnode *pMnode); - #ifdef __cplusplus } #endif diff --git a/tests/system-test/0-others/information_schema.py b/tests/system-test/0-others/information_schema.py index a4c18d2938..27a3c73fb7 100644 --- a/tests/system-test/0-others/information_schema.py +++ b/tests/system-test/0-others/information_schema.py @@ -249,8 +249,8 @@ class TDTestCase: tdSql.checkEqual(tdSql.queryResult[0][8],"") tdSql.error('alter dnode 1 "activeCode" ""') tdSql.error('alter dnode 1 "activeCode"') - tdSql.Execute('alter all dnodes "activeCode" ""') - tdSql.Execute('alter all dnodes "activeCode"') + tdSql.execute('alter all dnodes "activeCode" ""') + tdSql.execute('alter all dnodes "activeCode"') tdSql.query(f'select active_code,c_active_code from information_schema.ins_dnodes') tdSql.checkEqual(tdSql.queryResult[0][0],"") tdSql.checkEqual(tdSql.queryResult[0][1],'') @@ -264,8 +264,8 @@ class TDTestCase: tdSql.error('alter dnode 1 "cActiveCode" "' + self.str510 + '"') tdSql.error('alter dnode 1 "cActiveCode" ""') tdSql.error('alter dnode 1 "cActiveCode"') - tdSql.Execute('alter all dnodes "cActiveCode" ""') - tdSql.Execute('alter all dnodes "cActiveCode"') + tdSql.execute('alter all dnodes "cActiveCode" ""') + tdSql.execute('alter all dnodes "cActiveCode"') tdSql.query(f'select active_code,c_active_code from information_schema.ins_dnodes') tdSql.checkEqual(tdSql.queryResult[0][0],"") tdSql.checkEqual(tdSql.queryResult[0][1],"") From cb9f784c33bca916ed5dbc5f21ed2c8b61995d94 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 17 Nov 2023 09:34:27 +0800 Subject: [PATCH 17/56] fix:add test cases --- tests/parallel_test/cases.task | 3 + .../{snodeRestart.py => snode_restart.py} | 48 ------------ .../8-stream/snode_restart_with_checkpoint.py | 78 +++++++++++++++++++ tests/system-test/8-stream/vnode_restart.py | 77 ++++++++++++++++++ 4 files changed, 158 insertions(+), 48 deletions(-) rename tests/system-test/8-stream/{snodeRestart.py => snode_restart.py} (58%) create mode 100644 tests/system-test/8-stream/snode_restart_with_checkpoint.py create mode 100644 tests/system-test/8-stream/vnode_restart.py diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 92eaec52b5..40eb926efb 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -20,6 +20,9 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/window_close_session_ext.py ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/partition_interval.py ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/pause_resume_test.py +#,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/vnode_restart.py +#,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/snode_restart.py +,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/snode_restart_with_checkpoint.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tbname_vgroup.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stbJoin.py diff --git a/tests/system-test/8-stream/snodeRestart.py b/tests/system-test/8-stream/snode_restart.py similarity index 58% rename from tests/system-test/8-stream/snodeRestart.py rename to tests/system-test/8-stream/snode_restart.py index d2b9062c6e..3657163ab0 100644 --- a/tests/system-test/8-stream/snodeRestart.py +++ b/tests/system-test/8-stream/snode_restart.py @@ -59,60 +59,12 @@ class TDTestCase: if data1 != data2: tdLog.info("num: %d, act data: %d, expect data: %d"%(i, data1, data2)) tdLog.exit("check data error!") - # tdSql.checkData(0, 0, '2016-01-01 08:00:07.000') - # tdSql.checkData(0, 1, 2000) # tdLog.debug("========sleep 500s========") # time.sleep(500) - # - # tdSql.query("drop snode on dnode 4") - # tdSql.query("drop stream if exists s1") - # tdSql.query("drop database test") tdLog.debug("case1 end") - def case2(self): - tdLog.debug("case2 start") - - updatecfgDict = {'checkpointInterval': 5} - print("===================: ", updatecfgDict) - - tdDnodes = cluster.dnodes - tdDnodes[0].stoptaosd() - tdDnodes[1].stoptaosd() - tdDnodes[2].stoptaosd() - tdDnodes[3].stoptaosd() - time.sleep(2) - tdDnodes[0].starttaosd() - tdDnodes[1].starttaosd() - tdDnodes[2].starttaosd() - tdDnodes[3].starttaosd() - tdLog.debug("taosd restart ok") - - os.system("taosBenchmark -y -B 1 -t 4 -S 1000 -n 1000 -i 1000 -v 2 &" ) - time.sleep(1) - tdSql.query("use test") - tdSql.query("create snode on dnode 4") - tdSql.query("create stream if not exists s1 trigger at_once ignore expired 0 ignore update 0 fill_history 1 into st1 as select sum(voltage) from meters partition by groupid interval(4s)") - tdLog.debug("create stream use snode and insert data") - time.sleep(10) - - tdDnodes[3].stoptaosd() - time.sleep(2) - tdDnodes[3].starttaosd() - tdLog.debug("snode restart ok") - - time.sleep(5) - os.system("kill -9 `pgrep taosBenchmark`") - - tdSql.query("select sum(voltage) from meters partition by groupid interval(4s)") - # tdSql.checkRows(1) - # - # tdSql.checkData(0, 0, '2016-01-01 08:00:07.000') - # tdSql.checkData(0, 1, 2000) - - tdLog.debug("case2 end") - def run(self): self.case1() diff --git a/tests/system-test/8-stream/snode_restart_with_checkpoint.py b/tests/system-test/8-stream/snode_restart_with_checkpoint.py new file mode 100644 index 0000000000..9567bbe439 --- /dev/null +++ b/tests/system-test/8-stream/snode_restart_with_checkpoint.py @@ -0,0 +1,78 @@ + +import taos +import sys +import time +import socket +import os +import threading +import math + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +from util.cluster import * + +class TDTestCase: + # updatecfgDict = {'checkpointInterval': 5} + # print("===================: ", updatecfgDict) + + def init(self, conn, logSql, replicaVar=1): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + + def case1(self): + tdLog.debug("========case1 start========") + + os.system("nohup taosBenchmark -y -B 1 -t 4 -S 1000 -n 1000 -i 1000 -v 2 > /dev/null 2>&1 &") + time.sleep(4) + tdSql.query("use test") + tdSql.query("create snode on dnode 4") + tdSql.query("create stream if not exists s1 trigger at_once ignore expired 0 ignore update 0 fill_history 1 into st1 as select _wstart,sum(voltage),groupid from meters partition by groupid interval(2s)") + tdLog.debug("========create stream useing snode and insert data ok========") + time.sleep(60) + + tdDnodes = cluster.dnodes + tdDnodes[3].stoptaosd() + time.sleep(2) + tdDnodes[3].starttaosd() + tdLog.debug("========snode restart ok========") + + time.sleep(30) + os.system("kill -9 `pgrep taosBenchmark`") + tdLog.debug("========stop insert ok========") + time.sleep(2) + + tdSql.query("select _wstart,sum(voltage),groupid from meters partition by groupid interval(2s) order by groupid,_wstart") + rowCnt = tdSql.getRows() + results = [] + for i in range(rowCnt): + results.append(tdSql.getData(i,1)) + + tdSql.query("select * from st1 order by groupid,_wstart") + tdSql.checkRows(rowCnt) + for i in range(rowCnt): + data1 = tdSql.getData(i,1) + data2 = results[i] + if data1 != data2: + tdLog.info("num: %d, act data: %d, expect data: %d"%(i, data1, data2)) + tdLog.exit("check data error!") + + # tdLog.debug("========sleep 500s========") + # time.sleep(500) + + tdLog.debug("case1 end") + + def run(self): + self.case1() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/8-stream/vnode_restart.py b/tests/system-test/8-stream/vnode_restart.py new file mode 100644 index 0000000000..a53432b77a --- /dev/null +++ b/tests/system-test/8-stream/vnode_restart.py @@ -0,0 +1,77 @@ + +import taos +import sys +import time +import socket +import os +import threading +import math + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +from util.cluster import * + +class TDTestCase: + updatecfgDict = {'checkpointInterval': 1100} + print("===================: ", updatecfgDict) + + def init(self, conn, logSql, replicaVar=1): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + + def case1(self): + tdLog.debug("========case1 start========") + + os.system("nohup taosBenchmark -y -B 1 -t 4 -S 1000 -n 1000 -i 1000 -v 2 > /dev/null 2>&1 &") + time.sleep(4) + tdSql.query("use test") + tdSql.query("create stream if not exists s1 trigger at_once ignore expired 0 ignore update 0 fill_history 1 into st1 as select _wstart,sum(voltage),groupid from meters partition by groupid interval(2s)") + tdLog.debug("========create stream useing snode and insert data ok========") + time.sleep(4) + + tdDnodes = cluster.dnodes + tdDnodes[2].stoptaosd() + time.sleep(2) + tdDnodes[2].starttaosd() + tdLog.debug("========vnode restart ok========") + + time.sleep(30) + os.system("kill -9 `pgrep taosBenchmark`") + tdLog.debug("========stop insert ok========") + time.sleep(2) + + tdSql.query("select _wstart,sum(voltage),groupid from meters partition by groupid interval(2s) order by groupid,_wstart") + rowCnt = tdSql.getRows() + results = [] + for i in range(rowCnt): + results.append(tdSql.getData(i,1)) + + tdSql.query("select * from st1 order by groupid,_wstart") + tdSql.checkRows(rowCnt) + for i in range(rowCnt): + data1 = tdSql.getData(i,1) + data2 = results[i] + if data1 != data2: + tdLog.info("num: %d, act data: %d, expect data: %d"%(i, data1, data2)) + tdLog.exit("check data error!") + + # tdLog.debug("========sleep 500s========") + # time.sleep(500) + + tdLog.debug("case1 end") + + def run(self): + self.case1() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From 60ad82d006188f32f8702620108b1ef39e13f8cf Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 17 Nov 2023 09:38:48 +0800 Subject: [PATCH 18/56] fix(cos/log): dump file name & line no --- source/common/CMakeLists.txt | 4 ++-- source/common/src/cos.c | 21 +++++++++++---------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/source/common/CMakeLists.txt b/source/common/CMakeLists.txt index c7831b2b6d..d3df1345df 100644 --- a/source/common/CMakeLists.txt +++ b/source/common/CMakeLists.txt @@ -60,8 +60,8 @@ if(${BUILD_S3}) find_library(S3_LIBRARY s3) find_library(CURL_LIBRARY curl $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH) find_library(XML2_LIBRARY xml2) - find_library(SSL_LIBRARY ssl $ENV{HOME}/.cos-local.2/lib64 NO_DEFAULT_PATH) - find_library(CRYPTO_LIBRARY crypto $ENV{HOME}/.cos-local.2/lib64 NO_DEFAULT_PATH) + find_library(SSL_LIBRARY ssl $ENV{HOME}/.cos-local.2/lib64 $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH) + find_library(CRYPTO_LIBRARY crypto $ENV{HOME}/.cos-local.2/lib64 $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH) target_link_libraries( common diff --git a/source/common/src/cos.c b/source/common/src/cos.c index 4cfc67fa94..fd462d5a11 100644 --- a/source/common/src/cos.c +++ b/source/common/src/cos.c @@ -66,11 +66,12 @@ static int should_retry() { return 0; } -static void s3PrintError(const char *func, S3Status status, char error_details[]) { +static void s3PrintError(const char *filename, int lineno, const char *funcname, S3Status status, + char error_details[]) { if (status < S3StatusErrorAccessDenied) { - uError("%s: %s", __func__, S3_get_status_name(status)); + uError("%s/%s:%d-%s: %s", filename, lineno, __func__, funcname, S3_get_status_name(status)); } else { - uError("%s: %s, %s", __func__, S3_get_status_name(status), error_details); + uError("%s/%s:%d-%s: %s, %s", filename, lineno, __func__, funcname, S3_get_status_name(status), error_details); } } @@ -437,7 +438,7 @@ static int try_get_parts_info(const char *bucketName, const char *key, UploadMan // printListMultipartHeader(data.allDetails); } } else { - s3PrintError(__func__, data.status, data.err_msg); + s3PrintError(__FILE__, __LINE__, __func__, data.status, data.err_msg); return -1; } @@ -498,7 +499,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { } while (S3_status_is_retryable(data.status) && should_retry()); if (data.status != S3StatusOK) { - s3PrintError(__func__, data.status, data.err_msg); + s3PrintError(__FILE__, __LINE__, __func__, data.status, data.err_msg); code = TAOS_SYSTEM_ERROR(EIO); } else if (data.contentLength) { uError("ERROR: %s Failed to read remaining %llu bytes from input", __func__, @@ -556,7 +557,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { } while (S3_status_is_retryable(manager.status) && should_retry()); if (manager.upload_id == 0 || manager.status != S3StatusOK) { - s3PrintError(__func__, manager.status, manager.err_msg); + s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg); code = TAOS_SYSTEM_ERROR(EIO); goto clean; } @@ -581,7 +582,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { partContentLength, 0, timeoutMsG, &partData); } while (S3_status_is_retryable(partData.put_object_data.status) && should_retry()); if (partData.put_object_data.status != S3StatusOK) { - s3PrintError(__func__, partData.put_object_data.status, partData.put_object_data.err_msg); + s3PrintError(__FILE__, __LINE__, __func__, partData.put_object_data.status, partData.put_object_data.err_msg); code = TAOS_SYSTEM_ERROR(EIO); goto clean; } @@ -609,7 +610,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { timeoutMsG, &manager); } while (S3_status_is_retryable(manager.status) && should_retry()); if (manager.status != S3StatusOK) { - s3PrintError(__func__, manager.status, manager.err_msg); + s3PrintError(__FILE__, __LINE__, __func__, manager.status, manager.err_msg); code = TAOS_SYSTEM_ERROR(EIO); goto clean; } @@ -722,7 +723,7 @@ static SArray *getListByPrefix(const char *prefix) { return data.objectArray; } } else { - s3PrintError(__func__, data.status, data.err_msg); + s3PrintError(__FILE__, __LINE__, __func__, data.status, data.err_msg); } taosArrayDestroyEx(data.objectArray, s3FreeObjectKey); @@ -741,7 +742,7 @@ void s3DeleteObjects(const char *object_name[], int nobject) { } while (S3_status_is_retryable(cbd.status) && should_retry()); if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) { - s3PrintError(__func__, cbd.status, cbd.err_msg); + s3PrintError(__FILE__, __LINE__, __func__, cbd.status, cbd.err_msg); } } } From 9d3ffa2c2cc9104599d18653f54f00a2de031609 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 17 Nov 2023 12:24:51 +0800 Subject: [PATCH 19/56] cos/print-error: fix error format --- source/common/src/cos.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/common/src/cos.c b/source/common/src/cos.c index fd462d5a11..684f437d05 100644 --- a/source/common/src/cos.c +++ b/source/common/src/cos.c @@ -69,9 +69,9 @@ static int should_retry() { static void s3PrintError(const char *filename, int lineno, const char *funcname, S3Status status, char error_details[]) { if (status < S3StatusErrorAccessDenied) { - uError("%s/%s:%d-%s: %s", filename, lineno, __func__, funcname, S3_get_status_name(status)); + uError("%s/%s:%d-%s: %s", __func__, filename, lineno, funcname, S3_get_status_name(status)); } else { - uError("%s/%s:%d-%s: %s, %s", filename, lineno, __func__, funcname, S3_get_status_name(status), error_details); + uError("%s/%s:%d-%s: %s, %s", __func__, filename, lineno, funcname, S3_get_status_name(status), error_details); } } @@ -885,7 +885,7 @@ long s3Size(const char *object_name) { } while (S3_status_is_retryable(cbd.status) && should_retry()); if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) { - uError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg); + s3PrintError(__FILE__, __LINE__, __func__, cbd.status, cbd.err_msg); } size = cbd.content_length; From 3ea79db6042495d04a97b1382d0c4b92ea948d70 Mon Sep 17 00:00:00 2001 From: slzhou Date: Fri, 17 Nov 2023 13:22:31 +0800 Subject: [PATCH 20/56] doc: insert into stb --- docs/en/12-taos-sql/05-insert.md | 49 +++++++++++++++++++++++++------- docs/zh/12-taos-sql/05-insert.md | 46 ++++++++++++++++++++++++------ 2 files changed, 76 insertions(+), 19 deletions(-) diff --git a/docs/en/12-taos-sql/05-insert.md b/docs/en/12-taos-sql/05-insert.md index 62c3e4ad8e..e5f502c563 100644 --- a/docs/en/12-taos-sql/05-insert.md +++ b/docs/en/12-taos-sql/05-insert.md @@ -5,7 +5,8 @@ description: This document describes how to insert data into TDengine. --- ## Syntax - +The writing of records supports two syntaxes, normal syntax and super table syntax. In the normal syntax, the table name immediately following `INSERT INTO` represents subtable names or regular table names. In the super table syntax, the table name immediately following `INSERT INTO` represents the super table name. +### Normal Syntax ```sql INSERT INTO tb_name @@ -20,6 +21,15 @@ INSERT INTO INSERT INTO tb_name [(field1_name, ...)] subquery ``` +### Super Table Syntax +```sql +INSERT INTO + stb1_name [(field1_name, ...)] + VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path + [stb2_name [(field1_name, ...)] + VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path + ...]; +``` **Timestamps** @@ -32,26 +42,34 @@ INSERT INTO tb_name [(field1_name, ...)] subquery **Syntax** -1. The USING clause automatically creates the specified subtable if it does not exist. If it's unknown whether the table already exists, the table can be created automatically while inserting using the SQL statement below. To use this functionality, a STable must be used as template and tag values must be provided. Any tags that you do not specify will be assigned a null value. +1. You can insert data into specified columns. Any columns in which you do not insert data will be assigned a null value. -2. You can insert data into specified columns. Any columns in which you do not insert data will be assigned a null value. +2. The VALUES clause inserts one or more rows of data into a table. -3. The VALUES clause inserts one or more rows of data into a table. +3. The FILE clause inserts tags or data from a comma-separates values (CSV) file. Do not include headers in your CSV files. -4. The FILE clause inserts tags or data from a comma-separates values (CSV) file. Do not include headers in your CSV files. +4. A single `INSERT ... VALUES` statement and `INSERT ... FILE` statement can write data to multiple tables. -5. A single `INSERT ... VALUES` statement and `INSERT ... FILE` statement can write data to multiple tables. - -6. The INSERT statement is fully parsed before being executed, so that if any element of the statement fails, the entire statement will fail. For example, the following statement will not create a table because the latter part of the statement is invalid: +5. The INSERT statement is fully parsed before being executed, so that if any element of the statement fails, the entire statement will fail. For example, the following statement will not create a table because the latter part of the statement is invalid: ```sql INSERT INTO d1001 USING meters TAGS('Beijing.Chaoyang', 2) VALUES('a'); ``` -7. However, an INSERT statement that writes data to multiple subtables can succeed for some tables and fail for others. This situation is caused because vnodes perform write operations independently of each other. One vnode failing to write data does not affect the ability of other vnodes to write successfully. +6. However, an INSERT statement that writes data to multiple subtables can succeed for some tables and fail for others. This situation is caused because vnodes perform write operations independently of each other. One vnode failing to write data does not affect the ability of other vnodes to write successfully. -8. Data from TDengine can be inserted into a specified table using the `INSERT ... subquery` statement. Arbitrary query statements are supported. This syntax can only be used for subtables and normal tables, and does not support automatic table creation. +**Normal Syntax** +1. The USING clause automatically creates the specified subtable if it does not exist. If it's unknown whether the table already exists, the table can be created automatically while inserting using the SQL statement below. To use this functionality, a STable must be used as template and tag values must be provided. Any tags that you do not specify will be assigned a null value. +2. Data from TDengine can be inserted into a specified table using the `INSERT ... subquery` statement. Arbitrary query statements are supported. This syntax can only be used for subtables and normal tables, and does not support automatic table creation. + +**Super Table Syntax** + +1. The tbname column must be included in the field_name list and represents the name of the child table. This column is of string type, and the use of the . character is not permitted in the tbname column. + +2. Tag columns are eligible for inclusion in the field_name list. If the specified child table doesn't exist, a new child table will be generated with the provided tag values. In the absence of specified tag values, the newly created table will have all NULL tag values. Existing child table tag values remain unchanged. + +3. Param binding is not supported. ## Insert a Record Single row or multiple rows specified with VALUES can be inserted into a specific table. A single row is inserted using the below statement. @@ -134,3 +152,14 @@ When writing data from a file, you can automatically create the specified subtab INSERT INTO d21001 USING meters TAGS ('California.SanFrancisco', 2) FILE '/tmp/csvfile_21001.csv' d21002 USING meters (groupId) TAGS (2) FILE '/tmp/csvfile_21002.csv'; ``` +## Super Table Syntax + +Automatically creating table and the table name is specified through the `tbname` column + +```sql +INSERT INTO meters(tbname, location, groupId, ts, current, phase) + values('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:34.630', 10.2, 219, 0.32) + values('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:35.779', 10.15, 217, 0.33) + values('d31002', NULL, 2, '2021-07-13 14:06:34.255', 10.15, 217, 0.33) +``` + diff --git a/docs/zh/12-taos-sql/05-insert.md b/docs/zh/12-taos-sql/05-insert.md index c03ad9bd8f..efcd5dd962 100644 --- a/docs/zh/12-taos-sql/05-insert.md +++ b/docs/zh/12-taos-sql/05-insert.md @@ -5,7 +5,9 @@ description: 写入数据的详细语法 --- ## 写入语法 +写入记录支持两种语法, 正常语法和超级表语法. 正常语法下, 紧跟INSERT INTO后名的表名是子表名或者普通表名. 超级表语法下, 紧跟INSERT INTO后名的表名是超级表名 +### 正常语法 ```sql INSERT INTO tb_name @@ -20,6 +22,15 @@ INSERT INTO INSERT INTO tb_name [(field1_name, ...)] subquery ``` +### 超级表语法 +```sql +INSERT INTO + stb1_name [(field1_name, ...)] + VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path + [stb2_name [(field1_name, ...)] + VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path + ...]; +``` **关于时间戳** @@ -32,26 +43,34 @@ INSERT INTO tb_name [(field1_name, ...)] subquery **语法说明** -1. USING 子句是自动建表语法。如果用户在写数据时并不确定某个表是否存在，此时可以在写入数据时使用自动建表语法来创建不存在的表，若该表已存在则不会建立新表。自动建表时，要求必须以超级表为模板，并写明数据表的 TAGS 取值。可以只是指定部分 TAGS 列的取值，未被指定的 TAGS 列将置为 NULL。 +1. 可以指定要插入值的列，对于未指定的列数据库将自动填充为 NULL。 -2. 可以指定要插入值的列，对于为指定的列数据库将自动填充为 NULL。 +2. VALUES 语法表示了要插入的一行或多行数据。 -3. VALUES 语法表示了要插入的一行或多行数据。 +3. FILE 语法表示数据来自于 CSV 文件（英文逗号分隔、英文单引号括住每个值），CSV 文件无需表头。 -4. FILE 语法表示数据来自于 CSV 文件（英文逗号分隔、英文单引号括住每个值），CSV 文件无需表头。 +4. `INSERT ... VALUES` 语句和 `INSERT ... FILE` 语句均可以在一条 INSERT 语句中同时向多个表插入数据。 -5. `INSERT ... VALUES` 语句和 `INSERT ... FILE` 语句均可以在一条 INSERT 语句中同时向多个表插入数据。 - -6. INSERT 语句是完整解析后再执行的，对如下语句，不会再出现数据错误但建表成功的情况： +5. INSERT 语句是完整解析后再执行的，对如下语句，不会再出现数据错误但建表成功的情况： ```sql INSERT INTO d1001 USING meters TAGS('Beijing.Chaoyang', 2) VALUES('a'); ``` +6. 对于向多个子表插入数据的情况，依然会有部分数据写入失败，部分数据写入成功的情况。这是因为多个子表可能分布在不同的 VNODE 上，客户端将 INSERT 语句完整解析后，将数据发往各个涉及的 VNODE 上，每个 VNODE 独立进行写入操作。如果某个 VNODE 因为某些原因（比如网络问题或磁盘故障）导致写入失败，并不会影响其他 VNODE 节点的写入。 -7. 对于向多个子表插入数据的情况，依然会有部分数据写入失败，部分数据写入成功的情况。这是因为多个子表可能分布在不同的 VNODE 上，客户端将 INSERT 语句完整解析后，将数据发往各个涉及的 VNODE 上，每个 VNODE 独立进行写入操作。如果某个 VNODE 因为某些原因（比如网络问题或磁盘故障）导致写入失败，并不会影响其他 VNODE 节点的写入。 +**正常语法说明** -8. 可以使用 `INSERT ... subquery` 语句将 TDengine 中的数据插入到指定表中。subquery 可以是任意的查询语句。此语法只能用于子表和普通表，且不支持自动建表。 +1. USING 子句是自动建表语法。如果用户在写数据时并不确定某个表是否存在，此时可以在写入数据时使用自动建表语法来创建不存在的表，若该表已存在则不会建立新表。自动建表时，要求必须以超级表为模板，并写明数据表的 TAGS 取值。可以只是指定部分 TAGS 列的取值，未被指定的 TAGS 列将置为 NULL。 +2. 可以使用 `INSERT ... subquery` 语句将 TDengine 中的数据插入到指定表中。subquery 可以是任意的查询语句。此语法只能用于子表和普通表，且不支持自动建表。 + +**超级表语法说明** + +1. 在 field_name 列表中必须指定 tbname 列，否则报错. tbname列是子表名, 类型是字符串. 其中字符不用转义, 不能包含点‘.‘ + +2. 在 field_name 列表中支持标签列，当子表已经存在时，指定标签值并不会触发标签值的修改；当子表不存在时会使用所指定的标签值建立子表. 如果没有指定任何标签列，则把所有标签列的值设置为NULL + +3. 不支持参数绑定写入 ## 插入一条记录 指定已经创建好的数据子表的表名，并通过 VALUES 关键字提供一行或多行数据，即可向数据库写入这些数据。例如，执行如下语句可以写入一行记录： @@ -134,3 +153,12 @@ INSERT INTO d21001 USING meters TAGS ('California.SanFrancisco', 2) FILE '/tmp/c INSERT INTO d21001 USING meters TAGS ('California.SanFrancisco', 2) FILE '/tmp/csvfile_21001.csv' d21002 USING meters (groupId) TAGS (2) FILE '/tmp/csvfile_21002.csv'; ``` +## 超级表语法 + +自动建表, 表名通过tbname列指定 +```sql +INSERT INTO meters(tbname, location, groupId, ts, current, phase) + values('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:34.630', 10.2, 219, 0.32) + values('d31001', 'California.SanFrancisco', 2, '2021-07-13 14:06:35.779', 10.15, 217, 0.33) + values('d31002', NULL, 2, '2021-07-13 14:06:34.255', 10.15, 217, 0.33) +``` From 8038fbaa047229f5472997021ae91645f27f90e1 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 17 Nov 2023 14:58:03 +0800 Subject: [PATCH 21/56] cos/multi: clear manager to init --- source/common/src/cos.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/source/common/src/cos.c b/source/common/src/cos.c index 684f437d05..aecba57e0a 100644 --- a/source/common/src/cos.c +++ b/source/common/src/cos.c @@ -233,7 +233,7 @@ typedef struct put_object_callback_data { #define MULTIPART_CHUNK_SIZE (768 << 20) // multipart is 768M -typedef struct UploadManager { +typedef struct { char err_msg[512]; S3Status status; uint64_t content_length; @@ -308,6 +308,7 @@ static int putObjectDataCallback(int bufferSize, char *buffer, void *callbackDat S3Status initial_multipart_callback(const char *upload_id, void *callbackData) { UploadManager *manager = (UploadManager *)callbackData; manager->upload_id = strdup(upload_id); + manager->status = S3StatusOK; return S3StatusOK; } @@ -509,15 +510,15 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { } else { uint64_t totalContentLength = contentLength; uint64_t todoContentLength = contentLength; - UploadManager manager; - manager.upload_id = 0; - manager.gb = 0; + UploadManager manager = {0}; + // manager.upload_id = 0; + // manager.gb = 0; // div round up int seq; uint64_t chunk_size = MULTIPART_CHUNK_SIZE >> 7; int totalSeq = (contentLength + chunk_size - 1) / chunk_size; - const int max_part_num = 1000; + const int max_part_num = 10000; if (totalSeq > max_part_num) { chunk_size = (contentLength + max_part_num - contentLength % max_part_num) / max_part_num; totalSeq = (contentLength + chunk_size - 1) / chunk_size; From 56badd0db07891b4aeb9b423068dced0252c8dbf Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 17 Nov 2023 15:12:40 +0800 Subject: [PATCH 22/56] cos/chunk-size: reduce to 64m from 768m --- source/common/src/cos.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/common/src/cos.c b/source/common/src/cos.c index aecba57e0a..b4e654c67a 100644 --- a/source/common/src/cos.c +++ b/source/common/src/cos.c @@ -231,7 +231,7 @@ typedef struct put_object_callback_data { int noStatus; } put_object_callback_data; -#define MULTIPART_CHUNK_SIZE (768 << 20) // multipart is 768M +#define MULTIPART_CHUNK_SIZE (64 << 20) // multipart is 768M typedef struct { char err_msg[512]; @@ -516,7 +516,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { // div round up int seq; - uint64_t chunk_size = MULTIPART_CHUNK_SIZE >> 7; + uint64_t chunk_size = MULTIPART_CHUNK_SIZE >> 3; int totalSeq = (contentLength + chunk_size - 1) / chunk_size; const int max_part_num = 10000; if (totalSeq > max_part_num) { From 88973c7464b2685c3d87be05c1d42ed410c4468a Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Fri, 17 Nov 2023 15:32:14 +0800 Subject: [PATCH 23/56] feat: skip tables out of limit --- source/libs/executor/inc/executorInt.h | 3 ++ source/libs/executor/src/scanoperator.c | 55 +++++++++++++++++++------ 2 files changed, 45 insertions(+), 13 deletions(-) diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index 331ce44366..41ffc26cd2 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -293,6 +293,9 @@ typedef struct STableMergeScanInfo { int32_t readIdx; SSDataBlock* pResBlock; SSampleExecInfo sample; // sample execution info + SSHashObj* mTableNumRows; // uid->num of table rows + SHashObj* mSkipTables; + int64_t mergeLimit; SSortExecInfo sortExecInfo; } STableMergeScanInfo; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index c47e14ad0d..4b2b7889d2 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -3255,6 +3255,24 @@ static SSDataBlock* getBlockForTableMergeScan(void* param) { pBlock->info.id.groupId = tableListGetTableGroupId(pInfo->base.pTableListInfo, pBlock->info.id.uid); + if (pInfo->mergeLimit != -1) { + int64_t nRows = 0; + void* pNum = tSimpleHashGet(pInfo->mTableNumRows, &pBlock->info.id.uid, sizeof(pBlock->info.id.uid)); + if (pNum == NULL) { + nRows = pBlock->info.rows; + } else { + nRows += *(int64_t*)pNum + pBlock->info.rows; + } + tSimpleHashPut(pInfo->mTableNumRows, &pBlock->info.id.uid, sizeof(pBlock->info.id.uid), &nRows, sizeof(nRows)); + if (nRows >= pInfo->mergeLimit) { + if (pInfo->mSkipTables == NULL) { + pInfo->mSkipTables = taosHashInit(pInfo->tableEndIndex - pInfo->tableStartIndex + 1, + taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_NO_LOCK); + } + int bSkip = 1; + taosHashPut(pInfo->mSkipTables, &pBlock->info.id.uid, sizeof(pBlock->info.id.uid), &bSkip, sizeof(bSkip)); + } + } pOperator->resultInfo.totalRows += pBlock->info.rows; pInfo->base.readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0; @@ -3314,22 +3332,20 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) { int32_t tableStartIdx = pInfo->tableStartIndex; int32_t tableEndIdx = pInfo->tableEndIndex; - bool hasLimit = pInfo->limitInfo.limit.limit != -1 || pInfo->limitInfo.limit.offset != -1; - int64_t mergeLimit = -1; - if (hasLimit) { - mergeLimit = pInfo->limitInfo.limit.limit + pInfo->limitInfo.limit.offset; - } + tSimpleHashClear(pInfo->mTableNumRows); + size_t szRow = blockDataGetRowSize(pInfo->pResBlock); - if (hasLimit) { - pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_SINGLESOURCE_SORT, -1, -1, - NULL, pTaskInfo->id.str, mergeLimit, szRow+8, tsPQSortMemThreshold * 1024* 1024); - } else { +// if (pInfo->mergeLimit != -1) { +// pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_SINGLESOURCE_SORT, -1, -1, +// NULL, pTaskInfo->id.str, pInfo->mergeLimit, szRow+8, tsPQSortMemThreshold * 1024* 1024); +// } else + { pInfo->sortBufSize = 2048 * pInfo->bufPageSize; int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_BLOCK_TS_MERGE, pInfo->bufPageSize, numOfBufPage, pInfo->pSortInputBlock, pTaskInfo->id.str, 0, 0, 0); - tsortSetMergeLimit(pInfo->pSortHandle, mergeLimit); + tsortSetMergeLimit(pInfo->pSortHandle, pInfo->mergeLimit); tsortSetAbortCheckFn(pInfo->pSortHandle, isTaskKilled, pOperator->pTaskInfo); } @@ -3341,7 +3357,8 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) { STableMergeScanSortSourceParam *param = taosMemoryCalloc(1, sizeof(STableMergeScanSortSourceParam)); param->pOperator = pOperator; STableKeyInfo* startKeyInfo = tableListGetInfo(pInfo->base.pTableListInfo, tableStartIdx); - pAPI->tsdReader.tsdReaderOpen(pHandle->vnode, &pInfo->base.cond, startKeyInfo, numOfTable, pInfo->pReaderBlock, (void**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo), false, NULL); + pAPI->tsdReader.tsdReaderOpen(pHandle->vnode, &pInfo->base.cond, startKeyInfo, numOfTable, pInfo->pReaderBlock, + (void**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo), false, &pInfo->mSkipTables); SSortSource* ps = taosMemoryCalloc(1, sizeof(SSortSource)); ps->param = param; @@ -3383,6 +3400,8 @@ int32_t stopGroupTableMergeScan(SOperatorInfo* pOperator) { pInfo->pSortHandle = NULL; resetLimitInfoForNextGroup(&pInfo->limitInfo); + taosHashCleanup(pInfo->mSkipTables); + pInfo->pSortHandle = NULL; return TSDB_CODE_SUCCESS; } @@ -3491,7 +3510,10 @@ void destroyTableMergeScanOperatorInfo(void* param) { taosArrayDestroy(pTableScanInfo->sortSourceParams); tsortDestroySortHandle(pTableScanInfo->pSortHandle); pTableScanInfo->pSortHandle = NULL; - + tSimpleHashCleanup(pTableScanInfo->mTableNumRows); + pTableScanInfo->mTableNumRows = NULL; + taosHashCleanup(pTableScanInfo->mSkipTables); + pTableScanInfo->pSortHandle = NULL; destroyTableScanBase(&pTableScanInfo->base, &pTableScanInfo->base.readerAPI); pTableScanInfo->pResBlock = blockDataDestroy(pTableScanInfo->pResBlock); @@ -3581,7 +3603,14 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN pInfo->pSortInfo = generateSortByTsInfo(pInfo->base.matchInfo.pList, pInfo->base.cond.order); pInfo->pSortInputBlock = createOneDataBlock(pInfo->pResBlock, false); initLimitInfo(pTableScanNode->scan.node.pLimit, pTableScanNode->scan.node.pSlimit, &pInfo->limitInfo); - + pInfo->mTableNumRows = tSimpleHashInit(1024, + taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT)); + pInfo->mergeLimit = -1; + bool hasLimit = pInfo->limitInfo.limit.limit != -1 || pInfo->limitInfo.limit.offset != -1; + if (hasLimit) { + pInfo->mergeLimit = pInfo->limitInfo.limit.limit + pInfo->limitInfo.limit.offset; + pInfo->mSkipTables = NULL; + } pInfo->pReaderBlock = createOneDataBlock(pInfo->pResBlock, false); int32_t rowSize = pInfo->pResBlock->info.rowSize; From 3b62d555ab64e9ddefa2891d3bb2c756e0c65584 Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 17 Nov 2023 15:50:08 +0800 Subject: [PATCH 24/56] enh: code optimization for active code --- include/common/tgrant.h | 2 ++ source/dnode/mnode/impl/src/mndDnode.c | 13 +++++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/include/common/tgrant.h b/include/common/tgrant.h index b18eaf8a6f..a5f3ab2e3f 100644 --- a/include/common/tgrant.h +++ b/include/common/tgrant.h @@ -31,6 +31,8 @@ extern "C" { #endif #define GRANT_HEART_BEAT_MIN 2 +#define GRANT_ACTIVE_CODE "activeCode" +#define GRANT_C_ACTIVE_CODE "cActiveCode" typedef enum { TSDB_GRANT_ALL, diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index d46f431ade..b0bffcc83e 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -790,7 +790,9 @@ static int32_t mndConfigDnode(SMnode *pMnode, SRpcMsg *pReq, SMCfgDnodeReq *pCfg if (cfgAll) { // alter all dnodes: if (!failRecord) failRecord = taosArrayInit(1, sizeof(int32_t)); if (failRecord) taosArrayPush(failRecord, &pDnode->id); - if (0 == cfgAllErr) cfgAllErr = terrno; // output 1st terrno. + if (0 == cfgAllErr || cfgAllErr == TSDB_CODE_GRANT_PAR_IVLD_ACTIVE) { + cfgAllErr = terrno; // output 1st or more specific error + } } } else { terrno = 0; // no action for dup active code @@ -806,7 +808,9 @@ static int32_t mndConfigDnode(SMnode *pMnode, SRpcMsg *pReq, SMCfgDnodeReq *pCfg if (cfgAll) { if (!failRecord) failRecord = taosArrayInit(1, sizeof(int32_t)); if (failRecord) taosArrayPush(failRecord, &pDnode->id); - if (0 == cfgAllErr) cfgAllErr = terrno; + if (0 == cfgAllErr || cfgAllErr == TSDB_CODE_GRANT_PAR_IVLD_ACTIVE) { + cfgAllErr = terrno; // output 1st or more specific error + } } } else { terrno = 0; @@ -1283,7 +1287,8 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { strcpy(dcfgReq.config, "supportvnodes"); snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag); - } else if (strncasecmp(cfgReq.config, "activeCode", 10) == 0 || strncasecmp(cfgReq.config, "cActiveCode", 11) == 0) { + } else if (strncasecmp(cfgReq.config, GRANT_ACTIVE_CODE, 10) == 0 || + strncasecmp(cfgReq.config, GRANT_C_ACTIVE_CODE, 11) == 0) { if (cfgReq.dnodeId != -1) { terrno = TSDB_CODE_INVALID_CFG; goto _err_out; @@ -1305,7 +1310,7 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { goto _err_out; } - strcpy(dcfgReq.config, opt == DND_ACTIVE_CODE ? "activeCode" : "cActiveCode"); + strcpy(dcfgReq.config, opt == DND_ACTIVE_CODE ? GRANT_ACTIVE_CODE : GRANT_C_ACTIVE_CODE); snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%s", cfgReq.value); if ((terrno = mndConfigDnode(pMnode, pReq, &cfgReq, opt)) != 0) { From e7593b4bcfb2a3a1608c034afbe7164fb717d441 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 17 Nov 2023 15:52:42 +0800 Subject: [PATCH 25/56] fix:testcase error --- tests/parallel_test/cases.task | 6 +++--- tests/system-test/2-query/db.py | 2 +- tests/system-test/7-tmq/tmqVnodeReplicate.py | 18 +++++++++--------- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 40eb926efb..7822e9614a 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -20,9 +20,9 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/window_close_session_ext.py ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/partition_interval.py ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/pause_resume_test.py -#,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/vnode_restart.py -#,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/snode_restart.py -,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/snode_restart_with_checkpoint.py +#,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/vnode_restart.py -N 4 +#,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/snode_restart.py -N 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/snode_restart_with_checkpoint.py -N 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tbname_vgroup.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stbJoin.py diff --git a/tests/system-test/2-query/db.py b/tests/system-test/2-query/db.py index 6870c59a0d..0246626e40 100644 --- a/tests/system-test/2-query/db.py +++ b/tests/system-test/2-query/db.py @@ -55,7 +55,7 @@ class TDTestCase: tdSql.checkData(0, 2, 0) tdSql.query("show dnode 1 variables like '%debugFlag'") - tdSql.checkRows(22) + tdSql.checkRows(23) tdSql.query("show dnode 1 variables like '____debugFlag'") tdSql.checkRows(2) diff --git a/tests/system-test/7-tmq/tmqVnodeReplicate.py b/tests/system-test/7-tmq/tmqVnodeReplicate.py index afb3319491..0ee11781ed 100644 --- a/tests/system-test/7-tmq/tmqVnodeReplicate.py +++ b/tests/system-test/7-tmq/tmqVnodeReplicate.py @@ -132,14 +132,15 @@ class TDTestCase: tmqCom.getStartConsumeNotifyFromTmqsim() tmqCom.getStartCommitNotifyFromTmqsim() - tdSql.query("select * from information_schema.ins_vnodes") - # tdLog.debug(tdSql.queryResult) - tdDnodes = cluster.dnodes - for result in tdSql.queryResult: - if result[2] == 'dbt' and result[3] == 'leader': - tdLog.debug("leader is %d"%(result[0] - 1)) - tdDnodes[result[0] - 1].stoptaosd() - break + tdSql.query("balance vgroup leader") + # tdSql.query("select * from information_schema.ins_vnodes") + # # tdLog.debug(tdSql.queryResult) + # tdDnodes = cluster.dnodes + # for result in tdSql.queryResult: + # if result[2] == 'dbt' and result[3] == 'leader': + # tdLog.debug("leader is %d"%(result[0] - 1)) + # tdDnodes[result[0] - 1].stoptaosd() + # break pInsertThread.join() expectRows = 1 @@ -158,7 +159,6 @@ class TDTestCase: tdLog.printNoPrefix("======== test case 1 end ...... ") def run(self): - tdSql.prepare() self.prepareTestEnv() self.tmqCase1() From f969579f7f4d2c409baf81121cdcc1321aae3918 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Chappyguoxy=E2=80=9D?= <“happy_guoxy@163.com”> Date: Fri, 17 Nov 2023 16:18:58 +0800 Subject: [PATCH 26/56] fix: reset wrong variable --- source/libs/executor/src/scanoperator.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 79bc38fa76..6acd8589fb 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -3403,7 +3403,7 @@ int32_t stopGroupTableMergeScan(SOperatorInfo* pOperator) { resetLimitInfoForNextGroup(&pInfo->limitInfo); taosHashCleanup(pInfo->mSkipTables); - pInfo->pSortHandle = NULL; + pInfo->mSkipTables = NULL; return TSDB_CODE_SUCCESS; } @@ -3515,7 +3515,7 @@ void destroyTableMergeScanOperatorInfo(void* param) { tSimpleHashCleanup(pTableScanInfo->mTableNumRows); pTableScanInfo->mTableNumRows = NULL; taosHashCleanup(pTableScanInfo->mSkipTables); - pTableScanInfo->pSortHandle = NULL; + pTableScanInfo->mSkipTables = NULL; destroyTableScanBase(&pTableScanInfo->base, &pTableScanInfo->base.readerAPI); pTableScanInfo->pResBlock = blockDataDestroy(pTableScanInfo->pResBlock); From abcd36b76de5f47da88a69947a8ea30d1365bb87 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 17 Nov 2023 16:26:22 +0800 Subject: [PATCH 27/56] jenkins: remove 50 --- Jenkinsfile2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 49d5f8884b..8ddee6dbbd 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -401,7 +401,7 @@ pipeline { } } stage('linux test') { - agent{label " slave1_50 || slave1_47 || slave1_48 || slave1_49 || slave1_52 || worker03 || slave215 || slave217 || slave219 "} + agent{label " slave1_47 || slave1_48 || slave1_49 || slave1_52 || worker03 || slave215 || slave217 || slave219 "} options { skipDefaultCheckout() } when { changeRequest() From 3241f9bda54678a2623c50fabc1377f620a261e4 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 17 Nov 2023 17:26:52 +0800 Subject: [PATCH 28/56] fix:memory leak --- source/dnode/snode/src/snode.c | 7 +++++++ source/dnode/vnode/src/tq/tq.c | 2 ++ source/libs/stream/src/stream.c | 1 - source/libs/stream/src/streamBackendRocksdb.c | 1 + tests/parallel_test/cases.task | 6 +++--- 5 files changed, 13 insertions(+), 4 deletions(-) diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 4863a1f7b7..007243192c 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -62,10 +62,14 @@ void sndEnqueueStreamDispatch(SSnode *pSnode, SRpcMsg *pMsg) { if (pTask) { SRpcMsg rsp = { .info = pMsg->info, .code = 0 }; streamProcessDispatchMsg(pTask, &req, &rsp); + tDeleteStreamDispatchReq(&req); streamMetaReleaseTask(pSnode->pMeta, pTask); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); return; + } else { + tDeleteStreamDispatchReq(&req); + return; } FAIL: @@ -450,14 +454,17 @@ int32_t sndProcessTaskDispatchReq(SSnode *pSnode, SRpcMsg *pMsg, bool exec) { SDecoder decoder; tDecoderInit(&decoder, (uint8_t *)msgBody, msgLen); tDecodeStreamDispatchReq(&decoder, &req); + tDecoderClear(&decoder); SStreamTask *pTask = streamMetaAcquireTask(pSnode->pMeta, req.streamId, req.taskId); if (pTask) { SRpcMsg rsp = {.info = pMsg->info, .code = 0}; streamProcessDispatchMsg(pTask, &req, &rsp); + tDeleteStreamDispatchReq(&req); streamMetaReleaseTask(pSnode->pMeta, pTask); return 0; } else { + tDeleteStreamDispatchReq(&req); return -1; } } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 6ba5529049..db045bdd2f 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1362,6 +1362,7 @@ int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg, bool exec) { if (pTask) { SRpcMsg rsp = {.info = pMsg->info, .code = 0}; streamProcessDispatchMsg(pTask, &req, &rsp); + tDeleteStreamDispatchReq(&req); streamMetaReleaseTask(pTq->pStreamMeta, pTask); return 0; } else { @@ -1600,6 +1601,7 @@ int32_t vnodeEnqueueStreamMsg(SVnode* pVnode, SRpcMsg* pMsg) { if (pTask != NULL) { SRpcMsg rsp = {.info = pMsg->info, .code = 0}; streamProcessDispatchMsg(pTask, &req, &rsp); + tDeleteStreamDispatchReq(&req); streamMetaReleaseTask(pTq->pStreamMeta, pTask); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 34b4677235..ab7951bb92 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -283,7 +283,6 @@ int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, S tmsgSendRsp(pRsp); } - tDeleteStreamDispatchReq(pReq); streamSchedExec(pTask); return 0; diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index c23483fffb..9699386fd4 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -519,6 +519,7 @@ void* streamBackendInit(const char* streamPath, int64_t chkpId, int32_t vgId) { if (err != NULL) { stError("failed to open rocksdb, path:%s, reason:%s", backendPath, err); taosMemoryFreeClear(err); + rocksdb_list_column_families_destroy(cfs, nCf); goto _EXIT; } } else { diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 7822e9614a..ca1cc704ae 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -20,9 +20,9 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/window_close_session_ext.py ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/partition_interval.py ,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/pause_resume_test.py -#,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/vnode_restart.py -N 4 -#,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/snode_restart.py -N 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/snode_restart_with_checkpoint.py -N 4 +#,,n,system-test,python3 ./test.py -f 8-stream/vnode_restart.py -N 4 +#,,n,system-test,python3 ./test.py -f 8-stream/snode_restart.py -N 4 +,,n,system-test,python3 ./test.py -f 8-stream/snode_restart_with_checkpoint.py -N 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tbname_vgroup.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stbJoin.py From 4c36d1ee113a8fc254e3607f9ffd969b55a5401f Mon Sep 17 00:00:00 2001 From: facetosea <25808407@qq.com> Date: Fri, 17 Nov 2023 14:27:56 +0800 Subject: [PATCH 29/56] fix: error msg when invalid operator --- source/libs/parser/src/parTranslater.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 1a65a29259..3e77feb0c1 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1439,8 +1439,10 @@ static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName); } - if (TSDB_CODE_SUCCESS != scalarGetOperatorResultType(pOp)) { - return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pOp->node.aliasName); + int32_t res = scalarGetOperatorResultType(pOp); + if (TSDB_CODE_SUCCESS != res) { + pCxt->errCode = res; + return DEAL_RES_CONTINUE; } return DEAL_RES_CONTINUE; From 9916e0d24cb5dff6baad304361e9b8432ef10f57 Mon Sep 17 00:00:00 2001 From: facetosea <25808407@qq.com> Date: Mon, 20 Nov 2023 09:55:59 +0800 Subject: [PATCH 30/56] build release version on windows --- cmake/cmake.define | 10 +++++++++- contrib/CMakeLists.txt | 32 +++++++++++++++++++++++++------- 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/cmake/cmake.define b/cmake/cmake.define index 44b36d0efa..7710c071eb 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -97,7 +97,15 @@ ENDIF() SET(JEMALLOC_ENABLED OFF) IF (TD_WINDOWS) MESSAGE("${Yellow} set compiler flag for Windows! ${ColourReset}") - SET(COMMON_FLAGS "/w /D_WIN32 /DWIN32 /Zi /MTd") + IF (${CMAKE_BUILD_TYPE} MATCHES "Release") + MESSAGE("${Green} will build Release version! ${ColourReset}") + SET(COMMON_FLAGS "/W3 /D_WIN32 /DWIN32 /Zi- /O2 /GL /MD") + + ELSE () + MESSAGE("${Green} will build Debug version! ${ColourReset}") + SET(COMMON_FLAGS "/w /D_WIN32 /DWIN32 /Zi /MTd") + ENDIF() + SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /MANIFEST:NO") # IF (MSVC AND (MSVC_VERSION GREATER_EQUAL 1900)) # SET(COMMON_FLAGS "${COMMON_FLAGS} /Wv:18") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index c5715bd53f..1b0a091e9d 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -317,7 +317,8 @@ if (${BUILD_WITH_ROCKSDB}) SET(CMAKE_BUILD_TYPE Release) endif() endif(${TD_LINUX}) - MESSAGE(STATUS "CXXXX STATUS CONFIG: " ${CMAKE_CXX_FLAGS}) + MESSAGE(STATUS "ROCKSDB CXXXX STATUS CONFIG: " ${CMAKE_CXX_FLAGS}) + MESSAGE(STATUS "ROCKSDB C STATUS CONFIG: " ${CMAKE_C_FLAGS}) if(${TD_DARWIN}) SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=maybe-uninitialized") @@ -329,8 +330,12 @@ if (${BUILD_WITH_ROCKSDB}) if (${TD_WINDOWS}) SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4244 /wd4819") + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /wd4244 /wd4819") option(WITH_JNI "" OFF) - option(WITH_MD_LIBRARY "build with MD" OFF) + if(CMAKE_C_FLAGS MATCHES "/MT" OR CMAKE_C_FLAGS MATCHES "/MTd") + message("Rocksdb build runtime lib use /MT or /MTd") + option(WITH_MD_LIBRARY "build with MD" OFF) + endif() set(SYSTEM_LIBS ${SYSTEM_LIBS} shlwapi.lib rpcrt4.lib) endif(${TD_WINDOWS}) @@ -361,9 +366,11 @@ if (${BUILD_WITH_ROCKSDB}) ) else() if (NOT ${TD_LINUX}) - MESSAGE(STATUS "CXXXX STATUS CONFIG: " ${CMAKE_CXX_FLAGS}) + MESSAGE(STATUS "ROCKSDB CXX STATUS CONFIG: " ${CMAKE_CXX_FLAGS}) + MESSAGE(STATUS "ROCKSDB C STATUS CONFIG: " ${CMAKE_C_FLAGS}) if(${TD_DARWIN}) SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=maybe-uninitialized") + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-error=maybe-uninitialized") endif(${TD_DARWIN}) if (${TD_DARWIN_ARM64}) @@ -372,8 +379,12 @@ if (${BUILD_WITH_ROCKSDB}) if (${TD_WINDOWS}) SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4244 /wd4819") + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /wd4244 /wd4819") option(WITH_JNI "" OFF) - option(WITH_MD_LIBRARY "build with MD" OFF) + if(CMAKE_C_FLAGS MATCHES "/MT" OR CMAKE_C_FLAGS MATCHES "/MTd") + message("Rocksdb build runtime lib use /MT or /MTd") + option(WITH_MD_LIBRARY "build with MD" OFF) + endif() set(SYSTEM_LIBS ${SYSTEM_LIBS} shlwapi.lib rpcrt4.lib) endif(${TD_WINDOWS}) @@ -456,7 +467,9 @@ endif(${BUILD_WITH_NURAFT}) # pthread if(${BUILD_PTHREAD}) - set(CMAKE_BUILD_TYPE debug) + if ("${CMAKE_BUILD_TYPE}" STREQUAL "") + SET(CMAKE_BUILD_TYPE Release) + endif() add_definitions(-DPTW32_STATIC_LIB) add_subdirectory(pthread EXCLUDE_FROM_ALL) set_target_properties(libpthreadVC3 PROPERTIES OUTPUT_NAME pthread) @@ -640,13 +653,18 @@ if(${BUILD_GEOS}) if(${TD_LINUX}) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS_REL}") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS_REL}") - IF ("${CMAKE_BUILD_TYPE}" STREQUAL "") + if ("${CMAKE_BUILD_TYPE}" STREQUAL "") SET(CMAKE_BUILD_TYPE Release) endif() endif(${TD_LINUX}) option(BUILD_SHARED_LIBS "Build GEOS with shared libraries" OFF) add_subdirectory(geos EXCLUDE_FROM_ALL) - unset(CMAKE_CXX_STANDARD CACHE) # undo libgeos's setting of global CMAKE_CXX_STANDARD + if (${TD_WINDOWS}) + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}") + else () + unset(CMAKE_CXX_STANDARD CACHE) # undo libgeos's setting of global CMAKE_CXX_STANDARD + endif(${TD_WINDOWS}) target_include_directories( geos_c PUBLIC $ From ec9c2bde696d8d9e965a5047c8b432bc0ad9bafa Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 20 Nov 2023 11:38:48 +0800 Subject: [PATCH 31/56] fix(cos/etags): initialize etags to NULL --- source/common/src/cos.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/source/common/src/cos.c b/source/common/src/cos.c index b4e654c67a..1dca5eda9b 100644 --- a/source/common/src/cos.c +++ b/source/common/src/cos.c @@ -537,7 +537,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { S3MultipartCommitHandler commit_handler = { {&responsePropertiesCallbackNull, &responseCompleteCallback}, &multipartPutXmlCallback, 0}; - manager.etags = (char **)taosMemoryMalloc(sizeof(char *) * totalSeq); + manager.etags = (char **)taosMemoryCalloc(totalSeq, sizeof(char *)); manager.next_etags_pos = 0; /* if (uploadId) { @@ -597,6 +597,10 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { char buf[256]; int n; for (i = 0; i < totalSeq; i++) { + if (!manager.etags[i]) { + code = TAOS_SYSTEM_ERROR(EIO); + goto clean; + } n = snprintf(buf, sizeof(buf), "%d" "%s", From d12bc412369467549565bec4819c1f9c355221a7 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 20 Nov 2023 14:30:25 +0800 Subject: [PATCH 32/56] tsdb/merge: clear file set before return 0 --- source/dnode/vnode/src/tsdb/tsdbMerge.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMerge.c b/source/dnode/vnode/src/tsdb/tsdbMerge.c index ef6aba1324..7babaa6e28 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMerge.c +++ b/source/dnode/vnode/src/tsdb/tsdbMerge.c @@ -576,7 +576,10 @@ static int32_t tsdbMerge(void *arg) { } } - if (skipMerge) return 0; + if (skipMerge) { + code = 0; + goto _exit; + } // do merge tsdbDebug("vgId:%d merge begin, fid:%d", TD_VID(tsdb->pVnode), merger->fid); From 96112fcd25bf969efbf5ecf48955fd730e21a95c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 20 Nov 2023 15:24:01 +0800 Subject: [PATCH 33/56] reverse --- source/libs/tfs/src/tfsTier.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/tfs/src/tfsTier.c b/source/libs/tfs/src/tfsTier.c index d4f228a537..911fdc52b7 100644 --- a/source/libs/tfs/src/tfsTier.c +++ b/source/libs/tfs/src/tfsTier.c @@ -112,7 +112,7 @@ int32_t tfsAllocDiskOnTier(STfsTier *pTier) { int32_t retId = -1; int64_t avail = 0; for (int32_t id = 0; id < TFS_MAX_DISKS_PER_TIER; ++id) { -#if 0 // round-robin +#if 1 // round-robin int32_t diskId = (pTier->nextid + id) % pTier->ndisk; STfsDisk *pDisk = pTier->disks[diskId]; From 2d7c4890da36ed62db39693c3250f2f9ef6f9ed8 Mon Sep 17 00:00:00 2001 From: dmchen Date: Mon, 20 Nov 2023 08:43:12 +0000 Subject: [PATCH 34/56] TD-27416 --- source/dnode/mnode/impl/src/mndStream.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index ee990956ff..02d401d924 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -1394,7 +1394,7 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { tNameFromString(&name, dropReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); // reuse this function for stream - auditRecord(pReq, pMnode->clusterId, "dropStream", name.dbname, name.tname, dropReq.sql, dropReq.sqlLen); + auditRecord(pReq, pMnode->clusterId, "dropStream", "", name.dbname, dropReq.sql, dropReq.sqlLen); sdbRelease(pMnode->pSdb, pStream); mndTransDrop(pTrans); From 905dbaf3b4a783151beb16b1058f2cddd2c4d302 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 20 Nov 2023 18:05:48 +0800 Subject: [PATCH 35/56] fix: update dmodule lib --- deps/arm/dm_static/libdmodule.a | Bin 628362 -> 628282 bytes deps/darwin/arm/dm_static/libdmodule.a | Bin 23792 -> 23792 bytes deps/darwin/x64/dm_static/libdmodule.a | Bin 23576 -> 23576 bytes deps/x86/dm_static/libdmodule.a | Bin 615154 -> 609650 bytes 4 files changed, 0 insertions(+), 0 deletions(-) diff --git a/deps/arm/dm_static/libdmodule.a b/deps/arm/dm_static/libdmodule.a index dd2afa9037ff005aff13b38bb04fe31418c420bd..fff594360670a1f9d9d931e148d3dc3395864cdd 100644 GIT binary patch literal 628282 zcmeEvd3;pG()KxX<_r_|Eeiz5gb)Z2NJ5ZBqD)w1RghifO4t+>*-;QA0TfY0h@#@2 zAS$ll4x$%uzwY~TuM6(D@AraN<*TZvdeV~&$bGNx_xk&U)93e_Ic3nyQ)W3+&KNOu z&VuQ4rye(J%2bj#@7P&W7R;FHkR(j!$(?730S9ZfTi34AT(>Sg@_Tk;wxo7x|DneO zT$dWfkLx_(R7ob4@QrXxhzb{Veh%ypux^M*E&63XOe9J9+JtqKuS-}@`TB$nly6Ab zNcqNuO_Xm+n5=vXVXE?Jgw2$1PMEHI3&NJlw<63?K0=tOd~3oi<=YTuE8mu|o$@(^ z?UnC9*irdh!cNL}Cd^YlpRhprE`){3cO^VR`EG>WmG42=Q~4tadnsQ;c$D%-6ZTd< zN?5FXAHu%M_ap4D{4s?*@HXXdC%i-XI|;Wazm@PV z~ZhzOVcTgdZyZ5#h(me?s`F@_PwCQ~qh1@GIrNCj3VEZwbFs{(Hh7 zl>d?NC*}W1_%G%65$;$1XTpCg{|n)-%Kt`qKzS!Xah-oDwg(73<%5L2@*%=n%7+P) zl&?)#NBO#h^^~tq*g*M)gpHJMOxQ&Eri97Lrx2zppGMeB`R0V_%C{hFseCKK4CNz) znaZ~&%u>D$VYc#Z3EL^3L)c#V4ul<*&n4`nd}qQu7ol<)ehf%J(7ct9(Dg{>mRiI6(PL zi*UE{j}Shp{9}ZVEB^%Hlgd9u_&4RBCVWQuX9=HE{&~U|lz)-%CFNfxd`0IQuC?uXCIzh4<+G%HAC2%U-T%5Zzyq;q0$? zl=QL)R}i)%%%Jp+gjs}ngzX3mo&7u8Iom5b65pNDE+ySwl-7~bdPVnV(9l(Mr~1QD zL8oqXdv98z6CG!ixoY^f$(oQp^l5`@GUCoOfrvvNI-VbZRdP{pR*Q@SIyK>Tp?SJ^PW$t;-~vHujG0XDoKyr)g}zi%K7y z^}jBDYV$5{!t1{-emHKCYHfEJ=kEE8p99`mE zP3^9JpZiU=a=(o~XkBcN^@Dsw{XV_UZwGizkN9=*quTEs=r{GTvWc^QW2w`#q6znz zetBK-nxeHN{Ul7(&xULl>u^6+$MloZc)Vr&OrG?W^%L}!=l#y!&i49TC)pu=?H%9O zjiu3^(qHx|_qVssdn(6ec-~Y}8u#~$Urm1>*Z$ru{iSkMF=duR_LaT7Gu7F?^MHHH z&I1AZ_2@T9zdrqj=(iUAhUqtnerwZj9r~?%OEkM|gyU?TTR~;HPVvEHw>%Iodzo!R z8pRt*W0~hp3C#~~Ycln5FTBnQDUH?y&yT3(OWt4Du3cmo z=aam@vJkN&oor{`q-VBp8k3zpDTZ%%mxM=}blMZjL0UPf`Gvb)5+I<&7M)&w=WfnpJ#wAXHo*(dhwuPk-Q9*WTi>QUUSe~X+| zCVS-{(D{n@D(cI6ht6shya!M^%e-!$IfB-Ww7sh*r}Np6&tw(f&^qbJ<@Ef*`#H(> znWwjP29&*=^GRRIS!ck?Z{UBE?#daAbgljEnF841@cJM-D)VT4^d(#S(%!WsqY2HK z4#F?Y7#VS#!IgQu&Piv0!!bzCX~xzvwzW`gCFDKXwzUiE51@HV@qqQ{EK2bOdb}@3 z>Aa9x(yU8tZIQ0Cq}E7o%jiiTxDDz9*TrYsz5hNS?HW6xzhj4-3%HJI?NfhNS5lp^ z_SH7KO&ZG)JbpuR)Ngb)7r*iOeF)h!#PS>S#&6xpZ*;b=7^Z%sGqkZ+{Kn^QKIe0K zBEJn&zfr#UP4yD`jdaCt)Rw7F`oL|l-zY!kH?BwLn)r=%4q}J+jq}-WT+W_rRA+ zWU-gi6Zvfj<@0=_eDRy=CG;EVI=>8fh1+6%u21?vy!egs$#37ueB*rEZ>+<9({lE_ z7r${n`%Uw?eVK1mXUuP!XSZqmHY8d7M&}#x+c2_mB>%F_>Nn<%--eOiSjs<1{WiAB zUhx~rVlSsB^4m$2&witP@tf);^c(4l->5CtkNJ&w@f+pG{KomV-&lwJrsZn*jq}-W znjiBU)fw}f=84}}-;Qr|UgbFJ(AiJ?Hi2xM%)e~2`i*(xw+W5CtkNJ&w@f+pG{KomV-&lwJrsZn*jq}-WnjiBU z)fw}f=84}}-}2iKj-k3=ReVbA=(7)>P3uw|XQWrCg7l_RFr--H3~TD4XCKL1;=c;WJlU%K1^EX4kML$K!l+soNrLFta}ze9>JFDfZb z_A`#(yeF_vHSX8&Dd)3KHQ)Hub$0cXI})66tHzP_EuVhT!^}meT-xRRfa6~;s*__N zKW|a2-{k&>^VufNH#X5+>1pXW&_ZD>E_{zvN}9o+xWxZ3rVRIkf^4puwudaK|6$TK#@ed_;rQGTX8 ziT5*iWF=vy<1~KXbBEBq)nGZBkRID>;$)r9>f?y|m}5D|$|2+OhK_i zd_0os;BhBR=wr@jA8US`k9E(F`IzbyAJaL)_DLn31!6wt7{xx;`otI9hV5gmhtCso z$5ySrn2$|el<)BQj{B9M-h@7;J>sx@OnSDDJ7zhEsZsSY$9;~g6q6BK$;T8!ck=n; zRB1!vt@wCyO&@oqxWVyQe60DlkDV6LOE?}^Xgrj;O!abnt)z2DoS#XD{CrihL-$Uk zLv~BsvDnJ(*nZY}xNa{kdP(*A$j|Y0@!5*~%zaZ^W&E_>C#v0L%3P*BCFbWm>i;jr zukblK(}`X}bNLyX%TLR>hxFK9GnYH2sh>;K&$Fm6^XPY0bbrNJ+%NL;V)FAyvSqRQ z`7Ek~=Q3gSxh#I>e4fjiU&GI(RBtKOMduvxGu1189$C%Lq!aTq>4=}TPVon~WBXa_ z;kw1o)#{7++0;e(;%Dw#f_fABdER0Ane^iPTu=R6rhZ;QJ~*F#SCF4p6SAMzlAp(t zEo;@!tEmq5GhsqMb3Xf7^K1C|QmXe-%gY&Mt%vIt zKUb?S=4VqE<%^%WZwcy6=;!kf)6b+A=Vv}s=pMY2_G4Zfe2(~!&l6e)?VH=Few-(6 zKG=EUH91c-Y`pyC1C#Tg`E{`)=ZYUjy0=uM@>;X*TL*;6UV4_N{dZ@c$#dw;5M5N! z#9dU;F;Kqyz_OR=*`7YCfjiO4tN@=WSA`(D!f zn&T<87a(1ZGh83PYvOYY+fW<#kYYpiHs0g1q8I+IgziAgta_;)>L=GnbH-`enChBS z*38^@M)x;`Dx<-YuWLC-wT~4xP){ z=G_>Bnth4g&&t}By6GM^X8(s|Ki}&ZyZ2;O`J9x{2R=t;x#beiyVIQf_kk*sf$ljxkxx{AynKbuV&%}r_zNgo@#j&lTFW&ob zeX$riU?87KYCMzuFtYfTUl&Jj(HOcK_o9Er^H~FDe`Ou2uL0(2RBfk^*2=mu>xcJ3 zvPa_NO{Dv==iM5!m)5K6EPomA1fBK#K9O}3l|5J)-ba32{0L%w{rFhV@9WXFtn2-i z-8IhVIqXZZiDKHKidURP6}P!HWBd)`E75!80Ig?|J=!M%^y|_u)vhrX|z|H}8v(mwj$T~d5?<}U*-<-3yzecU5TaoFrd5`R3moMJW2J8Clz z^O?tn`Pe+`o_Lqcpmhcf}`PSicm z_>9*3cbNy)0pueie6yp?;6%a4ZnjtU zXG&-LdCxrPIg0GhX8XxkI`(r8rQ4i!C&%8yA5xd!_n0|Wfw?E|lMYos#ZftDoAX~8 z$0hBP#1WcfvR@r)-yt3D3&#b*xN}`BCfoat_Q%-B^EkHtdmo>8pQ#hwFEIe?UiX=* z&%f;4>GX_mJLcc-9LsjvYebK9X>CS(M)>nDw68AX{g~Gg)x&#CiSm4pd9Zp*s7yrl z%pO<9{-bzWxfAc#u=n6|D-HjHy=N!w6>;%=i&Op>;zd;)JB&T4xPKULW?FF*2u$ z>GK;E-tL}!_NI5WIDhkbc8k4djXA`gHROMN&#GZR`C=mZ)b>T9zF(H0?^4&UvNCQItjrYMsny746l8fldz9s-{}=*E@)pTa9?RmGlF=&;{GzfhCciAd;O&M^}pNC?&?FS zpFZzn=8tWcynp?p&o*?fNHjm*`t|qc2gjJ|WqAkT`EhkH&6{myweRnNQa%(+nF z;v%{ar}M6p@ccvXaO7N5b&i)DJ~nq=dL}TFK5O|8Y7t>)Hc^sQe!Nz$CTl- zYKMJZA=@R6a-R-WuY=D89(KKaw`tp3OykMt1^$fp@5X|T(e38*ur!YHLvvPsqJ8Kj z$UjeC)eB5KHTu8(`tN^z^WM(VUw(L@H*1WZv|#4^1@yg^sWayf8yOioeddCNoIYae zl*qt|^PPTECrzI?Gcw?~(VR+pv*u16MuU3H+^JLhjO^#knmu*q$k8L|3I{(UhxVsj z%AY!A*4&Booso%?S>2gVJpW9YN#CFCJe|K%D?g`9oIZn^nm;Qtck09`k?bjvNsH!B zofkP}?yNH+XHJ|uFEWkO=S`nEd1@qkp40ERei1HMtz31@v*ym4d1_?xw5gL%pSOVa z#OnG9+L<(K%A%@v64h^M{9gTHPt0g)pY85HV#Kf!5fc9V`O^b`df-nF{ON%|J@BUo z{`A0~9{AG(e|q3g5B%wYKRxiL2mZhCK%Vw8dtUZDxf5ZA(qgja~vFoHu>me7XdRoH=!FT)hX= zn>=Fy6<}Tdsj{P*{u#3;&YwPM23aP$vHoedJ8ReYTIk5^dAX4(v*`9`=B)X2A0&To zEi!NZ#JTeq%tkX#Oi#1r>c~==KX>BHd68MCoHB3f{K)iqbfJWTd2}O_H)Hywywj%6 zJbn7id3k5entS@Z*%K#E%^THk`b_G`%!xDdsQ**v&YnAc-qgH#b0_D`p0VIm))l8q zO6*U)sX5jAea5^~Bhx0%oHBztVU0o!|A?8hXU#PuAY*Up{3CJ0c>23RQejxG0O|hm z&_98&&kAy_SnJKc#0J9jhYkpW0GF;Q`EL=4Bx>O{r*Vo)-}lgh;q2pMY}@lWGf-7) zxE;MnLIdHPO~eL+K8=8;v>zPH>{t+EbJ-9yc2QO88fJa2;(PiHv5u=+>#MXbXSP-t zrSD@Q98M=^1kjTt=4(@`^Jdad{3kx@FrV~Kt~QXR(!Z0rKvo!ezd&jT=~qZ;kbZ*{ zfph>;j<`L^aUteIbXDvDF`!};Vo=3_5PcPgLky`n4r18tM*Rz9l|ZcR7Ku0wVqF)V znGLa_it`~hcAu55E`*rk%J5{B(Z`6{5veYZu9vl3>)-IKgaK}me^cTqpX((&8 zqxR@s8`*x9jB8p_1$zf0QVB6dU%06hz*UkT5z8dqJv;ydp?ggAI>@)D(F~Z<0E(A{+LG)B<_YzLFP#R17Hx zQXkZkul;E>VuCTV(I{q3e;SR36sF0*JrUP78pCXDs@=(Hbmn`pk_*v1y4z?Gnm`8| zosB$nu+b9ap@WUqBM%*HbQL6Yu+i01#hNq|o5g^}>D(Eam>h~_y7qP&=Qr-#k;Eo! zSmXY)sUX;6s!|7VP(;aIreq#^)p#J+rJF+I9*xJG6{~9;k2dN$p(^#ns=CfMb#>Re zPCAxS8b=z>xGPr4%qn(PRmt6^WQLZUeF>Lr*yOc;nvzZ4sAAs?(Pku;GW0IoP^PTKrNl@(Q7368@e&l8wsJXLgF@2`l>!YZk;-u;_nUGq+DpFvCMh9} zCs%ADJ#9R>l3itln{2f^ZM7{>OLiO5o}D5GfN(fSyw7}FV#H3*No6C(c`5a4ok9~> z;(kiQTC7AafY`VeW@i{;lUg%KGmzB~Vrs2R>5X+DD*~~Zmc_o8lCEX3@1?Y;#S=A< zg?*1iwmFa$McEcwGxj~oMMK!=NPI^c!p=q_tisL~6X&O$HOGk+#l}YBSQ=X|OE)T# z@1vpdvUFo1d6MLB3m8LEXij-q-^OIN#LL=OO5+)*1DK{Qjq0}YVzio{DG9;8k14~wwQcWBQF1KOxE^7rakDd ztCDF#ki9V(`Y(nk4Dw`>S+#_GB(qAl$=qhM$&;Ma30#wjxTz(W1)NKNjB+9*1o+fRTumS=2YHhrDYXUnxg=$m`x^=L zkDVb$yI4rs?OxVYJZ*B>SFwZ;9tTkLD%p$kWwIAHw8>ta zFO$8vp-uKIUnYAG*}hEn;)XWai}PiQhqYP)BT~FLU#56*zD)7re3|0K`!dCg_hpJ$ zRpm<~8(*e)@xDy);(cjk<4dYveVI0W_GBj|RrU9&uhTqv7|1FM=(PP{7kO8vZCT1h zZw?&;(922juAUU{>Phjgo)quuNw%vedGFY+o)quuN%5|p?0qIPQC&S*E7EB@SvnKQ zTBx>vxP;6V+gGLJ2YH`??FGTOX;Ki3nlL868 zfh;{VbP3iI<&w17)2BEog+c5JYI|3$SC?E@$>lYsefeY{nJ<0WkkUKUj8-f95)H-m zB^rwBOEeVMmuM)iFVT?ImuRT1oM|+gMMH6YiH72)d~v83`T`^RgyQfEOrAjS0lLZ%iPri7^3fLh2kF7=sp2bX))p4up6@0L~ol zw3Kb8r~5Ot5)ShtL9)c+6!vHe$BL8|#8KWXDjE*+u8uNAZfD8}hr3D{u0BP`4YZ@B z3`ZH>$im?Pu`*G2l9ZVyWjMH`2;mVSMfN5tFLqapV!0@C*hvwhiae#D*vGwB6dx2t zo-wdO6rs}BeM3~<5fxsO(mAc*822a1{kP=SqESj=Le_AoMh%1AW|R~TXAn(xk6;}k z{N$gaCULZZcBwl+w2u|-lUZAcD)Q2nb02TWa_;k;RuS5uc`;D0P(_$=;A00FBkJ-nqAriVxg;Sc=;-frTNS0?M@G0^*<776ZhY^0xNjqcuOn#1uTPfz5hi@M~B&BdzHZHQ# zN$b#;s5~&H-S-*g>GNi@)`60$XUXPm_`Iu+kIlRA@ED=HE%bsQHwq(DVh^=)#!JuQ zOgdaRPJVjFos>(s%k&|A?N)$GlO`LnGlinS9xY0;I5_xIOrp%h` zq(_k#K`n!jm*XUzOf{pyIWG_!4!=gUwaeywL%%X#Q@A@REK>eY1QO6n(i^$2(NRT4 zEWN3_J53COn5tKx7_sy;m)&EIp6ThDi=$_HD?PxqK-o4bMj*CVi2M3{dkRE-EoM3LN z)1Owi!_hB5N`vMbS4%l{`-eM3!h=-$UTEe!N$kdOmt9NR==XY-*muX zanRMKA)0IGG$&~ol@Euxme%3|ZWv?9CCpVtrD$<4X^ztvhr@h7Pl1dk+A^t|uh75e zvg|la0c5Xoon}0(!{PTS2WjP61k&X)iCi}j4045%*y1x8OfJUvqTz5~sw7M1U&~&x zhrO0Z@xxvlF4eYKg!Ga`gXQeBR?Rk28SzG|<{VOKF+fU(#EX@JKnq9>F}qu}oJFOw zBds!Cj+KtYmCod9V^5{6THirpTdv8MsMfXRdGxixcAPL@=mMc<3O!5cLZORB!^ZRfPLVq``;eg|M`%dv&_w$Q;AI@&_6Ep)Pl zJiA-@woqUTU2LH%PZDF!5w_3`g3QNAcb5WTI9!V;UQg&vKxVV!DM`i#*Ae!m1g<7< zDS;~pTr5OIzYNngGpioeAgyF(pDL}KK{VN2K{SQkn8Hn@@M6Z0XSTy2$=;NiV+-wV zp#!fw?OJ9>TgV+xHJY4G)=*`3wuL-f$hU<8yS6U2P-qKXZQ%%8=w=JuZJ~!P^t6Q| zZK0Pf6xqU2ws5p9^tOelEfm{AA6w{a3;iHSj)&gifEmQ^M}T&)~%DM4jKHc;wi*pa!Ep)RtqR09F9_DtzE9)JeLNc$qTUS)5F7yYfk6kJ0d*q!)>A!VZL0`GK&Cemha*UgPaV$ZGBJ8%@LMgHtxUHBa+sJeLZ6j&L z&0sb=;$}987`@ianjv$uFi7EMvuDk7+%}LfWNvn{XyUBpwoMm_PoPIox9Mqsst$+e z#@cGP3vIPu9BZpXTw5LE+RBYrNnTUC!cgKr&VpTgBCO$ zA9Dt}S($oH%NgWyh8d=u!7Ad)J!goFQXmUA);UA<_=Hnj&ao~pcXNu%IZj2K;&O($ zW2hDLiX!KD6?6CmLd7^!&K)T%f$+2k$6l+-rJk_G@_>{x(!G}Y=8va@h4ia4SB?`o zquptgq;sNuMmY)6=Rtu3?&l=PTnsn?-px#(X&ZxxXZAoKa0%5P$bt(3UV!a0E(inz zeR0-+e)}#I3Zf(tX>2&4M{%Q59Ot<#Wpn(9SXCZlzp3~|fJr4)Y@QX`j!@RV!OGKQfx0lk{p#vyhW5%_Le={M(3ztN~%Nn@2Q7$Pvs{kzOQ{77t4} z%o8abW~WK?XNXqM16LbHXo71~Z{j?nf(I|%J4G*@URp`C^13C$N;Ahe6n zLZMxS9wD@w(C$Ke2<<8KNTI!i770B{=+Q!Z3&pXdSol6d`wHzRw7<|}gboloQ0O3` zgM|(eI#lSfLXQ(VOz3c-#|s@HbfnNxLPrZ7BXq3LaY9cJdZN&igpL>b7ojH$oglPC z=tQBDgiaPZMd(zarwBb&=rp0zg`Ot#bfGhZo*{Im&{;xf3!Ni$uF!cx=L=mR^h}{= z30)|3k8dX3O)gx=oX<{h2ACfZlU)Gy;tacLhl#)fY5D1w+nqx z=nkO|34K`TUxn@zx=ZM8p^pfCROn+u9~b(B&?kjHCG>AXpBDOz&}W4{C-ixtF9>~6 z=u1Lh7W#_NSB1VN^mUn&)Z$b~`@ZRLO0xlo`@Bl%84+sHj0m6VJKy5%BKwUsRKz%?1Ktn(yKx04?KvO_6 zAO(;LNCPwjGzX*uS^!!CS^+Wu5kMxOH6RPn29OPC3up((0kj8n0CWW80y+UY1M&d* zfC4}lKp~(j;0QoBKzBe7Ku^GtfL?$iz)^ss0lfiHKrx^Xpf8{wpg-UkzyQENz#zb2 zz!1Ptz_Ebi0K)*o0mlPI07e2v0Y(GH0LB8w0Zss%2sjBa9`F~y$$$xf62L^jB*0|A z6u?x#DS%S}(*V-}rvXj}%mADLmR>L05=0R18xD_3b+k$JKzq$oq#QX zt$@1#cLVML+zYr5a6jMyz&5~kz=MDtfQJAN1O5uw3D^bL4R{3bDBv-`)^uY0)9p!t)uhR{-tw#+7#}DnSw4SqQ6SLKYgk}R$o262sr7VH z(M_8=d!7^MESmHT?f7hUgo?>iW>D+R^IoOoBujvoX zJAPbi@A|lS&?olyG&c##do;JUHuk>HpSR7N68O+RrHiTYBk_Ff4&Y-ggNu&9Co0xO zji0L66yjbL@qQ}snTmKJ75H4mTqu8`VpoV?s#pZ^D;4`e{947K5Wi7z6vS^;9IqYw z&cBj7wmpzp@470g1?q)%_4KX-&F5(DAc*Z% zJQiYydKi=AA$F`cj4q=BS^Oab!dw+6LhPi<(;;?NaSp^h6&FFwug9+jysTx~hXR!^ zP`QiBJ;*;Ja(vbVrv6pq1-m%CMax<+L2)V6oAV+-wV zA=ef<*+ORsGK02s)#VsKFLLv)rxa6rzAe~y__*x%N$cu6VrqL7Z+D^1**-5r=vR%OhTPU%G$#$73 zcGXjD+fM0kb^BCXZJJ%d>9#P#7G~MPY`cOvb~|(JcIMe-=3B1>a~IgcnYOUVuKH}d zr;F{JrM6IN3uShhWp>VTTUcS+cCMXso}IJO7S6YYRkm<}EnH~Vw%QghvV}FaaIr0{ zwQE~v=ak#RCAM&>EnIF3SJ=V^Te#XDxod3UT3fiz7AkFFqrFaV;ExKKNxq5SaTwu7 zTe!*Y+|71vx574`-yW6qjGdx#Z(Bfusq1!oc(>SfZMEyV%NFjnh5Kybep`6Ju5Fv0 z^N{Veowfy!*g22d!eh4Zxb3JX?3^cU;VE1An=L$T3(wfXv$pWO-GvuyLtnIWUb4%) zYzwbIpm60lotg!E>Fbb+&i%#`FRvgv_a8tzvU!iz`53(3#7mCO1GqfBM1tl(NO-N$ zc~Jc#&P&6~mQLxDsK$=mP^4jAOU{!*d97OYuc|&T<4uuY^U>)kn)iy`!9V%2+~obdb3}mkb3VB3I6OH<<{j{Lm!4ZwaPo{fBZ0n zS(C#~0o$QJRTd5pA==tyFV7+lZvy$H7TzE6u%&PYY4)etgQ6YFr07a1%41m(E6Nt8 zut){=A888Y-bE?l@B>8gPVyyIW53`fB$v{;BkAT*CYRwT@tPQ8zma{3LCPEqcLtcZsB={(t_Oul|bOQn_^jAEa_%5T+? z^yIA8Il`msy4m@iTl3tM!kzW)yh?3&UeNI!dD&VOq_+IghgigES3n~di!q(s&!8H! z!v(wFF*O%Fa#4gebleIaeS$Tb6g*xps%;lMfhCEy3!dZ$*Fcso|AMFXlD-Uim-H+u zQlG}xp^sst5wB~guy`T~a?K6u3Lg;a2@MMMQBmQ*Jge~FixL)A)#GMRt~6gbW+SoT z@MfZz@DCH0!cALKv~cW3N@${vFX|_gA7ROh}WRhN36d zFXn`%M3dbJ(G(_>!d`>QNd=e>7kY&!_wS?5E}Vck4QCgY zK*B&5PP7NQa1tcU`NGMNFy{-WK#IteD4fcn#(br%@Dxbxk#{OzQv|X)K$^yrfr7ga zaXQ3ZDxStS8T6Gs6;J0)guV`>;tXD@^aaBZ%ASE;WUq=d*&_PxPe}VdYZQ4SdspF{ z)V`+cbFqDC*XKdP0xz83o?K;Sb>RXoM6((#oyiTG1zvd86{N`_9GMFTQ9JxKO^Ayg zmfUqJp8b@F8&q7((}2DMsp1ldn^io=ld5!^EL_@FMA`@`PSf~dOiPCoo79r*%MKSV z>lf?Xa*bZ9wxSc|pu~A65*rSe5XHP*KpXx8^B@(jLsY0PuAd59nr|~{PvN)RIK6%nU`5*xyT*ao|>WDE)YkZ zC{>je(%`fwJN;qgj&F#`Cv?7aX$I$YY)Y*|J$IrQsZu9&Ac*15>&)dkFKm#PKQdOJ zz}~R)xv%Ewc;lp2Kr--~Tt`R1iYW`n%) zilu1s2bTl4SH4ZsMs+;ms#-JtD0r1F(6 z&poevhssyEJT1NQ7gfG4W*3IwT^W1sY_}ETYxdc@$Gwl*?p{5o$68YM%6Cnm|1~>d zx!2r7m2v1XipsBYW3Af_xE+U>?yeG^W*fYGf%Gm=UP?2C`-u_jeF)FZ9crrT?J^%M z-@Ky~l|@}%Nx|}`OhU1IXD3*`XGdwT+4JW*ULQ?((P;E_7fK!bOhP|#XRv(VPOY?m z%)e;l7#W{n`MsvZu`cf+!SXkElwt(ViTR-0jxn|yzg zNK<@VxqP-m51%ww@0VT~OKf!SJI-co74>=)(uPpdr24UZ>-kp2bQ*fYT? zzBJL~diz3BE!sj^UisDuMVo}#JHhFbQ2RI4yw_-cOY0x*sXVxQ&Ku!4eJ3qA#qlb& z$LGLXm2l6iMuhX8ItQk2To&PtWnAyMQ|ZsmZG!IJbNX=yy}Ow5PjT8wU~KAbo;ZxU z)s3=fFqRf|5N6#{r+D6pwjOxz7H>Fdms1vRBSq&4Q?ayW0J~OlRxRy$Yl3RrR*2WYj2T7ziVmHCSl%O+Gj@sW%ui%HzfbDrA5yQvu7z;k+9y& z-mj{cRw3%$`#beMYwG=$>+PGs1n=^m>T-`c=6D6UR{1Ce&KT)?n5W4ghcgn_i#Xhfa`8|;jGi0ff2NRHujx+UTzo*M ze1_BWE8R0}#lVVsMRjO3@#Hx``2jli(WF=wVagxafvsmB!q$%CjIC$aW8OR=2hOfn zAYw@kqDKUWcduS~k54Fr_LI`Y=dEQpZ_!xEaf=*Y zvj&`HHkVdEZQYK2*m3CiPYZhueCORt4bp*FQrJ(!#7Ar74^9jdf2?s?huqIo7==r< z@DH)VrQ&21E_FE_g-fNd?6Yo}IF>5nWr2!L)Qb3I1S?LoDmu~SufL)2NiM&I(qdz@ z7@uuXY>ZWGjJuhZ>haSZcdRV55vXA-YS1c%n*PVC7;059)aAFQD0eK(hQR~17%wTb zJkTmOPeAa>92UU6>SoOUMJ>JHk_iB~it9${1egw~x{nX=4WH3Kz1dd4qbZdZYKr64KeA4*I)j?fO@*e1X|HhIK}}pr3uQWkn&i4{E|;PvQX_0w^Pqcl9qyOuM|uuf#edJZMXY~PO-ty1b7#xC_0K z<~c*1W0MQ329U+BwKca9ZBa*brXH|rjY_(;bPJ@~adl1*!@Jj^$=aC8^yumoNX0#E z75dD4Rb6rB^rH5)$J~CAuf>m2{W>+;K3Y3K{YXvGs>$sys?5AR@eQ{GCJAn?B zR1G(*<@#yI>u|^W*P-r>rnPoVO)IW|8wZ%2YVI9q(%J5aG(4!vV2S}%&C>s}4zIDf zLp5E8zO))?Ok#uRzoAua>Nw4$w{wZm z&mP5qvo^+mRF#%*_=TOJA8^*>8-7VFZ7nh_y165S+ZDNf!*A6z&sqDeYlJtoQO{ZP zt!o5hC6})Lv!Mf0Dim4A@S8P_TG$#f!lqcWYqmy=VC+oPLK|M{^pmT|I)?A6Y21g1 z5!T0=UGr_k2*%DtEzHkG3kwxl$M8$4(((--iCMBH-|)uDMD@?k)cO^re&x#)gU;Hr z#~mZh)>ebg8cH>iu`^K{n={bHRz=n^{5DM!FBxHmHq0X(F@mu)Q44jomWEafhHqMx z7BT#$SlXJ1;fVJqfaQ=Brk#!9Jtfo;9)@2$YtPM~P)@2&OSedAW8I91wY(>^F z{CrKL7G4Y(VOy-(H7^E?VC+oP!jDO4VV@%F82%ScqZY~nMwk|Bc1>Bp2*%DtEo`F$ z2LC*y$U27qwyM>9!yAhd)%8#ab?s7Q9m7ASY3vWj2={8EzzXLz_;J%CL^%)!+0&d~F&2KNwP)t&P$sl(|MQ zRwinJV>-&1`p}fwIke#z%r~41hOedgXS^b$DmuTh^1P+dPgrrhB2ug9o0`slLzRDK zDDF~3Y85R?2LG($tBN@(RXR4eNz=QgBDF|youaMFd|f`&(a$xCn-upcBDac;Y61RS z#ZmOU$v@*2Ayv^JykCn`Jje5#BZLk54JD;-;+r3`+uaAP>F0aV*O?vn!C$_J?}tS(!#bugx*O%*vpb zU0XT7oj9Ol38u{`GzF<>G_ds;xD`7-CdRwtmmmw6&u^nxcLg?iJ# z*wb!2W`Ne`V??~?+`pi`_9vr^MZMqL#yX2q8!2-v{(BHpOKHzYGHYzM z=gvW{_QrIo&0HN0vCDs#O@+~qD*&4Pop!(D;&U@#bxUCG$lqN&>1>Ri{1(s`T^-`inOu5YwX6o58R7Q=^FxT z!FsLMxxVayUP`$(=J}Vno_nd0YOQzu+A?+OUZ#aoE;n|O4}xCO6-L+J;Hv3uu2kKm ztEzNi7pR{pb+$I+?cXjD)w?60Dc+XA)mn!ctDtvx;2KS9bFJ}l)uV;c*@OMNl?dL3w7M=>V!+Z zMO9mx?sUA>G@N^za#|`aSr{*-nt8iXZFfg(nDg(9u|-=<-&3}_z8op&zxvyun^XE- zrYl|xdmjsWo$q#i+4zHA!98&{s3j@)y1Jn^DKyM|Q1r&sGV{FY{V}a9qt)6l55)4e z87_6Z=}y{%hHIXtO-6R8%Yt6kLnb%789efw`>;uR$^EPArNs3IIv<?NpC z8?9cZcH`U<^g6v_=0~$vV~Y8&xf-5YzOIQn1O;!HoPsws$Lso5yiT`tIM8eMcVu{N z{-GMQZI;FQ-TiH2Zpu5c{2uR`#I*Nf&fKHfUg&+J81#<)z@#4cq2Y#6?cVT@AbPR= zttAb-EN|__@hva`%-PP%`NZ|oKUF1i?Ox-QR6TZj6FxKZBh8#dHGEy@e(vgF()+9X zh24_(nfqm&Tp9R^4BQy_+Qh!QCE=c`gm)z2-B?1<8}W@+;f>N0pf~zklSC`cJK;MM zewy{Oy6I);yg*;?4}OnAo|!Mu^*3Q3+gIxcnqA(BKdM&zPL^rr*cOjY7!r_nSVn{aLkw-jtRw(L3eersS!=7;f6H z;JmEgGMo3+Al zv(Gi$obwDfcctOxoo~1Ws|*7rC(y}PrtP9X0 zFE`bvUt%h2`T))R)b?6qYK}5#msYD|eXKwmG9{bJTo&+~TdOzD-WHb!yoRgk&&Nxu z*RF$=>vf8g3T=t}bA>i!y4JL#nv}d=|C?QtlbUP%up!>qW*1Rs)9JTGCujt{mRFi; zy;hy|(KoFfxg_7REa+ui6;q2`ZJd>Pjp=)nE2&Ff*0nLEHrK_n3Q&=kdc8?%R$=sz zMf+BzN%3;J7}MHsjI*8^XyR)g;uoGs{Pyf@U#^$^Zo(G{mCw=mM++}cL4H*DK1_pr zo8vS0C;ig{e|q3g5B%wYKRxiL2mZhC0A6R=yFXHDrrSE;52k6S*W_!Kt*L_)TeD{C z|2I?Qgd~@rdfbwaH26o#nMn>;q9Jc%{X98J@tI0WgwA3)v{2|Gp>l5-S}gn$q2~x) zDzsE+ncVY)mI0OnRsiUK*`S}$d4QFG^8u>>7XU5Rtos%Ls|vt0st=i{H>7g1#AO6BoKNS(qAF%gtQCr7!n_c^aS8r zz>k0x@}kIJBM`a@($#?L0TqCa0#ySUXgy)Va3_?-p41bweDK5XE-5{P_7sX?K0)}C zg-#GUMd(za(}d0zI!EZ4LS^uM8E{{Q+LxjBF)%&`#;3n!NIyQj?%xgRKEVBeX8^AN zbVNRaM1RVZi+>IHC!h=-S_8NOPyyHsxJw|k9nynWJdTm$=9!p)@ydDQ~@@wyLh5c~|z6+2*i=(T^&@OB?&9%+$j_3LHbS$7wq+{4s@Vs5ckoAf?(nVAr&>JEVs*+p zA-|JYj36xG*3Xxe0-^G`dcRQku0oFxiYMZOt?yoAY+cA}Fb)V|R;oEc!>dSIL|67V&=34fJH{pd6L2J2;`_GK&fvC*1fjaTFT z<%qA>05$?P1MURe3wRK)3-AO0@3Q@uA-xH}m;QZ?=bt0-TY$#%Umz_<(AAChVo2)& zR{=E0-vmiF;kzN}?u-4_#}4a1jl35D8uH(U^Z{Tm;A?>HzPbUFdE&~IfQtZ^0ImVt z0@wn00I&nF8}OunOqEorFH>l1p;420~D z?EH}Ig(2)9;`|Weu|(exPEit5L&Z{3BBczQl=LH^n25tbE6KCW^)n@nXNi=W zDD-e-QZ2ydC%G^SzkU8XIPS!l! z;1E;O*Rr0amL}3EtX2EqF6%aAtxVBQxEt(+_3MIMm8fA0l1fExDve+=N}Q8aOdNY) zC3yIyqK!v*ACK^d9OXleYuw&shEN~cFT+UciR21M8vs`W^q6ujBz?-)L(3*4-UzrA zpf><|lG%bpJwP;v5%w-5Q+mkqRV7S!J!ztitHOwmpTf^1WCcX ztg&xM9uC7YLq$H=1C3ZE8rXQNN+8}!1eE-!*j6kQ_V*PY5&szB(Ic$1!IFYKz{9 ziid<#nHM{s$3%P$p!0YSq>ltFlQX3XxG_t3bOHltZDuU44vrITrD!Khc`4pT?y$bfJUb4AyokM(5;c_ z1|({Lx*5`D0qN7BcVj%B!X4H+)vLFq((h8C8lrV0);{aAfyU8z+vBa($O*&DEna*a z)wr+~UAPON;WYk+ZyWM7s>VNk=wnC1M+<$Xz~h&A2eHE55oJx)yz~)m+0Oiq;tyHP zdcA_(8J8(I|Jf%$tgT`x)lVp545ABGxvVo^)|Nj}bfySJ1hr35>hk!B6~9@K@c<3 z4uvv5MGx~2`wQkAI)=D`nKwgBM65uU>~s3Us%}V)SWQ?*ttAfu9I+H@1-4)p>?u0f zm!iUBKgKE@xa;>bg%@wdeclS;(i(Oti7C-R%G3<2*mf~t#3TN}rnC4+KgVSssxY<0 zYd*Y&X@P!N-IgAn6F+n4E(p1s)$NJf`b$!eb1E3xB-O z5kfI4n9^8;7$NJZ=VM#IXvy(Qw+M_}iPSSuC`NIz@L0w&qCOTgmI;>hbkV_*IbHY} zLa}_X$Y)8)Y@u_6V!_W7e!kEJLeCV6NM;3s1eZnUh-QK#MZ%bOu1vpbXRGs(s7I{$ zuz0A4zxX3o!e_wvvx}a)5E1df0tIyO$M4fWAa5T)@60y^q!Tv+ z^fl?Nkn|$`SxEl`><9b;H~?5K(NsDv+~L0)lro6C|7px^ES5JBiddE`yhONzzT-rc z!=BJe^fQD;gk}oO7TQ*5JE4ehh<=?U1yQYLJnJgC-GtVRVTfLK+=_~3u~5XSno$ao z2{8%L2yqAz2(brIXPnf7-NBAJIFgn~?j)gzE{H3LD2OEzMR-&~H_K3lHUn-0+z!B9 zSZFJxdjR(W@UkPcO(Yp#_|5j170X2f{yp>_@ki|reWd?=g&zLC0PkZ*_@C_p7?l&H zU3+Zo9buy6+9PzZ_MdijpKUKiVxrk9|rXSMeo+`K+K7MT+v6v_@|cm zCq%t|iw~v=pBMGPMjsn@qP<;!-GD~`@sE^GB2feM-ymt6eg=|;>*pXn4|oCaB0#VE z^cAND_E(Xof&EQLe+Ot_*GH~HJOuckpup#V`1t-c67@Okdq^7Qaq#nhLZZG>-4E$! zz^?$kY{bWBWHsUphd75di4z;neOQNxZ8(|PQ4Qy?Vrz{dp5eTOlNQccSW7rp;Y5YA z)MzP-^OGIP4ic+;oON)@!R*(q`eyVm;ru+*#MG;cyHPyh{EVNN@iS8ApiVxWb&sPm zoo9!dQt?~=yC|ody-tR`NYp+5RqVl^3CQ^A$HMRlF#lEw#J;}1K#=zJRkVII>3gXB z23StfpPmF(0xkqxEFjfqO17MJ;=X`^X@g*IZ<2=&>)0^=*x=qzEIdZ&K%tm;*hn$` zu-%<5xw7f`X9$n!G)wqH&7i}GaPfCwI-l;v;Az0pNTnf4W788*IgHq(F9YylL_hv8 zryqy71=B;%bn%xpakn%wwmAC8c5H=$Lyu_>@s<4m)<;q>3N`(YB@h3-_ziFXu-q4WR{+ihoCjDbAihW!E&Evb ze`m)3?>*pRBo6;LV;?UMeUSO19&Kx`NrZSjsxFcdIa}yrp-Y7R;SZ^Lx3n4ya}D5P zK>QtUIT9}cTnaeUldC?m{%<_19x8k#+yLXZJUxrW?{(YJqxdaPZ{#uV#I8(_I#d8FnG!C}y*iU7+N3s_fo(uZRG{Oe*-`^DBT!ZH{Io*h@ zsnUi#eOloq!l@!RWd@(vwe*{Ds>l&$%!eq>hhH&IEp@~!Jm_Xhfi^<1dn2yHQ8?Ve zIf&(m;&26`ICjqCqy`+>&?$)v(yBN#WbFkZEB=S9{WpYK3uP-s6xx(E8HH3RziXUL%PB*lEhHl zLJ!3)ZRstBsOFVIzSx9Aa~I*^-LAsptb<(>2WabY*vFK#uL$hX!1U}V_5Amf6cZCy z2N(lH_!6mqqR=T!rMEIw{RoSp)=cq!&Hltm$l4u4m}9aeLmfn4_DfujP`mSBPW`7z=Yzj-gwV8_CSmb1(X+mqRfVkP<$IS+_oX*3fh?^V#?Prn_Z3)&a z!WzZg=q8q9Egt?n`A;9Q#!D-*()<$PCkd4i*DL2~k|L|lm-Xh)lavKQ&l0*+sCrf1 z8Gm7L72K&8vDZPW0Bi(o0_dq)FATOI@h*TqiD6r+KKk}YX#^Gf`zybPY!yY&r_boBIul5k(vyAHkNh`+?vN7y~6 z>;r&4(CU-y7m_H(Tip{G9hJo)GTeu$J3%@(S!lxh{2S32rjZoyB+8?iTFqH8!b-)8 zjKp;H#q$q=SW30A!(X-1%kTQEg|FjqouLTJp-iSe<{1tv5*ejcq_z_c*olaU9TfW| zBK9cJkzV*?gvV(dSBY@QDUyCFQ>#ztY1QmY7A=wOulz`fnt55G4LcFJu_lTo9d_br z0v5uOnUXtOsKgMd-8!C$EHfT|?alfM6=FD6E?R_7x=3wZg?1B)I0<`-B;{zKQK5Z= z%8n_mp$`%Z{IQa!@k=AWPNzytAiVVPH8{n^A^rPQ{xi^g4)8KSmxUfzaZdH$N8W~@ zG<&5laHoLf2s!8B`&W{RfyW_effT{{#TXSID(^=dI{>=?j|243{Q@MlO)n&WLn7`1 zef{8}UNY!M4fV75`XPM%2F7)=Cj9spNB1C6Km4Izy3>z)>_(n``c}V>@e&gM4)_r8 zDIos!)2~R>Z{o#&co#1$e7vdf@AAZ1_X8dV=*M;SbGlC>@dbc>2k$LNdjKB-^kcJM zLHYskBj6{%KLP&|kdec((7jKuz%ej^huv}G93Qtn0s5PfB;aZ=@Vp56N5mZ*C$ItH;5401h^4! z6X0gRX230gTLHHLZU@`}xD&7iuoZ9@;BLS@fO`S=35Wv`1F);Xnb=cDh|N|+6zvlv z74h&S;jvjDHl8dg*f>grpD1*)&?!PMz#`BumRydhcC~=C&|0)|gcb`uT__$eLQNZ*8Nm30uSTaS#JmR5n5Ms{9U2Gz|at*d+kT)&&L3LTks>KYtcJhzt=<3 zcf*yCHUe${B)nt18F_lusPB#MKw`oVglTBF7b^DwG*IZL#Po|>@!!?e&y4*a?%q5w zwydldt?tu_=zaRG`n+E6dl&aPO*fS$IVY(MC!zbaRH`aD)v2V?Rh68iTioO%m7LI- zNL9`N4T6X`%b+q3ip=1M3W%sU04lRMfddYxsEFcl0f+bdeZOz-+LcPB19;ybFWqOa zz1CiP*WPP=>zluCwRwrzfQwc+Qt}|6cHHE)?bqwK+*|GUR8!2CJxKh;t(>#OOrEhh z9JOQ4j$-%jvGX}Q3LKN5HRa^a>6}wG_kn7sWlk=fmO0gMvzZeyr(RCBoL;$)vva#nubk=q zWxb29;diO#EU-IUy;qIzSMfm=A6D@@Dx7pFjJ4+v)%eFMM9+HuTs7yQWgzbPYc>9@ z3U{skd(}Rz;-6Ifi;B;v_?!yoyxsBZX{T&ec~+8VsP;@1&YXXjYRXvB0}xUroKB$@t5x=e4@2dEH6(3dchbmma_!HIs zOvT4lxVZ5N)&54sCsnw~-h~f|8$F-Vwa=;eHx*C4MRTO$87iKs;yYA)w~Fsm@oW_? zpuR}87pr)giXT$(BPw30;x#H>tKtnRepJPetN2M3Z&L9yD&DN(ttx(A#V@G%B^AG{ z;#XC?Q^mVg{DzA6sCb`>52*N%3O6siUD*xFAJMfxRPo0u{zS!}sra}Gw>i79*{#fO zUUs|kKd9$Fs`zIW|4YSZRs5?8w>dvea=+W0-Prt0HGYSR?^5wSD!xy}vsFA-#Sf@> zo(k6`yhyc|sCc=GAGX4H1hKgPjZk5Afr(g7$yxy~xt2=Yf}|bs40qU7F`k|5kysz; zUM8()!HzR_tlCkWfY?m&1DC8_T!9!*aRp*OlTzEN^<1-K-HzfAjLo+15exk_cFn%B zt23y$rT@ozif4Mbor`k{?EJJH&)D%+J3hjWx7qP_J9Zl*aaY|&N}4S3TN!(fxULa9 z?_?^+tzCRr2ZQO6oEmp=W{#(>mw%an`$@V2c*e4d00aeJmi>=Dn`us#Po+T8ySt0(96|F3hI zw=Yf3W#^n}p2U4?wvy{CITKn#Htsf#g7j_z&&lDm8$1d#2rBg@Ul432*1KUYtA9aj z*zOE$OuMakt#x6)q}-$xtTE}7mF}`#K8|ZG^1%NmgJIgK9jJ_@)zfZ3OJ;f@Py0R=2lISPcSu zSHZAmO_ZzO+HK@JzNzzT1P8+QdW!J8Jw?`P5AHJTZBG;}5zZ0K&{v9n>h}dN!tY&h zx;<;m-c}IRrJrN{ZmTIVhW12JLmM5de~tY@#~QV3dV9@?u)d9VkMOi4@f&u(YiC?7 z^OBpa@f9juAL9m;H>lB-;4UWqtr}fg{5#b?t-_^6*K@dV^t(6PFML#mi#&g-+Q(J6 ztmC3YdzwVg@#BKUOHbLu+H)tmnBnrptJKB)XgwQB21m+ z9J+A}3CQbNvU|nVB{9@xYhSgaaMQY-Z`e_sfZ*(S>ff~f2X?$_$7^=HZb!lB6Ycy- zc6`u|PqyPz?C7A|uEB8=pzCE^hxn6%%Qva`X%()Q`8m~&Q|O;Sh3|$$SJu2o&wQT> z*V%kXwck?Fu1<8@yW1K+qAsq;`2*ElXY&_UOF;0Ka4f9eW1}KC6^;<*NYa2L%qDp( zgIRlu!EUHI5u zgI;;DDSvKH2lV!SvprLK%-&zpjLH_&>)`Q$*2Ic1CW+60-9Bc=PSSFxwRaBbW2`-E z$4NU1A=n`I*dVvr6uO~>E`#h&443T5-OxpH*dJ#-SL~QXBb^9DQ}BTG+_K}g9VJ$D znTVclXrvngaka86fQMYlzD;xF+U0Lo&8;1-Wd3e7x~}SifIuA}}LU28Y1I7)Pl zbvtHs#kGU5>#rR_9%ESAvR&`}G4=ct6@RM2 zZK`b;wa1m=f2}S@ayg&UwZB*4s_{>&_D?EYL;f$SxuX1ERr@y;|E|LIrW=eMKduw@n;JiThu%djrzn8qU}p>+UkVx8?`uYr zb8~;|tE`8uu!Lyp2u`@{$R55{-O}YIuH$T5?&ol%p;#s^{!XvVT)lGO=#{n0lcDys zhlRDdY}mbW)aaF|t5=R1y>9Vxam?*e?!4t@DamN|h(EfSBhxE!t@mT9xuMF9O8;(+ zJ#Jfi>M6TH)}-EVS4~Ex-tV+p&-bcuqtdfflWnQ@`MNG8>Z|VkFlX~B)nyf;- z@;>WzYRugBObNf0c@F?X@3z zhKg@f@k|xpuHrjXe5Z=yjaCcRJ_y*TOp!V z5=sr%SZjhB^`5cEbz)gz0O3?|g5m-9t-pzk_AKF>)7GkI-D>A{zxC*#hSsWH)7Doa zuP}+Qz&Bd^<9M{*694UMSnt)Z+B59BzSD#(t0%rNasPjdpO>K7c0J&gp2SyJs}|Ll z9lFb?kX`>uyKBmh`jR<2KM9_{&w3`mHp!)*0NZya9BlKH5-#q>y_+!A(~XC3+Pm2t zcjf1gR||ZzJ>jW0S<%khKSPb}-28W{@i?ab`_#qh%8|VN|3)6?1X94O^n$LpdyQ(Y zzH_gt;{=6vZOM`HLc8wdD6t^{mj9^?shG~h(*+_)J<)?U*X`5lqL(wBE@e4O;IflT zJsun>ai{m~8oXao;R+qs!MM`XS^lGV{lsh9itjXH_9h0$dgturJ9xgtlMdgZwQsG7 zUR{1qTU#bw^phb2F2gVL9qd^h7}Il$1?-T-Qa!?=kFfUJ>?poU9NMF;^T zdh^5I-R-wEmyTK`+$G+=%l9SssObspw`~rS0b$yr%I7Sp1~>ZNlNjIRZb&Eo?bH0# zL~rr@oue(-PL8C0S@5i{7Yt9@cW6E5?K}z9lGQ||C8Z@4mOY(xd&!<)E3@Z5JAbSl zzsZiF9Uo`M6+2$G{z$sx*a#{m{4u@{O8i=8}|Gs+VM$te9(?hv12(; z_^m1&a*k8D9Z8~H;nq%@+ZArd5#{65Z^vmI{&!Ggr@yYDJJK-W5~!1V{J z9BFm9Tl!dJI?s}MWA=}_?>NQTXVm4h zDn6&8U7P0mvu_c<+^#?S4mG;lY5P_s?tSWUQ7)+aUIQLH@;u%Cd=)QH(eB^Z+h+V`mD9^ZW}(M53vL$vGu_}0v45$W8;L)mn)}86jcV>5`$^T@OZHQ$xwC9rIB3s5 z>YBUG{UW z%~N%`Q}4H{(S3V8f7f@Z(LH^?M>Qq&>T$nc?FQ20?!V7bm*=YZeic8U;s;ecPsNdr z!Y|Ram#T2L;g_rC-ornv+ACDF`x3uOjjvYG?pgdgHNIYj`x(1d%-xNDOxJ!|#amRo zT}8V~@vo`TU5Z_C=AOmBu50dK{9e`E!T9~Exr6b^(^&n!?mN=!_)m4sosa)gwZBr~ z&c}bPn)@LCt!nOy{CBFkp3hy8|4EJRk^C>JxnuHYRCA5szp3``D%?%^sXc-^743<# zlp3qYotD2{UEFv1J5_TR=I>U`J(<5(HFs!!mTKE&2VbxtH?~s^-ql&sVKo zbLpxCsh2Til0*PCKW%e;%8L+tco|Qc#Dd+s`xn-Z&UH}D&DT*7gYSBieFOk z4i&$w;#XAss)}Dz@lF-*Qt@sTzpmmpRQ#rj_o#TUiub8_zlsm2_@IgpsrW4wA6D_( zDt<@B@2dDc6~C|IBPu?s;ty2(p^86J@y9AYrs7Xj{HcmRQ}O32{!+zXsrZD7zgF={ z6@RDVQ!4&m#XqR{w2FUJ@lPuLS;c2md{)KhRQ#)oe^c@AD*i*oQ>FKQnu>2xadOT2 zak}(p>tWAP@dGM;P{s39JYU5NRJ>5di&VT=#miK@T*WI?yh_EZRlG*UYgN2X#p_kP zLB$(Y{HTf_Q}N>}enQ1hs`x1tZ&LBID&DN(Eh^rs;^$SoUBxe1VZwIZP?H2vDfAoG za=hXY%T)Ye^+U;!b;#6gN`-oVMOlEJ+n2g^pBOQfB3> zPr;57;J?w%FWb?+&Yqi4msmZaSV;V?e({Xmd#fE2@8==I$uCQ&yB{NIdyx(3l3iW4<9&9#-;SE(qz2}3*1lp#P5zplYarL{yqnfV-b~l6 z&y(#aJ&9{ZT)pn9=wFoF`%5ZZ5%6oO$s*n3ih$oxqbsdlAK*?l@6)ySt8lfz2UTm= z4t!XRCsz`sFnczBVn zwd)^VrpA}6aBYNq1$yKw(DN!?b8WRQ_Fho`0a?IBkooSzNzzoMPp#^{^xLE3Q0w>Me#_pQfT+CGxFmY!7!n zQZsU#O2l;`?Me_=d%Qru@Iot$&*|i9#JZia*Syt^kFaCTj%I)9eH_nwSL}G1M|)P1 z(X{6#I4SW^0lRi(>>g!T#k7g*k$tIP?Xn<=b2$lzA{&vtV~;qI1AC5h8ScmMGEqLa z`k$QYyH@^4!uc&iDM!-JXX=`3-d)N55;eY5g_E@(vYHKHl8+S!tKkvzchNqpXvcdJ zLp5vXV*AQ=F6Qrf`sDU6|5-IkF`1aR?W;K1Eneo7zf9@Nh6_^h=XQQWc7XU*9ix2&|ARD4FA zvw5zj@<`6cC%v7)d7*A_21k%!pLo@VLJYDS3*9aMJM1g0lU9yd*WHGk9D(EIh`a0u zI;9Co4Vo2!n1Dz!f>zh#>|TxSs-3UdQBr_b-4$!uv|~Ga>nU@2zjU;Gwu8l6h=J^(rXy-=+L$6fvMfx5&`uS-+%-7a_){c`r+IwmlC+A;1ju%stU@Gs79!G!A*MPi0#S2xuNQFB} zoxl+O3f<5S46jn-aWHXY#EKije_XdaOt@LxQC2&I90?jfuiK9U4VR+ZLE}i7+Kub& zGPT>zehwf4YVJ^2aP!4`bnSyGjx$j_|Euh0U!dZJ zDqf`G#VTH+;-xBHrsCx)en`cO|1W#WXGNv{L!|Ag54Xlwt9Xry*IHpPs9x>XS~t_S z%Q7c0l(q{FE-F0pl)Zo?2K&jR?n&?S_0}b@;~6{J5>75qk3vYY4Wb8$C23FBosY1m z+-Aqy?INeFd_{tbuxtO-?$wt((#{RF^qjSGdt>X@ z!B1H0S6VOe9UVPR?!A|IR+pz7{SGJpg|_+87s}@=NxSvUQobg68!0cl(ey`i4D9_h zmR-J}Ct$ap({}8fbB(=bzT5j5MA$DSVXK>JH3>8Jj-5N9rv4|U?&j2ozXtDr)91e8 z%TqVu4=;#+H=Vn>vVU%C^YXdJ*LNP++-aPQ%(3|Du(5M+?tJvQ>KJ`{XUEH%O@G+& zeaZRXCmIF3PUBp2 ze`Tk!x4ZxOe^q1mU>|^}sovh$Zl2?$hjxaa_~Kp*5V+ag-T&eq(p<-Eex9S;<*T@& zICJd)(Ftpf8|?se1l$64+n-_}-QLvs;FZSqp)QT>fsOi3eSh=tOYQHiw0GU#^Daas z53!gA);F)$*ZvcRjc^bOIKB{Qh@gG-p2mrhQ70Bz-P#0(`n~Zb~Obau8AI- zJG{a^t5@o)4>S(8Ro!jWHn({@Slvc*ZLPkJ^B!~~{>5o;Duqwp+|}t+u~fj>msnqG zd>Bq|G{XA+{?=xrsa|*v2;u<#?X)!po$Dh#I!Bv!BmP&y>U&q;hNu~Dd8+D$d z8yfN2I^U=_clVm%>h3=8#0aeHZ|v#P_DYklg&Qz%_PVHH=yn^`8;#AKb&iu&3p@NZ zJDu9ZMA&(AYd4yOmHqvd8))T>Tv=)Gp*A`DtHYU9uV^yRyxS=3tZlCF>+rObT5D`R(Pw09cV}Z@ZTH}^4K1dq(LA`!p5XQE zbYN?5Y((|w>F5=mes${-z(Z=E}Z< zyUhUJj6aXx2*cJY38w)?JL{XAb-T2E88K`F>QXM?l5dh|clTmv1!djcY3e`9y7C zXiN>Zj0ZN?fX%_^s;yDqs;}~G@hzhZNvQO7#R6?|?Ow$~fY7z;G^>I6BC$&JSL z%If~E&wH|J!(zbApTR)t$JbYPn!ehjQ}nb`@u=#ywz-d2#yLJWK1avB-Nxqiq1MP_ zC|9-;)y-`%t+)+9)4(X#qtDdKZ|+`=-#og?_q6knQ>0&K!^+y4PhI>VfM0wLc;oq| zeY#&zg;(@*yNTyuungnj=bEdWEucO!#W+I%8cx=@p7FKO>DRVSc~5#=yju!tj6bffQ@~o zHQ2=5Mw;+6okxrn9;>!+rMbJkxf;O~dJ*qNRRb}Ql&J2P6Z9l%!NP3V_vx|i_tdWf zR1(jy`oN)ER#uzBrFsI{r22J-N&HHyj=7B1Du5_DO|)WuNiny!3}>N6IccxHPc(-Q zN6+>K^stdL_=;dNqTcw4_KUc^y}`g~rG+pKTCd8<-rrq~u&z^2v>r6-gk7BB)3Ekd zAMhDzgzH-?8;<-ON)uhUqBl20#C1mmB^b5nl&|ct!_-#jndXc@|U1Ay|1`vLAy0%?3 zyaKD!>OIx+99p4AXs)i{C2>wlyRx-GT4%HzC8+pH0|Uyp#)qvsn!suOdVSUT2-HAv zAXli9rVxw?s`$S8&ehHR-JNZ~sgFG-aeZgSxE)+~CdTg49i*W3>(0n*?yefn#C+PR zKO;WJG{onahp0OyDmsmN;Nek+BzLyGZTt$J?`L=?o+UA|)y3H96&qI^eQX*7!k62# z!HHhanmN1Hktd)6S0iX`RdD5?+3?Z8V_Fqlxwf*&K}0jp641*kpK;(nr!hoVOofkU zyukPN;{l9LF*rbVK6`lj1JDW82`+6HP{B+YuZ+|6tz9EyU;uytBGkaCClw<2B+v2k zi=X1KaiHm0d0@ z_S+i(lS;aw%KA#g)$@n(N?Q%EKwH2~1M3Gnh5&uA;_JFMUORYyur`;qaKz`}mC*?o z@Hl*?&d~|&1JQus=dorbw%8+h&Ywah;(>UIQH${OA?E5@Vp-{YAi5C`-)&@#j^K&L zrRPoZ(dj0K7N@HRjnHApR&F&S@Ybyq&5&wO9Elz^D*kVh~mLzwxKsU z{umfo!L4Y9g1_m493S8x&+GfU*Zek@40)ZZ7_{n`5ISX zpMac!cj|ekZo22xm4|)Rsp8V9@tZH)G;`%;7oM^4h?(V6|0PJ>e9KL} zXL|<^cyhY8dCSecr#|wvcg@^#)2Y8$U%KVyQ!n_shrQH0okMxb4&P@jclF+XdT<#% ze`4bv^nBlce^|4l=S$+Ax12il-ul_I_nn!UDP6d4WANI+#vuN;>%8w~9!_D4UK}s+ zS8nRHW^1|m(z%0EjkBlz>)F$G<*weDN4)%=*Up{^7b4r6V7iJ#w@GG}&T)1`l!p5x^F4zFC+ws%4^ql(8 zQ zExo6n^OU>J;jy2*>0!a?!TSsn9?{$Ey>I=#GtYkZ%g)T)hi};!I(Ow9Ug6zO{+7GZ z_suuIYXcv3fHyus>sy|@#MXD-{H`}{+%~gu2LI!71XIzs_x{|uA-)mTL+_Ofrw^We z_uU5q)Vq2w@YPP=J^1WW>0suJ#;SK_=1gg3*
ddZvlk~14KXJ(eqv<=bIPoMkx z!Q_}d{q(`?h3L2KiBCUs;T*pAQxD$5?|s%OJc>^`wXrmO>PtCxryudQrRCGl{__h@ zzi{`3({H)!g=!%` z9L_DwrfSvvWSA@G3*juT6z3MRrD8s;lrM#o%e6u^sAbF5FkhGo3zbT_Vm;G?{dnZy z@YqxzlvFecRH<(`*qhqZ<7TVV;Z$il%+1c%&~rFd$QHx7@?3Vd5Da8ewPJ22oG(Klv6zA|JrP9JY-k?;d;T^M;nP3ns%2poaV!Ga3T-oPY zdT%J~-w*qOG|&15?%rc-@TkD6K=t4wR?hCOuY>g8pGMcm_N8)#axK7NDlrPRYA(EY zp-@>4bJ@9Ep;Rj5j~Ro^AjV*RDx53SCJPJ2U}!j%FH9~>hxvusd1w@>#V}i4p34=> zVRovVDW~fXsOMSsN1Yvjpw&j|71dk5xNFVG#=-H{S1=Q9AcktouH{+R) zdQ@MkRK`Rt%mw2qOhm4ftyU-brI@32{0-tSwXhf#^Q8jyN~PLxhR_!lmgd7S7|M*J zH8;Nim|2*sh1E1ho;tFA$V52BH$fbxq!glIb@yPW8N^+Js81?fs8tC#VRcz$sW3NP zyBNgns;4#^*ViCyH^Z&;YX26W-D<(0t@iZ?>8-v#zVK8PuQb0*MHt>SS2CCz9MbyW zY~h<{=E|1pwUe&%-0rzBU*f=;-iR>fkC>moO@XP5ej^i>*}1R%=6J;iY1}c9B0emV#`K|ABaxtGK_C zodpmN8E0ME-KbG{UfZ2KSWjb&`|+>Jm5PT#n>Kf_-4{G5xVpQ!7TjT`f%`#Nax;8` z$KDx0#=*jG23*_0h2S!jFQ$u6Ti5;|?%Q{#Bb+-VJ2jHM!^Vy6%ez~R=q^4DH$)E? z;>HbwXj=!OFqxwfoW2muEtE=pzqm_d2Yv=SO?^t>-x3T2L;b;--OJ4tXd%~v+tvaI z)2j~z>(C0;8{fEd1_&bl-r#QGPWz^Tpw<3p`%JtTEE(wUpT3C~>%2L67hgNL%WlRQ zkZbFOA9W!}zA=cFzNA0->UjOA^^Cpoqi+L#zpdMQZ=!$l=QsQDwI-w9zxi0a{gd$@ zg(?YvQ;@=<5#`0ymFDi{`p{VVI`QabYTLu1u>jRnVU~C)7#=dSQwU`kWIBLdvjHe6 zujlKR4>kZ#4ZvZxG7IEaoSrkHU{DawU#bumr?Aemb6kHh_c2!vff*O3FNXPIb*4I> z%@vNJyF-~mU#e1=Cc2s|mjKAECrx9y)d=EYaj}#Sv$z8xM4l1WBx9B>kw*ZLnJVJbfZ^vrO>$e1o55NYg&e0U+36sY!Fz(@Vu=%* z3=KNbO%IId%^W3x09B2M@NEyZ5y8%1ER&!Qk2+l}1LcDInX47EK&tqS1$-V>cMb$J zSPgEyHCOhpz^avnYUM$cm%OY@SLUtmWu;uZXmyl*DWk7dP^K_%6gE3I zT?mH2NdO081VhqBp}rcjsytV9sBy2sUmW3#e#30RVS_5{5{7b*4Cf zsTzO{%T}i6L*tw((`dL3;K1Klw18%*Uzq}fe5Gb4?nYwiS7%GH2s013E9c53`;6{0 zGmK{w03aIKxXj)g@g}wMJO+WQaGs(HuFqmAf%_ui9U6tofB#@70Qp_ttZ%I~ z&YIJ}{@K>GR1FM9t$NA0iNRsxIHu=kY5^GdO8SwtM}kYgK0Mw}O4pYvV>-Zq23UtE zQULU@1m-f}N(7u3*AjE@w7>ASRFzB+DDp+hQNSPpIL7%(af#exh!HR1R*d^paVa>n z1ay#Kk?}~R>Z45MUGE6uYcQ~GlVbqi zN?-%{7IWZ0fY3_8$OPUVOS?EXO@eak^~8i3mUfvFHv)458*~*pGh<_u4og5nwn!GI zE9HgxYG~}XW(K%~wy&#o$VX2|LdUEzn-M<_cwa1Yu)6--;KJF$g8k2M22gattUALE0ylgigfvH}&DhLtUF1(3Kd z*T+Tx8eXixc>*4Xu>6OZZgokI_hB-DWPmKdww9`3?qgzYE2Uh%q0g>z!|#CmyPI> zQ>al$Poo*!+aW+b*J_EulOyZsGF+GkK`EpU2=h3Qar{Quh+vVWxm;2HZn}cfrIbY9_`l z)Qb2qO0<;XDD43Cl%{~z!^;PoTkvKLuN-XDLl|Vhd=oGxi?u2^kaD&L)qsRR2*pO> zBH<)iX~(c7eiqmqP)_nYkX9;&XcNMVV>`$$F`C2s8G+_}0qu>0MR%-N?pQ)n$iI(iy+O~d=}CfYmW+su|{Gl#F;bJ!f0kCrqQCm zlg?^}WWmDn)D&5gh=3V?1!5b^)br&_8RFeE4?x-1F?;PRSvaX(QE|-Y4-*NCy`O>* zzfghDI}b**3V2P)EhuP@cROy!I9EMt4D}e48ue}G_X_j zzW6Bex*Rtu1*t(SmdTDq!uU2Y*mXPrlFsgpSSp_dld%L81`1SO?iA0B+ko(Ku~3=^ z06~8urZ=3D1WjxZ;sP~Z$-jjX830&RX?0980Q4@n0Y<k(=6B?I` z#u`!8m8M~5YxQLMRL@tlyjeN#%MPk7Dm?;$I#o7T9!RingMJPaus^J24M5XjP0dPG4 zqxh6`eAy)Tz7~T|Zfzl->ob)LG4S#%XeI_9yf3(ec_<*E5ekN=Saeml%RNJb8L^QI zbH%A5c-VWOe>EPr3fT~2ei14dFuca(gNaKoE%gOUOKDTi_6IAp)LDbAx_;;>;D%}q zP!<*zArSQk>u@Bc<4x2&4$NPlnzhEx(NG*LMU-Y%+gdh-w)=h3*g zxu-sQQ2Z2dYTy~z0m^l_mD+OvExAIq%EsMQxJg>0Bu*3U>nqUF;P;?H>kr^>r3x7C z3aYfhs-d=wYV(Fw7V_wNu%@nL{j>NS>5-t*u3%9YW)~O$IGeqv4CO(p z0mkm?7Aeu@TKXym9vmxhnhPlSO-?97#CEYeTZ4343&sGa zki{pV{x_DGBVh7iTT7~$uTzr@rvIvz6PKE){mgQ*$hBzCf;M}yG?RnI1OHK}3G`Je z&rM${S0Lq~0dF!zeQeFSV=NTtUdpn)*u%hYS$P+2P4iGM?sQY;!tr4Fe$YxtYeS|;xlKc z;pKDb%4V~^Sl>^4Nf|;1k!mD-cb6hCuzm2H+S@PHcOVTy?8*Y*M@I%nQ+oyTkKYF= zc_RG0Xd9>%fVh^)V$PZbW&B9hpei(3Cm0z^$PK@#Eim z0ERDNc(7roy^(xtQdgDA<*N(T`5;%Tl+us9@kqQ05f#{XaGd3-G)!FmbZF||u5Vu^ zdW0Sc)gY35d4fKD15&&0uG{7Qz`KJfY3{BxRyQ|W5E(b%S+TuII6BfP0LZ9!naY-` zW`1aKZGNRv09F9OiE&SHp*A%h3=YcfKoXt@CdifUE2V(~O+V`l`s&sbx2!Dzr4-b% zIq<-cOQ53Aznvjai_b0_JS7iU|ZVp-uwOj|B~Qslmzs z?!{m(-PqdcH&qA+6ctFVRz2I;3U}b17V3xmY}r=Afy$I*Pz;%=4j}o z>O1h`H4JcVwE@qp%Z=btt_r>2B!r;D79ZMq78B_B-UVp!92HLkT|&ex&cT+BVTNVT zEKoHGFPUMgK`Go$14?;|>DAKd?#el-H_2$tby42|RDlF> z?E@%KS%B2uhNrb`*s(@^cs$j-9KvY~=-EIe(mds$Z^M$nA}9soXj=^?roy$0;Y|CX zIg`a-##zN%HsGh;H0@nW`3%)jLis#8y3iuV)rCo*N?-*i$267Z%H^4&lqs;OU@n$% zHCK_p2JjpPai#!JOjCaY?~2*^67fZ{8;tr8-@ISY*6~T`x>mmm!=9CQs-U4uVj8Ys zjfdg(%Hv@QyIm_~a1ir#0G1wfW*nv@yES<5n43(d!W#3NAO;_E*Cx!7t)Ieg`CWr2 zx=G?&{8K~xsr@lX;Y9)idM9cyK%r5FI%E=k(3M&h4GBC%DF~)`a$T?mo0x^gfL16Ds~~`J6Ycd> z2A22gJJ1BSfCYLnBQv%#V@hK8Nk%iBgK?r@dL|cOXb0jY7id;?Q&?rl-N*$7)WA&F z(s6;)%wxHQ!SD!GV)F%P9YLW3xO+$@Q-{!;5Uzb$?88)plu6Ze>M;#917A9xpL~|4cAzKO5{Jh|2cdu& z8X3U@>}xJA%m>j)3X4DDMF{~=zfpXqh@8$XlA7WOVv1-68pLvNV}V}Lt^+maW=#YK z69EMjW`&5|DwsJSwXqRsKs0H?DVp?%UK2LGlIf;OAk5r3 z=$&$rM#fy$C^W2@$tg6x0<85E{DzL>SVji3ASBsZy4DYAKAo=hrI$eL+SNf#PA&>? z#z0)}w>TUQ5EOt!#j0aQ%V|cT$LP}lzb?|!KiTJ6*f8y0j_G? zq7)?b0fev*7^Hd|#^20nimrS#?3H#`)*4V(6G#o5V!LxGwZ02P7fhAm?9@3Rru?Qp zm6lHURz)LgNdmK!ai!jwuw4xPDKjHP!L9r!(tjJv~ zC}hBhBtTgr6SU*==Db+F7o0*jcS;z3DA{zU&TYiGMbS-V3R(=wWChNG^{WtoV+$d) zQE-f#Kc8X_!4BT63~|MZbp@lY{kS-09qpKWoE{g zszsxe&>NEvrVX1cHXs_WgEK&GgeT$Nfrjhd&?q#u)D&|qGqM4{(^81<>~8KLu zY*jF~Lt7+~POL1E*0j;IYGq5)v}mD7)z%pG!Qg6qj#%nEG#enOK!Es|p|c~9C?>{7 z#=xqNj1Qe3JU=**0nL~g9Usn&PfQGr4Gq@^Mn<7tLL?egv*GGx(2o*u{VXh7Bltek zy0t{_xB*x$7QGGeo0|OyABpyMAn8aHufYR={KVkc`Jqt=+Oh0Mhm?z&w!MEzDPoKs^T46PeM8%*4dV$ng02 z%<$;==HU-1D7;N_8KO{ zRis&1-QI&1i>~8qW`;CkSXiLC^xgnk2IB}WmZ{OK23TE7k^&Q_y_V0;q+<#ohZ(AXlRM$I z9mfX~!-5qKu=zSTjZSCaEo;CjprklUo`J}gPOOoY+pSY%T=59A@wgvC0tR7!H}V0s z+A-^4f=rQ_>OpwHV3jNIA+$INEJ$$n2$RL3c_R=(Fkhz@HLxn)4pY1kSL|Xi!1sjmYKCtp#pctHS*;n}@TW zq7D*<9hgX{tESPZSW7>lA4bx&8mPENLx7eGftmmgi*Zx>3E-=)nLl&-?Af#Q_M+dp zTBaz4-yV3V4+DO9oetIW^gM@8R{9M56ax5^C;IdUt-JfNK;Ya5%vTLV=%JnrmoW?4 z5)fD zN?|KCuN>gw1z0M80cMNf(sMv0P%h;ojf&}aTE5*F82NY4(0`_@ST#641nlW_-sh9E zcO*Pg%7@Nl&7vphXp41Vc!Bs1(i)U=n6q2NN>a3VHM;~3&VAA-dkW! zlMC%=<0y63q`-0yJ3*h6ahA|H;SbnN1QJRWlbuRHl=Ou$ibQ6f3an`87J+XWpaLpQ zCj#=wS624rJY_5Idjd;-2)F<(Tk-)rlkmipSJS z&Y2p>dbja?LlYUuFXI!Vql4$ihccr!L+yw9z)!5mQ^P@xfe`o?+C=@G0LwGfEq>!yL z@VFcSPXf_uVr*=Rv>x78Te~-q#t#BSG0Z?T8-+ABF*ckT1KT=|ZumGSo3;R5GiJU4 z5=xH6BO_QSre2K#!9X}+X&M&P<)IahVmIcYV}g!7hY$vkN5c@tZxc;K zkSr8I*v$bolgo(=M|AH@IxJ%?BSaIZl|oDk>DCH+Rpbth!OuhG`H#-yT)H(&+(yVd(8J$+{v~I%mh!6}H3&~AtOk+TycgJYv(BM`KP zG63?Ck)iXbkDtfF90TMJLA?OsD>E`hvq#MJiu*NAY7%0AjRj9qHOc&lp`^?Y4j+oj z2Lrg)Le27Eumr3HJOUZ8kev!h%eU$)>rg$-=OLe&*EZT(75jlI#ARspAw3wEgi&v6 z0U0?!=8S#lV>u(v+Bi>dFV@~fya*RAJOm-saz<6ewwskKRSDh^Y=MT|Fei-lhMa*! zalzbvYtY%i|IzR&vk%%*F=nI+KHh>LCtT8qLhp zog^n=4Dmk}VHUzeWSaf~@K<8UDG-}8+z?7apI_iv@B$_Pg-8Qn($S%MYi&}RCRQ@L zj1jrD5Nu3+XE#_2p(e&)K*<%wLHWA_VHwK`oI@AQ(UGA@)XQBYgootii2^L^R0xy? z`oFlcbx=?5UVc1M1ob1AntR@;fx+s24VEr23I}!k2rbTj-5a%N*&aRN(eWt&9ugqD zR(20CBNjAI#s$tixAQ^<$}wFG!QMw`6HExiXp`xNhX&7Q#6_6~8Co(Js)F;QX554g z4I@mwk6F_qrzL<5X3orfRVYjb#|I}$M3ZxK2)e<*f`=uSOKtgl1?oVV3CZry*GhDN zNQMGUcqoulF^nU|(nIRa%N0|DP$N3cMEr#7#th%ac$FvR$$N~j!P-_iCaz|1I0JDM z_yoACT%FG)`9M@XvVhi*nq>h9#$dYT^YL0GcvOMtRT;4+8?`xX1!qLyaeWRdZaH@e zfwhJRBS7#53kw!4CTmcIZ?7~Sh_!+MJzDHKRa&m3K@nhONRPu454=w5%y)K?(~9x0 zKyEF#cgngbIE6fX01vY>uxut763Sg*mHal7*^!7~9&9(=F%Yv9=LjH)ZR5sHuIP?X zB0Q?7mnF*s{apEy`w(=OZ=ellU?DZQ^utoLH`M0B-{mQo@D?7{5l{ph^Y>IFX;?+1T{L;c!&M*UzL=;)X18TA`Ks#~k0bu<5= zBc8_h(_2Q}j{a4B2YW{Sj{1t$*TR*EUyGl`sDLyO)?i5^^2bC#u?j0NHPhp8REeN{ z{~AHjB0kryAsZ%=2}Be{x(=J!{H4m#CVVJS3>rZQ+?Km^ufK*qc+y!v;XGs9 z_Ho`b)`93Viix2pIgavNgH!P_4oxOW{e4a*Vq z()wNSJ3zj`KXEn@s0ztaP&1Pd*CGrAT+Os3@D{JMLQ)ZpvELW5ct&B1jU0(>UNk=ay7tdjoN)ZvF#4DW2v|7ijI+Tc^*vz53)746 ze@Ch)8s6h`E>T+4BgKGZY2&U7bJTSKV5q$zy*54Lxaq8bu^D;;heE36Y;w&F5km$i zDG8>@24|@<3?!kzYBJ@-1erJ}?#(Eaw3!7U9~RRDOS*!aaB`=mbh#ri|J3_4kG(Sr zS(QqXf)Mgq8Hku&3HtFkbRWh;^1x81G3QmBj44v{Ku3C1k|r1Az~#_-XddHKtWCl1 z+P%nYFidvmb_R!Yh)D>;x!T3_`YO;)lE;huB4AG6y@v6WbPius^h_z^k1^gDB3Krq z2v`FLa-j&-Nq{Hrhfl4+g=y?8R+v6f=9t$;tLZRc%Oz*#bi=3ZA8NC2Jl!Jn?gj*1 z7dJwA8FoP@VH4+Nzeuye3@+H10Q?B6!5ws{?+@-koS6Bp+S_tqr7Dt>eo_b1;wp@t zGTuAk>jRLahacsj^guUfwoa;)o0!08c?c>83p_Xyp@_j$kadDdp3VU&Dxn4y5pbTv zIxrWv3Y>j1bQ6S$fVm7N=cfXASuvL#{5~x48z2%ydis;zc?>8}>Qdmw(WAmHsx66I~>7D&P>j76?>jD0L&Z5n9+U z!6BQ6BiVC9h!Hu1!JcWJku_p9<{2P`!{Cua{9>+C)*(NF%@G(S1i<#Q)a2;-G4L=> zao__Enm`k6tYL~O67G~z1UJ6M