From 33143e90c831de915b819f5721417f5dbf17a7df Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Wed, 12 Apr 2023 09:17:31 +0800 Subject: [PATCH 01/35] enhance: add udf stub expiration of 10s after creation --- source/libs/function/src/tudf.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 2269ad7f6a..92dbdda496 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -343,7 +343,7 @@ typedef struct SUdfcFuncStub { char udfName[TSDB_FUNC_NAME_LEN + 1]; UdfcFuncHandle handle; int32_t refCount; - int64_t lastRefTime; + int64_t createTime; } SUdfcFuncStub; typedef struct SUdfcProxy { @@ -982,15 +982,15 @@ int32_t acquireUdfFuncHandle(char *udfName, UdfcFuncHandle *pHandle) { if (stubIndex != -1) { SUdfcFuncStub *foundStub = taosArrayGet(gUdfcProxy.udfStubs, stubIndex); UdfcFuncHandle handle = foundStub->handle; - if (handle != NULL && ((SUdfcUvSession *)handle)->udfUvPipe != NULL) { + int64_t currUs = taosGetTimestampUs(); + if (handle != NULL && ((SUdfcUvSession *)handle)->udfUvPipe != NULL && (currUs - foundStub->createTime) < 10 * 1000 * 1000) { *pHandle = foundStub->handle; ++foundStub->refCount; - foundStub->lastRefTime = taosGetTimestampUs(); uv_mutex_unlock(&gUdfcProxy.udfStubsMutex); return 0; } else { - fnInfo("invalid handle for %s, refCount: %d, last ref time: %" PRId64 ". remove it from cache", udfName, - foundStub->refCount, foundStub->lastRefTime); + fnInfo("invalid handle for %s, refCount: %d, create time: %" PRId64 ". remove it from cache", udfName, + foundStub->refCount, foundStub->createTime); taosArrayRemove(gUdfcProxy.udfStubs, stubIndex); } } @@ -1001,7 +1001,7 @@ int32_t acquireUdfFuncHandle(char *udfName, UdfcFuncHandle *pHandle) { strncpy(stub.udfName, udfName, TSDB_FUNC_NAME_LEN); stub.handle = *pHandle; ++stub.refCount; - stub.lastRefTime = taosGetTimestampUs(); + stub.createTime = taosGetTimestampUs(); taosArrayPush(gUdfcProxy.udfStubs, &stub); taosArraySort(gUdfcProxy.udfStubs, compareUdfcFuncSub); } else { @@ -1046,14 +1046,14 @@ int32_t cleanUpUdfs() { fnInfo("tear down udf. udf name: %s, handle: %p, ref count: %d", stub->udfName, stub->handle, stub->refCount); doTeardownUdf(stub->handle); } else { - fnInfo("udf still in use. udf name: %s, ref count: %d, last ref time: %" PRId64 ", handle: %p", stub->udfName, - stub->refCount, stub->lastRefTime, stub->handle); + fnInfo("udf still in use. udf name: %s, ref count: %d, create time: %" PRId64 ", handle: %p", stub->udfName, + stub->refCount, stub->createTime, stub->handle); UdfcFuncHandle handle = stub->handle; if (handle != NULL && ((SUdfcUvSession *)handle)->udfUvPipe != NULL) { taosArrayPush(udfStubs, stub); } else { - fnInfo("udf invalid handle for %s, refCount: %d, last ref time: %" PRId64 ". remove it from cache", - stub->udfName, stub->refCount, stub->lastRefTime); + fnInfo("udf invalid handle for %s, refCount: %d, create time: %" PRId64 ". remove it from cache", + stub->udfName, stub->refCount, stub->createTime); } } ++i; From fe718f60ee1120d0d7bd7618d76dd99655974d4d Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Wed, 12 Apr 2023 12:15:09 +0800 Subject: [PATCH 02/35] fix: add expired udfc func stub to track the expired --- source/libs/function/src/tudf.c | 54 +++++++++++++++++++++++++++------ 1 file changed, 45 insertions(+), 9 deletions(-) diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 92dbdda496..53b2cf4aca 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -363,6 +363,7 @@ typedef struct SUdfcProxy { uv_mutex_t udfStubsMutex; SArray *udfStubs; // SUdfcFuncStub + SArray *expiredUdfStubs; //SUdfcFuncStub uv_mutex_t udfcUvMutex; int8_t initialized; @@ -983,15 +984,22 @@ int32_t acquireUdfFuncHandle(char *udfName, UdfcFuncHandle *pHandle) { SUdfcFuncStub *foundStub = taosArrayGet(gUdfcProxy.udfStubs, stubIndex); UdfcFuncHandle handle = foundStub->handle; int64_t currUs = taosGetTimestampUs(); - if (handle != NULL && ((SUdfcUvSession *)handle)->udfUvPipe != NULL && (currUs - foundStub->createTime) < 10 * 1000 * 1000) { - *pHandle = foundStub->handle; - ++foundStub->refCount; - uv_mutex_unlock(&gUdfcProxy.udfStubsMutex); - return 0; + bool expired = (currUs - foundStub->createTime) >= 10 * 1000 * 1000; + if (!expired) { + if (handle != NULL && ((SUdfcUvSession *)handle)->udfUvPipe != NULL) { + *pHandle = foundStub->handle; + ++foundStub->refCount; + uv_mutex_unlock(&gUdfcProxy.udfStubsMutex); + return 0; + } else { + fnInfo("invalid handle for %s, refCount: %d, create time: %" PRId64 ". remove it from cache", udfName, + foundStub->refCount, foundStub->createTime); + taosArrayRemove(gUdfcProxy.udfStubs, stubIndex); + } } else { - fnInfo("invalid handle for %s, refCount: %d, create time: %" PRId64 ". remove it from cache", udfName, - foundStub->refCount, foundStub->createTime); taosArrayRemove(gUdfcProxy.udfStubs, stubIndex); + taosArrayPush(gUdfcProxy.expiredUdfStubs, foundStub); + taosArraySort(gUdfcProxy.expiredUdfStubs, compareUdfcFuncSub); } } *pHandle = NULL; @@ -1017,13 +1025,17 @@ void releaseUdfFuncHandle(char *udfName) { SUdfcFuncStub key = {0}; strncpy(key.udfName, udfName, TSDB_FUNC_NAME_LEN); SUdfcFuncStub *foundStub = taosArraySearch(gUdfcProxy.udfStubs, &key, compareUdfcFuncSub, TD_EQ); - if (!foundStub) { + SUdfcFuncStub *expiredStub = taosArraySearch(gUdfcProxy.expiredUdfStubs, &key, compareUdfcFuncSub, TD_EQ); + if (!foundStub && !expiredStub) { uv_mutex_unlock(&gUdfcProxy.udfStubsMutex); return; } - if (foundStub->refCount > 0) { + if (foundStub != NULL && foundStub->refCount > 0) { --foundStub->refCount; } + if (expiredStub != NULL && expiredStub->refCount > 0) { + --expiredStub->refCount; + } uv_mutex_unlock(&gUdfcProxy.udfStubsMutex); } @@ -1060,6 +1072,28 @@ int32_t cleanUpUdfs() { } taosArrayDestroy(gUdfcProxy.udfStubs); gUdfcProxy.udfStubs = udfStubs; + + SArray *expiredUdfStubs = taosArrayInit(16, sizeof(SUdfcFuncStub)); + while (i < taosArrayGetSize(gUdfcProxy.expiredUdfStubs)) { + SUdfcFuncStub *stub = taosArrayGet(gUdfcProxy.expiredUdfStubs, i); + if (stub->refCount == 0) { + fnInfo("tear down udf. expired. udf name: %s, handle: %p, ref count: %d", stub->udfName, stub->handle, stub->refCount); + doTeardownUdf(stub->handle); + } else { + fnInfo("udf still in use. expired. udf name: %s, ref count: %d, create time: %" PRId64 ", handle: %p", stub->udfName, + stub->refCount, stub->createTime, stub->handle); + UdfcFuncHandle handle = stub->handle; + if (handle != NULL && ((SUdfcUvSession *)handle)->udfUvPipe != NULL) { + taosArrayPush(expiredUdfStubs, stub); + } else { + fnInfo("udf invalid handle for %s, expired. refCount: %d, create time: %" PRId64 ". remove it from cache", + stub->udfName, stub->refCount, stub->createTime); + } + } + ++i; + } + taosArrayDestroy(gUdfcProxy.udfStubs); + gUdfcProxy.expiredUdfStubs = expiredUdfStubs; uv_mutex_unlock(&gUdfcProxy.udfStubsMutex); return 0; } @@ -1663,6 +1697,7 @@ int32_t udfcOpen() { uv_barrier_wait(&proxy->initBarrier); uv_mutex_init(&proxy->udfStubsMutex); proxy->udfStubs = taosArrayInit(8, sizeof(SUdfcFuncStub)); + proxy->expiredUdfStubs = taosArrayInit(8, sizeof(SUdfcFuncStub)); uv_mutex_init(&proxy->udfcUvMutex); fnInfo("udfc initialized") return 0; } @@ -1679,6 +1714,7 @@ int32_t udfcClose() { uv_thread_join(&udfc->loopThread); uv_mutex_destroy(&udfc->taskQueueMutex); uv_barrier_destroy(&udfc->initBarrier); + taosArrayDestroy(udfc->expiredUdfStubs); taosArrayDestroy(udfc->udfStubs); uv_mutex_destroy(&udfc->udfStubsMutex); uv_mutex_destroy(&udfc->udfcUvMutex); From 99587db7a10b7a6d5aeb28c615c4becaf6e90584 Mon Sep 17 00:00:00 2001 From: slzhou Date: Fri, 14 Apr 2023 07:58:25 +0800 Subject: [PATCH 03/35] fix: udf handle expired after 10s --- source/libs/function/src/tudf.c | 30 ++++++++++++++++-------------- source/libs/function/src/udfd.c | 2 +- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 53b2cf4aca..f13235f24b 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -960,7 +960,7 @@ int32_t udfcOpen(); int32_t udfcClose(); int32_t acquireUdfFuncHandle(char *udfName, UdfcFuncHandle *pHandle); -void releaseUdfFuncHandle(char *udfName); +void releaseUdfFuncHandle(char *udfName, UdfcFuncHandle handle); int32_t cleanUpUdfs(); bool udfAggGetEnv(struct SFunctionNode *pFunc, SFuncExecEnv *pEnv); @@ -992,11 +992,12 @@ int32_t acquireUdfFuncHandle(char *udfName, UdfcFuncHandle *pHandle) { uv_mutex_unlock(&gUdfcProxy.udfStubsMutex); return 0; } else { - fnInfo("invalid handle for %s, refCount: %d, create time: %" PRId64 ". remove it from cache", udfName, + fnInfo("udf invalid handle for %s, refCount: %d, create time: %" PRId64 ". remove it from cache", udfName, foundStub->refCount, foundStub->createTime); taosArrayRemove(gUdfcProxy.udfStubs, stubIndex); } } else { + fnInfo("udf handle expired for %s, will setup udf. move it to expired list", udfName); taosArrayRemove(gUdfcProxy.udfStubs, stubIndex); taosArrayPush(gUdfcProxy.expiredUdfStubs, foundStub); taosArraySort(gUdfcProxy.expiredUdfStubs, compareUdfcFuncSub); @@ -1020,7 +1021,7 @@ int32_t acquireUdfFuncHandle(char *udfName, UdfcFuncHandle *pHandle) { return code; } -void releaseUdfFuncHandle(char *udfName) { +void releaseUdfFuncHandle(char *udfName, UdfcFuncHandle handle) { uv_mutex_lock(&gUdfcProxy.udfStubsMutex); SUdfcFuncStub key = {0}; strncpy(key.udfName, udfName, TSDB_FUNC_NAME_LEN); @@ -1030,10 +1031,10 @@ void releaseUdfFuncHandle(char *udfName) { uv_mutex_unlock(&gUdfcProxy.udfStubsMutex); return; } - if (foundStub != NULL && foundStub->refCount > 0) { + if (foundStub != NULL && foundStub->handle == handle && foundStub->refCount > 0) { --foundStub->refCount; } - if (expiredStub != NULL && expiredStub->refCount > 0) { + if (expiredStub != NULL && expiredStub->handle == handle && expiredStub->refCount > 0) { --expiredStub->refCount; } uv_mutex_unlock(&gUdfcProxy.udfStubsMutex); @@ -1046,7 +1047,8 @@ int32_t cleanUpUdfs() { } uv_mutex_lock(&gUdfcProxy.udfStubsMutex); - if (gUdfcProxy.udfStubs == NULL || taosArrayGetSize(gUdfcProxy.udfStubs) == 0) { + if ((gUdfcProxy.udfStubs == NULL || taosArrayGetSize(gUdfcProxy.udfStubs) == 0) && + (gUdfcProxy.expiredUdfStubs == NULL || taosArrayGetSize(gUdfcProxy.expiredUdfStubs) == 0)) { uv_mutex_unlock(&gUdfcProxy.udfStubsMutex); return TSDB_CODE_SUCCESS; } @@ -1092,7 +1094,7 @@ int32_t cleanUpUdfs() { } ++i; } - taosArrayDestroy(gUdfcProxy.udfStubs); + taosArrayDestroy(gUdfcProxy.expiredUdfStubs); gUdfcProxy.expiredUdfStubs = expiredUdfStubs; uv_mutex_unlock(&gUdfcProxy.udfStubsMutex); return 0; @@ -1109,7 +1111,7 @@ int32_t callUdfScalarFunc(char *udfName, SScalarParam *input, int32_t numOfCols, code = doCallUdfScalarFunc(handle, input, numOfCols, output); if (code != TSDB_CODE_SUCCESS) { fnError("udfc scalar function execution failure"); - releaseUdfFuncHandle(udfName); + releaseUdfFuncHandle(udfName, handle); return code; } @@ -1123,7 +1125,7 @@ int32_t callUdfScalarFunc(char *udfName, SScalarParam *input, int32_t numOfCols, code = TSDB_CODE_UDF_INVALID_OUTPUT_TYPE; } } - releaseUdfFuncHandle(udfName); + releaseUdfFuncHandle(udfName, handle); return code; } @@ -1156,7 +1158,7 @@ bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo *pResult SUdfInterBuf buf = {0}; if ((udfCode = doCallUdfAggInit(handle, &buf)) != 0) { fnError("udfAggInit error. step doCallUdfAggInit. udf code: %d", udfCode); - releaseUdfFuncHandle(pCtx->udfName); + releaseUdfFuncHandle(pCtx->udfName, handle); return false; } if (buf.bufLen <= session->bufSize) { @@ -1165,10 +1167,10 @@ bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo *pResult udfRes->interResNum = buf.numOfResult; } else { fnError("udfc inter buf size %d is greater than function bufSize %d", buf.bufLen, session->bufSize); - releaseUdfFuncHandle(pCtx->udfName); + releaseUdfFuncHandle(pCtx->udfName, handle); return false; } - releaseUdfFuncHandle(pCtx->udfName); + releaseUdfFuncHandle(pCtx->udfName, handle); freeUdfInterBuf(&buf); return true; } @@ -1225,7 +1227,7 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) { taosArrayDestroy(pTempBlock->pDataBlock); taosMemoryFree(pTempBlock); - releaseUdfFuncHandle(pCtx->udfName); + releaseUdfFuncHandle(pCtx->udfName, handle); freeUdfInterBuf(&newState); return udfCode; } @@ -1270,7 +1272,7 @@ int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock *pBlock) { freeUdfInterBuf(&resultBuf); int32_t numOfResults = functionFinalizeWithResultBuf(pCtx, pBlock, udfRes->finalResBuf); - releaseUdfFuncHandle(pCtx->udfName); + releaseUdfFuncHandle(pCtx->udfName, handle); return udfCallCode == 0 ? numOfResults : udfCallCode; } diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index aa72309c62..61c6f92e2f 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -591,7 +591,7 @@ SUdf *udfdNewUdf(const char *udfName) { SUdf *udfdGetOrCreateUdf(const char *udfName) { uv_mutex_lock(&global.udfsMutex); SUdf **pUdfHash = taosHashGet(global.udfsHash, udfName, strlen(udfName)); - int64_t currTime = taosGetTimestampSec(); + int64_t currTime = taosGetTimestampMs(); bool expired = false; if (pUdfHash) { expired = currTime - (*pUdfHash)->lastFetchTime > 10 * 1000; // 10s From 430457e5f834f77af1422b651fbd7e9c3b52416d Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Fri, 14 Apr 2023 20:02:46 +0800 Subject: [PATCH 04/35] fix: copy version from main branch of tmqDelete-1ctb.py --- tests/system-test/7-tmq/tmqDelete-1ctb.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/system-test/7-tmq/tmqDelete-1ctb.py b/tests/system-test/7-tmq/tmqDelete-1ctb.py index 4b8c8c8629..aa9c8d25d0 100644 --- a/tests/system-test/7-tmq/tmqDelete-1ctb.py +++ b/tests/system-test/7-tmq/tmqDelete-1ctb.py @@ -238,7 +238,7 @@ class TDTestCase: if self.snapshot == 0: consumerId = 2 - expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"]) + expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"]) * 2 elif self.snapshot == 1: consumerId = 3 expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1 + 1/4)) @@ -324,7 +324,7 @@ class TDTestCase: if self.snapshot == 0: consumerId = 4 - expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"]) + expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"]) * 2 elif self.snapshot == 1: consumerId = 5 expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1 - 1/4 + 1/4 + 3/4)) From 1a0c9f31bc105b7e8e1f8ffeb10b539be7b7a1e2 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 17 Apr 2023 10:34:34 +0800 Subject: [PATCH 05/35] enhance: refactor cleanup udf function --- source/libs/function/src/tudf.c | 72 +++++++++++++++++++-------------- 1 file changed, 42 insertions(+), 30 deletions(-) diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index f13235f24b..8c8b99a6f8 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -968,6 +968,8 @@ bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo *pRes int32_t udfAggProcess(struct SqlFunctionCtx *pCtx); int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock *pBlock); +void cleanupNotExpiredUdfs(); +void cleanupExpiredUdfs(); int compareUdfcFuncSub(const void *elem1, const void *elem2) { SUdfcFuncStub *stub1 = (SUdfcFuncStub *)elem1; SUdfcFuncStub *stub2 = (SUdfcFuncStub *)elem2; @@ -1040,18 +1042,32 @@ void releaseUdfFuncHandle(char *udfName, UdfcFuncHandle handle) { uv_mutex_unlock(&gUdfcProxy.udfStubsMutex); } -int32_t cleanUpUdfs() { - int8_t initialized = atomic_load_8(&gUdfcProxy.initialized); - if (!initialized) { - return TSDB_CODE_SUCCESS; +void cleanupExpiredUdfs() { + int32_t i = 0; + SArray *expiredUdfStubs = taosArrayInit(16, sizeof(SUdfcFuncStub)); + while (i < taosArrayGetSize(gUdfcProxy.expiredUdfStubs)) { + SUdfcFuncStub *stub = taosArrayGet(gUdfcProxy.expiredUdfStubs, i); + if (stub->refCount == 0) { + fnInfo("tear down udf. expired. udf name: %s, handle: %p, ref count: %d", stub->udfName, stub->handle, stub->refCount); + doTeardownUdf(stub->handle); + } else { + fnInfo("udf still in use. expired. udf name: %s, ref count: %d, create time: %" PRId64 ", handle: %p", stub->udfName, + stub->refCount, stub->createTime, stub->handle); + UdfcFuncHandle handle = stub->handle; + if (handle != NULL && ((SUdfcUvSession *)handle)->udfUvPipe != NULL) { + taosArrayPush(expiredUdfStubs, stub); + } else { + fnInfo("udf invalid handle for %s, expired. refCount: %d, create time: %" PRId64 ". remove it from cache", + stub->udfName, stub->refCount, stub->createTime); + } + } + ++i; } + taosArrayDestroy(gUdfcProxy.expiredUdfStubs); + gUdfcProxy.expiredUdfStubs = expiredUdfStubs; +} - uv_mutex_lock(&gUdfcProxy.udfStubsMutex); - if ((gUdfcProxy.udfStubs == NULL || taosArrayGetSize(gUdfcProxy.udfStubs) == 0) && - (gUdfcProxy.expiredUdfStubs == NULL || taosArrayGetSize(gUdfcProxy.expiredUdfStubs) == 0)) { - uv_mutex_unlock(&gUdfcProxy.udfStubsMutex); - return TSDB_CODE_SUCCESS; - } +void cleanupNotExpiredUdfs() { SArray *udfStubs = taosArrayInit(16, sizeof(SUdfcFuncStub)); int32_t i = 0; while (i < taosArrayGetSize(gUdfcProxy.udfStubs)) { @@ -1074,28 +1090,24 @@ int32_t cleanUpUdfs() { } taosArrayDestroy(gUdfcProxy.udfStubs); gUdfcProxy.udfStubs = udfStubs; +} - SArray *expiredUdfStubs = taosArrayInit(16, sizeof(SUdfcFuncStub)); - while (i < taosArrayGetSize(gUdfcProxy.expiredUdfStubs)) { - SUdfcFuncStub *stub = taosArrayGet(gUdfcProxy.expiredUdfStubs, i); - if (stub->refCount == 0) { - fnInfo("tear down udf. expired. udf name: %s, handle: %p, ref count: %d", stub->udfName, stub->handle, stub->refCount); - doTeardownUdf(stub->handle); - } else { - fnInfo("udf still in use. expired. udf name: %s, ref count: %d, create time: %" PRId64 ", handle: %p", stub->udfName, - stub->refCount, stub->createTime, stub->handle); - UdfcFuncHandle handle = stub->handle; - if (handle != NULL && ((SUdfcUvSession *)handle)->udfUvPipe != NULL) { - taosArrayPush(expiredUdfStubs, stub); - } else { - fnInfo("udf invalid handle for %s, expired. refCount: %d, create time: %" PRId64 ". remove it from cache", - stub->udfName, stub->refCount, stub->createTime); - } - } - ++i; +int32_t cleanUpUdfs() { + int8_t initialized = atomic_load_8(&gUdfcProxy.initialized); + if (!initialized) { + return TSDB_CODE_SUCCESS; } - taosArrayDestroy(gUdfcProxy.expiredUdfStubs); - gUdfcProxy.expiredUdfStubs = expiredUdfStubs; + + uv_mutex_lock(&gUdfcProxy.udfStubsMutex); + if ((gUdfcProxy.udfStubs == NULL || taosArrayGetSize(gUdfcProxy.udfStubs) == 0) && + (gUdfcProxy.expiredUdfStubs == NULL || taosArrayGetSize(gUdfcProxy.expiredUdfStubs) == 0)) { + uv_mutex_unlock(&gUdfcProxy.udfStubsMutex); + return TSDB_CODE_SUCCESS; + } + + cleanupNotExpiredUdfs(); + cleanupExpiredUdfs(); + uv_mutex_unlock(&gUdfcProxy.udfStubsMutex); return 0; } From a2d75a327cb68a4f59c76c6d2b9a2150880b6eee Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 18 Apr 2023 08:28:12 +0800 Subject: [PATCH 06/35] enhance: udf output column reserve capacity --- source/libs/function/src/udfd.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index 61c6f92e2f..5034af2f82 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -688,6 +688,8 @@ void udfdProcessCallRequest(SUvUdfWork *uvUdf, SUdfRequest *request) { output.colMeta.type = udf->outputType; output.colMeta.precision = 0; output.colMeta.scale = 0; + udfColEnsureCapacity(&output, call->block.info.rows); + SUdfDataBlock input = {0}; convertDataBlockToUdfDataBlock(&call->block, &input); code = udf->scriptPlugin->udfScalarProcFunc(&input, &output, udf->scriptUdfCtx); From 9581dcae7f322716c0dcf187788d42bc520e7300 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 18 Apr 2023 14:17:22 +0800 Subject: [PATCH 07/35] enh: add param for to_unixtimestamp to return timestamp type --- source/libs/function/src/builtins.c | 27 ++++++++++++++++++++++++--- source/libs/scalar/src/sclfunc.c | 3 ++- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index a293f45238..fe98a1dd53 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1933,14 +1933,35 @@ static int32_t translateToIso8601(SFunctionNode* pFunc, char* pErrBuf, int32_t l } static int32_t translateToUnixtimestamp(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { - if (1 != LIST_LENGTH(pFunc->pParameterList)) { + int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList); + int16_t resType = TSDB_DATA_TYPE_BIGINT; + + if (1 != numOfParams && 2 != numOfParams) { return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); } - if (!IS_STR_DATA_TYPE(((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type)) { + uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; + if (!IS_STR_DATA_TYPE(para1Type)) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } + if (2 == numOfParams) { + uint8_t para2Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 1))->resType.type; + if (!IS_INTEGER_TYPE(para2Type)) { + return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + } + + SValueNode* pValue = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 1); + if (pValue->datum.i == 1) { + resType = TSDB_DATA_TYPE_TIMESTAMP; + } else if (pValue->datum.i == 0) { + resType = TSDB_DATA_TYPE_BIGINT; + } else { + return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, + "TO_UNIXTIMESTAMP function second parameter should be 0/1"); + } + } + // add database precision as param uint8_t dbPrec = pFunc->node.resType.precision; int32_t code = addDbPrecisonParam(&pFunc->pParameterList, dbPrec); @@ -1948,7 +1969,7 @@ static int32_t translateToUnixtimestamp(SFunctionNode* pFunc, char* pErrBuf, int return code; } - pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT}; + pFunc->node.resType = (SDataType){.bytes = tDataTypes[resType].bytes, .type = resType}; return TSDB_CODE_SUCCESS; } diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 24b25cec80..835264eabe 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -1124,7 +1124,8 @@ _end: int32_t toUnixtimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { int32_t type = GET_PARAM_TYPE(pInput); int64_t timePrec; - GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[1]), pInput[1].columnData->pData); + int32_t idx = (inputNum == 2) ? 1 : 2; + GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[idx]), pInput[idx].columnData->pData); for (int32_t i = 0; i < pInput[0].numOfRows; ++i) { if (colDataIsNull_s(pInput[0].columnData, i)) { From 56ab728a4213b9767a4548e9bd24d27cdbf89a8d Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 18 Apr 2023 14:18:06 +0800 Subject: [PATCH 08/35] add test cases --- tests/system-test/2-query/To_unixtimestamp.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/tests/system-test/2-query/To_unixtimestamp.py b/tests/system-test/2-query/To_unixtimestamp.py index 8ee2007450..00a8ed84de 100644 --- a/tests/system-test/2-query/To_unixtimestamp.py +++ b/tests/system-test/2-query/To_unixtimestamp.py @@ -26,7 +26,7 @@ class TDTestCase: 'c1':'int', 'c2':'float', 'c3':'binary(20)' - + } # structure of tag self.tag_dict = { @@ -60,7 +60,7 @@ class TDTestCase: if tb_type == 'ntb' or tb_type == 'ctb': tdSql.checkRows(len(values_list)) elif tb_type == 'stb': - tdSql.checkRows(len(self.values_list)*tb_num) + tdSql.checkRows(len(self.values_list)*tb_num) for time in ['2020-01-32T08:00:00','2020-13-32T08:00:00','acd']: tdSql.query(f"select to_unixtimestamp('{time}') from {tbname}") if tb_type == 'ntb' or tb_type == 'ctb': @@ -74,7 +74,7 @@ class TDTestCase: if tb_type == 'ntb' or tb_type == 'ctb': tdSql.checkRows(len(values_list)) elif tb_type == 'stb': - tdSql.checkRows(len(values_list)*tb_num) + tdSql.checkRows(len(values_list)*tb_num) for time in self.error_param: tdSql.error(f"select to_unixtimestamp({time}) from {tbname}") def timestamp_change_check_ntb(self): @@ -95,9 +95,20 @@ class TDTestCase: self.data_check(f'{self.stbname}_{i}',self.values_list,'ctb') self.data_check(self.stbname,self.values_list,'stb',self.tbnum) tdSql.execute(f'drop database {self.dbname}') + def timestamp_change_return_type(self): + tdSql.query(f"select to_unixtimestamp('1970-01-01 08:00:00+08:00', 0);") + tdSql.checkEqual(tdSql.queryResult[0][0], 0) + tdSql.query(f"select to_unixtimestamp('1970-01-01 08:00:00+08:00', 1);") + tdSql.checkEqual(tdSql.queryResult[0][0], datetime.datetime(1970, 1, 1, 8, 0, 0)) + tdSql.error(f"select to_unixtimestamp('1970-01-01 08:00:00+08:00', 2);") + tdSql.error(f"select to_unixtimestamp('1970-01-01 08:00:00+08:00', 1.5);") + tdSql.error(f"select to_unixtimestamp('1970-01-01 08:00:00+08:00', 'abc');") + tdSql.error(f"select to_unixtimestamp('1970-01-01 08:00:00+08:00', true);") + tdSql.error(f"select to_unixtimestamp('1970-01-01 08:00:00+08:00', 1, 3);") def run(self): # sourcery skip: extract-duplicate-method self.timestamp_change_check_ntb() self.timestamp_change_check_stb() + self.timestamp_change_return_type() def stop(self): tdSql.close() tdLog.success(f"{__file__} successfully executed") From b2cca108a349ae4a5fa10eee560734f04a35a56d Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 18 Apr 2023 14:29:04 +0800 Subject: [PATCH 09/35] add documentations --- docs/en/12-taos-sql/10-function.md | 10 ++++++++-- docs/zh/12-taos-sql/10-function.md | 10 ++++++++-- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/docs/en/12-taos-sql/10-function.md b/docs/en/12-taos-sql/10-function.md index 8dfa9c2851..5c1a833e05 100644 --- a/docs/en/12-taos-sql/10-function.md +++ b/docs/en/12-taos-sql/10-function.md @@ -459,12 +459,17 @@ TO_JSON(str_literal) #### TO_UNIXTIMESTAMP ```sql -TO_UNIXTIMESTAMP(expr) +TO_UNIXTIMESTAMP(expr [, return_timestamp]) + +return_timestamp: { + 0 + | 1 +} ``` **Description**: UNIX timestamp converted from a string of date/time format -**Return value type**: BIGINT +**Return value type**: BIGINT, TIMESTAMP **Applicable column types**: VARCHAR and NCHAR @@ -476,6 +481,7 @@ TO_UNIXTIMESTAMP(expr) - The input string must be compatible with ISO8601/RFC3339 standard, NULL will be returned if the string can't be converted - The precision of the returned timestamp is same as the precision set for the current data base in use +- return_timestamp indicates whether the returned value type is TIMESTAMP or not. If this parameter set to 1, function will return TIMESTAMP type. Otherwise function will return BIGINT type. If parameter is omitted, default return value type is BIGINT. ### Time and Date Functions diff --git a/docs/zh/12-taos-sql/10-function.md b/docs/zh/12-taos-sql/10-function.md index 94f8052051..50e82e6b90 100644 --- a/docs/zh/12-taos-sql/10-function.md +++ b/docs/zh/12-taos-sql/10-function.md @@ -459,12 +459,17 @@ TO_JSON(str_literal) #### TO_UNIXTIMESTAMP ```sql -TO_UNIXTIMESTAMP(expr) +TO_UNIXTIMESTAMP(expr [, return_timestamp]) + +return_timestamp: { + 0 + | 1 +} ``` **功能说明**：将日期时间格式的字符串转换成为 UNIX 时间戳。 -**返回结果数据类型**：BIGINT。 +**返回结果数据类型**：BIGINT, TIMESTAMP。 **应用字段**：VARCHAR, NCHAR。 @@ -476,6 +481,7 @@ TO_UNIXTIMESTAMP(expr) - 输入的日期时间字符串须符合 ISO8601/RFC3339 标准，无法转换的字符串格式将返回 NULL。 - 返回的时间戳精度与当前 DATABASE 设置的时间精度一致。 +- return_timestamp 指定函数返回值是否为时间戳类型，设置为1时返回 TIMESTAMP 类型，设置为0时返回 BIGINT 类型。如不指定缺省返回 BIGINT 类型。 ### 时间和日期函数 From 1d2764ebdb50b66673219672a2062dadae846730 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 18 Apr 2023 15:36:50 +0800 Subject: [PATCH 10/35] feat: the fill value clause supports constant expressions --- source/libs/parser/inc/sql.y | 4 +- source/libs/parser/src/parTranslater.c | 22 +- source/libs/parser/src/sql.c | 1292 ++++++++++++------------ 3 files changed, 678 insertions(+), 640 deletions(-) diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index f99976e0df..d7a6baaffe 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -1047,8 +1047,8 @@ sliding_opt(A) ::= SLIDING NK_LP duration_literal(B) NK_RP. fill_opt(A) ::= . { A = NULL; } fill_opt(A) ::= FILL NK_LP fill_mode(B) NK_RP. { A = createFillNode(pCxt, B, NULL); } -fill_opt(A) ::= FILL NK_LP VALUE NK_COMMA literal_list(B) NK_RP. { A = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, B)); } -fill_opt(A) ::= FILL NK_LP VALUE_F NK_COMMA literal_list(B) NK_RP. { A = createFillNode(pCxt, FILL_MODE_VALUE_F, createNodeListNode(pCxt, B)); } +fill_opt(A) ::= FILL NK_LP VALUE NK_COMMA expression_list(B) NK_RP. { A = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, B)); } +fill_opt(A) ::= FILL NK_LP VALUE_F NK_COMMA expression_list(B) NK_RP. { A = createFillNode(pCxt, FILL_MODE_VALUE_F, createNodeListNode(pCxt, B)); } %type fill_mode { EFillMode } %destructor fill_mode { } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index e29be53bb6..cbe7f5668c 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1310,7 +1310,8 @@ static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) { } static EDealRes haveVectorFunction(SNode* pNode, void* pContext) { - if (isAggFunc(pNode) || isIndefiniteRowsFunc(pNode) || isWindowPseudoColumnFunc(pNode) || isInterpPseudoColumnFunc(pNode)) { + if (isAggFunc(pNode) || isIndefiniteRowsFunc(pNode) || isWindowPseudoColumnFunc(pNode) || + isInterpPseudoColumnFunc(pNode)) { *((bool*)pContext) = true; return DEAL_RES_END; } @@ -2911,6 +2912,11 @@ static int32_t convertFillValue(STranslateContext* pCxt, SDataType dt, SNodeList if (TSDB_CODE_SUCCESS == code) { code = scalarCalculateConstants(pCaseFunc, &pCell->pNode); } + if (TSDB_CODE_SUCCESS == code && QUERY_NODE_VALUE != nodeType(pCell->pNode)) { + code = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Fill value is just a constant"); + } else if (TSDB_CODE_SUCCESS != code) { + code = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Filled data type mismatch"); + } return code; } @@ -2927,9 +2933,9 @@ static int32_t checkFillValues(STranslateContext* pCxt, SFillNode* pFill, SNodeL if (fillNo >= LIST_LENGTH(pFillValues->pNodeList)) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Filled values number mismatch"); } - if (TSDB_CODE_SUCCESS != - convertFillValue(pCxt, ((SExprNode*)pProject)->resType, pFillValues->pNodeList, fillNo)) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Filled data type mismatch"); + int32_t code = convertFillValue(pCxt, ((SExprNode*)pProject)->resType, pFillValues->pNodeList, fillNo); + if (TSDB_CODE_SUCCESS != code) { + return code; } ++fillNo; } @@ -5715,6 +5721,14 @@ static int32_t translateDropCGroup(STranslateContext* pCxt, SDropCGroupStmt* pSt static int32_t translateAlterLocal(STranslateContext* pCxt, SAlterLocalStmt* pStmt) { // The statement is executed directly on the client without constructing a message. + if ('\0' != pStmt->value[0]) { + return TSDB_CODE_SUCCESS; + } + char* p = strchr(pStmt->config, ' '); + if (NULL != p) { + *p = 0; + strcpy(pStmt->value, p + 1); + } return TSDB_CODE_SUCCESS; } diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index 9ad5bcf644..a38e2368dd 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -217,331 +217,343 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (2905) +#define YY_ACTTAB_COUNT (3030) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 2116, 1881, 2010, 499, 432, 1877, 500, 1758, 431, 2102, - /* 10 */ 670, 2051, 46, 44, 1646, 1722, 2102, 2008, 640, 2098, - /* 20 */ 393, 504, 1495, 1520, 39, 38, 2098, 501, 45, 43, - /* 30 */ 42, 41, 40, 1576, 1791, 1493, 2134, 2010, 1520, 132, - /* 40 */ 131, 130, 129, 128, 127, 126, 125, 124, 2084, 384, - /* 50 */ 669, 591, 2007, 640, 2275, 2094, 2100, 374, 244, 1571, - /* 60 */ 28, 1944, 2094, 2100, 375, 19, 663, 652, 371, 2281, - /* 70 */ 184, 638, 1501, 663, 2276, 617, 1942, 628, 140, 1869, - /* 80 */ 507, 2115, 107, 500, 1758, 2151, 36, 296, 169, 2117, - /* 90 */ 673, 2119, 2120, 668, 168, 663, 1734, 758, 141, 9, + /* 0 */ 2116, 209, 2010, 1881, 432, 502, 1883, 1765, 431, 2102, + /* 10 */ 670, 2010, 48, 46, 1646, 1722, 2098, 2008, 640, 2098, + /* 20 */ 393, 1522, 1495, 384, 41, 40, 2007, 640, 47, 45, + /* 30 */ 44, 43, 42, 1576, 1791, 1493, 2134, 180, 1520, 132, + /* 40 */ 131, 130, 129, 128, 127, 126, 125, 124, 2084, 1931, + /* 50 */ 669, 591, 2094, 2100, 2275, 2094, 2100, 388, 398, 1571, + /* 60 */ 30, 1937, 1939, 663, 499, 19, 663, 500, 1758, 2281, + /* 70 */ 184, 168, 1501, 1734, 2276, 617, 1520, 628, 140, 1869, + /* 80 */ 507, 2115, 107, 500, 1758, 2151, 38, 296, 169, 2117, + /* 90 */ 673, 2119, 2120, 668, 655, 663, 2176, 758, 141, 9, /* 100 */ 15, 735, 734, 733, 732, 403, 1884, 731, 730, 144, /* 110 */ 725, 724, 723, 722, 721, 720, 719, 157, 715, 714, /* 120 */ 713, 402, 401, 710, 709, 708, 707, 706, 592, 2241, - /* 130 */ 398, 2219, 1320, 1937, 1939, 123, 1578, 1579, 122, 121, + /* 130 */ 62, 2219, 1320, 1938, 1939, 123, 1578, 1579, 122, 121, /* 140 */ 120, 119, 118, 117, 116, 115, 114, 1311, 695, 694, /* 150 */ 693, 1315, 692, 1317, 1318, 691, 688, 2216, 1326, 685, - /* 160 */ 1328, 1329, 682, 679, 177, 652, 1551, 1561, 2280, 1409, - /* 170 */ 1410, 2275, 1577, 1580, 1944, 653, 1892, 630, 182, 2212, - /* 180 */ 2213, 356, 138, 2217, 358, 1993, 1496, 2279, 1494, 1942, - /* 190 */ 1720, 2276, 2278, 133, 287, 288, 516, 39, 38, 286, - /* 200 */ 537, 45, 43, 42, 41, 40, 278, 62, 703, 155, + /* 160 */ 1328, 1329, 682, 679, 504, 652, 1551, 1561, 2280, 277, + /* 170 */ 501, 2275, 1577, 1580, 1944, 653, 1892, 630, 182, 2212, + /* 180 */ 2213, 356, 138, 2217, 1409, 1410, 1496, 2279, 1494, 1942, + /* 190 */ 1720, 2276, 2278, 133, 287, 288, 1989, 41, 40, 286, + /* 200 */ 537, 47, 45, 44, 43, 42, 278, 52, 703, 155, /* 210 */ 154, 700, 699, 698, 152, 1499, 1500, 1794, 1550, 1553, /* 220 */ 1554, 1555, 1556, 1557, 1558, 1559, 1560, 665, 661, 1569, - /* 230 */ 1570, 1572, 1573, 1574, 1575, 2, 46, 44, 425, 1169, - /* 240 */ 1522, 341, 62, 1518, 393, 49, 1495, 62, 611, 93, + /* 230 */ 1570, 1572, 1573, 1574, 1575, 2, 48, 46, 425, 1169, + /* 240 */ 192, 341, 62, 1518, 393, 51, 1495, 62, 87, 93, /* 250 */ 469, 2116, 616, 483, 350, 2275, 482, 1576, 177, 1493, - /* 260 */ 406, 670, 427, 423, 405, 45, 43, 42, 41, 40, - /* 270 */ 615, 184, 452, 50, 484, 2276, 617, 454, 1171, 1994, - /* 280 */ 1174, 1175, 180, 1571, 555, 554, 553, 2134, 1723, 19, - /* 290 */ 106, 545, 137, 549, 1931, 1520, 1501, 548, 1605, 2084, + /* 260 */ 406, 670, 427, 423, 405, 47, 45, 44, 43, 42, + /* 270 */ 615, 184, 452, 1888, 484, 2276, 617, 454, 1171, 1994, + /* 280 */ 1174, 1175, 187, 1571, 555, 554, 553, 2134, 1723, 19, + /* 290 */ 106, 545, 137, 549, 551, 550, 1501, 548, 1605, 2084, /* 300 */ 103, 669, 547, 552, 366, 365, 1521, 591, 546, 123, /* 310 */ 2275, 1639, 122, 121, 120, 119, 118, 117, 116, 115, /* 320 */ 114, 758, 359, 101, 15, 2281, 184, 430, 2280, 429, /* 330 */ 2276, 617, 2115, 1191, 442, 1190, 2151, 653, 1892, 110, /* 340 */ 2117, 673, 2119, 2120, 668, 1522, 663, 1885, 226, 143, /* 350 */ 438, 150, 2175, 2204, 1606, 133, 428, 387, 2200, 187, - /* 360 */ 1578, 1579, 542, 480, 1519, 1192, 474, 473, 472, 471, + /* 360 */ 1578, 1579, 542, 480, 652, 1192, 474, 473, 472, 471, /* 370 */ 468, 467, 466, 465, 464, 460, 459, 458, 457, 340, - /* 380 */ 449, 448, 447, 209, 444, 443, 357, 502, 277, 1765, - /* 390 */ 1551, 1561, 2280, 338, 187, 2275, 1577, 1580, 1650, 187, - /* 400 */ 555, 554, 553, 705, 1520, 1938, 1939, 545, 137, 549, - /* 410 */ 1496, 2279, 1494, 548, 606, 2276, 2277, 1868, 547, 552, - /* 420 */ 366, 365, 1354, 1355, 546, 187, 1708, 1264, 35, 391, + /* 380 */ 449, 448, 447, 177, 444, 443, 357, 653, 1892, 638, + /* 390 */ 1551, 1561, 2280, 338, 187, 2275, 1577, 1580, 165, 187, + /* 400 */ 555, 554, 553, 358, 1993, 189, 1895, 545, 137, 549, + /* 410 */ 1496, 2279, 1494, 548, 606, 2276, 2277, 2070, 547, 552, + /* 420 */ 366, 365, 1354, 1355, 546, 2279, 1708, 1264, 37, 391, /* 430 */ 1600, 1601, 1602, 1603, 1604, 1608, 1609, 1610, 1611, 1499, - /* 440 */ 1500, 1552, 1550, 1553, 1554, 1555, 1556, 1557, 1558, 1559, + /* 440 */ 1500, 1974, 1550, 1553, 1554, 1555, 1556, 1557, 1558, 1559, /* 450 */ 1560, 665, 661, 1569, 1570, 1572, 1573, 1574, 1575, 2, - /* 460 */ 12, 46, 44, 227, 1266, 1495, 167, 1521, 211, 393, - /* 470 */ 2116, 1495, 502, 1833, 1765, 1674, 578, 652, 1493, 172, - /* 480 */ 631, 705, 1576, 476, 1493, 533, 529, 525, 521, 224, - /* 490 */ 2219, 39, 38, 277, 1745, 45, 43, 42, 41, 40, - /* 500 */ 612, 607, 600, 1523, 1523, 66, 2134, 1883, 1571, 551, - /* 510 */ 550, 1501, 1520, 610, 19, 1501, 2215, 2098, 2084, 639, + /* 460 */ 12, 48, 46, 227, 1266, 1495, 413, 1521, 610, 393, + /* 470 */ 2116, 1495, 652, 196, 167, 1674, 516, 1520, 1493, 172, + /* 480 */ 631, 1833, 1576, 476, 1493, 533, 529, 525, 521, 224, + /* 490 */ 253, 41, 40, 277, 2134, 47, 45, 44, 43, 42, + /* 500 */ 612, 607, 600, 1523, 611, 66, 2134, 2103, 1571, 12, + /* 510 */ 1191, 10, 1190, 84, 19, 1501, 83, 2098, 2084, 639, /* 520 */ 669, 1501, 603, 602, 1672, 1673, 1675, 1676, 1677, 88, - /* 530 */ 39, 38, 222, 12, 45, 43, 42, 41, 40, 2134, - /* 540 */ 758, 1944, 2084, 200, 199, 543, 758, 1744, 381, 15, - /* 550 */ 1552, 2115, 1607, 2094, 2100, 2151, 1942, 49, 110, 2117, - /* 560 */ 673, 2119, 2120, 668, 663, 663, 475, 1262, 166, 514, - /* 570 */ 181, 2003, 2204, 316, 39, 38, 387, 2200, 45, 43, - /* 580 */ 42, 41, 40, 1426, 1427, 1578, 1579, 314, 73, 186, - /* 590 */ 1743, 72, 62, 609, 560, 2084, 1191, 2230, 1190, 221, - /* 600 */ 215, 62, 87, 639, 220, 12, 512, 10, 2219, 570, - /* 610 */ 207, 495, 493, 490, 696, 1551, 1561, 396, 360, 1425, - /* 620 */ 1428, 1577, 1580, 240, 213, 162, 33, 1887, 1192, 1496, - /* 630 */ 717, 1494, 372, 1894, 2214, 1496, 1612, 1494, 2084, 563, - /* 640 */ 1942, 2067, 653, 1892, 557, 653, 1892, 1944, 1742, 239, - /* 650 */ 62, 255, 193, 637, 386, 2003, 2102, 541, 1499, 1500, - /* 660 */ 189, 540, 1942, 55, 1499, 1500, 2098, 1550, 1553, 1554, + /* 530 */ 41, 40, 222, 91, 47, 45, 44, 43, 42, 1868, + /* 540 */ 758, 211, 1192, 200, 199, 502, 758, 1765, 609, 15, + /* 550 */ 1650, 2115, 51, 2094, 2100, 2151, 1520, 62, 110, 2117, + /* 560 */ 673, 2119, 2120, 668, 663, 663, 475, 2051, 166, 514, + /* 570 */ 181, 2003, 2204, 316, 41, 40, 387, 2200, 47, 45, + /* 580 */ 44, 43, 42, 1426, 1427, 1578, 1579, 314, 73, 186, + /* 590 */ 657, 72, 2176, 718, 560, 1854, 456, 2230, 1745, 221, + /* 600 */ 215, 62, 705, 705, 220, 455, 512, 14, 13, 570, + /* 610 */ 207, 495, 493, 490, 244, 1551, 1561, 396, 385, 1425, + /* 620 */ 1428, 1577, 1580, 240, 213, 162, 165, 653, 1892, 1496, + /* 630 */ 1177, 1494, 372, 1894, 1894, 1496, 1519, 1494, 616, 563, + /* 640 */ 1942, 2275, 653, 1892, 557, 57, 2084, 1519, 1944, 239, + /* 650 */ 62, 255, 44, 43, 42, 371, 615, 184, 1499, 1500, + /* 660 */ 436, 2276, 617, 1942, 1499, 1500, 1523, 1550, 1553, 1554, /* 670 */ 1555, 1556, 1557, 1558, 1559, 1560, 665, 661, 1569, 1570, - /* 680 */ 1572, 1573, 1574, 1575, 2, 46, 44, 1581, 109, 70, - /* 690 */ 1944, 2116, 69, 393, 1879, 1495, 2084, 397, 653, 1892, - /* 700 */ 1715, 631, 2094, 2100, 388, 1942, 1576, 1741, 1493, 187, - /* 710 */ 718, 32, 1854, 663, 628, 140, 436, 39, 38, 653, - /* 720 */ 1892, 45, 43, 42, 41, 40, 1834, 2134, 81, 80, - /* 730 */ 435, 87, 1571, 191, 164, 729, 727, 437, 639, 2084, - /* 740 */ 1875, 669, 39, 38, 187, 1501, 45, 43, 42, 41, - /* 750 */ 40, 653, 1892, 187, 339, 2084, 1888, 421, 1974, 1989, - /* 760 */ 419, 415, 411, 408, 428, 1177, 1740, 653, 1892, 446, - /* 770 */ 758, 1519, 2115, 47, 653, 1892, 2151, 2116, 1739, 110, - /* 780 */ 2117, 673, 2119, 2120, 668, 461, 663, 670, 648, 1767, - /* 790 */ 2003, 181, 462, 2204, 34, 1643, 1738, 387, 2200, 1714, - /* 800 */ 39, 38, 187, 192, 45, 43, 42, 41, 40, 1578, - /* 810 */ 1579, 653, 1892, 2134, 2084, 275, 2212, 627, 2231, 134, - /* 820 */ 626, 569, 2275, 628, 140, 2084, 2084, 669, 1896, 515, - /* 830 */ 1523, 653, 1892, 196, 567, 1685, 565, 615, 184, 1551, - /* 840 */ 1561, 1989, 2276, 617, 2084, 1577, 1580, 39, 38, 1889, - /* 850 */ 420, 45, 43, 42, 41, 40, 1867, 142, 2115, 1496, - /* 860 */ 2175, 1494, 2151, 653, 1892, 110, 2117, 673, 2119, 2120, - /* 870 */ 668, 1944, 663, 84, 242, 249, 83, 2295, 241, 2204, - /* 880 */ 363, 245, 165, 387, 2200, 194, 1943, 634, 1499, 1500, - /* 890 */ 1895, 1550, 1553, 1554, 1555, 1556, 1557, 1558, 1559, 1560, - /* 900 */ 665, 661, 1569, 1570, 1572, 1573, 1574, 1575, 2, 46, - /* 910 */ 44, 2116, 655, 456, 2176, 653, 1892, 393, 1737, 1495, - /* 920 */ 1736, 670, 455, 2238, 183, 2212, 2213, 253, 138, 2217, - /* 930 */ 1576, 2116, 1493, 587, 591, 1174, 1175, 2275, 90, 345, - /* 940 */ 1662, 667, 370, 657, 571, 2176, 364, 2134, 362, 361, - /* 950 */ 1733, 539, 2281, 184, 1586, 1735, 1571, 2276, 617, 2084, - /* 960 */ 1520, 669, 653, 1892, 653, 1892, 2084, 2134, 2084, 1501, - /* 970 */ 91, 1275, 541, 42, 41, 40, 540, 2279, 254, 2084, - /* 980 */ 632, 669, 636, 2103, 1274, 703, 155, 154, 700, 699, - /* 990 */ 698, 152, 2115, 2098, 758, 2077, 2151, 47, 2084, 110, - /* 1000 */ 2117, 673, 2119, 2120, 668, 2116, 663, 2078, 14, 13, - /* 1010 */ 1732, 2295, 2115, 2204, 1731, 670, 2151, 387, 2200, 332, - /* 1020 */ 2117, 673, 2119, 2120, 668, 666, 663, 654, 2169, 2094, - /* 1030 */ 2100, 39, 38, 1578, 1579, 45, 43, 42, 41, 40, - /* 1040 */ 663, 2134, 591, 385, 1642, 2275, 485, 1989, 620, 653, - /* 1050 */ 1892, 165, 660, 2084, 591, 669, 664, 2275, 2084, 1894, - /* 1060 */ 2281, 184, 2084, 1551, 1561, 2276, 617, 291, 697, 1577, - /* 1070 */ 1580, 1935, 2281, 184, 2224, 1639, 396, 2276, 617, 1717, - /* 1080 */ 1718, 653, 1892, 1496, 165, 1494, 2115, 653, 1892, 1279, - /* 1090 */ 2151, 198, 1894, 170, 2117, 673, 2119, 2120, 668, 650, - /* 1100 */ 663, 576, 1278, 628, 140, 651, 1552, 52, 1619, 3, - /* 1110 */ 243, 701, 1499, 1500, 1935, 1550, 1553, 1554, 1555, 1556, + /* 680 */ 1572, 1573, 1574, 1575, 2, 48, 46, 1581, 109, 70, + /* 690 */ 1944, 2116, 69, 393, 619, 1495, 1744, 381, 653, 1892, + /* 700 */ 1715, 631, 1552, 1221, 187, 1942, 1576, 1619, 1493, 187, + /* 710 */ 2219, 34, 729, 727, 628, 140, 437, 41, 40, 653, + /* 720 */ 1892, 47, 45, 44, 43, 42, 578, 2134, 81, 80, + /* 730 */ 435, 541, 1571, 191, 164, 540, 2215, 446, 639, 2084, + /* 740 */ 1222, 669, 41, 40, 2084, 1501, 47, 45, 44, 43, + /* 750 */ 42, 653, 1892, 187, 339, 142, 697, 421, 2175, 1935, + /* 760 */ 419, 415, 411, 408, 428, 12, 243, 653, 1892, 461, + /* 770 */ 758, 1501, 2115, 49, 653, 1892, 2151, 2116, 639, 110, + /* 780 */ 2117, 673, 2119, 2120, 668, 462, 663, 670, 637, 1767, + /* 790 */ 2003, 181, 515, 2204, 36, 396, 87, 387, 2200, 1714, + /* 800 */ 41, 40, 187, 165, 47, 45, 44, 43, 42, 1578, + /* 810 */ 1579, 1894, 360, 2134, 193, 275, 2212, 627, 2231, 134, + /* 820 */ 626, 1887, 2275, 628, 140, 2084, 696, 669, 648, 1944, + /* 830 */ 2003, 653, 1892, 1468, 1469, 1685, 386, 615, 184, 1551, + /* 840 */ 1561, 1989, 2276, 617, 1942, 1577, 1580, 41, 40, 1889, + /* 850 */ 717, 47, 45, 44, 43, 42, 1867, 590, 2115, 1496, + /* 860 */ 1275, 1494, 2151, 653, 1892, 110, 2117, 673, 2119, 2120, + /* 870 */ 668, 1944, 663, 1274, 242, 1944, 1989, 2295, 241, 2204, + /* 880 */ 363, 245, 397, 387, 2200, 194, 1943, 634, 1499, 1500, + /* 890 */ 1942, 1550, 1553, 1554, 1555, 1556, 1557, 1558, 1559, 1560, + /* 900 */ 665, 661, 1569, 1570, 1572, 1573, 1574, 1575, 2, 48, + /* 910 */ 46, 2116, 1743, 1174, 1175, 653, 1892, 393, 1742, 1495, + /* 920 */ 198, 670, 1877, 2238, 183, 2212, 2213, 2067, 138, 2217, + /* 930 */ 1576, 2116, 1493, 587, 591, 485, 1523, 2275, 90, 345, + /* 940 */ 1662, 667, 370, 1643, 571, 1741, 364, 2134, 362, 361, + /* 950 */ 1740, 539, 2281, 184, 1879, 569, 1571, 2276, 617, 2084, + /* 960 */ 2084, 669, 653, 1892, 653, 1892, 2084, 2134, 567, 1501, + /* 970 */ 565, 701, 541, 1875, 1935, 702, 540, 55, 1935, 2084, + /* 980 */ 632, 669, 636, 2219, 1896, 703, 155, 154, 700, 699, + /* 990 */ 698, 152, 2115, 2084, 758, 2077, 2151, 49, 2084, 110, + /* 1000 */ 2117, 673, 2119, 2120, 668, 2116, 663, 2078, 54, 2214, + /* 1010 */ 3, 2295, 2115, 2204, 1520, 670, 2151, 387, 2200, 332, + /* 1020 */ 2117, 673, 2119, 2120, 668, 666, 663, 654, 2169, 653, + /* 1030 */ 1892, 41, 40, 1578, 1579, 47, 45, 44, 43, 42, + /* 1040 */ 399, 2134, 591, 1739, 249, 2275, 1586, 291, 165, 653, + /* 1050 */ 1892, 153, 1520, 2084, 591, 669, 1894, 2275, 653, 1892, + /* 1060 */ 2281, 184, 1279, 1551, 1561, 2276, 617, 650, 1607, 1577, + /* 1070 */ 1580, 232, 2281, 184, 230, 1278, 651, 2276, 617, 2224, + /* 1080 */ 1639, 653, 1892, 1496, 254, 1494, 2115, 1738, 653, 1892, + /* 1090 */ 2151, 2084, 439, 170, 2117, 673, 2119, 2120, 668, 297, + /* 1100 */ 663, 576, 1737, 628, 140, 440, 400, 1736, 56, 146, + /* 1110 */ 1733, 135, 1499, 1500, 1732, 1550, 1553, 1554, 1555, 1556, /* 1120 */ 1557, 1558, 1559, 1560, 665, 661, 1569, 1570, 1572, 1573, - /* 1130 */ 1574, 1575, 2, 46, 44, 653, 1892, 653, 1892, 1730, - /* 1140 */ 1729, 393, 399, 1495, 618, 2296, 1728, 2116, 591, 2070, - /* 1150 */ 165, 2275, 1727, 297, 1576, 400, 1493, 670, 1894, 2251, - /* 1160 */ 153, 623, 439, 1726, 487, 1725, 2281, 184, 1870, 252, - /* 1170 */ 310, 2276, 617, 1921, 2116, 440, 702, 1468, 1469, 1935, - /* 1180 */ 1571, 74, 232, 2134, 670, 230, 598, 2084, 2084, 146, - /* 1190 */ 573, 135, 572, 1501, 2084, 2084, 616, 669, 413, 2275, - /* 1200 */ 2084, 590, 1597, 544, 185, 2212, 2213, 1781, 138, 2217, - /* 1210 */ 2134, 2084, 148, 2084, 615, 184, 2244, 54, 758, 2276, - /* 1220 */ 617, 15, 2084, 234, 669, 1260, 233, 272, 2115, 556, - /* 1230 */ 82, 1504, 2151, 153, 2116, 110, 2117, 673, 2119, 2120, - /* 1240 */ 668, 1503, 663, 604, 670, 236, 619, 2295, 235, 2204, - /* 1250 */ 1774, 1772, 225, 387, 2200, 2115, 153, 1578, 1579, 2151, - /* 1260 */ 64, 711, 110, 2117, 673, 2119, 2120, 668, 238, 663, - /* 1270 */ 2134, 237, 558, 561, 2295, 64, 2204, 259, 621, 266, - /* 1280 */ 387, 2200, 2084, 1240, 669, 2105, 2135, 1551, 1561, 1998, - /* 1290 */ 1463, 1221, 404, 1577, 1580, 1759, 703, 155, 154, 700, - /* 1300 */ 699, 698, 152, 14, 13, 153, 1764, 1496, 48, 1494, - /* 1310 */ 1932, 284, 2234, 1466, 71, 2115, 151, 1671, 153, 2151, - /* 1320 */ 629, 53, 169, 2117, 673, 2119, 2120, 668, 1222, 663, - /* 1330 */ 48, 1768, 1670, 64, 261, 48, 1499, 1500, 2107, 1550, + /* 1130 */ 1574, 1575, 2, 48, 46, 2084, 420, 487, 1870, 1731, + /* 1140 */ 1730, 393, 35, 1495, 618, 2296, 1729, 2116, 591, 623, + /* 1150 */ 2084, 2275, 1612, 1728, 1576, 2084, 1493, 670, 2084, 2251, + /* 1160 */ 1552, 620, 2084, 1727, 1726, 1725, 2281, 184, 74, 543, + /* 1170 */ 310, 2276, 617, 1921, 2116, 234, 236, 1504, 233, 235, + /* 1180 */ 1571, 544, 148, 2134, 670, 1781, 598, 2084, 2084, 153, + /* 1190 */ 252, 1262, 1642, 1501, 2084, 2084, 238, 669, 1552, 237, + /* 1200 */ 573, 2084, 572, 1260, 185, 2212, 2213, 556, 138, 2217, + /* 1210 */ 2134, 2084, 2084, 2084, 153, 50, 50, 82, 758, 1774, + /* 1220 */ 259, 15, 2084, 1772, 669, 1717, 1718, 664, 2115, 1834, + /* 1230 */ 14, 13, 2151, 153, 2116, 110, 2117, 673, 2119, 2120, + /* 1240 */ 668, 558, 663, 1503, 670, 561, 1463, 2295, 50, 2204, + /* 1250 */ 2105, 284, 660, 387, 2200, 2115, 71, 1578, 1579, 2151, + /* 1260 */ 151, 153, 110, 2117, 673, 2119, 2120, 668, 2244, 663, + /* 1270 */ 2134, 1466, 1671, 1670, 2295, 64, 2204, 261, 50, 1735, + /* 1280 */ 387, 2200, 2084, 225, 669, 272, 604, 1551, 1561, 2135, + /* 1290 */ 635, 266, 404, 1577, 1580, 1998, 703, 155, 154, 700, + /* 1300 */ 699, 698, 152, 2107, 1759, 1423, 1932, 1496, 289, 1494, + /* 1310 */ 2234, 50, 711, 645, 677, 2115, 151, 293, 1305, 2151, + /* 1320 */ 1768, 153, 169, 2117, 673, 2119, 2120, 668, 1764, 663, + /* 1330 */ 1507, 629, 1613, 136, 1240, 1562, 1499, 1500, 151, 1550, /* 1340 */ 1553, 1554, 1555, 1556, 1557, 1558, 1559, 1560, 665, 661, - /* 1350 */ 1569, 1570, 1572, 1573, 1574, 1575, 2, 271, 274, 390, - /* 1360 */ 389, 677, 635, 2242, 151, 1423, 153, 2116, 289, 1509, - /* 1370 */ 136, 645, 151, 293, 712, 1305, 1, 670, 5, 2269, - /* 1380 */ 1576, 753, 1502, 407, 1507, 412, 354, 309, 1446, 441, - /* 1390 */ 1613, 304, 1562, 624, 1506, 197, 1238, 2116, 1523, 1999, - /* 1400 */ 445, 478, 450, 2134, 1518, 463, 1571, 670, 1991, 2223, - /* 1410 */ 470, 477, 479, 488, 489, 2084, 202, 669, 1332, 1501, - /* 1420 */ 486, 1336, 201, 1343, 491, 492, 204, 1341, 494, 156, - /* 1430 */ 496, 1524, 497, 2134, 4, 498, 505, 506, 1526, 508, - /* 1440 */ 212, 1521, 509, 214, 659, 2084, 1525, 669, 2115, 510, - /* 1450 */ 1527, 511, 2151, 217, 513, 110, 2117, 673, 2119, 2120, - /* 1460 */ 668, 219, 663, 85, 86, 2116, 1194, 2295, 517, 2204, - /* 1470 */ 223, 536, 534, 387, 2200, 670, 535, 538, 2115, 344, - /* 1480 */ 2060, 1882, 2151, 2057, 229, 110, 2117, 673, 2119, 2120, - /* 1490 */ 668, 1878, 663, 112, 577, 575, 231, 2295, 89, 2204, - /* 1500 */ 158, 2134, 159, 387, 2200, 149, 1880, 1876, 160, 2056, - /* 1510 */ 246, 161, 582, 2084, 580, 669, 305, 581, 585, 250, - /* 1520 */ 1453, 588, 8, 605, 2250, 248, 643, 595, 2249, 586, - /* 1530 */ 257, 601, 614, 1510, 2235, 1505, 2245, 376, 608, 2226, - /* 1540 */ 265, 173, 596, 594, 260, 268, 2115, 267, 593, 622, - /* 1550 */ 2151, 2298, 625, 110, 2117, 673, 2119, 2120, 668, 269, - /* 1560 */ 663, 377, 1513, 1515, 2274, 2179, 139, 2204, 270, 1639, - /* 1570 */ 1522, 387, 2200, 2116, 2220, 661, 1569, 1570, 1572, 1573, - /* 1580 */ 1574, 1575, 279, 670, 633, 380, 1528, 96, 2004, 306, - /* 1590 */ 641, 642, 2018, 2116, 2017, 2016, 307, 646, 273, 383, - /* 1600 */ 98, 647, 308, 670, 61, 100, 1893, 2185, 102, 2134, - /* 1610 */ 1936, 311, 754, 1855, 2076, 755, 675, 757, 51, 346, - /* 1620 */ 2075, 2084, 347, 669, 2074, 315, 300, 335, 320, 2134, - /* 1630 */ 313, 334, 78, 2071, 409, 1486, 410, 1487, 324, 190, - /* 1640 */ 414, 2084, 416, 669, 2069, 417, 418, 2068, 355, 2066, - /* 1650 */ 422, 2065, 424, 2064, 2115, 79, 426, 1449, 2151, 1448, - /* 1660 */ 2030, 110, 2117, 673, 2119, 2120, 668, 2029, 663, 2028, - /* 1670 */ 433, 434, 2027, 2177, 2115, 2204, 1400, 2116, 2151, 387, - /* 1680 */ 2200, 110, 2117, 673, 2119, 2120, 668, 670, 663, 2026, - /* 1690 */ 1982, 1981, 1979, 656, 145, 2204, 1978, 1977, 1980, 387, - /* 1700 */ 2200, 2116, 1976, 1975, 1973, 1972, 1971, 195, 451, 1970, - /* 1710 */ 453, 670, 1984, 2134, 1969, 1968, 1967, 1966, 1965, 1964, - /* 1720 */ 1963, 1962, 1961, 1960, 1959, 2084, 1958, 669, 1957, 1956, - /* 1730 */ 1955, 1954, 1953, 1952, 1983, 147, 1951, 2134, 1950, 1949, - /* 1740 */ 1948, 1947, 481, 1946, 1945, 1797, 203, 1402, 342, 2084, - /* 1750 */ 1796, 669, 1795, 205, 206, 343, 1793, 1276, 2115, 1280, - /* 1760 */ 1754, 1176, 2151, 218, 2024, 111, 2117, 673, 2119, 2120, - /* 1770 */ 668, 178, 663, 1753, 2047, 2037, 2025, 1272, 2116, 2204, - /* 1780 */ 2002, 1871, 2115, 2203, 2200, 76, 2151, 77, 670, 111, - /* 1790 */ 2117, 673, 2119, 2120, 668, 208, 663, 216, 2104, 210, - /* 1800 */ 1792, 2116, 179, 2204, 503, 1790, 518, 658, 2200, 520, - /* 1810 */ 519, 670, 1788, 522, 2134, 1214, 523, 1786, 524, 526, - /* 1820 */ 1784, 527, 528, 530, 2116, 532, 2084, 1771, 669, 1770, - /* 1830 */ 1750, 531, 1873, 1348, 670, 1347, 1872, 2134, 1263, 1261, - /* 1840 */ 726, 1259, 1258, 1257, 1256, 1255, 728, 2116, 1250, 2084, - /* 1850 */ 1252, 669, 1782, 1251, 1249, 367, 1775, 670, 1773, 671, - /* 1860 */ 2134, 228, 368, 2151, 369, 559, 111, 2117, 673, 2119, - /* 1870 */ 2120, 668, 2084, 663, 669, 63, 562, 1749, 564, 1748, - /* 1880 */ 2204, 566, 2115, 2134, 349, 2200, 2151, 1747, 568, 111, - /* 1890 */ 2117, 673, 2119, 2120, 668, 2084, 663, 669, 113, 1473, - /* 1900 */ 1475, 27, 1472, 2204, 2046, 2115, 1459, 1455, 2201, 2151, - /* 1910 */ 2116, 67, 326, 2117, 673, 2119, 2120, 668, 1457, 663, - /* 1920 */ 670, 2036, 1477, 583, 2023, 2021, 2280, 20, 2115, 1687, - /* 1930 */ 56, 17, 2151, 6, 29, 170, 2117, 673, 2119, 2120, - /* 1940 */ 668, 7, 663, 589, 256, 584, 2134, 597, 258, 599, - /* 1950 */ 59, 382, 60, 373, 163, 613, 1669, 171, 2084, 251, - /* 1960 */ 669, 262, 30, 263, 1661, 264, 21, 65, 92, 2105, - /* 1970 */ 31, 1707, 1708, 22, 1702, 2116, 1701, 378, 1706, 1705, - /* 1980 */ 379, 1636, 1635, 2022, 276, 667, 2020, 2297, 2019, 2001, - /* 1990 */ 58, 2115, 94, 95, 174, 2151, 2116, 282, 333, 2117, - /* 2000 */ 673, 2119, 2120, 668, 283, 663, 670, 23, 644, 2116, - /* 2010 */ 1667, 2134, 285, 290, 68, 2000, 97, 292, 295, 670, - /* 2020 */ 103, 13, 24, 2084, 1511, 669, 1588, 99, 1587, 11, - /* 2030 */ 1543, 1598, 2134, 2154, 175, 1566, 1564, 392, 662, 188, - /* 2040 */ 57, 1563, 672, 674, 2084, 2134, 669, 18, 37, 16, - /* 2050 */ 394, 25, 676, 1535, 1333, 26, 2115, 2084, 395, 669, - /* 2060 */ 2151, 579, 678, 332, 2117, 673, 2119, 2120, 668, 1330, - /* 2070 */ 663, 680, 2170, 681, 683, 1327, 684, 2115, 686, 761, - /* 2080 */ 1321, 2151, 687, 689, 333, 2117, 673, 2119, 2120, 668, - /* 2090 */ 2115, 663, 1319, 303, 2151, 690, 2116, 333, 2117, 673, - /* 2100 */ 2119, 2120, 668, 1325, 663, 104, 670, 1324, 298, 176, - /* 2110 */ 105, 1342, 1323, 75, 1338, 751, 747, 743, 739, 301, - /* 2120 */ 2116, 1322, 1212, 704, 1246, 1245, 1244, 1270, 1243, 1242, - /* 2130 */ 670, 1241, 2134, 1239, 1237, 1236, 1235, 1233, 716, 299, - /* 2140 */ 1232, 1231, 1230, 1229, 2084, 1228, 669, 1227, 1265, 1267, - /* 2150 */ 1224, 1223, 1218, 1220, 1219, 1789, 2134, 1217, 736, 108, - /* 2160 */ 737, 1787, 294, 738, 740, 742, 1785, 744, 2084, 1783, - /* 2170 */ 669, 748, 1769, 746, 741, 752, 745, 574, 750, 1166, - /* 2180 */ 749, 2151, 1746, 2116, 328, 2117, 673, 2119, 2120, 668, - /* 2190 */ 302, 663, 756, 670, 649, 1497, 312, 759, 760, 1721, - /* 2200 */ 1721, 2115, 1721, 1721, 1721, 2151, 1721, 2116, 317, 2117, - /* 2210 */ 673, 2119, 2120, 668, 1721, 663, 1721, 670, 1721, 2134, - /* 2220 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 281, - /* 2230 */ 1721, 2084, 1721, 669, 280, 1721, 1721, 1721, 1721, 1721, - /* 2240 */ 1721, 1721, 1721, 2134, 1721, 1721, 1721, 1721, 1721, 1721, - /* 2250 */ 1721, 1721, 1721, 2116, 247, 2084, 1721, 669, 1721, 1721, - /* 2260 */ 1721, 1721, 1721, 670, 2115, 1721, 1721, 1721, 2151, 2116, - /* 2270 */ 1721, 318, 2117, 673, 2119, 2120, 668, 1721, 663, 670, - /* 2280 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2115, 2134, - /* 2290 */ 1721, 1721, 2151, 1721, 1721, 319, 2117, 673, 2119, 2120, - /* 2300 */ 668, 2084, 663, 669, 1721, 2134, 1721, 1721, 1721, 1721, - /* 2310 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2084, 1721, 669, - /* 2320 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2116, - /* 2330 */ 1721, 1721, 1721, 1721, 2115, 1721, 1721, 1721, 2151, 670, - /* 2340 */ 1721, 325, 2117, 673, 2119, 2120, 668, 1721, 663, 1721, - /* 2350 */ 2115, 1721, 1721, 2116, 2151, 1721, 1721, 329, 2117, 673, - /* 2360 */ 2119, 2120, 668, 670, 663, 2134, 1721, 1721, 1721, 1721, - /* 2370 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2084, 1721, 669, - /* 2380 */ 2116, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2134, - /* 2390 */ 670, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2116, - /* 2400 */ 1721, 2084, 1721, 669, 1721, 1721, 1721, 1721, 1721, 670, - /* 2410 */ 2115, 1721, 1721, 1721, 2151, 1721, 2134, 321, 2117, 673, - /* 2420 */ 2119, 2120, 668, 1721, 663, 1721, 1721, 1721, 2084, 1721, - /* 2430 */ 669, 1721, 1721, 1721, 2115, 2134, 1721, 1721, 2151, 1721, - /* 2440 */ 1721, 330, 2117, 673, 2119, 2120, 668, 2084, 663, 669, - /* 2450 */ 2116, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, - /* 2460 */ 670, 2115, 1721, 1721, 1721, 2151, 2116, 1721, 322, 2117, - /* 2470 */ 673, 2119, 2120, 668, 1721, 663, 670, 1721, 1721, 1721, - /* 2480 */ 2115, 1721, 1721, 1721, 2151, 1721, 2134, 331, 2117, 673, - /* 2490 */ 2119, 2120, 668, 1721, 663, 1721, 2116, 1721, 2084, 1721, - /* 2500 */ 669, 1721, 2134, 1721, 1721, 1721, 670, 1721, 1721, 1721, - /* 2510 */ 1721, 1721, 1721, 1721, 2084, 1721, 669, 2116, 1721, 1721, - /* 2520 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 670, 1721, 1721, - /* 2530 */ 1721, 2115, 2134, 1721, 1721, 2151, 1721, 1721, 323, 2117, - /* 2540 */ 673, 2119, 2120, 668, 2084, 663, 669, 2115, 1721, 1721, - /* 2550 */ 1721, 2151, 1721, 2134, 336, 2117, 673, 2119, 2120, 668, - /* 2560 */ 1721, 663, 1721, 2116, 1721, 2084, 1721, 669, 1721, 1721, - /* 2570 */ 1721, 1721, 1721, 670, 1721, 1721, 1721, 2115, 1721, 1721, - /* 2580 */ 1721, 2151, 1721, 1721, 337, 2117, 673, 2119, 2120, 668, - /* 2590 */ 1721, 663, 1721, 1721, 1721, 1721, 1721, 1721, 2115, 2134, - /* 2600 */ 1721, 1721, 2151, 1721, 1721, 2128, 2117, 673, 2119, 2120, - /* 2610 */ 668, 2084, 663, 669, 1721, 1721, 1721, 1721, 1721, 1721, - /* 2620 */ 1721, 1721, 1721, 2116, 1721, 1721, 1721, 1721, 1721, 1721, - /* 2630 */ 1721, 1721, 1721, 670, 1721, 1721, 2116, 1721, 1721, 1721, - /* 2640 */ 1721, 1721, 1721, 1721, 2115, 1721, 670, 1721, 2151, 1721, - /* 2650 */ 1721, 2127, 2117, 673, 2119, 2120, 668, 1721, 663, 2134, - /* 2660 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, - /* 2670 */ 1721, 2084, 2134, 669, 1721, 1721, 1721, 1721, 1721, 1721, - /* 2680 */ 1721, 1721, 1721, 1721, 2084, 1721, 669, 2116, 1721, 1721, - /* 2690 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 670, 1721, 1721, - /* 2700 */ 1721, 1721, 1721, 1721, 2115, 1721, 1721, 1721, 2151, 1721, - /* 2710 */ 1721, 2126, 2117, 673, 2119, 2120, 668, 2115, 663, 1721, - /* 2720 */ 1721, 2151, 1721, 2134, 351, 2117, 673, 2119, 2120, 668, - /* 2730 */ 1721, 663, 1721, 2116, 1721, 2084, 1721, 669, 1721, 1721, - /* 2740 */ 1721, 1721, 1721, 670, 1721, 1721, 1721, 1721, 2116, 1721, - /* 2750 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 670, 1721, - /* 2760 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2115, 2134, - /* 2770 */ 1721, 1721, 2151, 1721, 1721, 352, 2117, 673, 2119, 2120, - /* 2780 */ 668, 2084, 663, 669, 2134, 1721, 1721, 1721, 1721, 1721, - /* 2790 */ 1721, 1721, 1721, 1721, 2116, 1721, 2084, 1721, 669, 1721, - /* 2800 */ 1721, 1721, 1721, 1721, 670, 1721, 1721, 1721, 1721, 2116, - /* 2810 */ 1721, 1721, 1721, 1721, 2115, 1721, 1721, 1721, 2151, 670, - /* 2820 */ 1721, 348, 2117, 673, 2119, 2120, 668, 1721, 663, 2115, - /* 2830 */ 2134, 1721, 1721, 2151, 1721, 1721, 353, 2117, 673, 2119, - /* 2840 */ 2120, 668, 2084, 663, 669, 2134, 1721, 1721, 1721, 1721, - /* 2850 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2084, 1721, 669, - /* 2860 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, - /* 2870 */ 1721, 1721, 1721, 1721, 1721, 671, 1721, 1721, 1721, 2151, - /* 2880 */ 1721, 1721, 328, 2117, 673, 2119, 2120, 668, 1721, 663, - /* 2890 */ 2115, 1721, 1721, 1721, 2151, 1721, 1721, 327, 2117, 673, - /* 2900 */ 2119, 2120, 668, 1721, 663, + /* 1350 */ 1569, 1570, 1572, 1573, 1574, 1575, 2, 274, 271, 390, + /* 1360 */ 389, 712, 1, 2242, 5, 407, 412, 2116, 309, 1509, + /* 1370 */ 753, 1332, 354, 1336, 1446, 304, 197, 670, 1343, 2269, + /* 1380 */ 1576, 624, 1502, 1238, 441, 1523, 1999, 445, 450, 478, + /* 1390 */ 1341, 621, 1518, 463, 1991, 156, 1506, 2116, 470, 477, + /* 1400 */ 479, 488, 1597, 2134, 489, 486, 1571, 670, 201, 2223, + /* 1410 */ 202, 491, 492, 1524, 497, 2084, 204, 669, 494, 1501, + /* 1420 */ 496, 4, 498, 505, 506, 1526, 508, 212, 1521, 509, + /* 1430 */ 214, 1525, 1527, 2134, 510, 513, 511, 217, 1194, 517, + /* 1440 */ 112, 534, 535, 219, 659, 2084, 85, 669, 2115, 86, + /* 1450 */ 223, 536, 2151, 538, 344, 110, 2117, 673, 2119, 2120, + /* 1460 */ 668, 575, 663, 2060, 89, 2116, 577, 2295, 305, 2204, + /* 1470 */ 1882, 229, 1878, 387, 2200, 670, 231, 158, 2115, 246, + /* 1480 */ 159, 1880, 2151, 1876, 160, 110, 2117, 673, 2119, 2120, + /* 1490 */ 668, 161, 663, 581, 2057, 580, 250, 2295, 1453, 2204, + /* 1500 */ 588, 2134, 605, 387, 2200, 2056, 585, 595, 643, 601, + /* 1510 */ 2250, 2235, 582, 2084, 2249, 669, 248, 149, 2245, 8, + /* 1520 */ 376, 257, 608, 614, 586, 260, 596, 594, 2226, 593, + /* 1530 */ 267, 265, 377, 1510, 625, 1505, 2298, 173, 622, 1639, + /* 1540 */ 268, 139, 1522, 633, 380, 279, 2115, 96, 1528, 2004, + /* 1550 */ 2151, 270, 646, 110, 2117, 673, 2119, 2120, 668, 306, + /* 1560 */ 663, 641, 1513, 1515, 2220, 2179, 647, 2204, 98, 642, + /* 1570 */ 307, 387, 2200, 2116, 2018, 661, 1569, 1570, 1572, 1573, + /* 1580 */ 1574, 1575, 1893, 670, 100, 61, 269, 2017, 2185, 102, + /* 1590 */ 2016, 383, 308, 2116, 675, 1936, 1855, 300, 311, 2274, + /* 1600 */ 754, 757, 273, 670, 755, 53, 335, 320, 346, 2134, + /* 1610 */ 347, 313, 315, 2076, 334, 324, 2075, 2074, 78, 2071, + /* 1620 */ 409, 2084, 410, 669, 1486, 1487, 190, 414, 2069, 2134, + /* 1630 */ 416, 417, 418, 2068, 355, 2066, 422, 2065, 2064, 426, + /* 1640 */ 424, 2084, 79, 669, 1449, 1448, 2030, 2029, 2028, 433, + /* 1650 */ 434, 2027, 2026, 1400, 2115, 1982, 1981, 1979, 2151, 1978, + /* 1660 */ 1977, 110, 2117, 673, 2119, 2120, 668, 145, 663, 1980, + /* 1670 */ 1976, 1975, 1973, 2177, 2115, 2204, 195, 2116, 2151, 387, + /* 1680 */ 2200, 110, 2117, 673, 2119, 2120, 668, 670, 663, 1972, + /* 1690 */ 1971, 1970, 453, 656, 1984, 2204, 451, 1969, 1968, 387, + /* 1700 */ 2200, 2116, 1967, 1966, 1965, 1964, 1963, 1962, 1961, 1960, + /* 1710 */ 1959, 670, 1958, 2134, 1957, 1956, 1955, 147, 1954, 1953, + /* 1720 */ 1952, 1983, 1951, 1950, 1949, 2084, 1948, 669, 1947, 1946, + /* 1730 */ 1945, 481, 342, 1402, 343, 1276, 1797, 2134, 203, 1280, + /* 1740 */ 1796, 205, 1795, 1793, 206, 1754, 2104, 179, 1176, 2084, + /* 1750 */ 218, 669, 1753, 2047, 2037, 2025, 2024, 2002, 2115, 1871, + /* 1760 */ 1272, 1214, 2151, 1792, 1790, 111, 2117, 673, 2119, 2120, + /* 1770 */ 668, 76, 663, 208, 503, 210, 77, 178, 2116, 2204, + /* 1780 */ 216, 518, 2115, 2203, 2200, 1788, 2151, 519, 670, 111, + /* 1790 */ 2117, 673, 2119, 2120, 668, 522, 663, 520, 523, 1786, + /* 1800 */ 524, 2116, 526, 2204, 528, 1784, 530, 658, 2200, 1771, + /* 1810 */ 1770, 670, 532, 1750, 2134, 527, 1873, 531, 1348, 1347, + /* 1820 */ 1872, 1263, 1261, 1259, 2116, 1258, 2084, 1257, 669, 1256, + /* 1830 */ 1255, 1252, 1782, 726, 670, 728, 1251, 2134, 1249, 1250, + /* 1840 */ 367, 1775, 1773, 368, 559, 369, 562, 2116, 63, 2084, + /* 1850 */ 1749, 669, 1748, 1747, 228, 568, 113, 670, 2046, 671, + /* 1860 */ 2134, 564, 566, 2151, 1473, 374, 111, 2117, 673, 2119, + /* 1870 */ 2120, 668, 2084, 663, 669, 1475, 1477, 29, 1472, 67, + /* 1880 */ 2204, 1459, 2115, 2134, 349, 2200, 2151, 1457, 375, 111, + /* 1890 */ 2117, 673, 2119, 2120, 668, 2084, 663, 669, 1455, 2036, + /* 1900 */ 583, 2023, 2021, 2204, 20, 2115, 2280, 31, 2201, 2151, + /* 1910 */ 2116, 599, 333, 2117, 673, 2119, 2120, 668, 251, 663, + /* 1920 */ 670, 17, 6, 1687, 256, 7, 589, 264, 2115, 597, + /* 1930 */ 58, 258, 2151, 163, 21, 333, 2117, 673, 2119, 2120, + /* 1940 */ 668, 22, 663, 2105, 584, 1669, 2134, 171, 373, 262, + /* 1950 */ 263, 32, 33, 65, 24, 1661, 92, 23, 2084, 1702, + /* 1960 */ 669, 1701, 1707, 1708, 378, 1706, 1705, 379, 276, 1636, + /* 1970 */ 1635, 60, 174, 2022, 2020, 2116, 2019, 2001, 95, 94, + /* 1980 */ 282, 25, 2000, 283, 285, 670, 1667, 290, 68, 99, + /* 1990 */ 644, 2115, 97, 295, 292, 2151, 2116, 13, 326, 2117, + /* 2000 */ 673, 2119, 2120, 668, 103, 663, 670, 26, 1511, 2116, + /* 2010 */ 1588, 2134, 1587, 1598, 175, 11, 2154, 188, 1566, 667, + /* 2020 */ 662, 1543, 39, 2084, 59, 669, 674, 676, 18, 1564, + /* 2030 */ 1338, 672, 2134, 395, 298, 1563, 16, 382, 27, 680, + /* 2040 */ 28, 613, 1535, 1333, 2084, 2134, 669, 678, 683, 1330, + /* 2050 */ 1327, 681, 684, 686, 689, 1321, 2115, 2084, 687, 669, + /* 2060 */ 2151, 1319, 690, 170, 2117, 673, 2119, 2120, 668, 104, + /* 2070 */ 663, 105, 1342, 1246, 704, 75, 1212, 2115, 1325, 1324, + /* 2080 */ 1245, 2151, 1244, 2116, 333, 2117, 673, 2119, 2120, 668, + /* 2090 */ 2115, 663, 1243, 670, 2151, 1242, 1241, 332, 2117, 673, + /* 2100 */ 2119, 2120, 668, 2116, 663, 1323, 2170, 1322, 1239, 1237, + /* 2110 */ 1236, 1235, 1270, 670, 299, 2297, 716, 1233, 1232, 2134, + /* 2120 */ 1231, 1230, 1229, 1228, 392, 1227, 1267, 1265, 1224, 1223, + /* 2130 */ 1220, 2084, 1219, 669, 1218, 1217, 1789, 736, 737, 2134, + /* 2140 */ 738, 1787, 740, 742, 394, 741, 1785, 745, 744, 746, + /* 2150 */ 1783, 2084, 748, 669, 749, 750, 1769, 752, 1166, 1746, + /* 2160 */ 302, 756, 1721, 759, 2115, 1497, 312, 760, 2151, 1721, + /* 2170 */ 1721, 333, 2117, 673, 2119, 2120, 668, 1721, 663, 1721, + /* 2180 */ 1721, 1721, 1721, 1721, 2115, 1721, 579, 1721, 2151, 1721, + /* 2190 */ 1721, 333, 2117, 673, 2119, 2120, 668, 1721, 663, 1721, + /* 2200 */ 2116, 1721, 1721, 1721, 761, 1721, 1721, 1721, 1721, 1721, + /* 2210 */ 670, 1721, 1721, 1721, 1721, 1721, 2116, 1721, 303, 1721, + /* 2220 */ 1721, 1721, 1721, 1721, 1721, 1721, 670, 1721, 1721, 1721, + /* 2230 */ 1721, 1721, 1721, 1721, 176, 1721, 2134, 1721, 1721, 1721, + /* 2240 */ 751, 747, 743, 739, 301, 1721, 2116, 1721, 2084, 1721, + /* 2250 */ 669, 1721, 2134, 1721, 1721, 1721, 670, 1721, 1721, 1721, + /* 2260 */ 1721, 1721, 1721, 1721, 2084, 1721, 669, 1721, 1721, 1721, + /* 2270 */ 1721, 1721, 1721, 1721, 1721, 1721, 2116, 1721, 1721, 1721, + /* 2280 */ 1721, 574, 2134, 1721, 108, 2151, 670, 294, 328, 2117, + /* 2290 */ 673, 2119, 2120, 668, 2084, 663, 669, 2115, 1721, 1721, + /* 2300 */ 1721, 2151, 1721, 1721, 317, 2117, 673, 2119, 2120, 668, + /* 2310 */ 1721, 663, 2134, 1721, 1721, 1721, 1721, 1721, 1721, 649, + /* 2320 */ 1721, 1721, 1721, 1721, 2084, 1721, 669, 2115, 1721, 1721, + /* 2330 */ 1721, 2151, 1721, 1721, 318, 2117, 673, 2119, 2120, 668, + /* 2340 */ 2116, 663, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2350 */ 670, 1721, 1721, 2116, 281, 1721, 1721, 2115, 1721, 280, + /* 2360 */ 1721, 2151, 1721, 670, 319, 2117, 673, 2119, 2120, 668, + /* 2370 */ 1721, 663, 1721, 1721, 1721, 1721, 2134, 1721, 1721, 247, + /* 2380 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2084, 2134, + /* 2390 */ 669, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2400 */ 1721, 2084, 1721, 669, 2116, 1721, 1721, 1721, 1721, 1721, + /* 2410 */ 1721, 1721, 1721, 1721, 670, 1721, 1721, 2116, 1721, 1721, + /* 2420 */ 1721, 2115, 1721, 1721, 1721, 2151, 1721, 670, 325, 2117, + /* 2430 */ 673, 2119, 2120, 668, 2115, 663, 1721, 1721, 2151, 1721, + /* 2440 */ 2134, 329, 2117, 673, 2119, 2120, 668, 1721, 663, 1721, + /* 2450 */ 1721, 1721, 2084, 2134, 669, 1721, 1721, 1721, 1721, 1721, + /* 2460 */ 1721, 1721, 1721, 1721, 1721, 2084, 1721, 669, 1721, 1721, + /* 2470 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2116, 1721, + /* 2480 */ 1721, 1721, 1721, 1721, 1721, 2115, 1721, 1721, 670, 2151, + /* 2490 */ 1721, 1721, 321, 2117, 673, 2119, 2120, 668, 2115, 663, + /* 2500 */ 1721, 1721, 2151, 1721, 1721, 330, 2117, 673, 2119, 2120, + /* 2510 */ 668, 1721, 663, 1721, 2134, 1721, 1721, 1721, 1721, 1721, + /* 2520 */ 1721, 1721, 1721, 1721, 1721, 1721, 2084, 1721, 669, 2116, + /* 2530 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 670, + /* 2540 */ 1721, 1721, 1721, 1721, 2116, 1721, 1721, 1721, 1721, 1721, + /* 2550 */ 1721, 1721, 1721, 1721, 670, 1721, 1721, 1721, 1721, 2115, + /* 2560 */ 1721, 1721, 1721, 2151, 1721, 2134, 322, 2117, 673, 2119, + /* 2570 */ 2120, 668, 1721, 663, 1721, 1721, 1721, 2084, 1721, 669, + /* 2580 */ 2134, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2590 */ 2116, 1721, 2084, 1721, 669, 1721, 1721, 1721, 1721, 1721, + /* 2600 */ 670, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2610 */ 2115, 1721, 1721, 1721, 2151, 1721, 1721, 331, 2117, 673, + /* 2620 */ 2119, 2120, 668, 1721, 663, 2115, 2134, 1721, 1721, 2151, + /* 2630 */ 1721, 1721, 323, 2117, 673, 2119, 2120, 668, 2084, 663, + /* 2640 */ 669, 2116, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2650 */ 1721, 670, 1721, 1721, 1721, 1721, 1721, 1721, 2116, 1721, + /* 2660 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 670, 1721, + /* 2670 */ 1721, 2115, 1721, 1721, 1721, 2151, 1721, 2134, 336, 2117, + /* 2680 */ 673, 2119, 2120, 668, 1721, 663, 1721, 2116, 1721, 2084, + /* 2690 */ 1721, 669, 1721, 1721, 2134, 1721, 1721, 670, 1721, 1721, + /* 2700 */ 1721, 1721, 1721, 1721, 1721, 1721, 2084, 1721, 669, 1721, + /* 2710 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2720 */ 1721, 1721, 2115, 2134, 1721, 1721, 2151, 1721, 1721, 337, + /* 2730 */ 2117, 673, 2119, 2120, 668, 2084, 663, 669, 1721, 2115, + /* 2740 */ 1721, 1721, 1721, 2151, 1721, 1721, 2128, 2117, 673, 2119, + /* 2750 */ 2120, 668, 1721, 663, 1721, 2116, 1721, 1721, 1721, 1721, + /* 2760 */ 1721, 1721, 1721, 1721, 1721, 670, 1721, 1721, 2115, 1721, + /* 2770 */ 1721, 1721, 2151, 1721, 2116, 2127, 2117, 673, 2119, 2120, + /* 2780 */ 668, 1721, 663, 1721, 670, 1721, 1721, 2116, 1721, 1721, + /* 2790 */ 1721, 2134, 1721, 1721, 1721, 1721, 1721, 670, 1721, 1721, + /* 2800 */ 1721, 1721, 1721, 2084, 1721, 669, 1721, 1721, 1721, 1721, + /* 2810 */ 2134, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2820 */ 1721, 1721, 2084, 2134, 669, 1721, 1721, 1721, 1721, 1721, + /* 2830 */ 1721, 1721, 1721, 1721, 1721, 2084, 2115, 669, 1721, 1721, + /* 2840 */ 2151, 1721, 1721, 2126, 2117, 673, 2119, 2120, 668, 1721, + /* 2850 */ 663, 1721, 1721, 1721, 1721, 2115, 1721, 1721, 1721, 2151, + /* 2860 */ 2116, 1721, 351, 2117, 673, 2119, 2120, 668, 2115, 663, + /* 2870 */ 670, 1721, 2151, 2116, 1721, 352, 2117, 673, 2119, 2120, + /* 2880 */ 668, 1721, 663, 670, 1721, 1721, 1721, 1721, 2116, 1721, + /* 2890 */ 1721, 1721, 1721, 1721, 1721, 1721, 2134, 1721, 670, 1721, + /* 2900 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2084, 2134, + /* 2910 */ 669, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2920 */ 1721, 2084, 1721, 669, 2134, 1721, 1721, 1721, 1721, 1721, + /* 2930 */ 1721, 1721, 1721, 1721, 2116, 1721, 2084, 1721, 669, 1721, + /* 2940 */ 1721, 2115, 1721, 1721, 670, 2151, 1721, 1721, 348, 2117, + /* 2950 */ 673, 2119, 2120, 668, 2115, 663, 1721, 1721, 2151, 1721, + /* 2960 */ 1721, 353, 2117, 673, 2119, 2120, 668, 1721, 663, 671, + /* 2970 */ 2134, 1721, 1721, 2151, 1721, 1721, 328, 2117, 673, 2119, + /* 2980 */ 2120, 668, 2084, 663, 669, 1721, 1721, 1721, 1721, 1721, + /* 2990 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 3000 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 3010 */ 1721, 1721, 1721, 1721, 1721, 2115, 1721, 1721, 1721, 2151, + /* 3020 */ 1721, 1721, 327, 2117, 673, 2119, 2120, 668, 1721, 663, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 333, 370, 383, 337, 400, 370, 340, 341, 404, 371, - /* 10 */ 343, 365, 12, 13, 14, 0, 371, 398, 399, 381, - /* 20 */ 20, 14, 22, 20, 8, 9, 381, 20, 12, 13, - /* 30 */ 14, 15, 16, 33, 0, 35, 369, 383, 20, 24, - /* 40 */ 25, 26, 27, 28, 29, 30, 31, 32, 381, 395, - /* 50 */ 383, 447, 398, 399, 450, 417, 418, 419, 412, 59, - /* 60 */ 44, 369, 417, 418, 419, 65, 428, 20, 376, 465, - /* 70 */ 466, 20, 72, 428, 470, 471, 384, 342, 343, 0, + /* 0 */ 333, 338, 383, 370, 400, 342, 371, 344, 404, 371, + /* 10 */ 343, 383, 12, 13, 14, 0, 381, 398, 399, 381, + /* 20 */ 20, 20, 22, 395, 8, 9, 398, 399, 12, 13, + /* 30 */ 14, 15, 16, 33, 0, 35, 369, 368, 20, 24, + /* 40 */ 25, 26, 27, 28, 29, 30, 31, 32, 381, 380, + /* 50 */ 383, 447, 417, 418, 450, 417, 418, 419, 379, 59, + /* 60 */ 44, 382, 383, 428, 337, 65, 428, 340, 341, 465, + /* 70 */ 466, 332, 72, 334, 470, 471, 20, 342, 343, 0, /* 80 */ 337, 414, 348, 340, 341, 418, 436, 437, 421, 422, - /* 90 */ 423, 424, 425, 426, 332, 428, 334, 97, 364, 39, + /* 90 */ 423, 424, 425, 426, 432, 428, 434, 97, 364, 39, /* 100 */ 100, 67, 68, 69, 70, 71, 372, 73, 74, 75, /* 110 */ 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, /* 120 */ 86, 87, 88, 89, 90, 91, 92, 93, 461, 462, - /* 130 */ 379, 420, 97, 382, 383, 21, 136, 137, 24, 25, + /* 130 */ 100, 420, 97, 382, 383, 21, 136, 137, 24, 25, /* 140 */ 26, 27, 28, 29, 30, 31, 32, 112, 113, 114, /* 150 */ 115, 116, 117, 118, 119, 120, 121, 446, 123, 124, - /* 160 */ 125, 126, 127, 128, 369, 20, 166, 167, 447, 166, - /* 170 */ 167, 450, 172, 173, 369, 342, 343, 442, 443, 444, - /* 180 */ 445, 376, 447, 448, 389, 390, 186, 466, 188, 384, - /* 190 */ 330, 470, 471, 360, 130, 131, 64, 8, 9, 135, + /* 160 */ 125, 126, 127, 128, 14, 20, 166, 167, 447, 168, + /* 170 */ 20, 450, 172, 173, 369, 342, 343, 442, 443, 444, + /* 180 */ 445, 376, 447, 448, 166, 167, 186, 466, 188, 384, + /* 190 */ 330, 470, 471, 360, 130, 131, 343, 8, 9, 135, /* 200 */ 367, 12, 13, 14, 15, 16, 59, 100, 129, 130, /* 210 */ 131, 132, 133, 134, 135, 215, 216, 0, 218, 219, /* 220 */ 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, /* 230 */ 230, 231, 232, 233, 234, 235, 12, 13, 181, 4, - /* 240 */ 20, 18, 100, 20, 20, 100, 22, 100, 20, 102, + /* 240 */ 387, 18, 100, 20, 20, 100, 22, 100, 350, 102, /* 250 */ 27, 333, 447, 30, 65, 450, 33, 33, 369, 35, /* 260 */ 400, 343, 205, 206, 404, 12, 13, 14, 15, 16, - /* 270 */ 465, 466, 49, 100, 51, 470, 471, 54, 43, 390, - /* 280 */ 45, 46, 368, 59, 67, 68, 69, 369, 0, 65, - /* 290 */ 100, 74, 75, 76, 380, 20, 72, 80, 109, 381, + /* 270 */ 465, 466, 49, 375, 51, 470, 471, 54, 43, 390, + /* 280 */ 45, 46, 252, 59, 67, 68, 69, 369, 0, 65, + /* 290 */ 100, 74, 75, 76, 355, 356, 72, 80, 109, 381, /* 300 */ 110, 383, 85, 86, 87, 88, 20, 447, 91, 21, /* 310 */ 450, 251, 24, 25, 26, 27, 28, 29, 30, 31, /* 320 */ 32, 97, 99, 348, 100, 465, 466, 185, 3, 187, @@ -550,378 +562,390 @@ static const YYCODETYPE yy_lookahead[] = { /* 350 */ 342, 433, 434, 435, 165, 360, 214, 439, 440, 252, /* 360 */ 136, 137, 367, 140, 20, 52, 143, 144, 145, 146, /* 370 */ 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, - /* 380 */ 157, 158, 159, 338, 161, 162, 163, 342, 168, 344, - /* 390 */ 166, 167, 447, 385, 252, 450, 172, 173, 14, 252, - /* 400 */ 67, 68, 69, 64, 20, 382, 383, 74, 75, 76, + /* 380 */ 157, 158, 159, 369, 161, 162, 163, 342, 343, 20, + /* 390 */ 166, 167, 447, 385, 252, 450, 172, 173, 369, 252, + /* 400 */ 67, 68, 69, 389, 390, 360, 377, 74, 75, 76, /* 410 */ 186, 466, 188, 80, 171, 470, 471, 0, 85, 86, - /* 420 */ 87, 88, 136, 137, 91, 252, 101, 35, 239, 240, + /* 420 */ 87, 88, 136, 137, 91, 3, 101, 35, 239, 240, /* 430 */ 241, 242, 243, 244, 245, 246, 247, 248, 249, 215, - /* 440 */ 216, 166, 218, 219, 220, 221, 222, 223, 224, 225, + /* 440 */ 216, 0, 218, 219, 220, 221, 222, 223, 224, 225, /* 450 */ 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, - /* 460 */ 236, 12, 13, 33, 72, 22, 351, 20, 338, 20, - /* 470 */ 333, 22, 342, 358, 344, 215, 111, 20, 35, 49, - /* 480 */ 343, 64, 33, 81, 35, 55, 56, 57, 58, 59, - /* 490 */ 420, 8, 9, 168, 333, 12, 13, 14, 15, 16, - /* 500 */ 257, 258, 259, 20, 20, 4, 369, 371, 59, 355, - /* 510 */ 356, 72, 20, 343, 65, 72, 446, 381, 381, 342, + /* 460 */ 236, 12, 13, 33, 72, 22, 49, 20, 343, 20, + /* 470 */ 333, 22, 20, 59, 351, 215, 64, 20, 35, 49, + /* 480 */ 343, 358, 33, 81, 35, 55, 56, 57, 58, 59, + /* 490 */ 59, 8, 9, 168, 369, 12, 13, 14, 15, 16, + /* 500 */ 257, 258, 259, 20, 20, 4, 369, 371, 59, 236, + /* 510 */ 20, 238, 22, 99, 65, 72, 102, 381, 381, 342, /* 520 */ 383, 72, 262, 263, 264, 265, 266, 267, 268, 99, - /* 530 */ 8, 9, 102, 236, 12, 13, 14, 15, 16, 369, - /* 540 */ 97, 369, 381, 141, 142, 13, 97, 333, 376, 100, - /* 550 */ 166, 414, 165, 417, 418, 418, 384, 100, 421, 422, - /* 560 */ 423, 424, 425, 426, 428, 428, 164, 35, 18, 392, + /* 530 */ 8, 9, 102, 102, 12, 13, 14, 15, 16, 0, + /* 540 */ 97, 338, 52, 141, 142, 342, 97, 344, 423, 100, + /* 550 */ 14, 414, 100, 417, 418, 418, 20, 100, 421, 422, + /* 560 */ 423, 424, 425, 426, 428, 428, 164, 365, 18, 392, /* 570 */ 433, 394, 435, 23, 8, 9, 439, 440, 12, 13, /* 580 */ 14, 15, 16, 136, 137, 136, 137, 37, 38, 452, - /* 590 */ 333, 41, 100, 423, 4, 381, 20, 460, 22, 169, - /* 600 */ 170, 100, 350, 342, 174, 236, 176, 238, 420, 19, - /* 610 */ 60, 61, 62, 63, 111, 166, 167, 361, 366, 172, - /* 620 */ 173, 172, 173, 33, 194, 369, 239, 375, 52, 186, - /* 630 */ 72, 188, 376, 377, 446, 186, 249, 188, 381, 49, - /* 640 */ 384, 0, 342, 343, 54, 342, 343, 369, 333, 59, - /* 650 */ 100, 168, 168, 392, 376, 394, 371, 129, 215, 216, - /* 660 */ 360, 133, 384, 360, 215, 216, 381, 218, 219, 220, + /* 590 */ 432, 41, 434, 357, 4, 359, 155, 460, 333, 169, + /* 600 */ 170, 100, 64, 64, 174, 164, 176, 1, 2, 19, + /* 610 */ 60, 61, 62, 63, 412, 166, 167, 361, 361, 172, + /* 620 */ 173, 172, 173, 33, 194, 369, 369, 342, 343, 186, + /* 630 */ 14, 188, 376, 377, 377, 186, 20, 188, 447, 49, + /* 640 */ 384, 450, 342, 343, 54, 360, 381, 20, 369, 59, + /* 650 */ 100, 168, 14, 15, 16, 376, 465, 466, 215, 216, + /* 660 */ 360, 470, 471, 384, 215, 216, 20, 218, 219, 220, /* 670 */ 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, /* 680 */ 231, 232, 233, 234, 235, 12, 13, 14, 138, 99, - /* 690 */ 369, 333, 102, 20, 370, 22, 381, 376, 342, 343, - /* 700 */ 178, 343, 417, 418, 419, 384, 33, 333, 35, 252, - /* 710 */ 357, 2, 359, 428, 342, 343, 360, 8, 9, 342, - /* 720 */ 343, 12, 13, 14, 15, 16, 358, 369, 178, 179, - /* 730 */ 180, 350, 59, 183, 168, 355, 356, 360, 342, 381, - /* 740 */ 370, 383, 8, 9, 252, 72, 12, 13, 14, 15, - /* 750 */ 16, 342, 343, 252, 204, 381, 375, 207, 0, 343, - /* 760 */ 210, 211, 212, 213, 214, 14, 333, 342, 343, 360, - /* 770 */ 97, 20, 414, 100, 342, 343, 418, 333, 333, 421, + /* 690 */ 369, 333, 102, 20, 272, 22, 333, 376, 342, 343, + /* 700 */ 178, 343, 166, 35, 252, 384, 33, 101, 35, 252, + /* 710 */ 420, 2, 355, 356, 342, 343, 360, 8, 9, 342, + /* 720 */ 343, 12, 13, 14, 15, 16, 111, 369, 178, 179, + /* 730 */ 180, 129, 59, 183, 168, 133, 446, 360, 342, 381, + /* 740 */ 72, 383, 8, 9, 381, 72, 12, 13, 14, 15, + /* 750 */ 16, 342, 343, 252, 204, 431, 378, 207, 434, 381, + /* 760 */ 210, 211, 212, 213, 214, 236, 130, 342, 343, 360, + /* 770 */ 97, 72, 414, 100, 342, 343, 418, 333, 342, 421, /* 780 */ 422, 423, 424, 425, 426, 360, 428, 343, 392, 345, - /* 790 */ 394, 433, 360, 435, 2, 4, 333, 439, 440, 277, - /* 800 */ 8, 9, 252, 387, 12, 13, 14, 15, 16, 136, - /* 810 */ 137, 342, 343, 369, 381, 443, 444, 445, 460, 447, - /* 820 */ 448, 21, 450, 342, 343, 381, 381, 383, 370, 360, - /* 830 */ 20, 342, 343, 59, 34, 101, 36, 465, 466, 166, - /* 840 */ 167, 343, 470, 471, 381, 172, 173, 8, 9, 360, - /* 850 */ 209, 12, 13, 14, 15, 16, 0, 431, 414, 186, - /* 860 */ 434, 188, 418, 342, 343, 421, 422, 423, 424, 425, - /* 870 */ 426, 369, 428, 99, 131, 370, 102, 433, 135, 435, - /* 880 */ 37, 360, 369, 439, 440, 387, 384, 400, 215, 216, - /* 890 */ 377, 218, 219, 220, 221, 222, 223, 224, 225, 226, + /* 790 */ 394, 433, 360, 435, 2, 361, 350, 439, 440, 277, + /* 800 */ 8, 9, 252, 369, 12, 13, 14, 15, 16, 136, + /* 810 */ 137, 377, 366, 369, 168, 443, 444, 445, 460, 447, + /* 820 */ 448, 375, 450, 342, 343, 381, 111, 383, 392, 369, + /* 830 */ 394, 342, 343, 197, 198, 101, 376, 465, 466, 166, + /* 840 */ 167, 343, 470, 471, 384, 172, 173, 8, 9, 360, + /* 850 */ 72, 12, 13, 14, 15, 16, 0, 48, 414, 186, + /* 860 */ 22, 188, 418, 342, 343, 421, 422, 423, 424, 425, + /* 870 */ 426, 369, 428, 35, 131, 369, 343, 433, 135, 435, + /* 880 */ 37, 360, 376, 439, 440, 387, 384, 400, 215, 216, + /* 890 */ 384, 218, 219, 220, 221, 222, 223, 224, 225, 226, /* 900 */ 227, 228, 229, 230, 231, 232, 233, 234, 235, 12, - /* 910 */ 13, 333, 432, 155, 434, 342, 343, 20, 333, 22, - /* 920 */ 333, 343, 164, 345, 443, 444, 445, 59, 447, 448, - /* 930 */ 33, 333, 35, 360, 447, 45, 46, 450, 195, 196, - /* 940 */ 101, 343, 199, 432, 201, 434, 103, 369, 105, 106, - /* 950 */ 333, 108, 465, 466, 14, 334, 59, 470, 471, 381, - /* 960 */ 20, 383, 342, 343, 342, 343, 381, 369, 381, 72, - /* 970 */ 102, 22, 129, 14, 15, 16, 133, 3, 168, 381, - /* 980 */ 360, 383, 360, 371, 35, 129, 130, 131, 132, 133, + /* 910 */ 13, 333, 333, 45, 46, 342, 343, 20, 333, 22, + /* 920 */ 387, 343, 370, 345, 443, 444, 445, 0, 447, 448, + /* 930 */ 33, 333, 35, 360, 447, 97, 20, 450, 195, 196, + /* 940 */ 101, 343, 199, 4, 201, 333, 103, 369, 105, 106, + /* 950 */ 333, 108, 465, 466, 370, 21, 59, 470, 471, 381, + /* 960 */ 381, 383, 342, 343, 342, 343, 381, 369, 34, 72, + /* 970 */ 36, 378, 129, 370, 381, 378, 133, 168, 381, 381, + /* 980 */ 360, 383, 360, 420, 370, 129, 130, 131, 132, 133, /* 990 */ 134, 135, 414, 381, 97, 400, 418, 100, 381, 421, - /* 1000 */ 422, 423, 424, 425, 426, 333, 428, 400, 1, 2, - /* 1010 */ 333, 433, 414, 435, 333, 343, 418, 439, 440, 421, - /* 1020 */ 422, 423, 424, 425, 426, 427, 428, 429, 430, 417, - /* 1030 */ 418, 8, 9, 136, 137, 12, 13, 14, 15, 16, - /* 1040 */ 428, 369, 447, 361, 253, 450, 97, 343, 44, 342, - /* 1050 */ 343, 369, 65, 381, 447, 383, 370, 450, 381, 377, - /* 1060 */ 465, 466, 381, 166, 167, 470, 471, 360, 378, 172, - /* 1070 */ 173, 381, 465, 466, 250, 251, 361, 470, 471, 136, - /* 1080 */ 137, 342, 343, 186, 369, 188, 414, 342, 343, 22, - /* 1090 */ 418, 387, 377, 421, 422, 423, 424, 425, 426, 360, - /* 1100 */ 428, 400, 35, 342, 343, 360, 166, 42, 101, 44, - /* 1110 */ 130, 378, 215, 216, 381, 218, 219, 220, 221, 222, + /* 1000 */ 422, 423, 424, 425, 426, 333, 428, 400, 42, 446, + /* 1010 */ 44, 433, 414, 435, 20, 343, 418, 439, 440, 421, + /* 1020 */ 422, 423, 424, 425, 426, 427, 428, 429, 430, 342, + /* 1030 */ 343, 8, 9, 136, 137, 12, 13, 14, 15, 16, + /* 1040 */ 361, 369, 447, 333, 370, 450, 14, 360, 369, 342, + /* 1050 */ 343, 44, 20, 381, 447, 383, 377, 450, 342, 343, + /* 1060 */ 465, 466, 22, 166, 167, 470, 471, 360, 165, 172, + /* 1070 */ 173, 104, 465, 466, 107, 35, 360, 470, 471, 250, + /* 1080 */ 251, 342, 343, 186, 168, 188, 414, 333, 342, 343, + /* 1090 */ 418, 381, 22, 421, 422, 423, 424, 425, 426, 360, + /* 1100 */ 428, 400, 333, 342, 343, 35, 360, 333, 101, 42, + /* 1110 */ 333, 44, 215, 216, 333, 218, 219, 220, 221, 222, /* 1120 */ 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, - /* 1130 */ 233, 234, 235, 12, 13, 342, 343, 342, 343, 333, - /* 1140 */ 333, 20, 361, 22, 472, 473, 333, 333, 447, 0, - /* 1150 */ 369, 450, 333, 360, 33, 360, 35, 343, 377, 345, - /* 1160 */ 44, 44, 22, 333, 97, 333, 465, 466, 0, 405, - /* 1170 */ 362, 470, 471, 365, 333, 35, 378, 197, 198, 381, - /* 1180 */ 59, 111, 104, 369, 343, 107, 345, 381, 381, 42, - /* 1190 */ 200, 44, 202, 72, 381, 381, 447, 383, 49, 450, - /* 1200 */ 381, 48, 215, 13, 443, 444, 445, 0, 447, 448, - /* 1210 */ 369, 381, 44, 381, 465, 466, 391, 101, 97, 470, - /* 1220 */ 471, 100, 381, 104, 383, 35, 107, 474, 414, 22, - /* 1230 */ 160, 35, 418, 44, 333, 421, 422, 423, 424, 425, - /* 1240 */ 426, 35, 428, 463, 343, 104, 272, 433, 107, 435, - /* 1250 */ 0, 0, 346, 439, 440, 414, 44, 136, 137, 418, - /* 1260 */ 44, 13, 421, 422, 423, 424, 425, 426, 104, 428, - /* 1270 */ 369, 107, 22, 22, 433, 44, 435, 44, 274, 457, - /* 1280 */ 439, 440, 381, 35, 383, 47, 369, 166, 167, 391, - /* 1290 */ 101, 35, 346, 172, 173, 341, 129, 130, 131, 132, - /* 1300 */ 133, 134, 135, 1, 2, 44, 343, 186, 44, 188, - /* 1310 */ 380, 44, 391, 101, 44, 414, 44, 101, 44, 418, - /* 1320 */ 449, 168, 421, 422, 423, 424, 425, 426, 72, 428, - /* 1330 */ 44, 0, 101, 44, 101, 44, 215, 216, 100, 218, + /* 1130 */ 233, 234, 235, 12, 13, 381, 209, 97, 0, 333, + /* 1140 */ 333, 20, 239, 22, 472, 473, 333, 333, 447, 44, + /* 1150 */ 381, 450, 249, 333, 33, 381, 35, 343, 381, 345, + /* 1160 */ 166, 44, 381, 333, 333, 333, 465, 466, 111, 13, + /* 1170 */ 362, 470, 471, 365, 333, 104, 104, 35, 107, 107, + /* 1180 */ 59, 13, 44, 369, 343, 0, 345, 381, 381, 44, + /* 1190 */ 405, 35, 253, 72, 381, 381, 104, 383, 166, 107, + /* 1200 */ 200, 381, 202, 35, 443, 444, 445, 22, 447, 448, + /* 1210 */ 369, 381, 381, 381, 44, 44, 44, 160, 97, 0, + /* 1220 */ 44, 100, 381, 0, 383, 136, 137, 370, 414, 358, + /* 1230 */ 1, 2, 418, 44, 333, 421, 422, 423, 424, 425, + /* 1240 */ 426, 22, 428, 35, 343, 22, 101, 433, 44, 435, + /* 1250 */ 47, 44, 65, 439, 440, 414, 44, 136, 137, 418, + /* 1260 */ 44, 44, 421, 422, 423, 424, 425, 426, 391, 428, + /* 1270 */ 369, 101, 101, 101, 433, 44, 435, 101, 44, 334, + /* 1280 */ 439, 440, 381, 346, 383, 474, 463, 166, 167, 369, + /* 1290 */ 101, 457, 346, 172, 173, 391, 129, 130, 131, 132, + /* 1300 */ 133, 134, 135, 100, 341, 101, 380, 186, 101, 188, + /* 1310 */ 391, 44, 13, 101, 44, 414, 44, 101, 101, 418, + /* 1320 */ 0, 44, 421, 422, 423, 424, 425, 426, 343, 428, + /* 1330 */ 188, 449, 101, 44, 35, 101, 215, 216, 44, 218, /* 1340 */ 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, - /* 1350 */ 229, 230, 231, 232, 233, 234, 235, 441, 467, 12, - /* 1360 */ 13, 44, 101, 462, 44, 101, 44, 333, 101, 22, - /* 1370 */ 44, 101, 44, 101, 13, 101, 451, 343, 254, 345, - /* 1380 */ 33, 50, 35, 416, 188, 49, 415, 101, 184, 388, - /* 1390 */ 101, 402, 101, 276, 188, 42, 35, 333, 20, 391, - /* 1400 */ 388, 165, 386, 369, 20, 342, 59, 343, 342, 345, - /* 1410 */ 388, 386, 386, 98, 354, 381, 342, 383, 101, 72, - /* 1420 */ 96, 101, 353, 101, 95, 352, 342, 101, 342, 101, - /* 1430 */ 342, 20, 335, 369, 48, 339, 335, 339, 20, 409, - /* 1440 */ 350, 20, 383, 350, 97, 381, 20, 383, 414, 344, - /* 1450 */ 20, 401, 418, 350, 344, 421, 422, 423, 424, 425, - /* 1460 */ 426, 350, 428, 350, 350, 333, 53, 433, 342, 435, - /* 1470 */ 350, 335, 347, 439, 440, 343, 347, 369, 414, 335, - /* 1480 */ 381, 369, 418, 381, 369, 421, 422, 423, 424, 425, - /* 1490 */ 426, 369, 428, 342, 413, 203, 369, 433, 100, 435, - /* 1500 */ 369, 369, 369, 439, 440, 411, 369, 369, 369, 381, - /* 1510 */ 348, 369, 408, 381, 191, 383, 409, 192, 383, 348, - /* 1520 */ 190, 342, 269, 261, 456, 407, 260, 381, 456, 406, - /* 1530 */ 396, 381, 177, 186, 391, 188, 391, 381, 381, 459, - /* 1540 */ 458, 456, 271, 270, 396, 454, 414, 455, 255, 273, - /* 1550 */ 418, 475, 275, 421, 422, 423, 424, 425, 426, 453, - /* 1560 */ 428, 278, 215, 216, 469, 433, 343, 435, 416, 251, - /* 1570 */ 20, 439, 440, 333, 420, 228, 229, 230, 231, 232, - /* 1580 */ 233, 234, 348, 343, 342, 344, 20, 348, 394, 396, - /* 1590 */ 381, 381, 381, 333, 381, 381, 396, 170, 468, 381, - /* 1600 */ 348, 393, 365, 343, 100, 348, 343, 438, 100, 369, - /* 1610 */ 381, 342, 36, 359, 0, 336, 373, 335, 403, 397, - /* 1620 */ 0, 381, 397, 383, 0, 331, 348, 410, 363, 369, - /* 1630 */ 349, 363, 42, 0, 35, 35, 208, 35, 363, 35, - /* 1640 */ 208, 381, 35, 383, 0, 35, 208, 0, 208, 0, - /* 1650 */ 35, 0, 22, 0, 414, 195, 35, 188, 418, 186, - /* 1660 */ 0, 421, 422, 423, 424, 425, 426, 0, 428, 0, - /* 1670 */ 182, 181, 0, 433, 414, 435, 47, 333, 418, 439, + /* 1350 */ 229, 230, 231, 232, 233, 234, 235, 467, 441, 12, + /* 1360 */ 13, 13, 451, 462, 254, 416, 49, 333, 101, 22, + /* 1370 */ 50, 101, 415, 101, 184, 402, 42, 343, 101, 345, + /* 1380 */ 33, 276, 35, 35, 388, 20, 391, 388, 386, 165, + /* 1390 */ 101, 274, 20, 342, 342, 101, 188, 333, 388, 386, + /* 1400 */ 386, 98, 215, 369, 354, 96, 59, 343, 353, 345, + /* 1410 */ 342, 95, 352, 20, 335, 381, 342, 383, 342, 72, + /* 1420 */ 342, 48, 339, 335, 339, 20, 409, 350, 20, 383, + /* 1430 */ 350, 20, 20, 369, 344, 344, 401, 350, 53, 342, + /* 1440 */ 342, 347, 347, 350, 97, 381, 350, 383, 414, 350, + /* 1450 */ 350, 335, 418, 369, 335, 421, 422, 423, 424, 425, + /* 1460 */ 426, 203, 428, 381, 100, 333, 413, 433, 409, 435, + /* 1470 */ 369, 369, 369, 439, 440, 343, 369, 369, 414, 348, + /* 1480 */ 369, 369, 418, 369, 369, 421, 422, 423, 424, 425, + /* 1490 */ 426, 369, 428, 192, 381, 191, 348, 433, 190, 435, + /* 1500 */ 342, 369, 261, 439, 440, 381, 383, 381, 260, 381, + /* 1510 */ 456, 391, 408, 381, 456, 383, 407, 411, 391, 269, + /* 1520 */ 381, 396, 381, 177, 406, 396, 271, 270, 459, 255, + /* 1530 */ 455, 458, 278, 186, 275, 188, 475, 456, 273, 251, + /* 1540 */ 454, 343, 20, 342, 344, 348, 414, 348, 20, 394, + /* 1550 */ 418, 416, 170, 421, 422, 423, 424, 425, 426, 396, + /* 1560 */ 428, 381, 215, 216, 420, 433, 393, 435, 348, 381, + /* 1570 */ 396, 439, 440, 333, 381, 228, 229, 230, 231, 232, + /* 1580 */ 233, 234, 343, 343, 348, 100, 453, 381, 438, 100, + /* 1590 */ 381, 381, 365, 333, 373, 381, 359, 348, 342, 469, + /* 1600 */ 36, 335, 468, 343, 336, 403, 410, 363, 397, 369, + /* 1610 */ 397, 349, 331, 0, 363, 363, 0, 0, 42, 0, + /* 1620 */ 35, 381, 208, 383, 35, 35, 35, 208, 0, 369, + /* 1630 */ 35, 35, 208, 0, 208, 0, 35, 0, 0, 35, + /* 1640 */ 22, 381, 195, 383, 188, 186, 0, 0, 0, 182, + /* 1650 */ 181, 0, 0, 47, 414, 0, 0, 0, 418, 0, + /* 1660 */ 0, 421, 422, 423, 424, 425, 426, 42, 428, 0, + /* 1670 */ 0, 0, 0, 433, 414, 435, 155, 333, 418, 439, /* 1680 */ 440, 421, 422, 423, 424, 425, 426, 343, 428, 0, - /* 1690 */ 0, 0, 0, 433, 42, 435, 0, 0, 0, 439, - /* 1700 */ 440, 333, 0, 0, 0, 0, 0, 155, 35, 0, - /* 1710 */ 155, 343, 0, 369, 0, 0, 0, 0, 0, 0, + /* 1690 */ 0, 0, 155, 433, 0, 435, 35, 0, 0, 439, + /* 1700 */ 440, 333, 0, 0, 0, 0, 0, 0, 0, 0, + /* 1710 */ 0, 343, 0, 369, 0, 0, 0, 42, 0, 0, /* 1720 */ 0, 0, 0, 0, 0, 381, 0, 383, 0, 0, - /* 1730 */ 0, 0, 0, 0, 0, 42, 0, 369, 0, 0, - /* 1740 */ 0, 0, 139, 0, 0, 0, 59, 22, 48, 381, - /* 1750 */ 0, 383, 0, 59, 59, 48, 0, 22, 414, 22, - /* 1760 */ 0, 14, 418, 177, 0, 421, 422, 423, 424, 425, - /* 1770 */ 426, 44, 428, 0, 0, 0, 0, 35, 333, 435, - /* 1780 */ 0, 0, 414, 439, 440, 39, 418, 39, 343, 421, - /* 1790 */ 422, 423, 424, 425, 426, 42, 428, 39, 47, 40, - /* 1800 */ 0, 333, 47, 435, 47, 0, 35, 439, 440, 39, - /* 1810 */ 49, 343, 0, 35, 369, 66, 49, 0, 39, 35, - /* 1820 */ 0, 49, 39, 35, 333, 39, 381, 0, 383, 0, - /* 1830 */ 0, 49, 0, 35, 343, 22, 0, 369, 35, 35, - /* 1840 */ 44, 35, 35, 35, 35, 35, 44, 333, 22, 381, - /* 1850 */ 35, 383, 0, 35, 35, 22, 0, 343, 0, 414, - /* 1860 */ 369, 107, 22, 418, 22, 51, 421, 422, 423, 424, - /* 1870 */ 425, 426, 381, 428, 383, 109, 35, 0, 35, 0, - /* 1880 */ 435, 35, 414, 369, 439, 440, 418, 0, 22, 421, - /* 1890 */ 422, 423, 424, 425, 426, 381, 428, 383, 20, 35, - /* 1900 */ 35, 100, 35, 435, 0, 414, 193, 35, 440, 418, - /* 1910 */ 333, 100, 421, 422, 423, 424, 425, 426, 22, 428, - /* 1920 */ 343, 0, 101, 22, 0, 0, 3, 44, 414, 101, - /* 1930 */ 168, 256, 418, 48, 100, 421, 422, 423, 424, 425, - /* 1940 */ 426, 48, 428, 175, 100, 168, 369, 98, 101, 96, - /* 1950 */ 44, 374, 44, 168, 189, 464, 101, 100, 381, 170, - /* 1960 */ 383, 100, 100, 44, 101, 47, 256, 3, 100, 47, - /* 1970 */ 44, 101, 101, 44, 35, 333, 35, 35, 35, 35, - /* 1980 */ 35, 101, 101, 0, 47, 343, 0, 473, 0, 0, - /* 1990 */ 44, 414, 100, 39, 47, 418, 333, 47, 421, 422, - /* 2000 */ 423, 424, 425, 426, 101, 428, 343, 100, 171, 333, - /* 2010 */ 101, 369, 100, 100, 100, 0, 39, 169, 47, 343, - /* 2020 */ 110, 2, 44, 381, 22, 383, 98, 100, 98, 237, - /* 2030 */ 22, 215, 369, 100, 47, 101, 101, 374, 100, 47, - /* 2040 */ 250, 101, 217, 111, 381, 369, 383, 256, 100, 100, - /* 2050 */ 374, 100, 35, 101, 101, 100, 414, 381, 35, 383, - /* 2060 */ 418, 1, 100, 421, 422, 423, 424, 425, 426, 101, - /* 2070 */ 428, 35, 430, 100, 35, 101, 100, 414, 35, 19, - /* 2080 */ 101, 418, 100, 35, 421, 422, 423, 424, 425, 426, - /* 2090 */ 414, 428, 101, 33, 418, 100, 333, 421, 422, 423, - /* 2100 */ 424, 425, 426, 122, 428, 100, 343, 122, 44, 49, - /* 2110 */ 100, 35, 122, 100, 22, 55, 56, 57, 58, 59, - /* 2120 */ 333, 122, 66, 65, 35, 35, 35, 72, 35, 35, - /* 2130 */ 343, 35, 369, 35, 35, 35, 35, 35, 94, 44, - /* 2140 */ 35, 35, 22, 35, 381, 35, 383, 35, 35, 72, - /* 2150 */ 35, 35, 22, 35, 35, 0, 369, 35, 35, 99, - /* 2160 */ 49, 0, 102, 39, 35, 39, 0, 35, 381, 0, - /* 2170 */ 383, 35, 0, 39, 49, 35, 49, 414, 39, 35, - /* 2180 */ 49, 418, 0, 333, 421, 422, 423, 424, 425, 426, - /* 2190 */ 22, 428, 21, 343, 134, 22, 22, 21, 20, 476, - /* 2200 */ 476, 414, 476, 476, 476, 418, 476, 333, 421, 422, - /* 2210 */ 423, 424, 425, 426, 476, 428, 476, 343, 476, 369, - /* 2220 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 169, - /* 2230 */ 476, 381, 476, 383, 174, 476, 476, 476, 476, 476, - /* 2240 */ 476, 476, 476, 369, 476, 476, 476, 476, 476, 476, - /* 2250 */ 476, 476, 476, 333, 194, 381, 476, 383, 476, 476, - /* 2260 */ 476, 476, 476, 343, 414, 476, 476, 476, 418, 333, - /* 2270 */ 476, 421, 422, 423, 424, 425, 426, 476, 428, 343, - /* 2280 */ 476, 476, 476, 476, 476, 476, 476, 476, 414, 369, - /* 2290 */ 476, 476, 418, 476, 476, 421, 422, 423, 424, 425, - /* 2300 */ 426, 381, 428, 383, 476, 369, 476, 476, 476, 476, - /* 2310 */ 476, 476, 476, 476, 476, 476, 476, 381, 476, 383, - /* 2320 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 333, - /* 2330 */ 476, 476, 476, 476, 414, 476, 476, 476, 418, 343, - /* 2340 */ 476, 421, 422, 423, 424, 425, 426, 476, 428, 476, - /* 2350 */ 414, 476, 476, 333, 418, 476, 476, 421, 422, 423, - /* 2360 */ 424, 425, 426, 343, 428, 369, 476, 476, 476, 476, - /* 2370 */ 476, 476, 476, 476, 476, 476, 476, 381, 476, 383, - /* 2380 */ 333, 476, 476, 476, 476, 476, 476, 476, 476, 369, - /* 2390 */ 343, 476, 476, 476, 476, 476, 476, 476, 476, 333, - /* 2400 */ 476, 381, 476, 383, 476, 476, 476, 476, 476, 343, - /* 2410 */ 414, 476, 476, 476, 418, 476, 369, 421, 422, 423, - /* 2420 */ 424, 425, 426, 476, 428, 476, 476, 476, 381, 476, - /* 2430 */ 383, 476, 476, 476, 414, 369, 476, 476, 418, 476, - /* 2440 */ 476, 421, 422, 423, 424, 425, 426, 381, 428, 383, - /* 2450 */ 333, 476, 476, 476, 476, 476, 476, 476, 476, 476, - /* 2460 */ 343, 414, 476, 476, 476, 418, 333, 476, 421, 422, - /* 2470 */ 423, 424, 425, 426, 476, 428, 343, 476, 476, 476, - /* 2480 */ 414, 476, 476, 476, 418, 476, 369, 421, 422, 423, - /* 2490 */ 424, 425, 426, 476, 428, 476, 333, 476, 381, 476, - /* 2500 */ 383, 476, 369, 476, 476, 476, 343, 476, 476, 476, - /* 2510 */ 476, 476, 476, 476, 381, 476, 383, 333, 476, 476, - /* 2520 */ 476, 476, 476, 476, 476, 476, 476, 343, 476, 476, - /* 2530 */ 476, 414, 369, 476, 476, 418, 476, 476, 421, 422, - /* 2540 */ 423, 424, 425, 426, 381, 428, 383, 414, 476, 476, - /* 2550 */ 476, 418, 476, 369, 421, 422, 423, 424, 425, 426, - /* 2560 */ 476, 428, 476, 333, 476, 381, 476, 383, 476, 476, - /* 2570 */ 476, 476, 476, 343, 476, 476, 476, 414, 476, 476, - /* 2580 */ 476, 418, 476, 476, 421, 422, 423, 424, 425, 426, - /* 2590 */ 476, 428, 476, 476, 476, 476, 476, 476, 414, 369, - /* 2600 */ 476, 476, 418, 476, 476, 421, 422, 423, 424, 425, - /* 2610 */ 426, 381, 428, 383, 476, 476, 476, 476, 476, 476, - /* 2620 */ 476, 476, 476, 333, 476, 476, 476, 476, 476, 476, - /* 2630 */ 476, 476, 476, 343, 476, 476, 333, 476, 476, 476, - /* 2640 */ 476, 476, 476, 476, 414, 476, 343, 476, 418, 476, - /* 2650 */ 476, 421, 422, 423, 424, 425, 426, 476, 428, 369, - /* 2660 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, - /* 2670 */ 476, 381, 369, 383, 476, 476, 476, 476, 476, 476, - /* 2680 */ 476, 476, 476, 476, 381, 476, 383, 333, 476, 476, - /* 2690 */ 476, 476, 476, 476, 476, 476, 476, 343, 476, 476, - /* 2700 */ 476, 476, 476, 476, 414, 476, 476, 476, 418, 476, - /* 2710 */ 476, 421, 422, 423, 424, 425, 426, 414, 428, 476, - /* 2720 */ 476, 418, 476, 369, 421, 422, 423, 424, 425, 426, - /* 2730 */ 476, 428, 476, 333, 476, 381, 476, 383, 476, 476, - /* 2740 */ 476, 476, 476, 343, 476, 476, 476, 476, 333, 476, - /* 2750 */ 476, 476, 476, 476, 476, 476, 476, 476, 343, 476, - /* 2760 */ 476, 476, 476, 476, 476, 476, 476, 476, 414, 369, - /* 2770 */ 476, 476, 418, 476, 476, 421, 422, 423, 424, 425, - /* 2780 */ 426, 381, 428, 383, 369, 476, 476, 476, 476, 476, - /* 2790 */ 476, 476, 476, 476, 333, 476, 381, 476, 383, 476, - /* 2800 */ 476, 476, 476, 476, 343, 476, 476, 476, 476, 333, - /* 2810 */ 476, 476, 476, 476, 414, 476, 476, 476, 418, 343, - /* 2820 */ 476, 421, 422, 423, 424, 425, 426, 476, 428, 414, - /* 2830 */ 369, 476, 476, 418, 476, 476, 421, 422, 423, 424, - /* 2840 */ 425, 426, 381, 428, 383, 369, 476, 476, 476, 476, - /* 2850 */ 476, 476, 476, 476, 476, 476, 476, 381, 476, 383, - /* 2860 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, - /* 2870 */ 476, 476, 476, 476, 476, 414, 476, 476, 476, 418, - /* 2880 */ 476, 476, 421, 422, 423, 424, 425, 426, 476, 428, - /* 2890 */ 414, 476, 476, 476, 418, 476, 476, 421, 422, 423, - /* 2900 */ 424, 425, 426, 476, 428, + /* 1730 */ 0, 139, 48, 22, 48, 22, 0, 369, 59, 22, + /* 1740 */ 0, 59, 0, 0, 59, 0, 47, 47, 14, 381, + /* 1750 */ 177, 383, 0, 0, 0, 0, 0, 0, 414, 0, + /* 1760 */ 35, 66, 418, 0, 0, 421, 422, 423, 424, 425, + /* 1770 */ 426, 39, 428, 42, 47, 40, 39, 44, 333, 435, + /* 1780 */ 39, 35, 414, 439, 440, 0, 418, 49, 343, 421, + /* 1790 */ 422, 423, 424, 425, 426, 35, 428, 39, 49, 0, + /* 1800 */ 39, 333, 35, 435, 39, 0, 35, 439, 440, 0, + /* 1810 */ 0, 343, 39, 0, 369, 49, 0, 49, 35, 22, + /* 1820 */ 0, 35, 35, 35, 333, 35, 381, 35, 383, 35, + /* 1830 */ 35, 35, 0, 44, 343, 44, 35, 369, 35, 22, + /* 1840 */ 22, 0, 0, 22, 51, 22, 35, 333, 109, 381, + /* 1850 */ 0, 383, 0, 0, 107, 22, 20, 343, 0, 414, + /* 1860 */ 369, 35, 35, 418, 35, 374, 421, 422, 423, 424, + /* 1870 */ 425, 426, 381, 428, 383, 35, 101, 100, 35, 100, + /* 1880 */ 435, 193, 414, 369, 439, 440, 418, 22, 374, 421, + /* 1890 */ 422, 423, 424, 425, 426, 381, 428, 383, 35, 0, + /* 1900 */ 22, 0, 0, 435, 44, 414, 3, 100, 440, 418, + /* 1910 */ 333, 96, 421, 422, 423, 424, 425, 426, 170, 428, + /* 1920 */ 343, 256, 48, 101, 100, 48, 175, 47, 414, 98, + /* 1930 */ 168, 101, 418, 189, 44, 421, 422, 423, 424, 425, + /* 1940 */ 426, 44, 428, 47, 168, 101, 369, 100, 168, 100, + /* 1950 */ 44, 100, 44, 3, 44, 101, 100, 256, 381, 35, + /* 1960 */ 383, 35, 101, 101, 35, 35, 35, 35, 47, 101, + /* 1970 */ 101, 44, 47, 0, 0, 333, 0, 0, 39, 100, + /* 1980 */ 47, 100, 0, 101, 100, 343, 101, 100, 100, 100, + /* 1990 */ 171, 414, 39, 47, 169, 418, 333, 2, 421, 422, + /* 2000 */ 423, 424, 425, 426, 110, 428, 343, 44, 22, 333, + /* 2010 */ 98, 369, 98, 215, 47, 237, 100, 47, 101, 343, + /* 2020 */ 100, 22, 100, 381, 250, 383, 111, 35, 256, 101, + /* 2030 */ 22, 217, 369, 35, 44, 101, 100, 374, 100, 35, + /* 2040 */ 100, 464, 101, 101, 381, 369, 383, 100, 35, 101, + /* 2050 */ 101, 100, 100, 35, 35, 101, 414, 381, 100, 383, + /* 2060 */ 418, 101, 100, 421, 422, 423, 424, 425, 426, 100, + /* 2070 */ 428, 100, 35, 35, 65, 100, 66, 414, 122, 122, + /* 2080 */ 35, 418, 35, 333, 421, 422, 423, 424, 425, 426, + /* 2090 */ 414, 428, 35, 343, 418, 35, 35, 421, 422, 423, + /* 2100 */ 424, 425, 426, 333, 428, 122, 430, 122, 35, 35, + /* 2110 */ 35, 35, 72, 343, 44, 473, 94, 35, 35, 369, + /* 2120 */ 35, 22, 35, 35, 374, 35, 72, 35, 35, 35, + /* 2130 */ 35, 381, 35, 383, 22, 35, 0, 35, 49, 369, + /* 2140 */ 39, 0, 35, 39, 374, 49, 0, 49, 35, 39, + /* 2150 */ 0, 381, 35, 383, 49, 39, 0, 35, 35, 0, + /* 2160 */ 22, 21, 476, 21, 414, 22, 22, 20, 418, 476, + /* 2170 */ 476, 421, 422, 423, 424, 425, 426, 476, 428, 476, + /* 2180 */ 476, 476, 476, 476, 414, 476, 1, 476, 418, 476, + /* 2190 */ 476, 421, 422, 423, 424, 425, 426, 476, 428, 476, + /* 2200 */ 333, 476, 476, 476, 19, 476, 476, 476, 476, 476, + /* 2210 */ 343, 476, 476, 476, 476, 476, 333, 476, 33, 476, + /* 2220 */ 476, 476, 476, 476, 476, 476, 343, 476, 476, 476, + /* 2230 */ 476, 476, 476, 476, 49, 476, 369, 476, 476, 476, + /* 2240 */ 55, 56, 57, 58, 59, 476, 333, 476, 381, 476, + /* 2250 */ 383, 476, 369, 476, 476, 476, 343, 476, 476, 476, + /* 2260 */ 476, 476, 476, 476, 381, 476, 383, 476, 476, 476, + /* 2270 */ 476, 476, 476, 476, 476, 476, 333, 476, 476, 476, + /* 2280 */ 476, 414, 369, 476, 99, 418, 343, 102, 421, 422, + /* 2290 */ 423, 424, 425, 426, 381, 428, 383, 414, 476, 476, + /* 2300 */ 476, 418, 476, 476, 421, 422, 423, 424, 425, 426, + /* 2310 */ 476, 428, 369, 476, 476, 476, 476, 476, 476, 134, + /* 2320 */ 476, 476, 476, 476, 381, 476, 383, 414, 476, 476, + /* 2330 */ 476, 418, 476, 476, 421, 422, 423, 424, 425, 426, + /* 2340 */ 333, 428, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2350 */ 343, 476, 476, 333, 169, 476, 476, 414, 476, 174, + /* 2360 */ 476, 418, 476, 343, 421, 422, 423, 424, 425, 426, + /* 2370 */ 476, 428, 476, 476, 476, 476, 369, 476, 476, 194, + /* 2380 */ 476, 476, 476, 476, 476, 476, 476, 476, 381, 369, + /* 2390 */ 383, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2400 */ 476, 381, 476, 383, 333, 476, 476, 476, 476, 476, + /* 2410 */ 476, 476, 476, 476, 343, 476, 476, 333, 476, 476, + /* 2420 */ 476, 414, 476, 476, 476, 418, 476, 343, 421, 422, + /* 2430 */ 423, 424, 425, 426, 414, 428, 476, 476, 418, 476, + /* 2440 */ 369, 421, 422, 423, 424, 425, 426, 476, 428, 476, + /* 2450 */ 476, 476, 381, 369, 383, 476, 476, 476, 476, 476, + /* 2460 */ 476, 476, 476, 476, 476, 381, 476, 383, 476, 476, + /* 2470 */ 476, 476, 476, 476, 476, 476, 476, 476, 333, 476, + /* 2480 */ 476, 476, 476, 476, 476, 414, 476, 476, 343, 418, + /* 2490 */ 476, 476, 421, 422, 423, 424, 425, 426, 414, 428, + /* 2500 */ 476, 476, 418, 476, 476, 421, 422, 423, 424, 425, + /* 2510 */ 426, 476, 428, 476, 369, 476, 476, 476, 476, 476, + /* 2520 */ 476, 476, 476, 476, 476, 476, 381, 476, 383, 333, + /* 2530 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 343, + /* 2540 */ 476, 476, 476, 476, 333, 476, 476, 476, 476, 476, + /* 2550 */ 476, 476, 476, 476, 343, 476, 476, 476, 476, 414, + /* 2560 */ 476, 476, 476, 418, 476, 369, 421, 422, 423, 424, + /* 2570 */ 425, 426, 476, 428, 476, 476, 476, 381, 476, 383, + /* 2580 */ 369, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2590 */ 333, 476, 381, 476, 383, 476, 476, 476, 476, 476, + /* 2600 */ 343, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2610 */ 414, 476, 476, 476, 418, 476, 476, 421, 422, 423, + /* 2620 */ 424, 425, 426, 476, 428, 414, 369, 476, 476, 418, + /* 2630 */ 476, 476, 421, 422, 423, 424, 425, 426, 381, 428, + /* 2640 */ 383, 333, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2650 */ 476, 343, 476, 476, 476, 476, 476, 476, 333, 476, + /* 2660 */ 476, 476, 476, 476, 476, 476, 476, 476, 343, 476, + /* 2670 */ 476, 414, 476, 476, 476, 418, 476, 369, 421, 422, + /* 2680 */ 423, 424, 425, 426, 476, 428, 476, 333, 476, 381, + /* 2690 */ 476, 383, 476, 476, 369, 476, 476, 343, 476, 476, + /* 2700 */ 476, 476, 476, 476, 476, 476, 381, 476, 383, 476, + /* 2710 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2720 */ 476, 476, 414, 369, 476, 476, 418, 476, 476, 421, + /* 2730 */ 422, 423, 424, 425, 426, 381, 428, 383, 476, 414, + /* 2740 */ 476, 476, 476, 418, 476, 476, 421, 422, 423, 424, + /* 2750 */ 425, 426, 476, 428, 476, 333, 476, 476, 476, 476, + /* 2760 */ 476, 476, 476, 476, 476, 343, 476, 476, 414, 476, + /* 2770 */ 476, 476, 418, 476, 333, 421, 422, 423, 424, 425, + /* 2780 */ 426, 476, 428, 476, 343, 476, 476, 333, 476, 476, + /* 2790 */ 476, 369, 476, 476, 476, 476, 476, 343, 476, 476, + /* 2800 */ 476, 476, 476, 381, 476, 383, 476, 476, 476, 476, + /* 2810 */ 369, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2820 */ 476, 476, 381, 369, 383, 476, 476, 476, 476, 476, + /* 2830 */ 476, 476, 476, 476, 476, 381, 414, 383, 476, 476, + /* 2840 */ 418, 476, 476, 421, 422, 423, 424, 425, 426, 476, + /* 2850 */ 428, 476, 476, 476, 476, 414, 476, 476, 476, 418, + /* 2860 */ 333, 476, 421, 422, 423, 424, 425, 426, 414, 428, + /* 2870 */ 343, 476, 418, 333, 476, 421, 422, 423, 424, 425, + /* 2880 */ 426, 476, 428, 343, 476, 476, 476, 476, 333, 476, + /* 2890 */ 476, 476, 476, 476, 476, 476, 369, 476, 343, 476, + /* 2900 */ 476, 476, 476, 476, 476, 476, 476, 476, 381, 369, + /* 2910 */ 383, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2920 */ 476, 381, 476, 383, 369, 476, 476, 476, 476, 476, + /* 2930 */ 476, 476, 476, 476, 333, 476, 381, 476, 383, 476, + /* 2940 */ 476, 414, 476, 476, 343, 418, 476, 476, 421, 422, + /* 2950 */ 423, 424, 425, 426, 414, 428, 476, 476, 418, 476, + /* 2960 */ 476, 421, 422, 423, 424, 425, 426, 476, 428, 414, + /* 2970 */ 369, 476, 476, 418, 476, 476, 421, 422, 423, 424, + /* 2980 */ 425, 426, 381, 428, 383, 476, 476, 476, 476, 476, + /* 2990 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 3000 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 3010 */ 476, 476, 476, 476, 476, 414, 476, 476, 476, 418, + /* 3020 */ 476, 476, 421, 422, 423, 424, 425, 426, 476, 428, }; #define YY_SHIFT_COUNT (761) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (2182) +#define YY_SHIFT_MAX (2185) static const unsigned short int yy_shift_ofst[] = { /* 0 */ 550, 0, 224, 0, 449, 449, 449, 449, 449, 449, /* 10 */ 449, 449, 449, 449, 449, 449, 673, 897, 897, 1121, /* 20 */ 897, 897, 897, 897, 897, 897, 897, 897, 897, 897, /* 30 */ 897, 897, 897, 897, 897, 897, 897, 897, 897, 897, - /* 40 */ 897, 897, 897, 897, 897, 897, 897, 897, 897, 457, - /* 50 */ 492, 142, 145, 147, 107, 173, 107, 145, 145, 1347, - /* 60 */ 1347, 1347, 107, 1347, 1347, 501, 107, 18, 447, 47, - /* 70 */ 47, 447, 235, 235, 3, 286, 7, 7, 47, 47, - /* 80 */ 47, 47, 47, 47, 47, 51, 47, 47, 132, 18, - /* 90 */ 47, 47, 228, 47, 18, 47, 51, 47, 51, 18, - /* 100 */ 47, 47, 18, 47, 18, 18, 18, 47, 339, 223, + /* 40 */ 897, 897, 897, 897, 897, 897, 897, 897, 897, 897, + /* 50 */ 897, 452, 457, 142, 145, 147, 30, 107, 30, 145, + /* 60 */ 145, 1347, 30, 1347, 1347, 501, 30, 56, 447, 344, + /* 70 */ 344, 447, 235, 235, 18, 286, 150, 150, 344, 344, + /* 80 */ 344, 344, 344, 344, 344, 369, 344, 344, 412, 56, + /* 90 */ 344, 344, 484, 344, 56, 344, 369, 344, 369, 56, + /* 100 */ 344, 344, 56, 344, 56, 56, 56, 344, 538, 223, /* 110 */ 189, 189, 333, 114, 443, 443, 443, 443, 443, 443, /* 120 */ 443, 443, 443, 443, 443, 443, 443, 443, 443, 443, - /* 130 */ 443, 443, 443, 843, 325, 3, 286, 392, 220, 220, - /* 140 */ 220, 417, 369, 369, 392, 344, 344, 344, 132, 365, - /* 150 */ 297, 18, 439, 18, 439, 439, 503, 558, 35, 35, - /* 160 */ 35, 35, 35, 35, 35, 35, 2060, 217, 288, 483, - /* 170 */ 522, 260, 313, 243, 384, 940, 576, 484, 890, 751, - /* 180 */ 528, 810, 824, 60, 974, 824, 1065, 791, 275, 1124, - /* 190 */ 1336, 1204, 1353, 1378, 1353, 1236, 1384, 1384, 1353, 1236, - /* 200 */ 1236, 1315, 1324, 1384, 1329, 1384, 1384, 1384, 1411, 1386, - /* 210 */ 1411, 1386, 1418, 132, 1421, 132, 1426, 1430, 132, 1426, - /* 220 */ 132, 132, 132, 1384, 132, 1413, 1413, 1411, 18, 18, - /* 230 */ 18, 18, 18, 18, 18, 18, 18, 18, 18, 1384, - /* 240 */ 1411, 439, 439, 439, 1292, 1398, 1418, 339, 1325, 1323, - /* 250 */ 1421, 339, 1330, 1384, 1378, 1378, 439, 1262, 1266, 439, - /* 260 */ 1262, 1266, 439, 439, 18, 1253, 1355, 1262, 1271, 1273, - /* 270 */ 1293, 1124, 1283, 1277, 1276, 1318, 344, 1550, 1384, 1426, - /* 280 */ 339, 339, 1566, 1266, 439, 439, 439, 439, 439, 1266, - /* 290 */ 439, 1427, 339, 503, 339, 344, 1504, 1508, 439, 558, - /* 300 */ 1384, 339, 1576, 1411, 2905, 2905, 2905, 2905, 2905, 2905, - /* 310 */ 2905, 2905, 2905, 34, 430, 15, 590, 734, 16, 839, + /* 130 */ 443, 443, 443, 843, 325, 18, 286, 392, 1, 1, + /* 140 */ 1, 539, 273, 273, 392, 627, 627, 627, 412, 615, + /* 150 */ 529, 56, 699, 56, 699, 699, 715, 778, 35, 35, + /* 160 */ 35, 35, 35, 35, 35, 35, 2185, 217, 288, 483, + /* 170 */ 522, 260, 313, 243, 536, 1032, 490, 646, 868, 616, + /* 180 */ 602, 916, 829, 60, 422, 829, 966, 939, 994, 1110, + /* 190 */ 1317, 1190, 1334, 1365, 1334, 1224, 1372, 1372, 1334, 1224, + /* 200 */ 1224, 1303, 1309, 1372, 1316, 1372, 1372, 1372, 1393, 1373, + /* 210 */ 1393, 1373, 1405, 412, 1408, 412, 1411, 1412, 412, 1411, + /* 220 */ 412, 412, 412, 1372, 412, 1385, 1385, 1393, 56, 56, + /* 230 */ 56, 56, 56, 56, 56, 56, 56, 56, 56, 1372, + /* 240 */ 1393, 699, 699, 699, 1258, 1364, 1405, 538, 1301, 1304, + /* 250 */ 1408, 538, 1308, 1372, 1365, 1365, 699, 1241, 1248, 699, + /* 260 */ 1241, 1248, 699, 699, 56, 1250, 1346, 1241, 1255, 1257, + /* 270 */ 1274, 1110, 1254, 1259, 1265, 1288, 627, 1522, 1372, 1411, + /* 280 */ 538, 538, 1528, 1248, 699, 699, 699, 699, 699, 1248, + /* 290 */ 699, 1382, 538, 715, 538, 627, 1485, 1489, 699, 778, + /* 300 */ 1372, 538, 1564, 1393, 3030, 3030, 3030, 3030, 3030, 3030, + /* 310 */ 3030, 3030, 3030, 34, 430, 15, 590, 734, 16, 839, /* 320 */ 79, 709, 792, 566, 856, 1023, 1023, 1023, 1023, 1023, /* 330 */ 1023, 1023, 1023, 1023, 1167, 743, 253, 253, 402, 57, - /* 340 */ 758, 774, 949, 1067, 800, 980, 64, 64, 959, 1007, - /* 350 */ 387, 959, 959, 959, 1149, 641, 1116, 1140, 1147, 1070, - /* 360 */ 1168, 1078, 1119, 1141, 1164, 532, 1190, 1207, 1250, 1251, - /* 370 */ 990, 1189, 1212, 868, 1216, 1231, 1233, 943, 1004, 1117, - /* 380 */ 1153, 1261, 1264, 1267, 1270, 1272, 1274, 1302, 1289, 1196, - /* 390 */ 1206, 987, 1291, 1238, 1286, 1317, 1320, 1322, 1326, 1328, - /* 400 */ 190, 1248, 1361, 1256, 1331, 1614, 1620, 1624, 1590, 1633, - /* 410 */ 1599, 1428, 1600, 1602, 1604, 1432, 1644, 1607, 1610, 1438, - /* 420 */ 1647, 1440, 1649, 1615, 1651, 1630, 1653, 1621, 1460, 1469, - /* 430 */ 1473, 1660, 1667, 1669, 1488, 1490, 1672, 1689, 1629, 1690, - /* 440 */ 1691, 1692, 1652, 1696, 1697, 1698, 1702, 1703, 1704, 1705, - /* 450 */ 1706, 1552, 1673, 1709, 1555, 1712, 1714, 1715, 1716, 1717, - /* 460 */ 1718, 1719, 1720, 1721, 1722, 1723, 1724, 1726, 1728, 1729, - /* 470 */ 1730, 1693, 1731, 1732, 1733, 1734, 1736, 1738, 1725, 1739, - /* 480 */ 1740, 1741, 1603, 1743, 1744, 1735, 1700, 1737, 1707, 1745, - /* 490 */ 1687, 1742, 1750, 1694, 1752, 1695, 1756, 1760, 1753, 1746, - /* 500 */ 1727, 1751, 1755, 1747, 1757, 1773, 1759, 1748, 1774, 1775, - /* 510 */ 1776, 1758, 1586, 1764, 1780, 1781, 1749, 1800, 1805, 1771, - /* 520 */ 1761, 1770, 1812, 1778, 1767, 1779, 1817, 1784, 1772, 1783, - /* 530 */ 1820, 1788, 1782, 1786, 1827, 1829, 1830, 1832, 1766, 1754, - /* 540 */ 1798, 1813, 1836, 1803, 1804, 1806, 1807, 1808, 1809, 1810, - /* 550 */ 1796, 1802, 1815, 1818, 1826, 1819, 1852, 1833, 1856, 1840, - /* 560 */ 1814, 1858, 1842, 1841, 1877, 1843, 1879, 1846, 1887, 1866, - /* 570 */ 1878, 1864, 1865, 1867, 1821, 1801, 1904, 1762, 1811, 1713, - /* 580 */ 1872, 1896, 1921, 1765, 1901, 1777, 1789, 1924, 1925, 1785, - /* 590 */ 1768, 1923, 1883, 1675, 1834, 1828, 1844, 1885, 1849, 1893, - /* 600 */ 1853, 1847, 1906, 1908, 1855, 1857, 1861, 1862, 1863, 1919, - /* 610 */ 1918, 1922, 1868, 1926, 1710, 1870, 1871, 1964, 1929, 1791, - /* 620 */ 1939, 1941, 1942, 1943, 1944, 1945, 1880, 1881, 1937, 1790, - /* 630 */ 1946, 1947, 1983, 1986, 1988, 1989, 1892, 1954, 1751, 1950, - /* 640 */ 1907, 1903, 1909, 1912, 1913, 1837, 1914, 2015, 1977, 1848, - /* 650 */ 1927, 1910, 1751, 1971, 1978, 1928, 1792, 1930, 2019, 2002, - /* 660 */ 1816, 1933, 1934, 1938, 1935, 1948, 1940, 1987, 1949, 1951, - /* 670 */ 1992, 1952, 2008, 1825, 1955, 1932, 1953, 2017, 2023, 1962, - /* 680 */ 1968, 2036, 1973, 1974, 2039, 1976, 1979, 2043, 1982, 1991, - /* 690 */ 2048, 1995, 1981, 1985, 1990, 1999, 2005, 2064, 2010, 2076, - /* 700 */ 2013, 2064, 2064, 2092, 2056, 2058, 2089, 2090, 2091, 2093, - /* 710 */ 2094, 2096, 2098, 2099, 2100, 2101, 2055, 2044, 2095, 2102, - /* 720 */ 2105, 2106, 2120, 2108, 2110, 2112, 2077, 1796, 2113, 1802, - /* 730 */ 2115, 2116, 2118, 2119, 2130, 2122, 2155, 2123, 2111, 2124, - /* 740 */ 2161, 2129, 2125, 2126, 2166, 2132, 2127, 2134, 2169, 2136, - /* 750 */ 2131, 2139, 2172, 2140, 2144, 2182, 2168, 2171, 2173, 2174, - /* 760 */ 2176, 2178, + /* 340 */ 441, 414, 838, 1040, 934, 636, 64, 64, 638, 606, + /* 350 */ 903, 638, 638, 638, 417, 927, 1007, 1070, 1067, 1057, + /* 360 */ 1138, 967, 1071, 1072, 1092, 1156, 1168, 1185, 1219, 1223, + /* 370 */ 1000, 1145, 1170, 431, 1171, 1172, 1176, 1089, 1117, 1105, + /* 380 */ 809, 1189, 1204, 1207, 1212, 1216, 1217, 1229, 1231, 1142, + /* 390 */ 1208, 1187, 1234, 1203, 1267, 1270, 1272, 1277, 1289, 1294, + /* 400 */ 190, 1299, 1348, 668, 1320, 1613, 1616, 1617, 1576, 1619, + /* 410 */ 1585, 1414, 1589, 1590, 1591, 1419, 1628, 1595, 1596, 1424, + /* 420 */ 1633, 1426, 1635, 1601, 1637, 1618, 1638, 1604, 1447, 1456, + /* 430 */ 1459, 1646, 1647, 1648, 1467, 1469, 1651, 1652, 1606, 1655, + /* 440 */ 1656, 1657, 1625, 1659, 1660, 1669, 1670, 1671, 1672, 1689, + /* 450 */ 1690, 1521, 1661, 1691, 1537, 1694, 1697, 1698, 1702, 1703, + /* 460 */ 1704, 1705, 1706, 1707, 1708, 1709, 1710, 1712, 1714, 1715, + /* 470 */ 1716, 1675, 1718, 1719, 1720, 1721, 1722, 1723, 1711, 1724, + /* 480 */ 1726, 1728, 1592, 1729, 1730, 1713, 1684, 1717, 1686, 1736, + /* 490 */ 1679, 1725, 1740, 1682, 1742, 1685, 1743, 1745, 1731, 1732, + /* 500 */ 1733, 1699, 1700, 1734, 1727, 1752, 1735, 1737, 1753, 1754, + /* 510 */ 1755, 1741, 1573, 1756, 1757, 1759, 1695, 1763, 1764, 1746, + /* 520 */ 1738, 1758, 1785, 1760, 1749, 1761, 1799, 1767, 1766, 1765, + /* 530 */ 1805, 1771, 1768, 1773, 1809, 1810, 1813, 1816, 1739, 1747, + /* 540 */ 1783, 1797, 1820, 1786, 1787, 1788, 1790, 1792, 1794, 1795, + /* 550 */ 1789, 1791, 1796, 1801, 1817, 1803, 1832, 1818, 1841, 1821, + /* 560 */ 1793, 1842, 1823, 1811, 1850, 1826, 1852, 1827, 1853, 1833, + /* 570 */ 1836, 1829, 1840, 1843, 1775, 1777, 1858, 1762, 1779, 1688, + /* 580 */ 1863, 1865, 1899, 1744, 1878, 1776, 1748, 1901, 1902, 1780, + /* 590 */ 1751, 1903, 1860, 1665, 1807, 1822, 1824, 1874, 1831, 1877, + /* 600 */ 1815, 1830, 1890, 1897, 1844, 1847, 1849, 1851, 1854, 1906, + /* 610 */ 1880, 1896, 1856, 1908, 1701, 1861, 1862, 1950, 1910, 1772, + /* 620 */ 1924, 1926, 1929, 1930, 1931, 1932, 1868, 1869, 1921, 1774, + /* 630 */ 1927, 1925, 1973, 1974, 1976, 1977, 1879, 1939, 1699, 1933, + /* 640 */ 1881, 1882, 1885, 1884, 1887, 1819, 1888, 1982, 1953, 1825, + /* 650 */ 1889, 1894, 1699, 1946, 1963, 1912, 1778, 1914, 1995, 1986, + /* 660 */ 1798, 1916, 1917, 1920, 1928, 1922, 1934, 1967, 1936, 1938, + /* 670 */ 1970, 1941, 1999, 1814, 1940, 1915, 1942, 1992, 1998, 1947, + /* 680 */ 1948, 2004, 1951, 1949, 2013, 1952, 1954, 2018, 1958, 1960, + /* 690 */ 2019, 1962, 1956, 1957, 1983, 1985, 1969, 1990, 1971, 2037, + /* 700 */ 1975, 1990, 1990, 2008, 2010, 2009, 2038, 2045, 2047, 2057, + /* 710 */ 2060, 2061, 2073, 2074, 2075, 2076, 2040, 2022, 2070, 2082, + /* 720 */ 2083, 2085, 2099, 2087, 2088, 2090, 2054, 1789, 2092, 1791, + /* 730 */ 2093, 2094, 2095, 2097, 2112, 2100, 2136, 2102, 2089, 2101, + /* 740 */ 2141, 2107, 2096, 2104, 2146, 2113, 2098, 2110, 2150, 2117, + /* 750 */ 2105, 2116, 2156, 2122, 2123, 2159, 2138, 2140, 2143, 2144, + /* 760 */ 2142, 2147, }; #define YY_REDUCE_COUNT (312) #define YY_REDUCE_MIN (-396) -#define YY_REDUCE_MAX (2476) +#define YY_REDUCE_MAX (2601) static const short yy_reduce_ofst[] = { /* 0 */ -140, 137, -82, 358, 444, 578, 814, 841, 1034, 1064, /* 10 */ 1132, 1240, 1260, 1344, 1368, 1445, 598, -333, 672, 1468, - /* 20 */ 901, 1491, 1514, 1577, 1642, 1663, 1676, 1763, 1787, 1850, - /* 30 */ 1874, 1920, 1936, 1996, 2020, 2047, 2066, 2117, 2133, 2163, - /* 40 */ 2184, 2230, 2290, 2303, 2354, 2400, 2415, 2461, 2476, 372, - /* 50 */ -195, -396, -265, 487, 595, 607, 701, 481, 761, -362, - /* 60 */ -355, 285, 749, 136, 612, -279, -55, 256, -346, -167, - /* 70 */ -5, -381, -334, -257, -205, -249, 45, 130, 300, 303, - /* 80 */ 356, 377, 409, 425, 432, 177, 469, 489, 252, -308, - /* 90 */ 521, 573, 170, 620, 172, 622, 261, 707, 396, 682, - /* 100 */ 739, 745, 278, 793, 715, 321, 781, 795, -266, 8, - /* 110 */ -350, -350, 115, -238, 161, 214, 257, 315, 374, 433, - /* 120 */ 445, 463, 585, 587, 617, 677, 681, 806, 807, 813, - /* 130 */ 819, 830, 832, -86, -289, -111, 23, 154, -289, 70, - /* 140 */ 188, -25, 480, 511, 380, 416, 498, 704, 381, -354, - /* 150 */ 426, 513, 690, 502, 733, 798, 808, 353, -369, -365, - /* 160 */ 324, 370, 458, 505, 686, 458, 764, 368, 621, 825, - /* 170 */ 753, 780, 906, 822, 917, 917, 946, 898, 954, 963, - /* 180 */ 930, 921, 871, 871, 891, 871, 916, 925, 917, 967, - /* 190 */ 971, 989, 1001, 1008, 1012, 1016, 1063, 1066, 1022, 1025, - /* 200 */ 1026, 1060, 1069, 1074, 1073, 1084, 1086, 1088, 1097, 1096, - /* 210 */ 1101, 1098, 1030, 1090, 1059, 1093, 1105, 1050, 1103, 1110, - /* 220 */ 1111, 1113, 1114, 1126, 1120, 1125, 1129, 1136, 1108, 1112, - /* 230 */ 1115, 1122, 1127, 1131, 1133, 1137, 1138, 1139, 1142, 1151, - /* 240 */ 1144, 1099, 1102, 1128, 1081, 1094, 1107, 1162, 1104, 1118, - /* 250 */ 1135, 1171, 1123, 1179, 1143, 1145, 1146, 1068, 1134, 1150, - /* 260 */ 1072, 1148, 1156, 1157, 917, 1080, 1082, 1085, 1092, 1091, - /* 270 */ 1106, 1152, 1076, 1095, 1130, 871, 1223, 1154, 1242, 1241, - /* 280 */ 1234, 1239, 1194, 1193, 1209, 1210, 1211, 1213, 1214, 1200, - /* 290 */ 1218, 1208, 1252, 1237, 1257, 1263, 1169, 1243, 1229, 1254, - /* 300 */ 1269, 1278, 1279, 1282, 1215, 1217, 1222, 1225, 1265, 1268, - /* 310 */ 1275, 1281, 1294, + /* 20 */ 901, 1491, 1514, 1577, 1642, 1663, 1676, 1750, 1770, 1867, + /* 30 */ 1883, 1913, 1943, 2007, 2020, 2071, 2084, 2145, 2196, 2211, + /* 40 */ 2257, 2308, 2325, 2354, 2422, 2441, 2454, 2527, 2540, 2555, + /* 50 */ 2601, 372, -195, -396, -265, 487, 595, 607, 701, 481, + /* 60 */ 761, -362, 191, -365, 136, -279, -55, 256, -372, -167, + /* 70 */ -5, -381, -273, -257, 14, -321, -337, 203, 45, 285, + /* 80 */ 300, 356, 377, 409, 425, 177, 432, 489, 446, 279, + /* 90 */ 521, 573, 125, 620, 321, 622, 396, 687, 436, 257, + /* 100 */ 707, 716, 460, 739, 434, 506, 679, 746, -266, 8, + /* 110 */ -350, -350, 123, -261, 265, 363, 579, 585, 612, 617, + /* 120 */ 710, 754, 769, 774, 777, 781, 806, 807, 813, 820, + /* 130 */ 830, 831, 832, -331, -289, -111, -249, -61, -289, 290, + /* 140 */ 563, -25, -338, 158, 357, -147, 498, 533, -102, 202, + /* 150 */ 324, 29, 378, 502, 593, 597, 808, 236, -367, 552, + /* 160 */ 584, 603, 614, 674, 857, 614, 785, 871, 945, 877, + /* 170 */ 811, 823, 937, 834, 920, 920, 946, 904, 963, 985, + /* 180 */ 926, 919, 882, 882, 890, 882, 917, 911, 920, 949, + /* 190 */ 957, 973, 996, 995, 999, 1002, 1051, 1052, 1010, 1013, + /* 200 */ 1014, 1050, 1055, 1068, 1060, 1074, 1076, 1078, 1079, 1083, + /* 210 */ 1088, 1085, 1017, 1077, 1046, 1080, 1090, 1035, 1087, 1091, + /* 220 */ 1093, 1096, 1099, 1097, 1100, 1094, 1095, 1116, 1084, 1101, + /* 230 */ 1102, 1103, 1107, 1108, 1111, 1112, 1114, 1115, 1122, 1098, + /* 240 */ 1119, 1082, 1113, 1124, 1053, 1106, 1059, 1131, 1104, 1109, + /* 250 */ 1123, 1148, 1118, 1158, 1120, 1127, 1126, 1054, 1125, 1128, + /* 260 */ 1058, 1129, 1139, 1141, 920, 1069, 1073, 1081, 1075, 1086, + /* 270 */ 1133, 1135, 1061, 1130, 1134, 882, 1198, 1144, 1201, 1200, + /* 280 */ 1197, 1199, 1155, 1163, 1180, 1188, 1193, 1206, 1209, 1174, + /* 290 */ 1210, 1173, 1220, 1227, 1236, 1239, 1150, 1221, 1214, 1237, + /* 300 */ 1256, 1249, 1268, 1266, 1202, 1196, 1211, 1213, 1244, 1251, + /* 310 */ 1252, 1262, 1281, }; static const YYACTIONTYPE yy_default[] = { /* 0 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, @@ -2454,8 +2478,8 @@ static const char *const yyRuleName[] = { /* 531 */ "sliding_opt ::= SLIDING NK_LP duration_literal NK_RP", /* 532 */ "fill_opt ::=", /* 533 */ "fill_opt ::= FILL NK_LP fill_mode NK_RP", - /* 534 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP", - /* 535 */ "fill_opt ::= FILL NK_LP VALUE_F NK_COMMA literal_list NK_RP", + /* 534 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA expression_list NK_RP", + /* 535 */ "fill_opt ::= FILL NK_LP VALUE_F NK_COMMA expression_list NK_RP", /* 536 */ "fill_mode ::= NONE", /* 537 */ "fill_mode ::= PREV", /* 538 */ "fill_mode ::= NULL", @@ -3663,8 +3687,8 @@ static const struct { { 396, -4 }, /* (531) sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ { 456, 0 }, /* (532) fill_opt ::= */ { 456, -4 }, /* (533) fill_opt ::= FILL NK_LP fill_mode NK_RP */ - { 456, -6 }, /* (534) fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ - { 456, -6 }, /* (535) fill_opt ::= FILL NK_LP VALUE_F NK_COMMA literal_list NK_RP */ + { 456, -6 }, /* (534) fill_opt ::= FILL NK_LP VALUE NK_COMMA expression_list NK_RP */ + { 456, -6 }, /* (535) fill_opt ::= FILL NK_LP VALUE_F NK_COMMA expression_list NK_RP */ { 463, -1 }, /* (536) fill_mode ::= NONE */ { 463, -1 }, /* (537) fill_mode ::= PREV */ { 463, -1 }, /* (538) fill_mode ::= NULL */ @@ -5351,10 +5375,10 @@ static YYACTIONTYPE yy_reduce( case 533: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ { yymsp[-3].minor.yy448 = createFillNode(pCxt, yymsp[-1].minor.yy46, NULL); } break; - case 534: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ + case 534: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA expression_list NK_RP */ { yymsp[-5].minor.yy448 = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, yymsp[-1].minor.yy432)); } break; - case 535: /* fill_opt ::= FILL NK_LP VALUE_F NK_COMMA literal_list NK_RP */ + case 535: /* fill_opt ::= FILL NK_LP VALUE_F NK_COMMA expression_list NK_RP */ { yymsp[-5].minor.yy448 = createFillNode(pCxt, FILL_MODE_VALUE_F, createNodeListNode(pCxt, yymsp[-1].minor.yy432)); } break; case 536: /* fill_mode ::= NONE */ From fdbb9cce48f66e8372c421cd9e28689da65f6e77 Mon Sep 17 00:00:00 2001 From: huolibo Date: Tue, 18 Apr 2023 16:08:22 +0800 Subject: [PATCH 11/35] fix: illegal accesses (#20960) --- source/client/src/clientJniConnector.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/source/client/src/clientJniConnector.c b/source/client/src/clientJniConnector.c index d2a9665eee..b613354751 100644 --- a/source/client/src/clientJniConnector.c +++ b/source/client/src/clientJniConnector.c @@ -1259,8 +1259,9 @@ JNIEXPORT jobject JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_schemalessInse int code = taos_errno(tres); if (code != TSDB_CODE_SUCCESS) { jniError("jobj:%p, conn:%p, code: 0x%x, msg:%s", jobj, taos, code, taos_errstr(tres)); + jobject jobject = createSchemalessResp(env, 0, code, taos_errstr(tres)); taos_free_result(tres); - return createSchemalessResp(env, 0, code, taos_errstr(tres)); + return jobject; } taos_free_result(tres); @@ -1286,8 +1287,9 @@ JNIEXPORT jobject JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_schemalessInse int code = taos_errno(tres); if (code != TSDB_CODE_SUCCESS) { jniError("jobj:%p, conn:%p, code: 0x%x, msg:%s", jobj, taos, code, taos_errstr(tres)); + jobject jobject = createSchemalessResp(env, 0, code, taos_errstr(tres)); taos_free_result(tres); - return createSchemalessResp(env, 0, code, taos_errstr(tres)); + return jobject; } taos_free_result(tres); @@ -1315,8 +1317,9 @@ JNIEXPORT jobject JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_schemalessInse int code = taos_errno(tres); if (code != TSDB_CODE_SUCCESS) { jniError("jobj:%p, conn:%p, code: 0x%x, msg:%s", jobj, taos, code, taos_errstr(tres)); + jobject jobject = createSchemalessResp(env, 0, code, taos_errstr(tres)); taos_free_result(tres); - return createSchemalessResp(env, 0, code, taos_errstr(tres)); + return jobject; } taos_free_result(tres); @@ -1343,8 +1346,9 @@ JNIEXPORT jobject JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_schemalessInse int code = taos_errno(tres); if (code != TSDB_CODE_SUCCESS) { jniError("jobj:%p, conn:%p, code: 0x%x, msg:%s", jobj, taos, code, taos_errstr(tres)); + jobject jobject = createSchemalessResp(env, 0, code, taos_errstr(tres)); taos_free_result(tres); - return createSchemalessResp(env, 0, code, taos_errstr(tres)); + return jobject; } taos_free_result(tres); From 01c3df9a556fc550c269e1d30bf1764026a2eda6 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 18 Apr 2023 17:12:49 +0800 Subject: [PATCH 12/35] fix test cases --- tests/system-test/2-query/To_unixtimestamp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/system-test/2-query/To_unixtimestamp.py b/tests/system-test/2-query/To_unixtimestamp.py index 00a8ed84de..424ebff6c5 100644 --- a/tests/system-test/2-query/To_unixtimestamp.py +++ b/tests/system-test/2-query/To_unixtimestamp.py @@ -98,8 +98,8 @@ class TDTestCase: def timestamp_change_return_type(self): tdSql.query(f"select to_unixtimestamp('1970-01-01 08:00:00+08:00', 0);") tdSql.checkEqual(tdSql.queryResult[0][0], 0) - tdSql.query(f"select to_unixtimestamp('1970-01-01 08:00:00+08:00', 1);") - tdSql.checkEqual(tdSql.queryResult[0][0], datetime.datetime(1970, 1, 1, 8, 0, 0)) + tdSql.query(f"select to_unixtimestamp('1970-01-01 00:00:00', 1);") + tdSql.checkData(0, 0, '1970-01-01 00:00:00') tdSql.error(f"select to_unixtimestamp('1970-01-01 08:00:00+08:00', 2);") tdSql.error(f"select to_unixtimestamp('1970-01-01 08:00:00+08:00', 1.5);") tdSql.error(f"select to_unixtimestamp('1970-01-01 08:00:00+08:00', 'abc');") From 6669c3a5ad392f242185f7a89b3fb2c2566ed766 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 18 Apr 2023 17:13:14 +0800 Subject: [PATCH 13/35] feat: last queries with tags output can be read from cache --- source/dnode/vnode/src/tsdb/tsdbCacheRead.c | 17 +++++------ source/libs/executor/src/cachescanoperator.c | 32 ++++++++++---------- source/libs/planner/src/planOptimizer.c | 21 ++++++++----- 3 files changed, 37 insertions(+), 33 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index 95981c2f08..f537b7d6e8 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -362,15 +362,6 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 p->ts = pCol->ts; p->colVal = pCol->colVal; singleTableLastTs = pCol->ts; - - // only set value for last row query - if (HASTYPE(pr->type, CACHESCAN_RETRIEVE_LAST_ROW)) { - if (taosArrayGetSize(pTableUidList) == 0) { - taosArrayPush(pTableUidList, &pKeyInfo->uid); - } else { - taosArraySet(pTableUidList, 0, &pKeyInfo->uid); - } - } } } else { SLastCol* p = taosArrayGet(pLastCols, slotId); @@ -417,6 +408,14 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 } } + if (0 == i) { + if (taosArrayGetSize(pTableUidList) == 0) { + taosArrayPush(pTableUidList, &pKeyInfo->uid); + } else { + taosArraySet(pTableUidList, 0, &pKeyInfo->uid); + } + } + tsdbCacheRelease(lruCache, h); } diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index f6fc332b37..61f024ebb7 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -45,13 +45,13 @@ static void destroyCacheScanOperator(void* param); static int32_t extractCacheScanSlotId(const SArray* pColMatchInfo, SExecTaskInfo* pTaskInfo, int32_t** pSlotIds); static int32_t removeRedundantTsCol(SLastRowScanPhysiNode* pScanNode, SColMatchInfo* pColMatchInfo); -#define SCAN_ROW_TYPE(_t) ((_t)? CACHESCAN_RETRIEVE_LAST : CACHESCAN_RETRIEVE_LAST_ROW) +#define SCAN_ROW_TYPE(_t) ((_t) ? CACHESCAN_RETRIEVE_LAST : CACHESCAN_RETRIEVE_LAST_ROW) SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SReadHandle* readHandle, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo) { - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; SCacheRowsScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SCacheRowsScanInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; tableListDestroy(pTableListInfo); @@ -91,7 +91,8 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe uint64_t suid = tableListGetSuid(pTableListInfo); code = tsdbCacherowsReaderOpen(pInfo->readHandle.vnode, pInfo->retrieveType, pList, totalTables, - taosArrayGetSize(pInfo->matchInfo.pList), suid, &pInfo->pLastrowReader, pTaskInfo->id.str); + taosArrayGetSize(pInfo->matchInfo.pList), suid, &pInfo->pLastrowReader, + pTaskInfo->id.str); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -114,7 +115,8 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe p->pCtx = createSqlFunctionCtx(p->pExprInfo, p->numOfExprs, &p->rowEntryInfoOffset); } - setOperatorInfo(pOperator, "CachedRowScanOperator", QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); + setOperatorInfo(pOperator, "CachedRowScanOperator", QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN, false, OP_NOT_OPENED, + pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock); pOperator->fpSet = @@ -123,7 +125,7 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe pOperator->cost.openCost = 0; return pOperator; - _error: +_error: pTaskInfo->code = code; destroyCacheScanOperator(pInfo); taosMemoryFree(pOperator); @@ -136,8 +138,8 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { } SCacheRowsScanInfo* pInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - STableListInfo* pTableList = pInfo->pTableList; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + STableListInfo* pTableList = pInfo->pTableList; uint64_t suid = tableListGetSuid(pTableList); int32_t size = tableListGetSize(pTableList); @@ -194,8 +196,8 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { pRes->info.rows = 1; SExprSupp* pSup = &pInfo->pseudoExprSup; - int32_t code = addTagPseudoColumnData(&pInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pRes, - pRes->info.rows, GET_TASKID(pTaskInfo), NULL); + int32_t code = addTagPseudoColumnData(&pInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pRes, + pRes->info.rows, GET_TASKID(pTaskInfo), NULL); if (code != TSDB_CODE_SUCCESS) { pTaskInfo->code = code; return NULL; @@ -217,7 +219,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { } STableKeyInfo* pList = NULL; - int32_t num = 0; + int32_t num = 0; int32_t code = tableListGetGroupList(pTableList, pInfo->currentGroupIndex, &pList, &num); if (code != TSDB_CODE_SUCCESS) { @@ -251,11 +253,9 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { pInfo->pRes->info.id.groupId = pKeyInfo->groupId; if (taosArrayGetSize(pInfo->pUidList) > 0) { - ASSERT((pInfo->retrieveType & CACHESCAN_RETRIEVE_LAST_ROW) == CACHESCAN_RETRIEVE_LAST_ROW); - pInfo->pRes->info.id.uid = *(tb_uid_t*)taosArrayGet(pInfo->pUidList, 0); - code = addTagPseudoColumnData(&pInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pInfo->pRes, pInfo->pRes->info.rows, - GET_TASKID(pTaskInfo), NULL); + code = addTagPseudoColumnData(&pInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pInfo->pRes, + pInfo->pRes->info.rows, GET_TASKID(pTaskInfo), NULL); if (code != TSDB_CODE_SUCCESS) { pTaskInfo->code = code; return NULL; @@ -325,7 +325,7 @@ int32_t removeRedundantTsCol(SLastRowScanPhysiNode* pScanNode, SColMatchInfo* pC return TSDB_CODE_SUCCESS; } - size_t size = taosArrayGetSize(pColMatchInfo->pList); + size_t size = taosArrayGetSize(pColMatchInfo->pList); SArray* pMatchInfo = taosArrayInit(size, sizeof(SColMatchItem)); for (int32_t i = 0; i < size; ++i) { diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 52bb03466c..d3a03b7b75 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1095,7 +1095,7 @@ static int32_t sortPriKeyOptGetSequencingNodesImpl(SLogicNode* pNode, bool group *pNotOptimize = false; return TSDB_CODE_SUCCESS; } - + switch (nodeType(pNode)) { case QUERY_NODE_LOGIC_PLAN_SCAN: { SScanLogicNode* pScan = (SScanLogicNode*)pNode; @@ -2139,7 +2139,7 @@ typedef struct SLastRowScanOptLastParaCkCxt { bool hasCol; } SLastRowScanOptLastParaCkCxt; -static EDealRes lastRowScanOptLastParaCheckImpl(SNode* pNode, void* pContext) { +static EDealRes lastRowScanOptLastParaIsTagImpl(SNode* pNode, void* pContext) { if (QUERY_NODE_COLUMN == nodeType(pNode)) { SLastRowScanOptLastParaCkCxt* pCxt = pContext; if (COLUMN_TYPE_TAG == ((SColumnNode*)pNode)->colType || COLUMN_TYPE_TBNAME == ((SColumnNode*)pNode)->colType) { @@ -2152,10 +2152,10 @@ static EDealRes lastRowScanOptLastParaCheckImpl(SNode* pNode, void* pContext) { return DEAL_RES_CONTINUE; } -static bool lastRowScanOptLastParaCheck(SNode* pExpr) { +static bool lastRowScanOptLastParaIsTag(SNode* pExpr) { SLastRowScanOptLastParaCkCxt cxt = {.hasTag = false, .hasCol = false}; - nodesWalkExpr(pExpr, lastRowScanOptLastParaCheckImpl, &cxt); - return !cxt.hasTag && cxt.hasCol; + nodesWalkExpr(pExpr, lastRowScanOptLastParaIsTagImpl, &cxt); + return cxt.hasTag && !cxt.hasCol; } static bool hasSuitableCache(int8_t cacheLastMode, bool hasLastRow, bool hasLast) { @@ -2195,15 +2195,19 @@ static bool lastRowScanOptMayBeOptimized(SLogicNode* pNode) { FOREACH(pFunc, ((SAggLogicNode*)pNode)->pAggFuncs) { SFunctionNode* pAggFunc = (SFunctionNode*)pFunc; if (FUNCTION_TYPE_LAST == pAggFunc->funcType) { - if (hasSelectFunc || !lastRowScanOptLastParaCheck(nodesListGetNode(pAggFunc->pParameterList, 0))) { + if (hasSelectFunc) { return false; } hasLastFunc = true; - } else if (FUNCTION_TYPE_SELECT_VALUE == pAggFunc->funcType || FUNCTION_TYPE_GROUP_KEY == pAggFunc->funcType) { + } else if (FUNCTION_TYPE_SELECT_VALUE == pAggFunc->funcType) { if (hasLastFunc) { return false; } hasSelectFunc = true; + } else if (FUNCTION_TYPE_GROUP_KEY == pAggFunc->funcType) { + if (!lastRowScanOptLastParaIsTag(nodesListGetNode(pAggFunc->pParameterList, 0))) { + return false; + } } else if (FUNCTION_TYPE_LAST_ROW != pAggFunc->funcType) { return false; } @@ -2282,6 +2286,8 @@ static int32_t lastRowScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogic nodesWalkExpr(nodesListGetNode(pFunc->pParameterList, 0), lastRowScanOptSetColDataType, &cxt); nodesListErase(pFunc->pParameterList, nodesListGetCell(pFunc->pParameterList, 1)); } + } else if (FUNCTION_TYPE_GROUP_KEY == funcType) { + nodesListMakeAppend(&cxt.pLastCols, nodesListGetNode(pFunc->pParameterList, 0)); } } @@ -2291,7 +2297,6 @@ static int32_t lastRowScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogic if (NULL != cxt.pLastCols) { cxt.doAgg = false; lastRowScanOptSetLastTargets(pScan->pScanCols, cxt.pLastCols); - NODES_DESTORY_LIST(pScan->pScanPseudoCols); lastRowScanOptSetLastTargets(pScan->node.pTargets, cxt.pLastCols); nodesClearList(cxt.pLastCols); } From 1431a313b18813e50cddcc064e694e036736d26a Mon Sep 17 00:00:00 2001 From: sunpeng Date: Tue, 18 Apr 2023 17:15:48 +0800 Subject: [PATCH 14/35] docs: fix for monitor doc (#20963) --- docs/zh/17-operation/10-monitor.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/17-operation/10-monitor.md b/docs/zh/17-operation/10-monitor.md index b31ada1b79..918642c3d6 100644 --- a/docs/zh/17-operation/10-monitor.md +++ b/docs/zh/17-operation/10-monitor.md @@ -41,7 +41,7 @@ chmod +x TDinsight.sh ## log 库 -TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db，可以在 taoskeeper 配置文件中修改，具体参考 [taoskeeper 文档](..../reference/taosKeeper)）。taoskeeper 启动后会自动创建 log 库，并将监控数据写入到该数据库中。 +TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db，可以在 taoskeeper 配置文件中修改，具体参考 [taoskeeper 文档](/reference/taosKeeper)）。taoskeeper 启动后会自动创建 log 库，并将监控数据写入到该数据库中。 ### cluster info 表 From 57e8c3c0915f7e9903592b0a40354a8ec3b1d796 Mon Sep 17 00:00:00 2001 From: sunpeng Date: Tue, 18 Apr 2023 17:48:19 +0800 Subject: [PATCH 15/35] docs: update title for monitor doc (#20965) --- docs/zh/17-operation/10-monitor.md | 39 +++++++++++++++--------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/docs/zh/17-operation/10-monitor.md b/docs/zh/17-operation/10-monitor.md index 918642c3d6..01a2257286 100644 --- a/docs/zh/17-operation/10-monitor.md +++ b/docs/zh/17-operation/10-monitor.md @@ -43,7 +43,7 @@ chmod +x TDinsight.sh TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db，可以在 taoskeeper 配置文件中修改，具体参考 [taoskeeper 文档](/reference/taosKeeper)）。taoskeeper 启动后会自动创建 log 库，并将监控数据写入到该数据库中。 -### cluster info 表 +### cluster\_info 表 `cluster_info` 表记录集群信息。 @@ -72,7 +72,7 @@ TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db， |protocol|INT||协议版本，目前为 1| |cluster\_id|NCHAR|TAG|cluster id| -### d info 表 +### d\_info 表 `d_info` 表记录 dnode 状态信息。 @@ -83,7 +83,7 @@ TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db， |dnode\_ep|NCHAR|TAG|dnode endpoint| |cluster\_id|NCHAR|TAG|cluster id| -### m info 表 +### m\_info 表 `m_info` 表记录 mnode 角色信息。 @@ -95,7 +95,7 @@ TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db， |mnode\_ep|NCHAR|TAG|master node endpoint| |cluster\_id|NCHAR|TAG|cluster id| -### dnodes info 表 +### dnodes\_info 表 `dnodes_info` 记录 dnode 信息。 @@ -137,7 +137,7 @@ TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db， |dnode\_ep|NCHAR|TAG|dnode endpoint| |cluster\_id|NCHAR|TAG|cluster id| -### data dir 表 +### data\_dir 表 `data_dir` 表记录 data 目录信息。 @@ -153,7 +153,7 @@ TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db， |dnode\_ep|NCHAR|TAG|dnode endpoint| |cluster\_id|NCHAR|TAG|cluster id| -### log dir 表 +### log\_dir 表 `log_dir` 表记录 log 目录信息。 @@ -168,7 +168,7 @@ TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db， |dnode\_ep|NCHAR|TAG|dnode endpoint| |cluster\_id|NCHAR|TAG|cluster id| -### tmp dir 表 +### temp\_dir 表 `temp_dir` 表记录 temp 目录信息。 @@ -183,7 +183,7 @@ TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db， |dnode\_ep|NCHAR|TAG|dnode endpoint| |cluster\_id|NCHAR|TAG|cluster id| -### vgroups info 表 +### vgroups\_info 表 `vgroups_info` 表记录虚拟节点组信息。 @@ -198,7 +198,7 @@ TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db， |dnode\_ep|NCHAR|TAG|dnode endpoint| |cluster\_id|NCHAR|TAG|cluster id| -### vnodes role 表 +### vnodes\_role 表 `vnodes_role` 表记录虚拟节点角色信息。 @@ -223,7 +223,7 @@ TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db， |dnode\_ep|NCHAR|TAG|dnode endpoint| |cluster\_id|NCHAR|TAG|cluster id| -### log summary 表 +### log\_summary 表 `log_summary` 记录日志统计信息。 @@ -238,7 +238,7 @@ TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db， |dnode\_ep|NCHAR|TAG|dnode endpoint| |cluster\_id|NCHAR|TAG|cluster id| -### grants info 表 +### grants\_info 表 `grants_info` 记录授权信息。 @@ -252,7 +252,7 @@ TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db， |dnode\_ep|NCHAR|TAG|dnode endpoint| |cluster\_id|NCHAR|TAG|cluster id| -### keeper monitor 表 +### keeper\_monitor 表 `keeper_monitor` 记录 taoskeeper 监控数据。 @@ -263,7 +263,7 @@ TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db， |mem|FLOAT||内存使用率| |identify|NCHAR|TAG|| -### taosadapter restful http request total 表 +### taosadapter\_restful\_http\_request\_total 表 `taosadapter_restful_http_request_total` 记录 taosadapter rest 请求信息，该表为 schemaless 方式创建的表，时间戳字段名为 `_ts`。 @@ -277,7 +277,7 @@ TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db， |request\_uri|NCHAR|TAG|request uri| |status\_code|NCHAR|TAG|status code| -### taosadapter restful http request fail 表 +### taosadapter\_restful\_http\_request\_fail 表 `taosadapter_restful_http_request_fail` 记录 taosadapter rest 请求失败信息，该表为 schemaless 方式创建的表，时间戳字段名为 `_ts`。 @@ -291,7 +291,7 @@ TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db， |request\_uri|NCHAR|TAG|request uri| |status\_code|NCHAR|TAG|status code| -### taosadapter restful http request in flight 表 +### taosadapter\_restful\_http\_request\_in\_flight 表 `taosadapter_restful_http_request_in_flight` 记录 taosadapter rest 实时请求信息，该表为 schemaless 方式创建的表，时间戳字段名为 `_ts`。 @@ -301,7 +301,7 @@ TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db， |guage|DOUBLE||监控指标值| |endpoint|NCHAR|TAG|taosadpater endpoint| -### taosadapter restful http request summary milliseconds 表 +### taosadapter\_restful\_http\_request\_summary\_milliseconds 表 `taosadapter_restful_http_request_summary_milliseconds` 记录 taosadapter rest 请求汇总信息，该表为 schemaless 方式创建的表，时间戳字段名为 `_ts`。 @@ -319,7 +319,7 @@ TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db， |request\_method|NCHAR|TAG|request method| |request\_uri|NCHAR|TAG|request uri| -### taosadapter system mem percent 表 +### taosadapter\_system\_mem\_percent 表 `taosadapter_system_mem_percent` 表记录 taosadapter 内存使用情况，该表为 schemaless 方式创建的表，时间戳字段名为 `_ts`。 @@ -329,11 +329,12 @@ TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db， |guage|DOUBLE||监控指标值| |endpoint|NCHAR|TAG|taosadpater endpoint| -### taosadapter system cpu percent 表 +### taosadapter\_system\_cpu\_percent 表 + +`taosadapter_system_cpu_percent` 表记录 taosadapter cpu 使用情况，该表为 schemaless 方式创建的表，时间戳字段名为 `_ts`。 |field|type|is\_tag|comment| |:----|:---|:-----|:------| |\_ts|TIMESTAMP||timestamp| |guage|DOUBLE||监控指标值| |endpoint|NCHAR|TAG|taosadpater endpoint| - From 230adc5433dc6697e7abf11a7b4458cd39880b94 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 18 Apr 2023 18:38:17 +0800 Subject: [PATCH 16/35] Update 23-perf.md --- docs/zh/12-taos-sql/23-perf.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/12-taos-sql/23-perf.md b/docs/zh/12-taos-sql/23-perf.md index d4ee0e178c..fc0b8072a7 100644 --- a/docs/zh/12-taos-sql/23-perf.md +++ b/docs/zh/12-taos-sql/23-perf.md @@ -69,7 +69,7 @@ TDengine 3.0 版本开始提供一个内置数据库 `performance_schema`，其 | 1 | consumer_id | BIGINT | 消费者的唯一 ID | | 2 | consumer_group | BINARY(192) | 消费者组 | | 3 | client_id | BINARY(192) | 用户自定义字符串，通过创建 consumer 时指定 client_id 来展示 | -| 4 | status | BINARY(20) | 消费者当前状态 | +| 4 | status | BINARY(20) | 消费者当前状态。消费者状态包括：ready(正常可用)、 lost(连接已丢失)、 rebalancing(消费者所属 vgroup 正在分配中)、unknown(未知状态)| | 5 | topics | BINARY(204) | 被订阅的 topic。若订阅多个 topic，则展示为多行 | | 6 | up_time | TIMESTAMP | 第一次连接 taosd 的时间 | | 7 | subscribe_time | TIMESTAMP | 上一次发起订阅的时间 | From 3b78d4382b68fe57daa0726a80f88a825bc767e4 Mon Sep 17 00:00:00 2001 From: cadem Date: Tue, 18 Apr 2023 19:03:45 +0800 Subject: [PATCH 17/35] add learner --- include/common/tmsg.h | 13 +- include/common/tmsgdef.h | 2 + include/dnode/mnode/mnode.h | 7 +- include/libs/sync/sync.h | 24 ++- include/util/tdef.h | 1 + source/common/src/tmsg.c | 36 ++++ source/dnode/mgmt/mgmt_dnode/inc/dmInt.h | 1 + source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 1 + source/dnode/mgmt/mgmt_dnode/src/dmInt.c | 1 + source/dnode/mgmt/mgmt_dnode/src/dmWorker.c | 3 + source/dnode/mgmt/mgmt_mnode/src/mmFile.c | 23 +- source/dnode/mgmt/mgmt_mnode/src/mmHandle.c | 18 +- source/dnode/mgmt/mgmt_mnode/src/mmInt.c | 10 +- source/dnode/mgmt/mgmt_vnode/inc/vmInt.h | 1 + source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 187 ++++++++++++++-- source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 3 + source/dnode/mgmt/node_mgmt/src/dmEnv.c | 61 ++++++ source/dnode/mgmt/node_util/inc/dmUtil.h | 4 + source/dnode/mnode/impl/inc/mndDef.h | 5 +- source/dnode/mnode/impl/inc/mndInt.h | 5 +- source/dnode/mnode/impl/src/mndMain.c | 8 + source/dnode/mnode/impl/src/mndMnode.c | 174 ++++++++++++--- source/dnode/mnode/impl/src/mndSync.c | 23 +- source/dnode/mnode/impl/src/mndVgroup.c | 125 +++++++++-- source/dnode/vnode/inc/vnode.h | 1 + source/dnode/vnode/src/vnd/vnodeCfg.c | 41 +++- source/dnode/vnode/src/vnd/vnodeOpen.c | 35 ++- source/dnode/vnode/src/vnd/vnodeSvr.c | 3 +- source/dnode/vnode/src/vnd/vnodeSync.c | 16 +- source/libs/sync/inc/syncIndexMgr.h | 11 +- source/libs/sync/inc/syncInt.h | 32 +-- source/libs/sync/inc/syncMessage.h | 1 + source/libs/sync/inc/syncPipeline.h | 2 + source/libs/sync/src/syncAppendEntries.c | 6 +- source/libs/sync/src/syncElection.c | 2 + source/libs/sync/src/syncIndexMgr.c | 22 +- source/libs/sync/src/syncMain.c | 224 +++++++++++++++----- source/libs/sync/src/syncPipeline.c | 19 +- source/libs/sync/src/syncRaftCfg.c | 38 +++- source/libs/sync/src/syncReplication.c | 4 +- source/libs/sync/src/syncSnapshot.c | 11 +- source/libs/sync/src/syncTimeout.c | 2 +- source/libs/sync/src/syncVoteMgr.c | 8 +- 43 files changed, 1014 insertions(+), 200 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index bb2450e8f7..b3b665d323 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1286,6 +1286,9 @@ typedef struct { int16_t hashSuffix; int32_t tsdbPageSize; int64_t reserved[8]; + int8_t learnerReplica; + int8_t learnerSelfIndex; + SReplica learnerReplicas[TSDB_MAX_LEARNER_REPLICA]; } SCreateVnodeReq; int32_t tSerializeSCreateVnodeReq(void* buf, int32_t bufLen, SCreateVnodeReq* pReq); @@ -1357,7 +1360,10 @@ typedef struct { int8_t replica; SReplica replicas[TSDB_MAX_REPLICA]; int64_t reserved[8]; -} SAlterVnodeReplicaReq; + int8_t learnerSelfIndex; + int8_t learnerReplica; + SReplica learnerReplicas[TSDB_MAX_LEARNER_REPLICA]; +} SAlterVnodeReplicaReq, SAlterVnodeTypeReq; int32_t tSerializeSAlterVnodeReplicaReq(void* buf, int32_t bufLen, SAlterVnodeReplicaReq* pReq); int32_t tDeserializeSAlterVnodeReplicaReq(void* buf, int32_t bufLen, SAlterVnodeReplicaReq* pReq); @@ -1629,7 +1635,10 @@ int32_t tDeserializeSCreateDropMQSNodeReq(void* buf, int32_t bufLen, SMCreateQno typedef struct { int8_t replica; SReplica replicas[TSDB_MAX_REPLICA]; -} SDCreateMnodeReq, SDAlterMnodeReq; + int8_t learnerReplica; + SReplica learnerReplicas[TSDB_MAX_LEARNER_REPLICA]; + int64_t lastIndex; +} SDCreateMnodeReq, SDAlterMnodeReq, SDAlterMnodeTypeReq; int32_t tSerializeSDCreateMnodeReq(void* buf, int32_t bufLen, SDCreateMnodeReq* pReq); int32_t tDeserializeSDCreateMnodeReq(void* buf, int32_t bufLen, SDCreateMnodeReq* pReq); diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 6cf6140815..9dca9ea322 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -83,6 +83,8 @@ enum { TD_DEF_MSG_TYPE(TDMT_DND_CONFIG_DNODE, "config-dnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_SYSTABLE_RETRIEVE, "dnode-retrieve", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_MAX_MSG, "dnd-max", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_DND_ALTER_MNODE_TYPE, "dnode-alter-mnode-type", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_DND_ALTER_VNODE_TYPE, "dnode-alter-vnode-type", NULL, NULL) TD_NEW_MSG_SEG(TDMT_MND_MSG) TD_DEF_MSG_TYPE(TDMT_MND_CONNECT, "connect", NULL, NULL) diff --git a/include/dnode/mnode/mnode.h b/include/dnode/mnode/mnode.h index cdb1642a5c..07a0ca952a 100644 --- a/include/dnode/mnode/mnode.h +++ b/include/dnode/mnode/mnode.h @@ -33,8 +33,11 @@ typedef struct { bool deploy; int8_t selfIndex; int8_t numOfReplicas; - SReplica replicas[TSDB_MAX_REPLICA]; + int8_t numOfTotalReplicas; + SReplica replicas[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; + int32_t nodeRoles[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; SMsgCb msgCb; + int64_t lastIndex; } SMnodeOpt; /* ------------------------ SMnode ------------------------ */ @@ -69,6 +72,8 @@ int32_t mndStart(SMnode *pMnode); */ void mndStop(SMnode *pMnode); +int32_t mndIsCatchUp(SMnode *pMnode); + /** * @brief Get mnode monitor info. * diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index fdb4506cf3..33cef538d2 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -55,6 +55,8 @@ extern "C" { #define SYNC_INDEX_INVALID -1 #define SYNC_TERM_INVALID -1 +#define SYNC_LEARNER_CATCHUP 10 + typedef enum { SYNC_STRATEGY_NO_SNAPSHOT = 0, SYNC_STRATEGY_STANDARD_SNAPSHOT = 1, @@ -76,19 +78,29 @@ typedef enum { TAOS_SYNC_STATE_CANDIDATE = 101, TAOS_SYNC_STATE_LEADER = 102, TAOS_SYNC_STATE_ERROR = 103, + TAOS_SYNC_STATE_LEARNER = 104, } ESyncState; +typedef enum { + TAOS_SYNC_ROLE_VOTER = 0, + TAOS_SYNC_ROLE_LEARNER = 1, + TAOS_SYNC_ROLE_ERROR = 2, +} ESyncRole; + typedef struct SNodeInfo { - int64_t clusterId; - int32_t nodeId; - uint16_t nodePort; - char nodeFqdn[TSDB_FQDN_LEN]; + int64_t clusterId; + int32_t nodeId; + uint16_t nodePort; + char nodeFqdn[TSDB_FQDN_LEN]; + ESyncRole nodeRole; } SNodeInfo; typedef struct SSyncCfg { + int32_t totalReplicaNum; int32_t replicaNum; int32_t myIndex; - SNodeInfo nodeInfo[TSDB_MAX_REPLICA]; + SNodeInfo nodeInfo[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; + SyncIndex lastIndex; } SSyncCfg; typedef struct SFsmCbMeta { @@ -155,6 +167,7 @@ typedef struct SSyncFSM { void (*FpBecomeLeaderCb)(const struct SSyncFSM* pFsm); void (*FpBecomeFollowerCb)(const struct SSyncFSM* pFsm); + void (*FpBecomeLearnerCb)(const struct SSyncFSM* pFsm); int32_t (*FpGetSnapshot)(const struct SSyncFSM* pFsm, SSnapshot* pSnapshot, void* pReaderParam, void** ppReader); void (*FpGetSnapshotInfo)(const struct SSyncFSM* pFsm, SSnapshot* pSnapshot); @@ -236,6 +249,7 @@ void syncStop(int64_t rid); void syncPreStop(int64_t rid); void syncPostStop(int64_t rid); int32_t syncPropose(int64_t rid, SRpcMsg* pMsg, bool isWeak, int64_t* seq); +int32_t syncIsCatchUp(int64_t rid); int32_t syncProcessMsg(int64_t rid, SRpcMsg* pMsg); int32_t syncReconfig(int64_t rid, SSyncCfg* pCfg); int32_t syncBeginSnapshot(int64_t rid, int64_t lastApplyIndex); diff --git a/include/util/tdef.h b/include/util/tdef.h index 2f86395dad..1e05903407 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -285,6 +285,7 @@ typedef enum ELogicConditionType { #define TSDB_DNODE_ROLE_VNODE 2 #define TSDB_MAX_REPLICA 5 +#define TSDB_MAX_LEARNER_REPLICA 10 #define TSDB_SYNC_LOG_BUFFER_SIZE 4096 #define TSDB_SYNC_LOG_BUFFER_RETENTION (TSDB_SYNC_LOG_BUFFER_SIZE >> 4) diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index d9802244b7..c785dff9f8 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -4129,6 +4129,12 @@ int32_t tSerializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *pR for (int32_t i = 0; i < 8; ++i) { if (tEncodeI64(&encoder, pReq->reserved[i]) < 0) return -1; } + if (tEncodeI8(&encoder, pReq->learnerReplica) < 0) return -1; + if (tEncodeI8(&encoder, pReq->learnerSelfIndex) < 0) return -1; + for (int32_t i = 0; i < TSDB_MAX_LEARNER_REPLICA; ++i) { + SReplica *pReplica = &pReq->learnerReplicas[i]; + if (tEncodeSReplica(&encoder, pReplica) < 0) return -1; + } tEndEncode(&encoder); @@ -4207,6 +4213,12 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq * for (int32_t i = 0; i < 8; ++i) { if (tDecodeI64(&decoder, &pReq->reserved[i]) < 0) return -1; } + if (tDecodeI8(&decoder, &pReq->learnerReplica) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->learnerSelfIndex) < 0) return -1; + for (int32_t i = 0; i < TSDB_MAX_LEARNER_REPLICA; ++i) { + SReplica *pReplica = &pReq->learnerReplicas[i]; + if (tDecodeSReplica(&decoder, pReplica) < 0) return -1; + } tEndDecode(&decoder); tDecoderClear(&decoder); @@ -4433,6 +4445,12 @@ int32_t tSerializeSAlterVnodeReplicaReq(void *buf, int32_t bufLen, SAlterVnodeRe for (int32_t i = 0; i < 8; ++i) { if (tEncodeI64(&encoder, pReq->reserved[i]) < 0) return -1; } + if (tEncodeI8(&encoder, pReq->learnerSelfIndex) < 0) return -1; + if (tEncodeI8(&encoder, pReq->learnerReplica) < 0) return -1; + for (int32_t i = 0; i < TSDB_MAX_LEARNER_REPLICA; ++i) { + SReplica *pReplica = &pReq->learnerReplicas[i]; + if (tEncodeSReplica(&encoder, pReplica) < 0) return -1; + } tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -4456,6 +4474,12 @@ int32_t tDeserializeSAlterVnodeReplicaReq(void *buf, int32_t bufLen, SAlterVnode for (int32_t i = 0; i < 8; ++i) { if (tDecodeI64(&decoder, &pReq->reserved[i]) < 0) return -1; } + if (tDecodeI8(&decoder, &pReq->learnerSelfIndex) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->learnerReplica) < 0) return -1; + for (int32_t i = 0; i < TSDB_MAX_LEARNER_REPLICA; ++i) { + SReplica *pReplica = &pReq->learnerReplicas[i]; + if (tDecodeSReplica(&decoder, pReplica) < 0) return -1; + } tEndDecode(&decoder); tDecoderClear(&decoder); @@ -4767,6 +4791,12 @@ int32_t tSerializeSDCreateMnodeReq(void *buf, int32_t bufLen, SDCreateMnodeReq * SReplica *pReplica = &pReq->replicas[i]; if (tEncodeSReplica(&encoder, pReplica) < 0) return -1; } + if (tEncodeI8(&encoder, pReq->learnerReplica) < 0) return -1; + for (int32_t i = 0; i < TSDB_MAX_LEARNER_REPLICA; ++i) { + SReplica *pReplica = &pReq->learnerReplicas[i]; + if (tEncodeSReplica(&encoder, pReplica) < 0) return -1; + } + if (tEncodeI64(&encoder, pReq->lastIndex) < 0) return -1; tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -4784,6 +4814,12 @@ int32_t tDeserializeSDCreateMnodeReq(void *buf, int32_t bufLen, SDCreateMnodeReq SReplica *pReplica = &pReq->replicas[i]; if (tDecodeSReplica(&decoder, pReplica) < 0) return -1; } + if (tDecodeI8(&decoder, &pReq->learnerReplica) < 0) return -1; + for (int32_t i = 0; i < TSDB_MAX_LEARNER_REPLICA; ++i) { + SReplica *pReplica = &pReq->learnerReplicas[i]; + if (tDecodeSReplica(&decoder, pReplica) < 0) return -1; + } + if (tDecodeI64(&decoder, &pReq->lastIndex) < 0) return -1; tEndDecode(&decoder); tDecoderClear(&decoder); diff --git a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h index 600a1f6829..6c40fa808f 100644 --- a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h +++ b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h @@ -32,6 +32,7 @@ typedef struct SDnodeMgmt { TdThread crashReportThread; SSingleWorker mgmtWorker; ProcessCreateNodeFp processCreateNodeFp; + ProcessAlterNodeTypeFp processAlterNodeTypeFp; ProcessDropNodeFp processDropNodeFp; SendMonitorReportFp sendMonitorReportFp; GetVnodeLoadsFp getVnodeLoadsFp; diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 228f301aec..1998269896 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -352,6 +352,7 @@ SArray *dmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_DND_CONFIG_DNODE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_SERVER_STATUS, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_SYSTABLE_RETRIEVE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_ALTER_MNODE_TYPE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; // Requests handled by MNODE if (dmSetMgmtHandle(pArray, TDMT_MND_GRANT, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c index 51df293ba7..09783a5ea9 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c @@ -48,6 +48,7 @@ static int32_t dmOpenMgmt(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { pMgmt->path = pInput->path; pMgmt->name = pInput->name; pMgmt->processCreateNodeFp = pInput->processCreateNodeFp; + pMgmt->processAlterNodeTypeFp = pInput->processAlterNodeTypeFp; pMgmt->processDropNodeFp = pInput->processDropNodeFp; pMgmt->sendMonitorReportFp = pInput->sendMonitorReportFp; pMgmt->getVnodeLoadsFp = pInput->getVnodeLoadsFp; diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index 0d3b423771..06b6221940 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -227,6 +227,9 @@ static void dmProcessMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { case TDMT_DND_DROP_SNODE: code = (*pMgmt->processDropNodeFp)(SNODE, pMsg); break; + case TDMT_DND_ALTER_MNODE_TYPE: + code = (*pMgmt->processAlterNodeTypeFp)(MNODE, pMsg); + break; case TDMT_DND_SERVER_STATUS: code = dmProcessServerRunStatus(pMgmt, pMsg); break; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmFile.c b/source/dnode/mgmt/mgmt_mnode/src/mmFile.c index f06669a610..cb0849f4b9 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmFile.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmFile.c @@ -24,12 +24,16 @@ static int32_t mmDecodeOption(SJson *pJson, SMnodeOpt *pOption) { if (code < 0) return -1; tjsonGetInt32ValueFromDouble(pJson, "selfIndex", pOption->selfIndex, code); if (code < 0) return 0; + tjsonGetInt32ValueFromDouble(pJson, "lastIndex", pOption->lastIndex, code); + if (code < 0) return 0; SJson *replicas = tjsonGetObjectItem(pJson, "replicas"); if (replicas == NULL) return 0; - pOption->numOfReplicas = tjsonGetArraySize(replicas); + pOption->numOfTotalReplicas = tjsonGetArraySize(replicas); - for (int32_t i = 0; i < pOption->numOfReplicas; ++i) { + pOption->numOfReplicas = 0; + + for (int32_t i = 0; i < pOption->numOfTotalReplicas; ++i) { SJson *replica = tjsonGetArrayItem(replicas, i); if (replica == NULL) return -1; @@ -40,6 +44,14 @@ static int32_t mmDecodeOption(SJson *pJson, SMnodeOpt *pOption) { if (code < 0) return -1; tjsonGetUInt16ValueFromDouble(replica, "port", pReplica->port, code); if (code < 0) return -1; + tjsonGetInt32ValueFromDouble(replica, "role", pOption->nodeRoles[i], code); + if (code < 0) return -1; + if(pOption->nodeRoles[i] == TAOS_SYNC_ROLE_VOTER){ + pOption->numOfReplicas++; + } + } + + for (int32_t i = 0; i < pOption->numOfTotalReplicas; ++i) { } return 0; @@ -112,14 +124,14 @@ _OVER: } static int32_t mmEncodeOption(SJson *pJson, const SMnodeOpt *pOption) { - if (pOption->deploy && pOption->numOfReplicas > 0) { + if (pOption->deploy && pOption->numOfTotalReplicas > 0) { if (tjsonAddDoubleToObject(pJson, "selfIndex", pOption->selfIndex) < 0) return -1; SJson *replicas = tjsonCreateArray(); if (replicas == NULL) return -1; if (tjsonAddItemToObject(pJson, "replicas", replicas) < 0) return -1; - for (int32_t i = 0; i < pOption->numOfReplicas; ++i) { + for (int32_t i = 0; i < pOption->numOfTotalReplicas; ++i) { SJson *replica = tjsonCreateObject(); if (replica == NULL) return -1; @@ -127,10 +139,13 @@ static int32_t mmEncodeOption(SJson *pJson, const SMnodeOpt *pOption) { if (tjsonAddDoubleToObject(replica, "id", pReplica->id) < 0) return -1; if (tjsonAddStringToObject(replica, "fqdn", pReplica->fqdn) < 0) return -1; if (tjsonAddDoubleToObject(replica, "port", pReplica->port) < 0) return -1; + if (tjsonAddDoubleToObject(replica, "role", pOption->nodeRoles[i]) < 0) return -1; if (tjsonAddItemToArray(replicas, replica) < 0) return -1; } } + if (tjsonAddDoubleToObject(pJson, "lastIndex", pOption->lastIndex) < 0) return -1; + if (tjsonAddDoubleToObject(pJson, "deployed", pOption->deploy) < 0) return -1; return 0; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index 76977dd4a8..76fe0ec2de 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -33,12 +33,24 @@ int32_t mmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) { return -1; } - SMnodeOpt option = {.deploy = true, .numOfReplicas = createReq.replica, .selfIndex = -1}; + SMnodeOpt option = {.deploy = true, .numOfReplicas = createReq.replica, + .numOfTotalReplicas = createReq.replica + createReq.learnerReplica, + .selfIndex = -1, .lastIndex = createReq.lastIndex}; + memcpy(option.replicas, createReq.replicas, sizeof(createReq.replicas)); - for (int32_t i = 0; i < option.numOfReplicas; ++i) { + for (int32_t i = 0; i < createReq.replica; ++i) { if (createReq.replicas[i].id == pInput->pData->dnodeId) { option.selfIndex = i; } + option.nodeRoles[i] = TAOS_SYNC_ROLE_VOTER; + } + + memcpy(&(option.replicas[createReq.replica]), createReq.learnerReplicas, sizeof(createReq.learnerReplicas)); + for (int32_t i = 0; i < createReq.learnerReplica; ++i) { + if (createReq.learnerReplicas[i].id == pInput->pData->dnodeId) { + option.selfIndex = createReq.replica + i; + } + option.nodeRoles[createReq.replica + i] = TAOS_SYNC_ROLE_LEARNER; } if (option.selfIndex == -1) { @@ -92,6 +104,8 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_VNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_VNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_CONFIG_DNODE_RSP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_ALTER_MNODE_TYPE_RSP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_ALTER_VNODE_TYPE_RSP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CONNECT, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_ACCT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c index e7f71ad420..05b59b9865 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c @@ -45,9 +45,11 @@ static void mmBuildOptionForDeploy(SMnodeMgmt *pMgmt, const SMgmtInputOpt *pInpu pOption->dnodeId = pMgmt->pData->dnodeId; pOption->selfIndex = 0; pOption->numOfReplicas = 1; + pOption->numOfTotalReplicas = 1; pOption->replicas[0].id = 1; pOption->replicas[0].port = tsServerPort; tstrncpy(pOption->replicas[0].fqdn, tsLocalFqdn, TSDB_FQDN_LEN); + pOption->lastIndex = SYNC_INDEX_INVALID; } static void mmBuildOptionForOpen(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) { @@ -123,9 +125,10 @@ static int32_t mmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { } tmsgReportStartup("mnode-worker", "initialized"); - if (option.numOfReplicas > 0) { + if (option.numOfTotalReplicas > 0) { option.deploy = true; option.numOfReplicas = 0; + option.numOfTotalReplicas = 0; if (mmWriteFile(pMgmt->path, &option) != 0) { dError("failed to write mnode file since %s", terrstr()); return -1; @@ -152,6 +155,10 @@ static void mmStop(SMnodeMgmt *pMgmt) { mndStop(pMgmt->pMnode); } +static int32_t mmSyncIsCatchUp(SMnodeMgmt *pMgmt) { + return mndIsCatchUp(pMgmt->pMnode); +} + SMgmtFunc mmGetMgmtFunc() { SMgmtFunc mgmtFunc = {0}; mgmtFunc.openFp = mmOpen; @@ -162,6 +169,7 @@ SMgmtFunc mmGetMgmtFunc() { mgmtFunc.dropFp = (NodeDropFp)mmProcessDropReq; mgmtFunc.requiredFp = mmRequire; mgmtFunc.getHandlesFp = mmGetMsgHandles; + mgmtFunc.isCatchUpFp = (NodeIsCatchUpFp)mmSyncIsCatchUp; return mgmtFunc; } diff --git a/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h b/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h index 4374ae363c..83fb331dbd 100644 --- a/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h +++ b/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h @@ -90,6 +90,7 @@ int32_t vmProcessDropVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t vmProcessAlterVnodeReplicaReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t vmProcessDisableVnodeWriteReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t vmProcessAlterHashRangeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); +int32_t vmProcessAlterVnodeTypeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); // vmFile.c int32_t vmGetVnodeListFromFile(SVnodeMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t *numOfVnodes); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index d61eb3ec03..42dfb873d3 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -131,15 +131,34 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { pCfg->tsdbPageSize = pCreate->tsdbPageSize * 1024; pCfg->standby = 0; - pCfg->syncCfg.myIndex = pCreate->selfIndex; - pCfg->syncCfg.replicaNum = pCreate->replica; + pCfg->syncCfg.replicaNum = 0; + pCfg->syncCfg.totalReplicaNum = 0; + memset(&pCfg->syncCfg.nodeInfo, 0, sizeof(pCfg->syncCfg.nodeInfo)); for (int32_t i = 0; i < pCreate->replica; ++i) { SNodeInfo *pNode = &pCfg->syncCfg.nodeInfo[i]; - pNode->nodeId = pCreate->replicas[i].id; - pNode->nodePort = pCreate->replicas[i].port; - tstrncpy(pNode->nodeFqdn, pCreate->replicas[i].fqdn, TSDB_FQDN_LEN); + pNode->nodeId = pCreate->replicas[pCfg->syncCfg.replicaNum].id; + pNode->nodePort = pCreate->replicas[pCfg->syncCfg.replicaNum].port; + pNode->nodeRole = TAOS_SYNC_ROLE_VOTER; + tstrncpy(pNode->nodeFqdn, pCreate->replicas[pCfg->syncCfg.replicaNum].fqdn, TSDB_FQDN_LEN); tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort); + pCfg->syncCfg.replicaNum++; + } + if(pCreate->selfIndex != -1){ + pCfg->syncCfg.myIndex = pCreate->selfIndex; + } + for (int32_t i = pCfg->syncCfg.replicaNum; i < pCreate->replica + pCreate->learnerReplica; ++i) { + SNodeInfo *pNode = &pCfg->syncCfg.nodeInfo[i]; + pNode->nodeId = pCreate->learnerReplicas[pCfg->syncCfg.totalReplicaNum].id; + pNode->nodePort = pCreate->learnerReplicas[pCfg->syncCfg.totalReplicaNum].port; + pNode->nodeRole = TAOS_SYNC_ROLE_LEARNER; + tstrncpy(pNode->nodeFqdn, pCreate->learnerReplicas[pCfg->syncCfg.totalReplicaNum].fqdn, TSDB_FQDN_LEN); + tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort); + pCfg->syncCfg.totalReplicaNum++; + } + pCfg->syncCfg.totalReplicaNum += pCfg->syncCfg.replicaNum; + if(pCreate->learnerSelfIndex != -1){ + pCfg->syncCfg.myIndex = pCreate->replica + pCreate->learnerSelfIndex; } } @@ -182,23 +201,35 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return -1; } - dInfo("vgId:%d, start to create vnode, page:%d pageSize:%d buffer:%d szPage:%d szBuf:%" PRIu64 + dInfo("vgId:%d, vnode management handle msgType:%s, start to create vnode, page:%d pageSize:%d buffer:%d szPage:%d szBuf:%" PRIu64 ", cacheLast:%d cacheLastSize:%d sstTrigger:%d tsdbPageSize:%d %d dbname:%s dbId:%" PRId64 ", days:%d keep0:%d keep1:%d keep2:%d tsma:%d precision:%d compression:%d minRows:%d maxRows:%d" ", wal fsync:%d level:%d retentionPeriod:%d retentionSize:%" PRId64 " rollPeriod:%d segSize:%" PRId64 - ", hash method:%d begin:%u end:%u prefix:%d surfix:%d replica:%d selfIndex:%d strict:%d", - req.vgId, req.pages, req.pageSize, req.buffer, req.pageSize * 1024, (uint64_t)req.buffer * 1024 * 1024, + ", hash method:%d begin:%u end:%u prefix:%d surfix:%d replica:%d selfIndex:%d " + "learnerReplica:%d learnerSelfIndex:%d strict:%d", + req.vgId, TMSG_INFO(pMsg->msgType), req.pages, req.pageSize, req.buffer, req.pageSize * 1024, + (uint64_t)req.buffer * 1024 * 1024, req.cacheLast, req.cacheLastSize, req.sstTrigger, req.tsdbPageSize, req.tsdbPageSize * 1024, req.db, req.dbUid, req.daysPerFile, req.daysToKeep0, req.daysToKeep1, req.daysToKeep2, req.isTsma, req.precision, req.compression, req.minRows, req.maxRows, req.walFsyncPeriod, req.walLevel, req.walRetentionPeriod, req.walRetentionSize, req.walRollPeriod, req.walSegmentSize, req.hashMethod, req.hashBegin, req.hashEnd, req.hashPrefix, - req.hashSuffix, req.replica, req.selfIndex, req.strict); + req.hashSuffix, req.replica, req.selfIndex, req.learnerReplica, req.learnerSelfIndex, req.strict); for (int32_t i = 0; i < req.replica; ++i) { dInfo("vgId:%d, replica:%d ep:%s:%u dnode:%d", req.vgId, i, req.replicas[i].fqdn, req.replicas[i].port, req.replicas[i].id); } + for (int32_t i = 0; i < req.learnerReplica; ++i) { + dInfo("vgId:%d, learnerReplica:%d ep:%s:%u dnode:%d", req.vgId, i, req.learnerReplicas[i].fqdn, + req.learnerReplicas[i].port, req.replicas[i].id); + } - SReplica *pReplica = &req.replicas[req.selfIndex]; + SReplica *pReplica = NULL; + if(req.selfIndex != -1){ + pReplica = &req.replicas[req.selfIndex]; + } + else{ + pReplica = &req.learnerReplicas[req.learnerSelfIndex]; + } if (pReplica->id != pMgmt->pData->dnodeId || pReplica->port != tsServerPort || strcmp(pReplica->fqdn, tsLocalFqdn) != 0) { terrno = TSDB_CODE_INVALID_MSG; @@ -275,7 +306,8 @@ _OVER: vnodeClose(pImpl); vnodeDestroy(path, pMgmt->pTfs); } else { - dInfo("vgId:%d, vnode is created", req.vgId); + dInfo("vgId:%d, vnode management handle msgType:%s, end to create vnode, vnode is created", + req.vgId, TMSG_INFO(pMsg->msgType)); } tFreeSCreateVnodeReq(&req); @@ -283,6 +315,106 @@ _OVER: return code; } +int32_t vmProcessAlterVnodeTypeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { + SAlterVnodeTypeReq req = {0}; + if (tDeserializeSAlterVnodeReplicaReq(pMsg->pCont, pMsg->contLen, &req) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + dInfo("vgId:%d, vnode management handle msgType:%s, start to process alter-node-type-request", + req.vgId, TMSG_INFO(pMsg->msgType)); + + SVnodeObj *pVnode = vmAcquireVnode(pMgmt, req.vgId); + if (pVnode == NULL) { + dError("vgId:%d, failed to alter hashrange since %s", req.vgId, terrstr()); + terrno = TSDB_CODE_VND_NOT_EXIST; + return -1; + } + + dInfo("vgId:%d, checking node catch up", req.vgId); + if(!vnodeIsCatchUp(pVnode->pImpl) == 0){ + return -1; + } + + dInfo("node:%s, catched up leader, continue to process alter-node-type-request", pMgmt->name); + + int32_t vgId = req.vgId; + dInfo("vgId:%d, start to alter vnode type replica:%d selfIndex:%d strict:%d", vgId, req.replica, req.selfIndex, + req.strict); + for (int32_t i = 0; i < req.replica; ++i) { + SReplica *pReplica = &req.replicas[i]; + dInfo("vgId:%d, replica:%d ep:%s:%u dnode:%d", vgId, i, pReplica->fqdn, pReplica->port, pReplica->id); + } + for (int32_t i = 0; i < req.learnerReplica; ++i) { + SReplica *pReplica = &req.learnerReplicas[i]; + dInfo("vgId:%d, learnerReplicas:%d ep:%s:%u dnode:%d", vgId, i, pReplica->fqdn, pReplica->port, pReplica->id); + } + + if (req.replica <= 0 || + (req.selfIndex < 0 && req.learnerSelfIndex <0)|| + req.selfIndex >= req.replica || req.learnerSelfIndex >= req.learnerReplica) { + terrno = TSDB_CODE_INVALID_MSG; + dError("vgId:%d, failed to alter replica since invalid msg", vgId); + return -1; + } + + SReplica *pReplica = NULL; + if(req.selfIndex > 0){ + pReplica = &req.replicas[req.selfIndex]; + } + else{ + pReplica = &req.learnerReplicas[req.learnerSelfIndex]; + } + + if (pReplica->id != pMgmt->pData->dnodeId || pReplica->port != tsServerPort || + strcmp(pReplica->fqdn, tsLocalFqdn) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + dError("vgId:%d, dnodeId:%d ep:%s:%u not matched with local dnode", vgId, pReplica->id, pReplica->fqdn, + pReplica->port); + return -1; + } + + dInfo("vgId:%d, start to close vnode", vgId); + SWrapperCfg wrapperCfg = { + .dropped = pVnode->dropped, + .vgId = pVnode->vgId, + .vgVersion = pVnode->vgVersion, + }; + tstrncpy(wrapperCfg.path, pVnode->path, sizeof(wrapperCfg.path)); + vmCloseVnode(pMgmt, pVnode, false); + + char path[TSDB_FILENAME_LEN] = {0}; + snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, vgId); + + dInfo("vgId:%d, start to alter vnode replica at %s", vgId, path); + if (vnodeAlterReplica(path, &req, pMgmt->pTfs) < 0) { + dError("vgId:%d, failed to alter vnode at %s since %s", vgId, path, terrstr()); + return -1; + } + + dInfo("vgId:%d, begin to open vnode", vgId); + SVnode *pImpl = vnodeOpen(path, pMgmt->pTfs, pMgmt->msgCb); + if (pImpl == NULL) { + dError("vgId:%d, failed to open vnode at %s since %s", vgId, path, terrstr()); + return -1; + } + + if (vmOpenVnode(pMgmt, &wrapperCfg, pImpl) != 0) { + dError("vgId:%d, failed to open vnode mgmt since %s", vgId, terrstr()); + return -1; + } + + if (vnodeStart(pImpl) != 0) { + dError("vgId:%d, failed to start sync since %s", vgId, terrstr()); + return -1; + } + + dInfo("vgId:%d, vnode management handle msgType:%s, end to process alter-node-type-request, vnode config is altered", + req.vgId, TMSG_INFO(pMsg->msgType)); + return 0; +} + int32_t vmProcessDisableVnodeWriteReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { SDisableVnodeWriteReq req = {0}; if (tDeserializeSDisableVnodeWriteReq(pMsg->pCont, pMsg->contLen, &req) != 0) { @@ -378,20 +510,35 @@ int32_t vmProcessAlterVnodeReplicaReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { } int32_t vgId = alterReq.vgId; - dInfo("vgId:%d, start to alter vnode replica:%d selfIndex:%d strict:%d", vgId, alterReq.replica, alterReq.selfIndex, - alterReq.strict); + dInfo("vgId:%d,vnode management handle msgType:%s, start to alter vnode replica:%d selfIndex:%d leanerReplica:%d " + "learnerSelfIndex:%d strict:%d", + vgId, TMSG_INFO(pMsg->msgType), alterReq.replica, alterReq.selfIndex, alterReq.learnerReplica, + alterReq.learnerSelfIndex, alterReq.strict); for (int32_t i = 0; i < alterReq.replica; ++i) { SReplica *pReplica = &alterReq.replicas[i]; dInfo("vgId:%d, replica:%d ep:%s:%u dnode:%d", vgId, i, pReplica->fqdn, pReplica->port, pReplica->port); } - - if (alterReq.replica <= 0 || alterReq.selfIndex < 0 || alterReq.selfIndex >= alterReq.replica) { + for (int32_t i = 0; i < alterReq.learnerReplica; ++i) { + SReplica *pReplica = &alterReq.learnerReplicas[i]; + dInfo("vgId:%d, learnerReplicas:%d ep:%s:%u dnode:%d", vgId, i, pReplica->fqdn, pReplica->port, pReplica->port); + } + + if (alterReq.replica <= 0 || + (alterReq.selfIndex < 0 && alterReq.learnerSelfIndex <0)|| + alterReq.selfIndex >= alterReq.replica || alterReq.learnerSelfIndex >= alterReq.learnerReplica) { terrno = TSDB_CODE_INVALID_MSG; dError("vgId:%d, failed to alter replica since invalid msg", vgId); return -1; } - SReplica *pReplica = &alterReq.replicas[alterReq.selfIndex]; + SReplica *pReplica = NULL; + if(alterReq.selfIndex != -1){ + pReplica = &alterReq.replicas[alterReq.selfIndex]; + } + else{ + pReplica = &alterReq.learnerReplicas[alterReq.learnerSelfIndex]; + } + if (pReplica->id != pMgmt->pData->dnodeId || pReplica->port != tsServerPort || strcmp(pReplica->fqdn, tsLocalFqdn) != 0) { terrno = TSDB_CODE_INVALID_MSG; @@ -425,7 +572,7 @@ int32_t vmProcessAlterVnodeReplicaReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return -1; } - dInfo("vgId:%d, close vnode", vgId); + dInfo("vgId:%d, begin to open vnode", vgId); SVnode *pImpl = vnodeOpen(path, pMgmt->pTfs, pMgmt->msgCb); if (pImpl == NULL) { dError("vgId:%d, failed to open vnode at %s since %s", vgId, path, terrstr()); @@ -442,7 +589,10 @@ int32_t vmProcessAlterVnodeReplicaReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return -1; } - dInfo("vgId:%d, vnode config is altered", vgId); + dInfo("vgId:%d, vnode management handle msgType:%s, end to alter vnode replica:%d selfIndex:%d leanerReplica:%d " + "learnerSelfIndex:%d strict:%d", + vgId, TMSG_INFO(pMsg->msgType), alterReq.replica, alterReq.selfIndex, alterReq.learnerReplica, + alterReq.learnerSelfIndex, alterReq.strict); return 0; } @@ -548,6 +698,7 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_TRIM, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_VNODE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_VNODE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_ALTER_VNODE_TYPE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_TIMEOUT_ELECTION, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index da08bd01ac..a7c4b2e70e 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -49,6 +49,9 @@ static void vmProcessMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { case TDMT_VND_ALTER_HASHRANGE: code = vmProcessAlterHashRangeReq(pMgmt, pMsg); break; + case TDMT_DND_ALTER_VNODE_TYPE: + code = vmProcessAlterVnodeTypeReq(pMgmt, pMsg); + break; default: terrno = TSDB_CODE_MSG_NOT_PROCESSED; dGError("msg:%p, not processed in vnode-mgmt queue", pMsg); diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c index acf96ad397..c53c21cd30 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c +++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c @@ -169,6 +169,8 @@ static int32_t dmProcessCreateNodeReq(EDndNodeType ntype, SRpcMsg *pMsg) { return -1; } + dInfo("start to process create-node-request"); + pWrapper = &pDnode->wrappers[ntype]; if (taosMkDir(pWrapper->path) != 0) { dmReleaseWrapper(pWrapper); @@ -198,6 +200,64 @@ static int32_t dmProcessCreateNodeReq(EDndNodeType ntype, SRpcMsg *pMsg) { return code; } +static int32_t dmProcessAlterNodeTypeReq(EDndNodeType ntype, SRpcMsg *pMsg) { + SDnode *pDnode = dmInstance(); + + SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, ntype); + if (pWrapper == NULL) { + dError("fail to process alter node type since node not exist"); + return -1; + } + dmReleaseWrapper(pWrapper); + + dInfo("node:%s, start to process alter-node-type-request", pWrapper->name); + + pWrapper = &pDnode->wrappers[ntype]; + + if(pWrapper->func.isCatchUpFp != NULL){ + dInfo("node:%s, checking node catch up", pWrapper->name); + if(!(*pWrapper->func.isCatchUpFp)(pWrapper->pMgmt) == 0){ + return -1; + } + } + + dInfo("node:%s, catched up leader, continue to process alter-node-type-request", pWrapper->name); + + taosThreadMutexLock(&pDnode->mutex); + + dInfo("node:%s, stopping node", pWrapper->name); + dmStopNode(pWrapper); + dInfo("node:%s, closing node", pWrapper->name); + dmCloseNode(pWrapper); + + pWrapper = &pDnode->wrappers[ntype]; + if (taosMkDir(pWrapper->path) != 0) { + dmReleaseWrapper(pWrapper); + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to create dir:%s since %s", pWrapper->path, terrstr()); + return -1; + } + + SMgmtInputOpt input = dmBuildMgmtInputOpt(pWrapper); + + dInfo("node:%s, start to create", pWrapper->name); + int32_t code = (*pWrapper->func.createFp)(&input, pMsg); + if (code != 0) { + dError("node:%s, failed to create since %s", pWrapper->name, terrstr()); + } else { + dInfo("node:%s, has been created", pWrapper->name); + code = dmOpenNode(pWrapper); + if (code == 0) { + code = dmStartNode(pWrapper); + } + pWrapper->deployed = true; + pWrapper->required = true; + } + + taosThreadMutexUnlock(&pDnode->mutex); + return code; +} + static int32_t dmProcessDropNodeReq(EDndNodeType ntype, SRpcMsg *pMsg) { SDnode *pDnode = dmInstance(); @@ -251,6 +311,7 @@ SMgmtInputOpt dmBuildMgmtInputOpt(SMgmtWrapper *pWrapper) { .name = pWrapper->name, .pData = &pWrapper->pDnode->data, .processCreateNodeFp = dmProcessCreateNodeReq, + .processAlterNodeTypeFp = dmProcessAlterNodeTypeReq, .processDropNodeFp = dmProcessDropNodeReq, .sendMonitorReportFp = dmSendMonitorReport, .getVnodeLoadsFp = dmGetVnodeLoads, diff --git a/source/dnode/mgmt/node_util/inc/dmUtil.h b/source/dnode/mgmt/node_util/inc/dmUtil.h index cfdea40477..000ce81207 100644 --- a/source/dnode/mgmt/node_util/inc/dmUtil.h +++ b/source/dnode/mgmt/node_util/inc/dmUtil.h @@ -89,6 +89,7 @@ typedef void (*SendMonitorReportFp)(); typedef void (*GetVnodeLoadsFp)(SMonVloadInfo *pInfo); typedef void (*GetMnodeLoadsFp)(SMonMloadInfo *pInfo); typedef void (*GetQnodeLoadsFp)(SQnodeLoad *pInfo); +typedef int32_t (*ProcessAlterNodeTypeFp)(EDndNodeType ntype, SRpcMsg *pMsg); typedef struct { int32_t dnodeId; @@ -112,6 +113,7 @@ typedef struct { SDnodeData *pData; SMsgCb msgCb; ProcessCreateNodeFp processCreateNodeFp; + ProcessAlterNodeTypeFp processAlterNodeTypeFp; ProcessDropNodeFp processDropNodeFp; SendMonitorReportFp sendMonitorReportFp; GetVnodeLoadsFp getVnodeLoadsFp; @@ -132,6 +134,7 @@ typedef int32_t (*NodeCreateFp)(const SMgmtInputOpt *pInput, SRpcMsg *pMsg); typedef int32_t (*NodeDropFp)(const SMgmtInputOpt *pInput, SRpcMsg *pMsg); typedef int32_t (*NodeRequireFp)(const SMgmtInputOpt *pInput, bool *required); typedef SArray *(*NodeGetHandlesFp)(); // array of SMgmtHandle +typedef bool (*NodeIsCatchUpFp)(void *pMgmt); typedef struct { NodeOpenFp openFp; @@ -142,6 +145,7 @@ typedef struct { NodeDropFp dropFp; NodeRequireFp requiredFp; NodeGetHandlesFp getHandlesFp; + NodeIsCatchUpFp isCatchUpFp; } SMgmtFunc; typedef struct { diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 3a4f06f6fa..cc295f0d24 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -215,6 +215,8 @@ typedef struct { bool syncRestore; int64_t stateStartTime; SDnodeObj* pDnode; + int32_t role; + SyncIndex lastIndex; } SMnodeObj; typedef struct { @@ -341,6 +343,7 @@ typedef struct { ESyncState syncState; bool syncRestore; bool syncCanRead; + ESyncRole nodeRole; } SVnodeGid; typedef struct { @@ -361,7 +364,7 @@ typedef struct { int8_t compact; int8_t isTsma; int8_t replica; - SVnodeGid vnodeGid[TSDB_MAX_REPLICA]; + SVnodeGid vnodeGid[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; void* pTsma; int32_t numOfCachedTables; } SVgObj; diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index ffb2443808..ec83f8a7e6 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -92,8 +92,11 @@ typedef struct { int64_t transSeq; TdThreadMutex lock; int8_t selfIndex; + int8_t numOfTotalReplicas; int8_t numOfReplicas; - SReplica replicas[TSDB_MAX_REPLICA]; + SReplica replicas[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; + ESyncRole nodeRoles[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; + SyncIndex lastIndex; } SSyncMgmt; typedef struct { diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 5c20887cf5..13ae4a11d5 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -491,7 +491,10 @@ static void mndSetOptions(SMnode *pMnode, const SMnodeOpt *pOption) { pMnode->selfDnodeId = pOption->dnodeId; pMnode->syncMgmt.selfIndex = pOption->selfIndex; pMnode->syncMgmt.numOfReplicas = pOption->numOfReplicas; + pMnode->syncMgmt.numOfTotalReplicas = pOption->numOfTotalReplicas; + pMnode->syncMgmt.lastIndex = pOption->lastIndex; memcpy(pMnode->syncMgmt.replicas, pOption->replicas, sizeof(pOption->replicas)); + memcpy(pMnode->syncMgmt.nodeRoles, pOption->nodeRoles, sizeof(pOption->nodeRoles)); } SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) { @@ -582,6 +585,11 @@ int32_t mndStart(SMnode *pMnode) { return mndInitTimer(pMnode); } +int32_t mndIsCatchUp(SMnode *pMnode) { + int64_t rid = pMnode->syncMgmt.sync; + return syncIsCatchUp(rid); +} + void mndStop(SMnode *pMnode) { mndSetStop(pMnode); mndSyncStop(pMnode); diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 50fab447e3..53baf843de 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -23,7 +23,7 @@ #include "mndTrans.h" #include "tmisce.h" -#define MNODE_VER_NUMBER 1 +#define MNODE_VER_NUMBER 2 #define MNODE_RESERVE_SIZE 64 static int32_t mndCreateDefaultMnode(SMnode *pMnode); @@ -53,6 +53,7 @@ int32_t mndInitMnode(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_CREATE_MNODE, mndProcessCreateMnodeReq); mndSetMsgHandle(pMnode, TDMT_DND_CREATE_MNODE_RSP, mndTransProcessRsp); + mndSetMsgHandle(pMnode, TDMT_DND_ALTER_MNODE_TYPE_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_MND_ALTER_MNODE, mndProcessAlterMnodeReq); mndSetMsgHandle(pMnode, TDMT_MND_ALTER_MNODE_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_MND_DROP_MNODE, mndProcessDropMnodeReq); @@ -126,6 +127,8 @@ static SSdbRaw *mndMnodeActionEncode(SMnodeObj *pObj) { SDB_SET_INT32(pRaw, dataPos, pObj->id, _OVER) SDB_SET_INT64(pRaw, dataPos, pObj->createdTime, _OVER) SDB_SET_INT64(pRaw, dataPos, pObj->updateTime, _OVER) + SDB_SET_INT32(pRaw, dataPos, pObj->role, _OVER) + SDB_SET_INT64(pRaw, dataPos, pObj->lastIndex, _OVER) SDB_SET_RESERVE(pRaw, dataPos, MNODE_RESERVE_SIZE, _OVER) terrno = 0; @@ -149,7 +152,7 @@ static SSdbRow *mndMnodeActionDecode(SSdbRaw *pRaw) { int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; - if (sver != MNODE_VER_NUMBER) { + if (sver != 1 && sver != 2) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; goto _OVER; } @@ -164,6 +167,10 @@ static SSdbRow *mndMnodeActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &pObj->id, _OVER) SDB_GET_INT64(pRaw, dataPos, &pObj->createdTime, _OVER) SDB_GET_INT64(pRaw, dataPos, &pObj->updateTime, _OVER) + if(sver >=2){ + SDB_GET_INT32(pRaw, dataPos, &pObj->role, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pObj->lastIndex, _OVER) + } SDB_GET_RESERVE(pRaw, dataPos, MNODE_RESERVE_SIZE, _OVER) terrno = 0; @@ -204,7 +211,9 @@ static int32_t mndMnodeActionDelete(SSdb *pSdb, SMnodeObj *pObj) { static int32_t mndMnodeActionUpdate(SSdb *pSdb, SMnodeObj *pOld, SMnodeObj *pNew) { mTrace("mnode:%d, perform update action, old row:%p new row:%p", pOld->id, pOld, pNew); + pOld->role = pNew->role; pOld->updateTime = pNew->updateTime; + pOld->lastIndex = pNew->lastIndex; mndReloadSyncConfig(pSdb->pMnode); return 0; @@ -302,6 +311,27 @@ static int32_t mndBuildCreateMnodeRedoAction(STrans *pTrans, SDCreateMnodeReq *p return 0; } +static int32_t mndBuildAlterMnodeTypeRedoAction(STrans *pTrans, + SDAlterMnodeTypeReq *pAlterMnodeTypeReq, SEpSet *pAlterMnodeTypeEpSet) { + int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, pAlterMnodeTypeReq); + void *pReq = taosMemoryMalloc(contLen); + tSerializeSDCreateMnodeReq(pReq, contLen, pAlterMnodeTypeReq); + + STransAction action = { + .epSet = *pAlterMnodeTypeEpSet, + .pCont = pReq, + .contLen = contLen, + .msgType = TDMT_DND_ALTER_MNODE_TYPE, + .acceptableCode = TSDB_CODE_MNODE_ALREADY_DEPLOYED, + }; + + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } + return 0; +} + static int32_t mndBuildAlterMnodeRedoAction(STrans *pTrans, SDCreateMnodeReq *pAlterReq, SEpSet *pAlterEpSet) { int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, pAlterReq); void *pReq = taosMemoryMalloc(contLen); @@ -347,6 +377,7 @@ static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; int32_t numOfReplicas = 0; + int32_t numOfLearnerReplicas = 0; SDCreateMnodeReq createReq = {0}; SEpSet createEpset = {0}; @@ -355,18 +386,29 @@ static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pMObj); if (pIter == NULL) break; - createReq.replicas[numOfReplicas].id = pMObj->id; - createReq.replicas[numOfReplicas].port = pMObj->pDnode->port; - memcpy(createReq.replicas[numOfReplicas].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN); + if(pMObj->role == TAOS_SYNC_ROLE_VOTER){ + createReq.replicas[numOfReplicas].id = pMObj->id; + createReq.replicas[numOfReplicas].port = pMObj->pDnode->port; + memcpy(createReq.replicas[numOfReplicas].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN); + numOfReplicas++; + } + else{ + createReq.learnerReplicas[numOfLearnerReplicas].id = pMObj->id; + createReq.learnerReplicas[numOfLearnerReplicas].port = pMObj->pDnode->port; + memcpy(createReq.learnerReplicas[numOfLearnerReplicas].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN); + numOfLearnerReplicas++; + } - numOfReplicas++; sdbRelease(pSdb, pMObj); } - createReq.replica = numOfReplicas + 1; - createReq.replicas[numOfReplicas].id = pDnode->id; - createReq.replicas[numOfReplicas].port = pDnode->port; - memcpy(createReq.replicas[numOfReplicas].fqdn, pDnode->fqdn, TSDB_FQDN_LEN); + createReq.replica = numOfReplicas; + createReq.learnerReplica = numOfLearnerReplicas + 1; + createReq.learnerReplicas[numOfLearnerReplicas].id = pDnode->id; + createReq.learnerReplicas[numOfLearnerReplicas].port = pDnode->port; + memcpy(createReq.learnerReplicas[numOfLearnerReplicas].fqdn, pDnode->fqdn, TSDB_FQDN_LEN); + + createReq.lastIndex = pObj->lastIndex; createEpset.inUse = 0; createEpset.numOfEps = 1; @@ -378,23 +420,78 @@ static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno return 0; } +static int32_t mndSetAlterMnodeTypeRedoActions(SMnode *pMnode, STrans *pTrans, SDnodeObj *pDnode, SMnodeObj *pObj) { + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; + SDAlterMnodeTypeReq alterReq = {0}; + SEpSet createEpset = {0}; + + while (1) { + SMnodeObj *pMObj = NULL; + pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pMObj); + if (pIter == NULL) break; + + if(pMObj->role == TAOS_SYNC_ROLE_VOTER){ + alterReq.replicas[alterReq.replica].id = pMObj->id; + alterReq.replicas[alterReq.replica].port = pMObj->pDnode->port; + memcpy(alterReq.replicas[alterReq.replica].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN); + alterReq.replica++; + } + else{ + alterReq.learnerReplicas[alterReq.learnerReplica].id = pMObj->id; + alterReq.learnerReplicas[alterReq.learnerReplica].port = pMObj->pDnode->port; + memcpy(alterReq.learnerReplicas[alterReq.learnerReplica].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN); + alterReq.learnerReplica++; + } + + sdbRelease(pSdb, pMObj); + } + + alterReq.replicas[alterReq.replica].id = pDnode->id; + alterReq.replicas[alterReq.replica].port = pDnode->port; + memcpy(alterReq.replicas[alterReq.replica].fqdn, pDnode->fqdn, TSDB_FQDN_LEN); + alterReq.replica++; + + alterReq.lastIndex = pObj->lastIndex; + + createEpset.inUse = 0; + createEpset.numOfEps = 1; + createEpset.eps[0].port = pDnode->port; + memcpy(createEpset.eps[0].fqdn, pDnode->fqdn, TSDB_FQDN_LEN); + + if (mndBuildAlterMnodeTypeRedoAction(pTrans, &alterReq, &createEpset) != 0) return -1; + + return 0; +} + static int32_t mndCreateMnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SMCreateMnodeReq *pCreate) { int32_t code = -1; - SMnodeObj mnodeObj = {0}; - mnodeObj.id = pDnode->id; - mnodeObj.createdTime = taosGetTimestampMs(); - mnodeObj.updateTime = mnodeObj.createdTime; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq, "create-mnode"); if (pTrans == NULL) goto _OVER; mndTransSetSerial(pTrans); mInfo("trans:%d, used to create mnode:%d", pTrans->id, pCreate->dnodeId); if (mndTrancCheckConflict(pMnode, pTrans) != 0) goto _OVER; + SMnodeObj mnodeObj = {0}; + mnodeObj.id = pDnode->id; + mnodeObj.createdTime = taosGetTimestampMs(); + mnodeObj.updateTime = mnodeObj.createdTime; + mnodeObj.role = TAOS_SYNC_ROLE_LEARNER; + mnodeObj.lastIndex = pMnode->applied; + if (mndSetCreateMnodeRedoActions(pMnode, pTrans, pDnode, &mnodeObj) != 0) goto _OVER; if (mndSetCreateMnodeRedoLogs(pMnode, pTrans, &mnodeObj) != 0) goto _OVER; - if (mndSetCreateMnodeCommitLogs(pMnode, pTrans, &mnodeObj) != 0) goto _OVER; + + SMnodeObj mnodeLeaderObj = {0}; + mnodeLeaderObj.id = pDnode->id; + mnodeLeaderObj.createdTime = taosGetTimestampMs(); + mnodeLeaderObj.updateTime = mnodeLeaderObj.createdTime; + mnodeLeaderObj.role = TAOS_SYNC_ROLE_VOTER; + mnodeLeaderObj.lastIndex = pMnode->applied + 1; + + if (mndSetAlterMnodeTypeRedoActions(pMnode, pTrans, pDnode, &mnodeLeaderObj) != 0) goto _OVER; + if (mndSetCreateMnodeCommitLogs(pMnode, pTrans, &mnodeLeaderObj) != 0) goto _OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; code = 0; @@ -514,6 +611,7 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode int32_t mndSetDropMnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj, bool force) { if (pObj == NULL) return 0; + pObj->lastIndex = pMnode->applied; if (mndSetDropMnodeRedoActions(pMnode, pTrans, pObj->pDnode, pObj, force) != 0) return -1; if (mndSetDropMnodeCommitLogs(pMnode, pTrans, pObj) != 0) return -1; return 0; @@ -730,7 +828,8 @@ static void mndReloadSyncConfig(SMnode *pMnode) { void *pIter = NULL; int32_t updatingMnodes = 0; int32_t readyMnodes = 0; - SSyncCfg cfg = {.myIndex = -1}; + SSyncCfg cfg = {.myIndex = -1, .lastIndex = 0,}; + SyncIndex maxIndex = 0; while (1) { pIter = sdbFetchAll(pSdb, SDB_MNODE, pIter, (void **)&pObj, &objStatus, false); @@ -745,26 +844,41 @@ static void mndReloadSyncConfig(SMnode *pMnode) { } if (objStatus == SDB_STATUS_READY || objStatus == SDB_STATUS_CREATING) { - SNodeInfo *pNode = &cfg.nodeInfo[cfg.replicaNum]; + SNodeInfo *pNode = &cfg.nodeInfo[cfg.totalReplicaNum]; pNode->nodeId = pObj->pDnode->id; pNode->clusterId = mndGetClusterId(pMnode); pNode->nodePort = pObj->pDnode->port; + pNode->nodeRole = pObj->role; tstrncpy(pNode->nodeFqdn, pObj->pDnode->fqdn, TSDB_FQDN_LEN); (void)tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort); mInfo("vgId:1, ep:%s:%u dnode:%d", pNode->nodeFqdn, pNode->nodePort, pNode->nodeId); if (pObj->pDnode->id == pMnode->selfDnodeId) { - cfg.myIndex = cfg.replicaNum; + cfg.myIndex = cfg.totalReplicaNum; + } + if(pNode->nodeRole == TAOS_SYNC_ROLE_VOTER){ + cfg.replicaNum++; + } + cfg.totalReplicaNum++; + if(pObj->lastIndex > cfg.lastIndex){ + cfg.lastIndex = pObj->lastIndex; } - cfg.replicaNum++; } + if (objStatus == SDB_STATUS_DROPPING) { + if(pObj->lastIndex > cfg.lastIndex){ + cfg.lastIndex = pObj->lastIndex; + } + } + + mInfo("vgId:1, mnode:%d, role:%d, lastIndex:%" PRId64, pObj->id, pObj->role, pObj->lastIndex); + sdbReleaseLock(pSdb, pObj, false); } - if (readyMnodes <= 0 || updatingMnodes <= 0) { - mInfo("vgId:1, mnode sync not reconfig since readyMnodes:%d updatingMnodes:%d", readyMnodes, updatingMnodes); - return; - } + //if (readyMnodes <= 0 || updatingMnodes <= 0) { + // mInfo("vgId:1, mnode sync not reconfig since readyMnodes:%d updatingMnodes:%d", readyMnodes, updatingMnodes); + // return; + //} if (cfg.myIndex == -1) { #if 1 @@ -777,12 +891,14 @@ static void mndReloadSyncConfig(SMnode *pMnode) { return; } - if (updatingMnodes > 0) { - mInfo("vgId:1, mnode sync reconfig, replica:%d myIndex:%d", cfg.replicaNum, cfg.myIndex); - for (int32_t i = 0; i < cfg.replicaNum; ++i) { + if (pMnode->syncMgmt.sync > 0) { + mInfo("vgId:1, mnode sync reconfig, totalReplica:%d replica:%d myIndex:%d", + cfg.totalReplicaNum, cfg.replicaNum, cfg.myIndex); + + for (int32_t i = 0; i < cfg.totalReplicaNum; ++i) { SNodeInfo *pNode = &cfg.nodeInfo[i]; - mInfo("vgId:1, index:%d, ep:%s:%u dnode:%d cluster:%" PRId64, i, pNode->nodeFqdn, pNode->nodePort, pNode->nodeId, - pNode->clusterId); + mInfo("vgId:1, index:%d, ep:%s:%u dnode:%d cluster:%" PRId64 " role:%d", i, pNode->nodeFqdn, pNode->nodePort, + pNode->nodeId, pNode->clusterId, pNode->nodeRole); } int32_t code = syncReconfig(pMnode->syncMgmt.sync, &cfg); diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 4965d5c34a..0a6df02f5f 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -237,6 +237,23 @@ static void mndBecomeFollower(const SSyncFSM *pFsm) { taosThreadMutexUnlock(&pMgmt->lock); } +static void mndBecomeLearner(const SSyncFSM *pFsm) { + SMnode *pMnode = pFsm->data; + SSyncMgmt *pMgmt = &pMnode->syncMgmt; + mInfo("vgId:1, become learner"); + + taosThreadMutexLock(&pMgmt->lock); + if (pMgmt->transId != 0) { + mInfo("vgId:1, become learner and post sem, trans:%d, failed to propose since not leader", pMgmt->transId); + pMgmt->transId = 0; + pMgmt->transSec = 0; + pMgmt->transSeq = 0; + pMgmt->errCode = TSDB_CODE_SYN_NOT_LEADER; + tsem_post(&pMgmt->syncSem); + } + taosThreadMutexUnlock(&pMgmt->lock); +} + static void mndBecomeLeader(const SSyncFSM *pFsm) { mInfo("vgId:1, become leader"); SMnode *pMnode = pFsm->data; @@ -278,6 +295,7 @@ SSyncFSM *mndSyncMakeFsm(SMnode *pMnode) { pFsm->FpReConfigCb = NULL; pFsm->FpBecomeLeaderCb = mndBecomeLeader; pFsm->FpBecomeFollowerCb = mndBecomeFollower; + pFsm->FpBecomeLearnerCb = mndBecomeLearner; pFsm->FpGetSnapshot = mndSyncGetSnapshot; pFsm->FpGetSnapshotInfo = mndSyncGetSnapshotInfo; pFsm->FpSnapshotStartRead = mndSnapshotStartRead; @@ -317,13 +335,16 @@ int32_t mndInitSync(SMnode *pMnode) { mInfo("vgId:1, start to open sync, replica:%d selfIndex:%d", pMgmt->numOfReplicas, pMgmt->selfIndex); SSyncCfg *pCfg = &syncInfo.syncCfg; + pCfg->totalReplicaNum = pMgmt->numOfTotalReplicas; pCfg->replicaNum = pMgmt->numOfReplicas; pCfg->myIndex = pMgmt->selfIndex; - for (int32_t i = 0; i < pMgmt->numOfReplicas; ++i) { + pCfg->lastIndex = pMgmt->lastIndex; + for (int32_t i = 0; i < pMgmt->numOfTotalReplicas; ++i) { SNodeInfo *pNode = &pCfg->nodeInfo[i]; pNode->nodeId = pMgmt->replicas[i].id; pNode->nodePort = pMgmt->replicas[i].port; tstrncpy(pNode->nodeFqdn, pMgmt->replicas[i].fqdn, sizeof(pNode->nodeFqdn)); + pNode->nodeRole = pMgmt->nodeRoles[i]; (void)tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort); mInfo("vgId:1, index:%d ep:%s:%u dnode:%d cluster:%" PRId64, i, pNode->nodeFqdn, pNode->nodePort, pNode->nodeId, pNode->clusterId); diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index ed1fddb63f..d674db5b4b 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -62,6 +62,7 @@ int32_t mndInitVgroup(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_VND_COMPACT_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_VND_DISABLE_WRITE_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_SYNC_FORCE_FOLLOWER_RSP, mndTransProcessRsp); + mndSetMsgHandle(pMnode, TDMT_DND_ALTER_VNODE_TYPE_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_MND_REDISTRIBUTE_VGROUP, mndProcessRedistributeVgroupMsg); mndSetMsgHandle(pMnode, TDMT_MND_SPLIT_VGROUP, mndProcessSplitVgroupMsg); @@ -203,7 +204,7 @@ static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOld, SVgObj *pNew) { pNew->compStorage = pOld->compStorage; pNew->pointsWritten = pOld->pointsWritten; pNew->compact = pOld->compact; - memcpy(pOld->vnodeGid, pNew->vnodeGid, TSDB_MAX_REPLICA * sizeof(SVnodeGid)); + memcpy(pOld->vnodeGid, pNew->vnodeGid, (TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA) * sizeof(SVnodeGid)); return 0; } @@ -244,8 +245,10 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg createReq.compression = pDb->cfg.compression; createReq.strict = pDb->cfg.strict; createReq.cacheLast = pDb->cfg.cacheLast; - createReq.replica = pVgroup->replica; + createReq.replica = 0; + createReq.learnerReplica = 0; createReq.selfIndex = -1; + createReq.learnerSelfIndex = -1; createReq.hashBegin = pVgroup->hashBegin; createReq.hashEnd = pVgroup->hashEnd; createReq.hashMethod = pDb->cfg.hashMethod; @@ -263,7 +266,15 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg createReq.tsdbPageSize = pDb->cfg.tsdbPageSize; for (int32_t v = 0; v < pVgroup->replica; ++v) { - SReplica *pReplica = &createReq.replicas[v]; + SReplica *pReplica = NULL; + + if(pVgroup->vnodeGid[v].nodeRole == TAOS_SYNC_ROLE_VOTER){ + pReplica = &createReq.replicas[createReq.replica]; + } + else{ + pReplica = &createReq.learnerReplicas[createReq.learnerReplica]; + } + SVnodeGid *pVgid = &pVgroup->vnodeGid[v]; SDnodeObj *pVgidDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); if (pVgidDnode == NULL) { @@ -275,21 +286,40 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg memcpy(pReplica->fqdn, pVgidDnode->fqdn, TSDB_FQDN_LEN); mndReleaseDnode(pMnode, pVgidDnode); - if (pDnode->id == pVgid->dnodeId) { - createReq.selfIndex = v; + if(pVgroup->vnodeGid[v].nodeRole == TAOS_SYNC_ROLE_VOTER){ + if (pDnode->id == pVgid->dnodeId) { + createReq.selfIndex = createReq.replica; + } + } + else{ + if (pDnode->id == pVgid->dnodeId) { + createReq.learnerSelfIndex = createReq.learnerReplica; + } + } + + if(pVgroup->vnodeGid[v].nodeRole == TAOS_SYNC_ROLE_VOTER){ + createReq.replica++; + } + else{ + createReq.learnerReplica++; } } - if (createReq.selfIndex == -1) { + if (createReq.selfIndex == -1 && createReq.learnerSelfIndex == -1) { terrno = TSDB_CODE_APP_ERROR; return NULL; } - mInfo("vgId:%d, build create vnode req, replica:%d selfIndex:%d strict:%d", createReq.vgId, createReq.replica, - createReq.selfIndex, createReq.strict); + mInfo("vgId:%d, build create vnode req, replica:%d selfIndex:%d learnerReplica:%d learnerSelfIndex:%d strict:%d", + createReq.vgId, createReq.replica, createReq.selfIndex, createReq.learnerReplica, + createReq.learnerReplica, createReq.strict); for (int32_t i = 0; i < createReq.replica; ++i) { mInfo("vgId:%d, replica:%d ep:%s:%u", createReq.vgId, i, createReq.replicas[i].fqdn, createReq.replicas[i].port); } + for (int32_t i = 0; i < createReq.learnerReplica; ++i) { + mInfo("vgId:%d, replica:%d ep:%s:%u", createReq.vgId, i, createReq.learnerReplicas[i].fqdn, + createReq.learnerReplicas[i].port); + } int32_t contLen = tSerializeSCreateVnodeReq(NULL, 0, &createReq); if (contLen < 0) { @@ -356,12 +386,24 @@ static void *mndBuildAlterVnodeReplicaReq(SMnode *pMnode, SDbObj *pDb, SVgObj *p SAlterVnodeReplicaReq alterReq = { .vgId = pVgroup->vgId, .strict = pDb->cfg.strict, - .replica = pVgroup->replica, + .replica = 0, + .learnerReplica = 0, .selfIndex = -1, + .learnerSelfIndex = -1, }; for (int32_t v = 0; v < pVgroup->replica; ++v) { - SReplica *pReplica = &alterReq.replicas[v]; + SReplica *pReplica = NULL; + + if(pVgroup->vnodeGid[v].nodeRole == TAOS_SYNC_ROLE_VOTER){ + pReplica = &alterReq.replicas[alterReq.replica]; + alterReq.replica++; + } + else{ + pReplica = &alterReq.learnerReplicas[alterReq.learnerReplica]; + alterReq.learnerReplica++; + } + SVnodeGid *pVgid = &pVgroup->vnodeGid[v]; SDnodeObj *pVgidDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); if (pVgidDnode == NULL) return NULL; @@ -371,18 +413,30 @@ static void *mndBuildAlterVnodeReplicaReq(SMnode *pMnode, SDbObj *pDb, SVgObj *p memcpy(pReplica->fqdn, pVgidDnode->fqdn, TSDB_FQDN_LEN); mndReleaseDnode(pMnode, pVgidDnode); - if (dnodeId == pVgid->dnodeId) { - alterReq.selfIndex = v; + if(pVgroup->vnodeGid[v].nodeRole == TAOS_SYNC_ROLE_VOTER){ + if (dnodeId == pVgid->dnodeId) { + alterReq.selfIndex = v; + } + } + else{ + if (dnodeId == pVgid->dnodeId) { + alterReq.learnerSelfIndex = v; + } } } - alterReq.replica = pVgroup->replica; - mInfo("vgId:%d, build alter vnode req, replica:%d selfIndex:%d strict:%d", alterReq.vgId, alterReq.replica, - alterReq.selfIndex, alterReq.strict); + + mInfo("vgId:%d, build alter vnode req, replica:%d selfIndex:%d learnerReplica:%d learnerSelfIndex:%d strict:%d", + alterReq.vgId, alterReq.replica, alterReq.selfIndex, alterReq.learnerReplica, + alterReq.learnerSelfIndex, alterReq.strict); for (int32_t i = 0; i < alterReq.replica; ++i) { mInfo("vgId:%d, replica:%d ep:%s:%u", alterReq.vgId, i, alterReq.replicas[i].fqdn, alterReq.replicas[i].port); } + for (int32_t i = 0; i < alterReq.learnerReplica; ++i) { + mInfo("vgId:%d, learnerReplica:%d ep:%s:%u", alterReq.vgId, i, + alterReq.learnerReplicas[i].fqdn, alterReq.learnerReplicas[i].port); + } - if (alterReq.selfIndex == -1) { + if (alterReq.selfIndex == -1 && alterReq.learnerSelfIndex == -1) { terrno = TSDB_CODE_APP_ERROR; return NULL; } @@ -1194,6 +1248,30 @@ int32_t mndAddAlterVnodeReplicaAction(SMnode *pMnode, STrans *pTrans, SDbObj *pD return 0; } +int32_t mndAddAlterVnodeTypeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, int32_t dnodeId) { + SDnodeObj *pDnode = mndAcquireDnode(pMnode, dnodeId); + if (pDnode == NULL) return -1; + + STransAction action = {0}; + action.epSet = mndGetDnodeEpset(pDnode); + mndReleaseDnode(pMnode, pDnode); + + int32_t contLen = 0; + void *pReq = mndBuildAlterVnodeReplicaReq(pMnode, pDb, pVgroup, dnodeId, &contLen); + if (pReq == NULL) return -1; + + action.pCont = pReq; + action.contLen = contLen; + action.msgType = TDMT_DND_ALTER_VNODE_TYPE; + + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } + + return 0; +} + static int32_t mndAddDisableVnodeWriteAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, int32_t dnodeId) { SDnodeObj *pDnode = mndAcquireDnode(pMnode, dnodeId); @@ -1953,18 +2031,33 @@ int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb mInfo("db:%s, vgId:%d, will add 2 vnodes, vn:0 dnode:%d", pVgroup->dbName, pVgroup->vgId, pVgroup->vnodeGid[0].dnodeId); + //add first if (mndAddVnodeToVgroup(pMnode, pTrans, &newVgroup, pArray) != 0) return -1; + + newVgroup.vnodeGid[0].nodeRole = TAOS_SYNC_ROLE_VOTER; + newVgroup.vnodeGid[1].nodeRole = TAOS_SYNC_ROLE_LEARNER; if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, &newVgroup, newVgroup.vnodeGid[0].dnodeId) != 0) return -1; + if (mndAddCreateVnodeAction(pMnode, pTrans, pNewDb, &newVgroup, &newVgroup.vnodeGid[1]) != 0) return -1; + newVgroup.vnodeGid[1].nodeRole = TAOS_SYNC_ROLE_VOTER; + if (mndAddAlterVnodeTypeAction(pMnode, pTrans, pNewDb, &newVgroup, newVgroup.vnodeGid[1].dnodeId) != 0) + return -1; + if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, &newVgroup) != 0) return -1; + //add second if (mndAddVnodeToVgroup(pMnode, pTrans, &newVgroup, pArray) != 0) return -1; + newVgroup.vnodeGid[0].nodeRole = TAOS_SYNC_ROLE_VOTER; + newVgroup.vnodeGid[1].nodeRole = TAOS_SYNC_ROLE_VOTER; + newVgroup.vnodeGid[2].nodeRole = TAOS_SYNC_ROLE_VOTER; + if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, &newVgroup, newVgroup.vnodeGid[0].dnodeId) != 0) return -1; if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, &newVgroup, newVgroup.vnodeGid[1].dnodeId) != 0) return -1; if (mndAddCreateVnodeAction(pMnode, pTrans, pNewDb, &newVgroup, &newVgroup.vnodeGid[2]) != 0) return -1; + if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, &newVgroup) != 0) return -1; } else if (newVgroup.replica == 3 && pNewDb->cfg.replications == 1) { mInfo("db:%s, vgId:%d, will remove 2 vnodes, vn:0 dnode:%d vn:1 dnode:%d vn:2 dnode:%d", pVgroup->dbName, diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index fb2c2f4be3..7dfaf1508d 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -68,6 +68,7 @@ void vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot); void vnodeGetInfo(SVnode *pVnode, const char **dbname, int32_t *vgId); int32_t vnodeProcessCreateTSma(SVnode *pVnode, void *pCont, uint32_t contLen); int32_t vnodeGetAllTableList(SVnode *pVnode, uint64_t uid, SArray *list); +int32_t vnodeIsCatchUp(SVnode *pVnode); int32_t vnodeGetCtbIdList(SVnode *pVnode, int64_t suid, SArray *list); int32_t vnodeGetCtbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg), void *arg); diff --git a/source/dnode/vnode/src/vnd/vnodeCfg.c b/source/dnode/vnode/src/vnd/vnodeCfg.c index c326c8bfac..a760fc03c2 100644 --- a/source/dnode/vnode/src/vnd/vnodeCfg.c +++ b/source/dnode/vnode/src/vnd/vnodeCfg.c @@ -57,6 +57,28 @@ int vnodeCheckCfg(const SVnodeCfg *pCfg) { return 0; } +const char* vnodeRoleToStr(ESyncRole role) { + switch (role) { + case TAOS_SYNC_ROLE_VOTER: + return "voter"; + case TAOS_SYNC_ROLE_LEARNER: + return "learner"; + default: + return "unknown"; + } +} + +const ESyncRole vnodeStrToRole(char* str) { + if(strcmp(str, "voter") == 0){ + return TAOS_SYNC_ROLE_VOTER; + } + if(strcmp(str, "learner") == 0){ + return TAOS_SYNC_ROLE_LEARNER; + } + + return TAOS_SYNC_ROLE_ERROR; +} + int vnodeEncodeConfig(const void *pObj, SJson *pJson) { const SVnodeCfg *pCfg = (SVnodeCfg *)pObj; @@ -117,6 +139,7 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) { if (tjsonAddIntegerToObject(pJson, "hashSuffix", pCfg->hashSuffix) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "syncCfg.replicaNum", pCfg->syncCfg.replicaNum) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "syncCfg.totalReplicaNum", pCfg->syncCfg.totalReplicaNum) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "syncCfg.myIndex", pCfg->syncCfg.myIndex) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "vndStats.stables", pCfg->vndStats.numOfSTables) < 0) return -1; @@ -128,9 +151,9 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) { SJson *nodeInfo = tjsonCreateArray(); if (nodeInfo == NULL) return -1; if (tjsonAddItemToObject(pJson, "syncCfg.nodeInfo", nodeInfo) < 0) return -1; - vDebug("vgId:%d, encode config, replicas:%d selfIndex:%d", pCfg->vgId, pCfg->syncCfg.replicaNum, - pCfg->syncCfg.myIndex); - for (int i = 0; i < pCfg->syncCfg.replicaNum; ++i) { + vDebug("vgId:%d, encode config, replicas:%d totalReplicas:%d selfIndex:%d", pCfg->vgId, pCfg->syncCfg.replicaNum, + pCfg->syncCfg.totalReplicaNum, pCfg->syncCfg.myIndex); + for (int i = 0; i < pCfg->syncCfg.totalReplicaNum; ++i) { SJson *info = tjsonCreateObject(); SNodeInfo *pNode = (SNodeInfo *)&pCfg->syncCfg.nodeInfo[i]; if (info == NULL) return -1; @@ -138,6 +161,7 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) { if (tjsonAddStringToObject(info, "nodeFqdn", pNode->nodeFqdn) < 0) return -1; if (tjsonAddIntegerToObject(info, "nodeId", pNode->nodeId) < 0) return -1; if (tjsonAddIntegerToObject(info, "clusterId", pNode->clusterId) < 0) return -1; + if (tjsonAddStringToObject(info, "nodeRole", vnodeRoleToStr(pNode->nodeRole)) < 0) return -1; if (tjsonAddItemToArray(nodeInfo, info) < 0) return -1; vDebug("vgId:%d, encode config, replica:%d ep:%s:%u dnode:%d", pCfg->vgId, i, pNode->nodeFqdn, pNode->nodePort, pNode->nodeId); @@ -235,6 +259,8 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { tjsonGetNumberValue(pJson, "syncCfg.replicaNum", pCfg->syncCfg.replicaNum, code); if (code < 0) return -1; + tjsonGetNumberValue(pJson, "syncCfg.totalReplicaNum", pCfg->syncCfg.totalReplicaNum, code); + if (code < 0) return -1; tjsonGetNumberValue(pJson, "syncCfg.myIndex", pCfg->syncCfg.myIndex, code); if (code < 0) return -1; @@ -251,10 +277,10 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { SJson *nodeInfo = tjsonGetObjectItem(pJson, "syncCfg.nodeInfo"); int arraySize = tjsonGetArraySize(nodeInfo); - if (arraySize != pCfg->syncCfg.replicaNum) return -1; + if (arraySize != pCfg->syncCfg.totalReplicaNum) return -1; - vDebug("vgId:%d, decode config, replicas:%d selfIndex:%d", pCfg->vgId, pCfg->syncCfg.replicaNum, - pCfg->syncCfg.myIndex); + vDebug("vgId:%d, decode config, replicas:%d totalReplicas:%d selfIndex:%d", pCfg->vgId, pCfg->syncCfg.replicaNum, + pCfg->syncCfg.totalReplicaNum, pCfg->syncCfg.myIndex); for (int i = 0; i < arraySize; ++i) { SJson *info = tjsonGetArrayItem(nodeInfo, i); SNodeInfo *pNode = &pCfg->syncCfg.nodeInfo[i]; @@ -266,6 +292,9 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { if (code < 0) return -1; tjsonGetNumberValue(info, "clusterId", pNode->clusterId, code); if (code < 0) return -1; + char role[10]; + tjsonGetStringValue(info, "nodeRole", role); + pNode->nodeRole = vnodeStrToRole(role); vDebug("vgId:%d, decode config, replica:%d ep:%s:%u dnode:%d", pCfg->vgId, i, pNode->nodeFqdn, pNode->nodePort, pNode->nodeId); } diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index c7d155be0d..b432e8ba4e 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -76,19 +76,41 @@ int32_t vnodeAlterReplica(const char *path, SAlterVnodeReplicaReq *pReq, STfs *p } SSyncCfg *pCfg = &info.config.syncCfg; - pCfg->myIndex = pReq->selfIndex; - pCfg->replicaNum = pReq->replica; + + pCfg->replicaNum = 0; + pCfg->totalReplicaNum = 0; memset(&pCfg->nodeInfo, 0, sizeof(pCfg->nodeInfo)); - vInfo("vgId:%d, save config while alter, replicas:%d selfIndex:%d", pReq->vgId, pCfg->replicaNum, pCfg->myIndex); for (int i = 0; i < pReq->replica; ++i) { SNodeInfo *pNode = &pCfg->nodeInfo[i]; pNode->nodeId = pReq->replicas[i].id; pNode->nodePort = pReq->replicas[i].port; tstrncpy(pNode->nodeFqdn, pReq->replicas[i].fqdn, sizeof(pNode->nodeFqdn)); + pNode->nodeRole = TAOS_SYNC_ROLE_VOTER; (void)tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort); vInfo("vgId:%d, replica:%d ep:%s:%u dnode:%d", pReq->vgId, i, pNode->nodeFqdn, pNode->nodePort, pNode->nodeId); + pCfg->replicaNum++; } + if(pReq->selfIndex != -1){ + pCfg->myIndex = pReq->selfIndex; + } + for (int i = pCfg->replicaNum; i < pReq->replica + pReq->learnerReplica; ++i) { + SNodeInfo *pNode = &pCfg->nodeInfo[i]; + pNode->nodeId = pReq->learnerReplicas[pCfg->totalReplicaNum].id; + pNode->nodePort = pReq->learnerReplicas[pCfg->totalReplicaNum].port; + pNode->nodeRole = TAOS_SYNC_ROLE_LEARNER; + tstrncpy(pNode->nodeFqdn, pReq->learnerReplicas[pCfg->totalReplicaNum].fqdn, sizeof(pNode->nodeFqdn)); + (void)tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort); + vInfo("vgId:%d, replica:%d ep:%s:%u dnode:%d", pReq->vgId, i, pNode->nodeFqdn, pNode->nodePort, pNode->nodeId); + pCfg->totalReplicaNum++; + } + pCfg->totalReplicaNum += pReq->replica; + if(pReq->learnerSelfIndex != -1){ + pCfg->myIndex = pReq->replica + pReq->learnerSelfIndex; + } + + vInfo("vgId:%d, save config while alter, replicas:%d totalReplicas:%d selfIndex:%d", + pReq->vgId, pCfg->replicaNum, pCfg->totalReplicaNum, pCfg->myIndex); info.config.syncCfg = *pCfg; ret = vnodeSaveInfo(dir, &info); @@ -181,6 +203,7 @@ int32_t vnodeAlterHashRange(const char *srcPath, const char *dstPath, SAlterVnod SSyncCfg *pCfg = &info.config.syncCfg; pCfg->myIndex = 0; pCfg->replicaNum = 1; + pCfg->totalReplicaNum = 1; memset(&pCfg->nodeInfo, 0, sizeof(pCfg->nodeInfo)); vInfo("vgId:%d, alter vnode replicas to 1", pReq->srcVgId); @@ -248,7 +271,7 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { // save vnode info on dnode ep changed bool updated = false; SSyncCfg *pCfg = &info.config.syncCfg; - for (int32_t i = 0; i < pCfg->replicaNum; ++i) { + for (int32_t i = 0; i < pCfg->totalReplicaNum; ++i) { SNodeInfo *pNode = &pCfg->nodeInfo[i]; if (tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort)) { updated = true; @@ -406,6 +429,10 @@ void vnodeClose(SVnode *pVnode) { // start the sync timer after the queue is ready int32_t vnodeStart(SVnode *pVnode) { return vnodeSyncStart(pVnode); } +int32_t vnodeIsCatchUp(SVnode *pVnode){ + return syncIsCatchUp(pVnode->sync); +} + void vnodeStop(SVnode *pVnode) {} int64_t vnodeGetSyncHandle(SVnode *pVnode) { return pVnode->sync; } diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index b62bf27def..1adb75ac88 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -1447,7 +1447,8 @@ int32_t vnodeProcessCreateTSma(SVnode *pVnode, void *pCont, uint32_t contLen) { } static int32_t vnodeProcessAlterConfirmReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { - vInfo("vgId:%d, alter replica confim msg is processed", TD_VID(pVnode)); + vInfo("vgId:%d, vnode management handle msgType:alter-confirm, alter replica confim msg is processed", + TD_VID(pVnode)); pRsp->msgType = TDMT_VND_ALTER_CONFIRM_RSP; pRsp->code = TSDB_CODE_SUCCESS; pRsp->pCont = NULL; diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index d4a394b584..bc69378487 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -571,6 +571,19 @@ static void vnodeBecomeFollower(const SSyncFSM *pFsm) { taosThreadMutexUnlock(&pVnode->lock); } +static void vnodeBecomeLearner(const SSyncFSM *pFsm) { + SVnode *pVnode = pFsm->data; + vInfo("vgId:%d, become learner", pVnode->config.vgId); + + taosThreadMutexLock(&pVnode->lock); + if (pVnode->blocked) { + pVnode->blocked = false; + vDebug("vgId:%d, become learner and post block", pVnode->config.vgId); + tsem_post(&pVnode->syncSem); + } + taosThreadMutexUnlock(&pVnode->lock); +} + static void vnodeBecomeLeader(const SSyncFSM *pFsm) { SVnode *pVnode = pFsm->data; vDebug("vgId:%d, become leader", pVnode->config.vgId); @@ -612,6 +625,7 @@ static SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) { pFsm->FpApplyQueueItems = vnodeApplyQueueItems; pFsm->FpBecomeLeaderCb = vnodeBecomeLeader; pFsm->FpBecomeFollowerCb = vnodeBecomeFollower; + pFsm->FpBecomeLearnerCb = vnodeBecomeLearner; pFsm->FpReConfigCb = NULL; pFsm->FpSnapshotStartRead = vnodeSnapshotStartRead; pFsm->FpSnapshotStopRead = vnodeSnapshotStopRead; @@ -644,7 +658,7 @@ int32_t vnodeSyncOpen(SVnode *pVnode, char *path) { SSyncCfg *pCfg = &syncInfo.syncCfg; vInfo("vgId:%d, start to open sync, replica:%d selfIndex:%d", pVnode->config.vgId, pCfg->replicaNum, pCfg->myIndex); - for (int32_t i = 0; i < pCfg->replicaNum; ++i) { + for (int32_t i = 0; i < pCfg->totalReplicaNum; ++i) { SNodeInfo *pNode = &pCfg->nodeInfo[i]; vInfo("vgId:%d, index:%d ep:%s:%u dnode:%d cluster:%" PRId64, pVnode->config.vgId, i, pNode->nodeFqdn, pNode->nodePort, pNode->nodeId, pNode->clusterId); diff --git a/source/libs/sync/inc/syncIndexMgr.h b/source/libs/sync/inc/syncIndexMgr.h index 4e6ab284f8..a4e83dc495 100644 --- a/source/libs/sync/inc/syncIndexMgr.h +++ b/source/libs/sync/inc/syncIndexMgr.h @@ -24,12 +24,13 @@ extern "C" { // SIndexMgr ----------------------------- typedef struct SSyncIndexMgr { - SRaftId (*replicas)[TSDB_MAX_REPLICA]; - SyncIndex index[TSDB_MAX_REPLICA]; - SyncTerm privateTerm[TSDB_MAX_REPLICA]; // for advanced function - int64_t startTimeArr[TSDB_MAX_REPLICA]; - int64_t recvTimeArr[TSDB_MAX_REPLICA]; + SRaftId (*replicas)[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; + SyncIndex index[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; + SyncTerm privateTerm[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; // for advanced function + int64_t startTimeArr[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; + int64_t recvTimeArr[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; int32_t replicaNum; + int32_t totalReplicaNum; SSyncNode *pNode; } SSyncIndexMgr; diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 6f2c1a1ad0..7d336c8313 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -54,13 +54,13 @@ typedef struct SSyncLogReplMgr SSyncLogReplMgr; #define MAX_CONFIG_INDEX_COUNT 256 typedef struct SRaftCfg { - SSyncCfg cfg; - int32_t batchSize; - int8_t isStandBy; - int8_t snapshotStrategy; - SyncIndex lastConfigIndex; - int32_t configIndexCount; - SyncIndex configIndexArr[MAX_CONFIG_INDEX_COUNT]; + SSyncCfg cfg; + int32_t batchSize; + int8_t isStandBy; + int8_t snapshotStrategy; + SyncIndex lastConfigIndex; + int32_t configIndexCount; + SyncIndex configIndexArr[MAX_CONFIG_INDEX_COUNT]; } SRaftCfg; typedef struct SRaftId { @@ -127,12 +127,13 @@ typedef struct SSyncNode { SRaftId myRaftId; int32_t peersNum; - SNodeInfo peersNodeInfo[TSDB_MAX_REPLICA]; - SEpSet peersEpset[TSDB_MAX_REPLICA]; - SRaftId peersId[TSDB_MAX_REPLICA]; + SNodeInfo peersNodeInfo[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; + SEpSet peersEpset[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; + SRaftId peersId[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; int32_t replicaNum; - SRaftId replicasId[TSDB_MAX_REPLICA]; + int32_t totalReplicaNum; + SRaftId replicasId[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; // raft algorithm SSyncFSM* pFsm; @@ -188,7 +189,7 @@ typedef struct SSyncNode { uint64_t heartbeatTimerCounter; // peer heartbeat timer - SSyncTimer peerHeartbeatTimerArr[TSDB_MAX_REPLICA]; + SSyncTimer peerHeartbeatTimerArr[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; // tools SSyncRespMgr* pSyncRespMgr; @@ -196,13 +197,13 @@ typedef struct SSyncNode { // restore state bool restoreFinish; // SSnapshot* pSnapshot; - SSyncSnapshotSender* senders[TSDB_MAX_REPLICA]; + SSyncSnapshotSender* senders[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; SSyncSnapshotReceiver* pNewNodeReceiver; // log replication mgr - SSyncLogReplMgr* logReplMgrs[TSDB_MAX_REPLICA]; + SSyncLogReplMgr* logReplMgrs[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; - SPeerState peerStates[TSDB_MAX_REPLICA]; + SPeerState peerStates[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; // is config changing bool changing; @@ -275,6 +276,7 @@ void syncNodeUpdateTerm(SSyncNode* pSyncNode, SyncTerm term); void syncNodeUpdateTermWithoutStepDown(SSyncNode* pSyncNode, SyncTerm term); void syncNodeStepDown(SSyncNode* pSyncNode, SyncTerm newTerm); void syncNodeBecomeFollower(SSyncNode* pSyncNode, const char* debugStr); +void syncNodeBecomeLearner(SSyncNode* pSyncNode, const char* debugStr); void syncNodeBecomeLeader(SSyncNode* pSyncNode, const char* debugStr); void syncNodeCandidate2Leader(SSyncNode* pSyncNode); void syncNodeFollower2Candidate(SSyncNode* pSyncNode); diff --git a/source/libs/sync/inc/syncMessage.h b/source/libs/sync/inc/syncMessage.h index c3566c7c82..f8c96d8be2 100644 --- a/source/libs/sync/inc/syncMessage.h +++ b/source/libs/sync/inc/syncMessage.h @@ -237,6 +237,7 @@ typedef struct SyncLeaderTransfer { typedef enum { SYNC_LOCAL_CMD_STEP_DOWN = 100, SYNC_LOCAL_CMD_FOLLOWER_CMT, + SYNC_LOCAL_CMD_LEARNER_CMT, } ESyncLocalCmd; typedef struct SyncLocalCmd { diff --git a/source/libs/sync/inc/syncPipeline.h b/source/libs/sync/inc/syncPipeline.h index d709e33cd4..02790732a2 100644 --- a/source/libs/sync/inc/syncPipeline.h +++ b/source/libs/sync/inc/syncPipeline.h @@ -56,6 +56,8 @@ typedef struct SSyncLogBuffer { int64_t size; TdThreadMutex mutex; TdThreadMutexAttr attr; + int64_t totalIndex; + bool isCatchup; } SSyncLogBuffer; // SSyncLogRepMgr diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 9ab545075c..b488455dba 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -137,8 +137,10 @@ int32_t syncNodeOnAppendEntries(SSyncNode* ths, const SRpcMsg* pRpcMsg) { pReply->term = pMsg->term; } - syncNodeStepDown(ths, pMsg->term); - resetElect = true; + if(ths->raftCfg.cfg.nodeInfo[ths->raftCfg.cfg.myIndex].nodeRole != TAOS_SYNC_ROLE_LEARNER){ + syncNodeStepDown(ths, pMsg->term); + resetElect = true; + } if (pMsg->dataLen < sizeof(SSyncRaftEntry)) { sError("vgId:%d, incomplete append entries received. prev index:%" PRId64 ", term:%" PRId64 ", datalen:%d", diff --git a/source/libs/sync/src/syncElection.c b/source/libs/sync/src/syncElection.c index 3699efbc59..86e28db90c 100644 --- a/source/libs/sync/src/syncElection.c +++ b/source/libs/sync/src/syncElection.c @@ -41,6 +41,8 @@ static int32_t syncNodeRequestVotePeers(SSyncNode* pNode) { int32_t ret = 0; for (int i = 0; i < pNode->peersNum; ++i) { + if(pNode->peersNodeInfo[i].nodeRole == TAOS_SYNC_ROLE_LEARNER) continue; + SRpcMsg rpcMsg = {0}; ret = syncBuildRequestVote(&rpcMsg, pNode->vgId); if (ret < 0) { diff --git a/source/libs/sync/src/syncIndexMgr.c b/source/libs/sync/src/syncIndexMgr.c index 7ecb9d7782..e3c3f63a4f 100644 --- a/source/libs/sync/src/syncIndexMgr.c +++ b/source/libs/sync/src/syncIndexMgr.c @@ -26,6 +26,7 @@ SSyncIndexMgr *syncIndexMgrCreate(SSyncNode *pNode) { pIndexMgr->replicas = &pNode->replicasId; pIndexMgr->replicaNum = pNode->replicaNum; + pIndexMgr->totalReplicaNum = pNode->totalReplicaNum; pIndexMgr->pNode = pNode; syncIndexMgrClear(pIndexMgr); @@ -35,6 +36,7 @@ SSyncIndexMgr *syncIndexMgrCreate(SSyncNode *pNode) { void syncIndexMgrUpdate(SSyncIndexMgr *pIndexMgr, SSyncNode *pNode) { pIndexMgr->replicas = &pNode->replicasId; pIndexMgr->replicaNum = pNode->replicaNum; + pIndexMgr->totalReplicaNum = pNode->totalReplicaNum; pIndexMgr->pNode = pNode; syncIndexMgrClear(pIndexMgr); } @@ -50,14 +52,14 @@ void syncIndexMgrClear(SSyncIndexMgr *pIndexMgr) { memset(pIndexMgr->privateTerm, 0, sizeof(pIndexMgr->privateTerm)); int64_t timeNow = taosGetTimestampMs(); - for (int i = 0; i < pIndexMgr->replicaNum; ++i) { + for (int i = 0; i < pIndexMgr->totalReplicaNum; ++i) { pIndexMgr->startTimeArr[i] = 0; pIndexMgr->recvTimeArr[i] = timeNow; } } void syncIndexMgrSetIndex(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId, SyncIndex index) { - for (int i = 0; i < pIndexMgr->replicaNum; ++i) { + for (int i = 0; i < pIndexMgr->totalReplicaNum; ++i) { if (syncUtilSameId(&((*(pIndexMgr->replicas))[i]), pRaftId)) { (pIndexMgr->index)[i] = index; return; @@ -69,7 +71,7 @@ void syncIndexMgrSetIndex(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId, Sync } SSyncLogReplMgr *syncNodeGetLogReplMgr(SSyncNode *pNode, SRaftId *pRaftId) { - for (int i = 0; i < pNode->replicaNum; i++) { + for (int i = 0; i < pNode->totalReplicaNum; i++) { if (syncUtilSameId(&pNode->replicasId[i], pRaftId)) { return pNode->logReplMgrs[i]; } @@ -80,7 +82,7 @@ SSyncLogReplMgr *syncNodeGetLogReplMgr(SSyncNode *pNode, SRaftId *pRaftId) { } SyncIndex syncIndexMgrGetIndex(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId) { - for (int i = 0; i < pIndexMgr->replicaNum; ++i) { + for (int i = 0; i < pIndexMgr->totalReplicaNum; ++i) { if (syncUtilSameId(&((*(pIndexMgr->replicas))[i]), pRaftId)) { SyncIndex idx = (pIndexMgr->index)[i]; return idx; @@ -93,7 +95,7 @@ SyncIndex syncIndexMgrGetIndex(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId) } void syncIndexMgrSetStartTime(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId, int64_t startTime) { - for (int i = 0; i < pIndexMgr->replicaNum; ++i) { + for (int i = 0; i < pIndexMgr->totalReplicaNum; ++i) { if (syncUtilSameId(&((*(pIndexMgr->replicas))[i]), pRaftId)) { (pIndexMgr->startTimeArr)[i] = startTime; return; @@ -105,7 +107,7 @@ void syncIndexMgrSetStartTime(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId, } int64_t syncIndexMgrGetStartTime(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId) { - for (int i = 0; i < pIndexMgr->replicaNum; ++i) { + for (int i = 0; i < pIndexMgr->totalReplicaNum; ++i) { if (syncUtilSameId(&((*(pIndexMgr->replicas))[i]), pRaftId)) { int64_t startTime = (pIndexMgr->startTimeArr)[i]; return startTime; @@ -118,7 +120,7 @@ int64_t syncIndexMgrGetStartTime(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftI } void syncIndexMgrSetRecvTime(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId, int64_t recvTime) { - for (int i = 0; i < pIndexMgr->replicaNum; ++i) { + for (int i = 0; i < pIndexMgr->totalReplicaNum; ++i) { if (syncUtilSameId(&((*(pIndexMgr->replicas))[i]), pRaftId)) { (pIndexMgr->recvTimeArr)[i] = recvTime; return; @@ -130,7 +132,7 @@ void syncIndexMgrSetRecvTime(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId, i } int64_t syncIndexMgrGetRecvTime(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId) { - for (int i = 0; i < pIndexMgr->replicaNum; ++i) { + for (int i = 0; i < pIndexMgr->totalReplicaNum; ++i) { if (syncUtilSameId(&((*(pIndexMgr->replicas))[i]), pRaftId)) { int64_t recvTime = (pIndexMgr->recvTimeArr)[i]; return recvTime; @@ -143,7 +145,7 @@ int64_t syncIndexMgrGetRecvTime(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId } void syncIndexMgrSetTerm(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId, SyncTerm term) { - for (int i = 0; i < pIndexMgr->replicaNum; ++i) { + for (int i = 0; i < pIndexMgr->totalReplicaNum; ++i) { if (syncUtilSameId(&((*(pIndexMgr->replicas))[i]), pRaftId)) { (pIndexMgr->privateTerm)[i] = term; return; @@ -155,7 +157,7 @@ void syncIndexMgrSetTerm(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId, SyncT } SyncTerm syncIndexMgrGetTerm(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId) { - for (int i = 0; i < pIndexMgr->replicaNum; ++i) { + for (int i = 0; i < pIndexMgr->totalReplicaNum; ++i) { if (syncUtilSameId(&((*(pIndexMgr->replicas))[i]), pRaftId)) { SyncTerm term = (pIndexMgr->privateTerm)[i]; return term; diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 966b3ed093..c4e7833245 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -141,6 +141,13 @@ int32_t syncReconfig(int64_t rid, SSyncCfg* pNewCfg) { SSyncNode* pSyncNode = syncNodeAcquire(rid); if (pSyncNode == NULL) return -1; + if(pSyncNode->raftCfg.lastConfigIndex >= pNewCfg->lastIndex){ + syncNodeRelease(pSyncNode); + sInfo("vgId:%d, no need Reconfig, current index:%" PRId64 ", new index:%" PRId64, pSyncNode->vgId, + pSyncNode->raftCfg.lastConfigIndex, pNewCfg->lastIndex); + return 0; + } + if (!syncNodeCheckNewConfig(pSyncNode, pNewCfg)) { syncNodeRelease(pSyncNode); terrno = TSDB_CODE_SYN_NEW_CONFIG_ERROR; @@ -149,12 +156,12 @@ int32_t syncReconfig(int64_t rid, SSyncCfg* pNewCfg) { } syncNodeUpdateNewConfigIndex(pSyncNode, pNewCfg); - syncNodeDoConfigChange(pSyncNode, pNewCfg, SYNC_INDEX_INVALID); + syncNodeDoConfigChange(pSyncNode, pNewCfg, pNewCfg->lastIndex); if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { syncNodeStopHeartbeatTimer(pSyncNode); - for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { + for (int32_t i = 0; i < TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA; ++i) { syncHbTimerInit(pSyncNode, &pSyncNode->peerHeartbeatTimerArr[i], pSyncNode->replicasId[i]); } @@ -315,8 +322,9 @@ int32_t syncBeginSnapshot(int64_t rid, int64_t lastApplyIndex) { } } - if (pSyncNode->replicaNum > 1) { - if (pSyncNode->state != TAOS_SYNC_STATE_LEADER && pSyncNode->state != TAOS_SYNC_STATE_FOLLOWER) { + if (pSyncNode->totalReplicaNum > 1) { + if (pSyncNode->state != TAOS_SYNC_STATE_LEADER && pSyncNode->state != TAOS_SYNC_STATE_FOLLOWER + && pSyncNode->state != TAOS_SYNC_STATE_LEARNER) { sNTrace(pSyncNode, "new-snapshot-index:%" PRId64 " candidate or unknown state, do not delete wal", lastApplyIndex); syncNodeRelease(pSyncNode); @@ -537,7 +545,8 @@ void syncGetRetryEpSet(int64_t rid, SEpSet* pEpSet) { SSyncNode* pSyncNode = syncNodeAcquire(rid); if (pSyncNode == NULL) return; - for (int32_t i = 0; i < pSyncNode->raftCfg.cfg.replicaNum; ++i) { + for (int32_t i = 0; i < pSyncNode->raftCfg.cfg.totalReplicaNum; ++i) { + if(pSyncNode->raftCfg.cfg.nodeInfo[i].nodeRole == TAOS_SYNC_ROLE_LEARNER) continue; SEp* pEp = &pEpSet->eps[i]; tstrncpy(pEp->fqdn, pSyncNode->raftCfg.cfg.nodeInfo[i].nodeFqdn, TSDB_FQDN_LEN); pEp->port = (pSyncNode->raftCfg.cfg.nodeInfo)[i].nodePort; @@ -564,6 +573,34 @@ int32_t syncPropose(int64_t rid, SRpcMsg* pMsg, bool isWeak, int64_t* seq) { return ret; } +int32_t syncIsCatchUp(int64_t rid) { + SSyncNode* pSyncNode = syncNodeAcquire(rid); + if (pSyncNode == NULL) { + sError("sync Node Acquire error since %d", errno); + return -1; + } + + while(1){ + if(pSyncNode->pLogBuf->totalIndex < 0 || pSyncNode->pLogBuf->commitIndex < 0 || + pSyncNode->pLogBuf->totalIndex < pSyncNode->pLogBuf->commitIndex || + pSyncNode->pLogBuf->totalIndex - pSyncNode->pLogBuf->commitIndex > SYNC_LEARNER_CATCHUP){ + sInfo("vgId:%d, Not catch up, wait one second, totalIndex:%" PRId64 " commitIndex:%" PRId64 " matchIndex:%" PRId64, + pSyncNode->vgId, pSyncNode->pLogBuf->totalIndex, pSyncNode->pLogBuf->commitIndex, + pSyncNode->pLogBuf->matchIndex); + taosSsleep(1); + } + else{ + sInfo("vgId:%d, Catch up, totalIndex:%" PRId64 " commitIndex:%" PRId64 " matchIndex:%" PRId64, + pSyncNode->vgId, pSyncNode->pLogBuf->totalIndex, pSyncNode->pLogBuf->commitIndex, + pSyncNode->pLogBuf->matchIndex); + break; + } + } + + syncNodeRelease(pSyncNode); + return 0; +} + int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak, int64_t* seq) { if (pSyncNode->state != TAOS_SYNC_STATE_LEADER) { terrno = TSDB_CODE_SYN_NOT_LEADER; @@ -655,6 +692,8 @@ static int32_t syncHbTimerStart(SSyncNode* pSyncNode, SSyncTimer* pSyncTimer) { pData->logicClock = pSyncTimer->logicClock; pData->execTime = tsNow + pSyncTimer->timerMS; + sTrace("vgId:%d, start hb timer, rid:%" PRId64 " addr:%" PRId64, pSyncNode->vgId, pData->rid, pData->destId.addr); + taosTmrReset(pSyncTimer->timerCb, pSyncTimer->timerMS / HEARTBEAT_TICK_NUM, (void*)(pData->rid), syncEnv()->pTimerManager, &pSyncTimer->pTimer); } else { @@ -719,7 +758,7 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { sInfo("vgId:%d, create a new raft config file", pSyncNode->vgId); pSyncNode->raftCfg.isStandBy = pSyncInfo->isStandBy; pSyncNode->raftCfg.snapshotStrategy = pSyncInfo->snapshotStrategy; - pSyncNode->raftCfg.lastConfigIndex = SYNC_INDEX_INVALID; + pSyncNode->raftCfg.lastConfigIndex = pSyncInfo->syncCfg.lastIndex; pSyncNode->raftCfg.batchSize = pSyncInfo->batchSize; pSyncNode->raftCfg.cfg = pSyncInfo->syncCfg; pSyncNode->raftCfg.configIndexCount = 1; @@ -736,7 +775,7 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { goto _error; } - if (pSyncInfo->syncCfg.replicaNum > 0 && syncIsConfigChanged(&pSyncNode->raftCfg.cfg, &pSyncInfo->syncCfg)) { + if (pSyncInfo->syncCfg.totalReplicaNum > 0 && syncIsConfigChanged(&pSyncNode->raftCfg.cfg, &pSyncInfo->syncCfg)) { sInfo("vgId:%d, use sync config from input options and write to cfg file", pSyncNode->vgId); pSyncNode->raftCfg.cfg = pSyncInfo->syncCfg; if (syncWriteCfgFile(pSyncNode) != 0) { @@ -753,8 +792,9 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { pSyncNode->vgId = pSyncInfo->vgId; SSyncCfg* pCfg = &pSyncNode->raftCfg.cfg; bool updated = false; - sInfo("vgId:%d, start to open sync node, replica:%d selfIndex:%d", pSyncNode->vgId, pCfg->replicaNum, pCfg->myIndex); - for (int32_t i = 0; i < pCfg->replicaNum; ++i) { + sInfo("vgId:%d, start to open sync node, totalReplicaNum:%d replicaNum:%d selfIndex:%d", + pSyncNode->vgId, pCfg->totalReplicaNum, pCfg->replicaNum, pCfg->myIndex); + for (int32_t i = 0; i < pCfg->totalReplicaNum; ++i) { SNodeInfo* pNode = &pCfg->nodeInfo[i]; if (tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort)) { updated = true; @@ -792,9 +832,9 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { } // init peersNum, peers, peersId - pSyncNode->peersNum = pSyncNode->raftCfg.cfg.replicaNum - 1; + pSyncNode->peersNum = pSyncNode->raftCfg.cfg.totalReplicaNum - 1; int32_t j = 0; - for (int32_t i = 0; i < pSyncNode->raftCfg.cfg.replicaNum; ++i) { + for (int32_t i = 0; i < pSyncNode->raftCfg.cfg.totalReplicaNum; ++i) { if (i != pSyncNode->raftCfg.cfg.myIndex) { pSyncNode->peersNodeInfo[j] = pSyncNode->raftCfg.cfg.nodeInfo[i]; syncUtilNodeInfo2EpSet(&pSyncNode->peersNodeInfo[j], &pSyncNode->peersEpset[j]); @@ -810,7 +850,8 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { // init replicaNum, replicasId pSyncNode->replicaNum = pSyncNode->raftCfg.cfg.replicaNum; - for (int32_t i = 0; i < pSyncNode->raftCfg.cfg.replicaNum; ++i) { + pSyncNode->totalReplicaNum = pSyncNode->raftCfg.cfg.totalReplicaNum; + for (int32_t i = 0; i < pSyncNode->raftCfg.cfg.totalReplicaNum; ++i) { if (!syncUtilNodeInfo2RaftId(&pSyncNode->raftCfg.cfg.nodeInfo[i], pSyncNode->vgId, &pSyncNode->replicasId[i])) { sError("vgId:%d, failed to determine raft member id, replica:%d", pSyncNode->vgId, i); goto _error; @@ -934,7 +975,7 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { pSyncNode->heartbeatTimerCounter = 0; // init peer heartbeat timer - for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { + for (int32_t i = 0; i < TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA; ++i) { syncHbTimerInit(pSyncNode, &(pSyncNode->peerHeartbeatTimerArr[i]), (pSyncNode->replicasId)[i]); } @@ -949,7 +990,7 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { pSyncNode->restoreFinish = false; // snapshot senders - for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { + for (int32_t i = 0; i < TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA; ++i) { SSyncSnapshotSender* pSender = snapshotSenderCreate(pSyncNode, i); if (pSender == NULL) return NULL; @@ -1059,14 +1100,19 @@ int32_t syncNodeRestore(SSyncNode* pSyncNode) { int32_t syncNodeStart(SSyncNode* pSyncNode) { // start raft - if (pSyncNode->replicaNum == 1) { - raftStoreNextTerm(pSyncNode); - syncNodeBecomeLeader(pSyncNode, "one replica start"); + if(pSyncNode->raftCfg.cfg.nodeInfo[pSyncNode->raftCfg.cfg.myIndex].nodeRole == TAOS_SYNC_ROLE_LEARNER){ + syncNodeBecomeLearner(pSyncNode, "first start"); + } + else{ + if (pSyncNode->replicaNum == 1) { + raftStoreNextTerm(pSyncNode); + syncNodeBecomeLeader(pSyncNode, "one replica start"); - // Raft 3.6.2 Committing entries from previous terms - syncNodeAppendNoop(pSyncNode); - } else { - syncNodeBecomeFollower(pSyncNode, "first start"); + // Raft 3.6.2 Committing entries from previous terms + syncNodeAppendNoop(pSyncNode); + } else { + syncNodeBecomeFollower(pSyncNode, "first start"); + } } int32_t ret = 0; @@ -1157,7 +1203,7 @@ void syncNodeClose(SSyncNode* pSyncNode) { syncLogBufferDestroy(pSyncNode->pLogBuf); pSyncNode->pLogBuf = NULL; - for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { + for (int32_t i = 0; i < TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA; ++i) { if (pSyncNode->senders[i] != NULL) { sDebug("vgId:%d, snapshot sender destroy while close, data:%p", pSyncNode->vgId, pSyncNode->senders[i]); @@ -1350,7 +1396,7 @@ inline bool syncNodeInConfig(SSyncNode* pNode, const SSyncCfg* pCfg) { bool b1 = false; bool b2 = false; - for (int32_t i = 0; i < pCfg->replicaNum; ++i) { + for (int32_t i = 0; i < pCfg->totalReplicaNum; ++i) { if (strcmp(pCfg->nodeInfo[i].nodeFqdn, pNode->myNodeInfo.nodeFqdn) == 0 && pCfg->nodeInfo[i].nodePort == pNode->myNodeInfo.nodePort) { b1 = true; @@ -1358,7 +1404,7 @@ inline bool syncNodeInConfig(SSyncNode* pNode, const SSyncCfg* pCfg) { } } - for (int32_t i = 0; i < pCfg->replicaNum; ++i) { + for (int32_t i = 0; i < pCfg->totalReplicaNum; ++i) { SRaftId raftId = { .addr = SYNC_ADDR(&pCfg->nodeInfo[i]), .vgId = pNode->vgId, @@ -1375,13 +1421,14 @@ inline bool syncNodeInConfig(SSyncNode* pNode, const SSyncCfg* pCfg) { } static bool syncIsConfigChanged(const SSyncCfg* pOldCfg, const SSyncCfg* pNewCfg) { - if (pOldCfg->replicaNum != pNewCfg->replicaNum) return true; + if (pOldCfg->totalReplicaNum != pNewCfg->totalReplicaNum) return true; if (pOldCfg->myIndex != pNewCfg->myIndex) return true; - for (int32_t i = 0; i < pOldCfg->replicaNum; ++i) { + for (int32_t i = 0; i < pOldCfg->totalReplicaNum; ++i) { const SNodeInfo* pOldInfo = &pOldCfg->nodeInfo[i]; const SNodeInfo* pNewInfo = &pNewCfg->nodeInfo[i]; if (strcmp(pOldInfo->nodeFqdn, pNewInfo->nodeFqdn) != 0) return true; if (pOldInfo->nodePort != pNewInfo->nodePort) return true; + if(pOldInfo->nodeRole != pNewInfo->nodeRole) return true; } return false; @@ -1418,8 +1465,10 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde } // log begin config change - sNInfo(pSyncNode, "begin do config change, from %d to %d, replicas:%d", pSyncNode->vgId, oldConfig.replicaNum, - pNewConfig->replicaNum); + sNInfo(pSyncNode, "begin do config change, from %d to %d, from %" PRId64 " to %" PRId64 ", replicas:%d", + pSyncNode->vgId, + oldConfig.totalReplicaNum, pNewConfig->totalReplicaNum, + oldConfig.lastIndex, pNewConfig->lastIndex); if (IamInNew) { pSyncNode->raftCfg.isStandBy = 0; // change isStandBy to normal @@ -1436,11 +1485,10 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde int32_t ret = 0; // save snapshot senders - int32_t oldReplicaNum = pSyncNode->replicaNum; - SRaftId oldReplicasId[TSDB_MAX_REPLICA]; + SRaftId oldReplicasId[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; memcpy(oldReplicasId, pSyncNode->replicasId, sizeof(oldReplicasId)); - SSyncSnapshotSender* oldSenders[TSDB_MAX_REPLICA]; - for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { + SSyncSnapshotSender* oldSenders[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; + for (int32_t i = 0; i < TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA; ++i) { oldSenders[i] = pSyncNode->senders[i]; sSTrace(oldSenders[i], "snapshot sender save old"); } @@ -1450,9 +1498,9 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde syncUtilNodeInfo2RaftId(&pSyncNode->myNodeInfo, pSyncNode->vgId, &pSyncNode->myRaftId); // init peersNum, peers, peersId - pSyncNode->peersNum = pSyncNode->raftCfg.cfg.replicaNum - 1; + pSyncNode->peersNum = pSyncNode->raftCfg.cfg.totalReplicaNum - 1; int32_t j = 0; - for (int32_t i = 0; i < pSyncNode->raftCfg.cfg.replicaNum; ++i) { + for (int32_t i = 0; i < pSyncNode->raftCfg.cfg.totalReplicaNum; ++i) { if (i != pSyncNode->raftCfg.cfg.myIndex) { pSyncNode->peersNodeInfo[j] = pSyncNode->raftCfg.cfg.nodeInfo[i]; syncUtilNodeInfo2EpSet(&pSyncNode->peersNodeInfo[j], &pSyncNode->peersEpset[j]); @@ -1465,7 +1513,8 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde // init replicaNum, replicasId pSyncNode->replicaNum = pSyncNode->raftCfg.cfg.replicaNum; - for (int32_t i = 0; i < pSyncNode->raftCfg.cfg.replicaNum; ++i) { + pSyncNode->totalReplicaNum = pSyncNode->raftCfg.cfg.totalReplicaNum; + for (int32_t i = 0; i < pSyncNode->raftCfg.cfg.totalReplicaNum; ++i) { syncUtilNodeInfo2RaftId(&pSyncNode->raftCfg.cfg.nodeInfo[i], pSyncNode->vgId, &pSyncNode->replicasId[i]); } @@ -1480,15 +1529,15 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde // reset snapshot senders // clear new - for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { + for (int32_t i = 0; i < TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA; ++i) { pSyncNode->senders[i] = NULL; } // reset new - for (int32_t i = 0; i < pSyncNode->replicaNum; ++i) { + for (int32_t i = 0; i < pSyncNode->totalReplicaNum; ++i) { // reset sender bool reset = false; - for (int32_t j = 0; j < TSDB_MAX_REPLICA; ++j) { + for (int32_t j = 0; j < TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA; ++j) { if (syncUtilSameId(&(pSyncNode->replicasId)[i], &oldReplicasId[j]) && oldSenders[j] != NULL) { sNTrace(pSyncNode, "snapshot sender reset for:%" PRId64 ", newIndex:%d, dnode:%d, %p", (pSyncNode->replicasId)[i].addr, i, DID(&pSyncNode->replicasId[i]), oldSenders[j]); @@ -1510,7 +1559,7 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde } // create new - for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { + for (int32_t i = 0; i < TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA; ++i) { if (pSyncNode->senders[i] == NULL) { pSyncNode->senders[i] = snapshotSenderCreate(pSyncNode, i); if (pSyncNode->senders[i] == NULL) { @@ -1525,7 +1574,7 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde } // free old - for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { + for (int32_t i = 0; i < TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA; ++i) { if (oldSenders[i] != NULL) { sSDebug(oldSenders[i], "snapshot sender destroy old, data:%p replica-index:%d", oldSenders[i], i); snapshotSenderDestroy(oldSenders[i]); @@ -1550,12 +1599,12 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde } else { // persist cfg syncWriteCfgFile(pSyncNode); - sNInfo(pSyncNode, "do not config change from %d to %d", oldConfig.replicaNum, pNewConfig->replicaNum); + sNInfo(pSyncNode, "do not config change from %d to %d", oldConfig.totalReplicaNum, pNewConfig->totalReplicaNum); } _END: // log end config change - sNInfo(pSyncNode, "end do config change, from %d to %d", oldConfig.replicaNum, pNewConfig->replicaNum); + sNInfo(pSyncNode, "end do config change, from %d to %d", oldConfig.totalReplicaNum, pNewConfig->totalReplicaNum); } // raft state change -------------- @@ -1635,6 +1684,27 @@ void syncNodeBecomeFollower(SSyncNode* pSyncNode, const char* debugStr) { syncNodeResetElectTimer(pSyncNode); } +void syncNodeBecomeLearner(SSyncNode* pSyncNode, const char* debugStr) { + pSyncNode->hbSlowNum = 0; + + // state change + pSyncNode->state = TAOS_SYNC_STATE_LEARNER; + + // trace log + sNTrace(pSyncNode, "become learner %s", debugStr); + + // call back + if (pSyncNode->pFsm != NULL && pSyncNode->pFsm->FpBecomeLearnerCb != NULL) { + pSyncNode->pFsm->FpBecomeLearnerCb(pSyncNode->pFsm); + } + + // min match index + pSyncNode->minMatchIndex = SYNC_INDEX_INVALID; + + // reset log buffer + syncLogBufferReset(pSyncNode->pLogBuf, pSyncNode); +} + // TLA+ Spec // \* Candidate i transitions to leader. // BecomeLeader(i) == @@ -1752,7 +1822,7 @@ void syncNodeCandidate2Leader(SSyncNode* pSyncNode) { bool syncNodeIsMnode(SSyncNode* pSyncNode) { return (pSyncNode->vgId == 1); } int32_t syncNodePeerStateInit(SSyncNode* pSyncNode) { - for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { + for (int32_t i = 0; i < TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA; ++i) { pSyncNode->peerStates[i].lastSendIndex = SYNC_INDEX_INVALID; pSyncNode->peerStates[i].lastSendTime = 0; } @@ -2039,7 +2109,7 @@ static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) { if (!syncIsInit()) return; SSyncNode* pNode = param; - if (pNode->replicaNum > 1) { + if (pNode->totalReplicaNum > 1) { if (atomic_load_64(&pNode->heartbeatTimerLogicClockUser) <= atomic_load_64(&pNode->heartbeatTimerLogicClock)) { SRpcMsg rpcMsg = {0}; int32_t code = syncBuildTimeout(&rpcMsg, SYNC_TIMEOUT_HEARTBEAT, atomic_load_64(&pNode->heartbeatTimerLogicClock), @@ -2074,7 +2144,7 @@ static void syncNodeEqPeerHeartbeatTimer(void* param, void* tmrId) { SSyncHbTimerData* pData = syncHbTimerDataAcquire(hbDataRid); if (pData == NULL) { - sError("hb timer get pData NULL, %" PRId64, hbDataRid); + sError("hb timer get pData NULL, rid:%" PRId64 " addr:%" PRId64, hbDataRid, pData->destId.addr); return; } @@ -2101,9 +2171,9 @@ static void syncNodeEqPeerHeartbeatTimer(void* param, void* tmrId) { return; } - // sTrace("vgId:%d, eq peer hb timer", pSyncNode->vgId); + sTrace("vgId:%d, eq peer hb timer, rid:%" PRId64 " addr:%" PRId64, pSyncNode->vgId, hbDataRid, pData->destId.addr); - if (pSyncNode->replicaNum > 1) { + if (pSyncNode->totalReplicaNum > 1) { int64_t timerLogicClock = atomic_load_64(&pSyncTimer->logicClock); int64_t msgLogicClock = atomic_load_64(&pData->logicClock); @@ -2130,6 +2200,7 @@ static void syncNodeEqPeerHeartbeatTimer(void* param, void* tmrId) { pSyncTimer->timeStamp = tsNow; // send msg + sTrace("vgId:%d, send heartbeat to dnode:%d", pSyncNode->vgId, DID(&(pSyncMsg->destId))); syncLogSendHeartbeat(pSyncNode, pSyncMsg, false, timerElapsed, pData->execTime); syncNodeSendHeartbeat(pSyncNode, &pSyncMsg->destId, &rpcMsg); } else { @@ -2212,7 +2283,7 @@ int32_t syncNodeAppend(SSyncNode* ths, SSyncRaftEntry* pEntry) { } bool syncNodeHeartbeatReplyTimeout(SSyncNode* pSyncNode) { - if (pSyncNode->replicaNum == 1) { + if (pSyncNode->totalReplicaNum == 1) { return false; } @@ -2237,7 +2308,7 @@ bool syncNodeHeartbeatReplyTimeout(SSyncNode* pSyncNode) { bool syncNodeSnapshotSending(SSyncNode* pSyncNode) { if (pSyncNode == NULL) return false; bool b = false; - for (int32_t i = 0; i < pSyncNode->replicaNum; ++i) { + for (int32_t i = 0; i < pSyncNode->totalReplicaNum; ++i) { if (pSyncNode->senders[i] != NULL && pSyncNode->senders[i]->start) { b = true; break; @@ -2328,13 +2399,17 @@ int32_t syncNodeOnHeartbeat(SSyncNode* ths, const SRpcMsg* pRpcMsg) { pMsgReply->startTime = ths->startTime; pMsgReply->timeStamp = tsMs; + sTrace( + "vgId:%d, heartbeat msg from dnode:%d, cluster:%d, Msgterm:%" PRId64 " currentTerm:%" PRId64, + ths->vgId, DID(&(pMsg->srcId)), CID(&(pMsg->srcId)), pMsg->term, currentTerm); + if (pMsg->term == currentTerm && ths->state != TAOS_SYNC_STATE_LEADER) { syncIndexMgrSetRecvTime(ths->pNextIndex, &(pMsg->srcId), tsMs); resetElect = true; ths->minMatchIndex = pMsg->minMatchIndex; - if (ths->state == TAOS_SYNC_STATE_FOLLOWER) { + if (ths->state == TAOS_SYNC_STATE_FOLLOWER || ths->state == TAOS_SYNC_STATE_LEARNER) { SRpcMsg rpcMsgLocalCmd = {0}; (void)syncBuildLocalCmd(&rpcMsgLocalCmd, ths->vgId); @@ -2356,7 +2431,7 @@ int32_t syncNodeOnHeartbeat(SSyncNode* ths, const SRpcMsg* pRpcMsg) { } } - if (pMsg->term >= currentTerm && ths->state != TAOS_SYNC_STATE_FOLLOWER) { + if (pMsg->term >= currentTerm && ths->state == TAOS_SYNC_STATE_LEADER) { SRpcMsg rpcMsgLocalCmd = {0}; (void)syncBuildLocalCmd(&rpcMsgLocalCmd, ths->vgId); @@ -2376,6 +2451,26 @@ int32_t syncNodeOnHeartbeat(SSyncNode* ths, const SRpcMsg* pRpcMsg) { } } + if (pMsg->term >= currentTerm && ths->state == TAOS_SYNC_STATE_LEARNER) { + SRpcMsg rpcMsgLocalCmd = {0}; + (void)syncBuildLocalCmd(&rpcMsgLocalCmd, ths->vgId); + + SyncLocalCmd* pSyncMsg = rpcMsgLocalCmd.pCont; + pSyncMsg->cmd = SYNC_LOCAL_CMD_LEARNER_CMT; + pSyncMsg->currentTerm = pMsg->term; + pSyncMsg->commitIndex = pMsg->commitIndex; + + if (ths->syncEqMsg != NULL && ths->msgcb != NULL) { + int32_t code = ths->syncEqMsg(ths->msgcb, &rpcMsgLocalCmd); + if (code != 0) { + sError("vgId:%d, sync enqueue step-down msg error, code:%d", ths->vgId, code); + rpcFreeCont(rpcMsgLocalCmd.pCont); + } else { + sTrace("vgId:%d, sync enqueue step-down msg, new-term:%" PRId64, ths->vgId, pSyncMsg->currentTerm); + } + } + } + // reply syncNodeSendMsgById(&pMsgReply->destId, ths, &rpcMsg); @@ -2439,7 +2534,20 @@ int32_t syncNodeOnLocalCmd(SSyncNode* ths, const SRpcMsg* pRpcMsg) { sError("vgId:%d, failed to commit raft log since %s. commit index:%" PRId64 "", ths->vgId, terrstr(), ths->commitIndex); } - } else { + } else if (pMsg->cmd == SYNC_LOCAL_CMD_LEARNER_CMT){ + if (syncLogBufferIsEmpty(ths->pLogBuf)) { + sError("vgId:%d, sync log buffer is empty.", ths->vgId); + return 0; + } + raftStoreSetTerm(ths, pMsg->currentTerm); + (void)syncNodeUpdateCommitIndex(ths, pMsg->commitIndex); + sTrace("vgId:%d, start to commit raft log in heartbeat. commit index:%" PRId64 "", ths->vgId, ths->commitIndex); + if (syncLogBufferCommit(ths->pLogBuf, ths, ths->commitIndex) < 0) { + sError("vgId:%d, failed to commit raft log since %s. commit index:%" PRId64 "", ths->vgId, terrstr(), + ths->commitIndex); + } + } + else { sError("error local cmd"); } @@ -2502,13 +2610,15 @@ const char* syncStr(ESyncState state) { return "error"; case TAOS_SYNC_STATE_OFFLINE: return "offline"; + case TAOS_SYNC_STATE_LEARNER: + return "learner"; default: return "unknown"; } } int32_t syncNodeUpdateNewConfigIndex(SSyncNode* ths, SSyncCfg* pNewCfg) { - for (int32_t i = 0; i < pNewCfg->replicaNum; ++i) { + for (int32_t i = 0; i < pNewCfg->totalReplicaNum; ++i) { SRaftId raftId = { .addr = SYNC_ADDR(&pNewCfg->nodeInfo[i]), .vgId = ths->vgId, @@ -2528,7 +2638,7 @@ bool syncNodeIsOptimizedOneReplica(SSyncNode* ths, SRpcMsg* pMsg) { } bool syncNodeInRaftGroup(SSyncNode* ths, SRaftId* pRaftId) { - for (int32_t i = 0; i < ths->replicaNum; ++i) { + for (int32_t i = 0; i < ths->totalReplicaNum; ++i) { if (syncUtilSameId(&((ths->replicasId)[i]), pRaftId)) { return true; } @@ -2538,7 +2648,7 @@ bool syncNodeInRaftGroup(SSyncNode* ths, SRaftId* pRaftId) { SSyncSnapshotSender* syncNodeGetSnapshotSender(SSyncNode* ths, SRaftId* pDestId) { SSyncSnapshotSender* pSender = NULL; - for (int32_t i = 0; i < ths->replicaNum; ++i) { + for (int32_t i = 0; i < ths->totalReplicaNum; ++i) { if (syncUtilSameId(pDestId, &((ths->replicasId)[i]))) { pSender = (ths->senders)[i]; } @@ -2548,7 +2658,7 @@ SSyncSnapshotSender* syncNodeGetSnapshotSender(SSyncNode* ths, SRaftId* pDestId) SSyncTimer* syncNodeGetHbTimer(SSyncNode* ths, SRaftId* pDestId) { SSyncTimer* pTimer = NULL; - for (int32_t i = 0; i < ths->replicaNum; ++i) { + for (int32_t i = 0; i < ths->totalReplicaNum; ++i) { if (syncUtilSameId(pDestId, &((ths->replicasId)[i]))) { pTimer = &((ths->peerHeartbeatTimerArr)[i]); } @@ -2558,7 +2668,7 @@ SSyncTimer* syncNodeGetHbTimer(SSyncNode* ths, SRaftId* pDestId) { SPeerState* syncNodeGetPeerState(SSyncNode* ths, const SRaftId* pDestId) { SPeerState* pState = NULL; - for (int32_t i = 0; i < ths->replicaNum; ++i) { + for (int32_t i = 0; i < ths->totalReplicaNum; ++i) { if (syncUtilSameId(pDestId, &((ths->replicasId)[i]))) { pState = &((ths->peerStates)[i]); } diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c index 6bebef77dc..81dbead506 100644 --- a/source/libs/sync/src/syncPipeline.c +++ b/source/libs/sync/src/syncPipeline.c @@ -242,6 +242,8 @@ int32_t syncLogBufferInitWithoutLock(SSyncLogBuffer* pBuf, SSyncNode* pNode) { // update startIndex pBuf->startIndex = takeDummy ? index : index + 1; + pBuf->isCatchup = false; + sInfo("vgId:%d, init sync log buffer. buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 ")", pNode->vgId, pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex); @@ -324,6 +326,15 @@ int32_t syncLogBufferAccept(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEnt goto _out; } + if(pNode->raftCfg.cfg.nodeInfo[pNode->raftCfg.cfg.myIndex].nodeRole == TAOS_SYNC_ROLE_LEARNER && + index > 0 && index > pBuf->totalIndex){ + pBuf->totalIndex = index; + sTrace("vgId:%d, update learner progress. index:%" PRId64 ", term:%" PRId64 ": prevterm:%" PRId64 + " != lastmatch:%" PRId64 ". log buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 ")", + pNode->vgId, pEntry->index, pEntry->term, prevTerm, lastMatchTerm, pBuf->startIndex, pBuf->commitIndex, + pBuf->matchIndex, pBuf->endIndex); + } + if (index - pBuf->startIndex >= pBuf->size) { sWarn("vgId:%d, out of buffer range. index:%" PRId64 ", term:%" PRId64 ". log buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 ")", @@ -483,7 +494,7 @@ _out: } int32_t syncFsmExecute(SSyncNode* pNode, SSyncFSM* pFsm, ESyncState role, SyncTerm term, SSyncRaftEntry* pEntry, - int32_t applyCode) { + int32_t applyCode) { if (pNode->replicaNum == 1 && pNode->restoreFinish && pNode->vgId != 1) { return 0; } @@ -957,7 +968,7 @@ void syncLogReplDestroy(SSyncLogReplMgr* pMgr) { } int32_t syncNodeLogReplInit(SSyncNode* pNode) { - for (int i = 0; i < TSDB_MAX_REPLICA; i++) { + for (int i = 0; i < TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA; i++) { ASSERT(pNode->logReplMgrs[i] == NULL); pNode->logReplMgrs[i] = syncLogReplCreate(); if (pNode->logReplMgrs[i] == NULL) { @@ -970,7 +981,7 @@ int32_t syncNodeLogReplInit(SSyncNode* pNode) { } void syncNodeLogReplDestroy(SSyncNode* pNode) { - for (int i = 0; i < TSDB_MAX_REPLICA; i++) { + for (int i = 0; i < TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA; i++) { syncLogReplDestroy(pNode->logReplMgrs[i]); pNode->logReplMgrs[i] = NULL; } @@ -1100,7 +1111,7 @@ int32_t syncLogBufferReset(SSyncLogBuffer* pBuf, SSyncNode* pNode) { pBuf->endIndex = pBuf->matchIndex + 1; // reset repl mgr - for (int i = 0; i < pNode->replicaNum; i++) { + for (int i = 0; i < pNode->totalReplicaNum; i++) { SSyncLogReplMgr* pMgr = pNode->logReplMgrs[i]; syncLogReplReset(pMgr); } diff --git a/source/libs/sync/src/syncRaftCfg.c b/source/libs/sync/src/syncRaftCfg.c index f780e255ce..88a57534f4 100644 --- a/source/libs/sync/src/syncRaftCfg.c +++ b/source/libs/sync/src/syncRaftCfg.c @@ -18,21 +18,45 @@ #include "syncUtil.h" #include "tjson.h" +const char* syncRoleToStr(ESyncRole role) { + switch (role) { + case TAOS_SYNC_ROLE_VOTER: + return "voter"; + case TAOS_SYNC_ROLE_LEARNER: + return "learner"; + default: + return "unknown"; + } +} + +const ESyncRole syncStrToRole(char* str) { + if(strcmp(str, "voter") == 0){ + return TAOS_SYNC_ROLE_VOTER; + } + if(strcmp(str, "learner") == 0){ + return TAOS_SYNC_ROLE_LEARNER; + } + + return TAOS_SYNC_ROLE_ERROR; +} + static int32_t syncEncodeSyncCfg(const void *pObj, SJson *pJson) { SSyncCfg *pCfg = (SSyncCfg *)pObj; + if (tjsonAddDoubleToObject(pJson, "totalReplicaNum", pCfg->totalReplicaNum) < 0) return -1; if (tjsonAddDoubleToObject(pJson, "replicaNum", pCfg->replicaNum) < 0) return -1; if (tjsonAddDoubleToObject(pJson, "myIndex", pCfg->myIndex) < 0) return -1; SJson *nodeInfo = tjsonCreateArray(); if (nodeInfo == NULL) return -1; if (tjsonAddItemToObject(pJson, "nodeInfo", nodeInfo) < 0) return -1; - for (int32_t i = 0; i < pCfg->replicaNum; ++i) { + for (int32_t i = 0; i < pCfg->totalReplicaNum; ++i) { SJson *info = tjsonCreateObject(); if (info == NULL) return -1; if (tjsonAddDoubleToObject(info, "nodePort", pCfg->nodeInfo[i].nodePort) < 0) return -1; if (tjsonAddStringToObject(info, "nodeFqdn", pCfg->nodeInfo[i].nodeFqdn) < 0) return -1; if (tjsonAddIntegerToObject(info, "nodeId", pCfg->nodeInfo[i].nodeId) < 0) return -1; if (tjsonAddIntegerToObject(info, "clusterId", pCfg->nodeInfo[i].clusterId) < 0) return -1; + if (tjsonAddStringToObject(info, "nodeRole", syncRoleToStr(pCfg->nodeInfo[i].nodeRole)) < 0) return -1; if (tjsonAddItemToArray(nodeInfo, info) < 0) return -1; } @@ -90,7 +114,8 @@ int32_t syncWriteCfgFile(SSyncNode *pNode) { if (taosRenameFile(file, realfile) != 0) goto _OVER; code = 0; - sInfo("vgId:%d, succeed to write sync cfg file:%s, len:%d", pNode->vgId, realfile, len); + sInfo("vgId:%d, succeed to write sync cfg file:%s, len:%d, lastConfigIndex:%" PRId64, pNode->vgId, + realfile, len, pNode->raftCfg.lastConfigIndex); _OVER: if (pJson != NULL) tjsonDelete(pJson); @@ -108,6 +133,7 @@ static int32_t syncDecodeSyncCfg(const SJson *pJson, void *pObj) { SSyncCfg *pCfg = (SSyncCfg *)pObj; int32_t code = 0; + tjsonGetInt32ValueFromDouble(pJson, "totalReplicaNum", pCfg->totalReplicaNum, code); tjsonGetInt32ValueFromDouble(pJson, "replicaNum", pCfg->replicaNum, code); if (code < 0) return -1; tjsonGetInt32ValueFromDouble(pJson, "myIndex", pCfg->myIndex, code); @@ -115,9 +141,9 @@ static int32_t syncDecodeSyncCfg(const SJson *pJson, void *pObj) { SJson *nodeInfo = tjsonGetObjectItem(pJson, "nodeInfo"); if (nodeInfo == NULL) return -1; - pCfg->replicaNum = tjsonGetArraySize(nodeInfo); + pCfg->totalReplicaNum = tjsonGetArraySize(nodeInfo); - for (int32_t i = 0; i < pCfg->replicaNum; ++i) { + for (int32_t i = 0; i < pCfg->totalReplicaNum; ++i) { SJson *info = tjsonGetArrayItem(nodeInfo, i); if (info == NULL) return -1; tjsonGetUInt16ValueFromDouble(info, "nodePort", pCfg->nodeInfo[i].nodePort, code); @@ -126,6 +152,10 @@ static int32_t syncDecodeSyncCfg(const SJson *pJson, void *pObj) { if (code < 0) return -1; tjsonGetNumberValue(info, "nodeId", pCfg->nodeInfo[i].nodeId, code); tjsonGetNumberValue(info, "clusterId", pCfg->nodeInfo[i].clusterId, code); + char role[10]; + code = tjsonGetStringValue(info, "nodeRole", role); + if(code < 0) return -1; + pCfg->nodeInfo[i].nodeRole = syncStrToRole(role); } return 0; diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index 8ac9a860e3..77e73402fa 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -66,10 +66,10 @@ int32_t syncNodeReplicate(SSyncNode* pNode) { } int32_t syncNodeReplicateWithoutLock(SSyncNode* pNode) { - if (pNode->state != TAOS_SYNC_STATE_LEADER || pNode->replicaNum == 1) { + if (pNode->state != TAOS_SYNC_STATE_LEADER || pNode->raftCfg.cfg.totalReplicaNum == 1) { return -1; } - for (int32_t i = 0; i < pNode->replicaNum; i++) { + for (int32_t i = 0; i < pNode->totalReplicaNum; i++) { if (syncUtilSameId(&pNode->replicasId[i], &pNode->myRaftId)) { continue; } diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index a9d0ddad17..763d4ec5d6 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -795,13 +795,18 @@ int32_t syncNodeOnSnapshot(SSyncNode *pSyncNode, const SRpcMsg *pRpcMsg) { return -1; } - if (pMsg->term > raftStoreGetTerm(pSyncNode)) { - syncNodeStepDown(pSyncNode, pMsg->term); + if(pSyncNode->raftCfg.cfg.nodeInfo[pSyncNode->raftCfg.cfg.myIndex].nodeRole != TAOS_SYNC_ROLE_LEARNER){ + if (pMsg->term > raftStoreGetTerm(pSyncNode)) { + syncNodeStepDown(pSyncNode, pMsg->term); + } + } + else{ + syncNodeUpdateTermWithoutStepDown(pSyncNode, pMsg->term); } // state, term, seq/ack int32_t code = 0; - if (pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER) { + if (pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER || pSyncNode->state == TAOS_SYNC_STATE_LEARNER) { if (pMsg->term == raftStoreGetTerm(pSyncNode)) { if (pMsg->seq == SYNC_SNAPSHOT_SEQ_PREP_SNAPSHOT) { syncLogRecvSyncSnapshotSend(pSyncNode, pMsg, "process seq pre-snapshot"); diff --git a/source/libs/sync/src/syncTimeout.c b/source/libs/sync/src/syncTimeout.c index a27be2853e..5ee67da9ab 100644 --- a/source/libs/sync/src/syncTimeout.c +++ b/source/libs/sync/src/syncTimeout.c @@ -58,7 +58,7 @@ static void syncNodeCleanConfigIndex(SSyncNode* ths) { static int32_t syncNodeTimerRoutine(SSyncNode* ths) { ths->tmrRoutineNum++; - if (ths->tmrRoutineNum % 60 == 0 && ths->replicaNum > 1) { + if (ths->tmrRoutineNum % 60 == 0 && ths->totalReplicaNum > 1) { sNInfo(ths, "timer routines"); } else { sNTrace(ths, "timer routines"); diff --git a/source/libs/sync/src/syncVoteMgr.c b/source/libs/sync/src/syncVoteMgr.c index 6bd9625276..83b0dde8e0 100644 --- a/source/libs/sync/src/syncVoteMgr.c +++ b/source/libs/sync/src/syncVoteMgr.c @@ -30,7 +30,7 @@ SVotesGranted *voteGrantedCreate(SSyncNode *pNode) { return NULL; } - pVotesGranted->replicas = &pNode->replicasId; + pVotesGranted->replicas = (void*)&pNode->replicasId; pVotesGranted->replicaNum = pNode->replicaNum; voteGrantedClearVotes(pVotesGranted); @@ -49,7 +49,7 @@ void voteGrantedDestroy(SVotesGranted *pVotesGranted) { } void voteGrantedUpdate(SVotesGranted *pVotesGranted, SSyncNode *pNode) { - pVotesGranted->replicas = &pNode->replicasId; + pVotesGranted->replicas = (void*)&pNode->replicasId; pVotesGranted->replicaNum = pNode->replicaNum; voteGrantedClearVotes(pVotesGranted); @@ -115,7 +115,7 @@ SVotesRespond *votesRespondCreate(SSyncNode *pNode) { return NULL; } - pVotesRespond->replicas = &pNode->replicasId; + pVotesRespond->replicas = (void*)&pNode->replicasId; pVotesRespond->replicaNum = pNode->replicaNum; pVotesRespond->term = 0; pVotesRespond->pNode = pNode; @@ -130,7 +130,7 @@ void votesRespondDestory(SVotesRespond *pVotesRespond) { } void votesRespondUpdate(SVotesRespond *pVotesRespond, SSyncNode *pNode) { - pVotesRespond->replicas = &pNode->replicasId; + pVotesRespond->replicas = (void*)&pNode->replicasId; pVotesRespond->replicaNum = pNode->replicaNum; pVotesRespond->term = 0; pVotesRespond->pNode = pNode; From 18b4694327231cf086b3796495ce4be743bf9b7a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 18 Apr 2023 19:22:52 +0800 Subject: [PATCH 18/35] Update 23-perf.md --- docs/en/12-taos-sql/23-perf.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/12-taos-sql/23-perf.md b/docs/en/12-taos-sql/23-perf.md index fc369ec663..43ff8e3091 100644 --- a/docs/en/12-taos-sql/23-perf.md +++ b/docs/en/12-taos-sql/23-perf.md @@ -69,7 +69,7 @@ Provides information about SQL queries currently running. Similar to SHOW QUERIE | 1 | consumer_id | BIGINT | Consumer ID | | 2 | consumer_group | BINARY(192) | Consumer group | | 3 | client_id | BINARY(192) | Client ID (user-defined) | -| 4 | status | BINARY(20) | Consumer status | +| 4 | status | BINARY(20) | Consumer status. All possible status include: ready(consumer is in normal state), lost(the connection between consumer and mnode is broken), rebalance(the redistribution of vgroups that belongs to current consumer is now in progress), unknown(consumer is in invalid state) | 5 | topics | BINARY(204) | Subscribed topic. Returns one row for each topic. | | 6 | up_time | TIMESTAMP | Time of first connection to TDengine Server | | 7 | subscribe_time | TIMESTAMP | Time of first subscription | From 586b3945d1571b2d0ca9320e80fbae147721849b Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Wed, 19 Apr 2023 13:09:11 +0800 Subject: [PATCH 19/35] feat: last queries with tags output can be read from cache --- source/dnode/vnode/src/tsdb/tsdbCacheRead.c | 10 ++++------ source/libs/planner/src/planOptimizer.c | 12 +++++------- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index f537b7d6e8..d6e819d84a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -408,12 +408,10 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 } } - if (0 == i) { - if (taosArrayGetSize(pTableUidList) == 0) { - taosArrayPush(pTableUidList, &pKeyInfo->uid); - } else { - taosArraySet(pTableUidList, 0, &pKeyInfo->uid); - } + if (taosArrayGetSize(pTableUidList) == 0) { + taosArrayPush(pTableUidList, &pKeyInfo->uid); + } else { + taosArraySet(pTableUidList, 0, &pKeyInfo->uid); } tsdbCacheRelease(lruCache, h); diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index d3a03b7b75..07ea110d7e 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -2195,7 +2195,7 @@ static bool lastRowScanOptMayBeOptimized(SLogicNode* pNode) { FOREACH(pFunc, ((SAggLogicNode*)pNode)->pAggFuncs) { SFunctionNode* pAggFunc = (SFunctionNode*)pFunc; if (FUNCTION_TYPE_LAST == pAggFunc->funcType) { - if (hasSelectFunc) { + if (hasSelectFunc || QUERY_NODE_VALUE == nodeType(nodesListGetNode(pAggFunc->pParameterList, 0))) { return false; } hasLastFunc = true; @@ -2241,7 +2241,7 @@ static EDealRes lastRowScanOptSetColDataType(SNode* pNode, void* pContext) { return DEAL_RES_CONTINUE; } -static void lastRowScanOptSetLastTargets(SNodeList* pTargets, SNodeList* pLastCols) { +static void lastRowScanOptSetLastTargets(SNodeList* pTargets, SNodeList* pLastCols, bool erase) { SNode* pTarget = NULL; WHERE_EACH(pTarget, pTargets) { bool found = false; @@ -2253,7 +2253,7 @@ static void lastRowScanOptSetLastTargets(SNodeList* pTargets, SNodeList* pLastCo break; } } - if (!found) { + if (!found && erase) { ERASE_NODE(pTargets); continue; } @@ -2286,8 +2286,6 @@ static int32_t lastRowScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogic nodesWalkExpr(nodesListGetNode(pFunc->pParameterList, 0), lastRowScanOptSetColDataType, &cxt); nodesListErase(pFunc->pParameterList, nodesListGetCell(pFunc->pParameterList, 1)); } - } else if (FUNCTION_TYPE_GROUP_KEY == funcType) { - nodesListMakeAppend(&cxt.pLastCols, nodesListGetNode(pFunc->pParameterList, 0)); } } @@ -2296,8 +2294,8 @@ static int32_t lastRowScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogic pScan->igLastNull = pAgg->hasLast ? true : false; if (NULL != cxt.pLastCols) { cxt.doAgg = false; - lastRowScanOptSetLastTargets(pScan->pScanCols, cxt.pLastCols); - lastRowScanOptSetLastTargets(pScan->node.pTargets, cxt.pLastCols); + lastRowScanOptSetLastTargets(pScan->pScanCols, cxt.pLastCols, true); + lastRowScanOptSetLastTargets(pScan->node.pTargets, cxt.pLastCols, false); nodesClearList(cxt.pLastCols); } pAgg->hasLastRow = false; From c22bd63ed40a1c476c06862cca6e84730ed350d7 Mon Sep 17 00:00:00 2001 From: cadem Date: Wed, 19 Apr 2023 15:11:16 +0800 Subject: [PATCH 20/35] vnode config compitible --- source/dnode/vnode/src/vnd/vnodeCfg.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeCfg.c b/source/dnode/vnode/src/vnd/vnodeCfg.c index a760fc03c2..7d7a76556c 100644 --- a/source/dnode/vnode/src/vnd/vnodeCfg.c +++ b/source/dnode/vnode/src/vnd/vnodeCfg.c @@ -293,8 +293,13 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { tjsonGetNumberValue(info, "clusterId", pNode->clusterId, code); if (code < 0) return -1; char role[10]; - tjsonGetStringValue(info, "nodeRole", role); - pNode->nodeRole = vnodeStrToRole(role); + code = tjsonGetStringValue(info, "nodeRole", role); + if(code > 0){ + pNode->nodeRole = vnodeStrToRole(role); + } + else{ + pNode->nodeRole = TAOS_SYNC_ROLE_VOTER; + } vDebug("vgId:%d, decode config, replica:%d ep:%s:%u dnode:%d", pCfg->vgId, i, pNode->nodeFqdn, pNode->nodePort, pNode->nodeId); } From 5367f25b4d53caa9e88dba8eea1dce8cd15fe73a Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Wed, 19 Apr 2023 16:55:08 +0800 Subject: [PATCH 21/35] feat: the having clause can be used after the partition by clause or window clause --- source/libs/parser/src/parTranslater.c | 5 +++-- source/libs/planner/src/planLogicCreater.c | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index c6e70798a7..25e92a55ec 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3035,12 +3035,13 @@ static int32_t translateSelectList(STranslateContext* pCxt, SSelectStmt* pSelect } static int32_t translateHaving(STranslateContext* pCxt, SSelectStmt* pSelect) { - if (NULL == pSelect->pGroupByList && NULL != pSelect->pHaving) { + if (NULL == pSelect->pGroupByList && NULL == pSelect->pPartitionByList && NULL == pSelect->pWindow && + NULL != pSelect->pHaving) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION); } pCxt->currClause = SQL_CLAUSE_HAVING; int32_t code = translateExpr(pCxt, &pSelect->pHaving); - if (TSDB_CODE_SUCCESS == code) { + if (TSDB_CODE_SUCCESS == code && (NULL != pSelect->pGroupByList || NULL != pSelect->pWindow)) { code = checkExprForGroupBy(pCxt, &pSelect->pHaving); } return code; diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index c9ee83a647..6544898be9 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -740,6 +740,13 @@ static int32_t createWindowLogicNodeFinalize(SLogicPlanContext* pCxt, SSelectStm code = createColumnByRewriteExprs(pWindow->pFuncs, &pWindow->node.pTargets); } + if (TSDB_CODE_SUCCESS == code && NULL != pSelect->pHaving) { + pWindow->node.pConditions = nodesCloneNode(pSelect->pHaving); + if (NULL == pWindow->node.pConditions) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + pSelect->hasAggFuncs = false; if (TSDB_CODE_SUCCESS == code) { @@ -1132,6 +1139,14 @@ static int32_t createPartitionLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pS } } + if (TSDB_CODE_SUCCESS == code && NULL != pSelect->pHaving && !pSelect->hasAggFuncs && NULL == pSelect->pGroupByList && + NULL == pSelect->pWindow) { + pPartition->node.pConditions = nodesCloneNode(pSelect->pHaving); + if (NULL == pPartition->node.pConditions) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + if (TSDB_CODE_SUCCESS == code) { *pLogicNode = (SLogicNode*)pPartition; } else { From 8b16f4b118840e9a19a6aac49e0a5cc63c0e9ca3 Mon Sep 17 00:00:00 2001 From: cadem Date: Wed, 19 Apr 2023 18:02:04 +0800 Subject: [PATCH 22/35] config compitible --- source/dnode/vnode/src/vnd/vnodeCfg.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeCfg.c b/source/dnode/vnode/src/vnd/vnodeCfg.c index 7d7a76556c..65f32b0a85 100644 --- a/source/dnode/vnode/src/vnd/vnodeCfg.c +++ b/source/dnode/vnode/src/vnd/vnodeCfg.c @@ -277,6 +277,9 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { SJson *nodeInfo = tjsonGetObjectItem(pJson, "syncCfg.nodeInfo"); int arraySize = tjsonGetArraySize(nodeInfo); + if(pCfg->syncCfg.totalReplicaNum == 0 && pCfg->syncCfg.replicaNum > 0){ + pCfg->syncCfg.totalReplicaNum = pCfg->syncCfg.replicaNum; + } if (arraySize != pCfg->syncCfg.totalReplicaNum) return -1; vDebug("vgId:%d, decode config, replicas:%d totalReplicas:%d selfIndex:%d", pCfg->vgId, pCfg->syncCfg.replicaNum, @@ -292,9 +295,10 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { if (code < 0) return -1; tjsonGetNumberValue(info, "clusterId", pNode->clusterId, code); if (code < 0) return -1; - char role[10]; + char role[10] = {0}; code = tjsonGetStringValue(info, "nodeRole", role); - if(code > 0){ + if (code < 0) return -1; + if(strlen(role) != 0){ pNode->nodeRole = vnodeStrToRole(role); } else{ From ca6ee940f742e8a25e2570c5e021cef28848cafd Mon Sep 17 00:00:00 2001 From: cadem Date: Wed, 19 Apr 2023 18:59:40 +0800 Subject: [PATCH 23/35] typo --- source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 42dfb873d3..e2e2323bdb 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -333,7 +333,7 @@ int32_t vmProcessAlterVnodeTypeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { } dInfo("vgId:%d, checking node catch up", req.vgId); - if(!vnodeIsCatchUp(pVnode->pImpl) == 0){ + if(vnodeIsCatchUp(pVnode->pImpl) != 0){ return -1; } From 66bacdb16523dc61ca3fdc737fea806cbaf051c6 Mon Sep 17 00:00:00 2001 From: cadem Date: Wed, 19 Apr 2023 20:13:32 +0800 Subject: [PATCH 24/35] vnode config compitible --- source/libs/sync/src/syncRaftCfg.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/source/libs/sync/src/syncRaftCfg.c b/source/libs/sync/src/syncRaftCfg.c index 88a57534f4..576b9d62f5 100644 --- a/source/libs/sync/src/syncRaftCfg.c +++ b/source/libs/sync/src/syncRaftCfg.c @@ -152,10 +152,15 @@ static int32_t syncDecodeSyncCfg(const SJson *pJson, void *pObj) { if (code < 0) return -1; tjsonGetNumberValue(info, "nodeId", pCfg->nodeInfo[i].nodeId, code); tjsonGetNumberValue(info, "clusterId", pCfg->nodeInfo[i].clusterId, code); - char role[10]; + char role[10] = {0}; code = tjsonGetStringValue(info, "nodeRole", role); if(code < 0) return -1; - pCfg->nodeInfo[i].nodeRole = syncStrToRole(role); + if(strlen(role) != 0){ + pCfg->nodeInfo[i].nodeRole = syncStrToRole(role); + } + else{ + pCfg->nodeInfo[i].nodeRole = TAOS_SYNC_ROLE_VOTER; + } } return 0; From d51070922b870948c2f63bdadf04f185efbd3612 Mon Sep 17 00:00:00 2001 From: cadem Date: Thu, 20 Apr 2023 09:45:32 +0800 Subject: [PATCH 25/35] message compitible --- source/common/src/tmsg.c | 38 ++++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index c785dff9f8..8c73a45832 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -4213,11 +4213,13 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq * for (int32_t i = 0; i < 8; ++i) { if (tDecodeI64(&decoder, &pReq->reserved[i]) < 0) return -1; } - if (tDecodeI8(&decoder, &pReq->learnerReplica) < 0) return -1; - if (tDecodeI8(&decoder, &pReq->learnerSelfIndex) < 0) return -1; - for (int32_t i = 0; i < TSDB_MAX_LEARNER_REPLICA; ++i) { - SReplica *pReplica = &pReq->learnerReplicas[i]; - if (tDecodeSReplica(&decoder, pReplica) < 0) return -1; + if (tDecodeIsEnd(&decoder)) { + if (tDecodeI8(&decoder, &pReq->learnerReplica) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->learnerSelfIndex) < 0) return -1; + for (int32_t i = 0; i < TSDB_MAX_LEARNER_REPLICA; ++i) { + SReplica *pReplica = &pReq->learnerReplicas[i]; + if (tDecodeSReplica(&decoder, pReplica) < 0) return -1; + } } tEndDecode(&decoder); @@ -4474,13 +4476,15 @@ int32_t tDeserializeSAlterVnodeReplicaReq(void *buf, int32_t bufLen, SAlterVnode for (int32_t i = 0; i < 8; ++i) { if (tDecodeI64(&decoder, &pReq->reserved[i]) < 0) return -1; } - if (tDecodeI8(&decoder, &pReq->learnerSelfIndex) < 0) return -1; - if (tDecodeI8(&decoder, &pReq->learnerReplica) < 0) return -1; - for (int32_t i = 0; i < TSDB_MAX_LEARNER_REPLICA; ++i) { - SReplica *pReplica = &pReq->learnerReplicas[i]; - if (tDecodeSReplica(&decoder, pReplica) < 0) return -1; + if (tDecodeIsEnd(&decoder)) { + if (tDecodeI8(&decoder, &pReq->learnerSelfIndex) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->learnerReplica) < 0) return -1; + for (int32_t i = 0; i < TSDB_MAX_LEARNER_REPLICA; ++i) { + SReplica *pReplica = &pReq->learnerReplicas[i]; + if (tDecodeSReplica(&decoder, pReplica) < 0) return -1; + } } - + tEndDecode(&decoder); tDecoderClear(&decoder); return 0; @@ -4814,12 +4818,14 @@ int32_t tDeserializeSDCreateMnodeReq(void *buf, int32_t bufLen, SDCreateMnodeReq SReplica *pReplica = &pReq->replicas[i]; if (tDecodeSReplica(&decoder, pReplica) < 0) return -1; } - if (tDecodeI8(&decoder, &pReq->learnerReplica) < 0) return -1; - for (int32_t i = 0; i < TSDB_MAX_LEARNER_REPLICA; ++i) { - SReplica *pReplica = &pReq->learnerReplicas[i]; - if (tDecodeSReplica(&decoder, pReplica) < 0) return -1; + if (tDecodeIsEnd(&decoder)) { + if (tDecodeI8(&decoder, &pReq->learnerReplica) < 0) return -1; + for (int32_t i = 0; i < TSDB_MAX_LEARNER_REPLICA; ++i) { + SReplica *pReplica = &pReq->learnerReplicas[i]; + if (tDecodeSReplica(&decoder, pReplica) < 0) return -1; + } + if (tDecodeI64(&decoder, &pReq->lastIndex) < 0) return -1; } - if (tDecodeI64(&decoder, &pReq->lastIndex) < 0) return -1; tEndDecode(&decoder); tDecoderClear(&decoder); From 55c6f115b05105d7c3f455c9902fc233d9a6d6de Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 18 Apr 2023 10:58:19 +0800 Subject: [PATCH 26/35] enh: add applied-index in sync node logging msg --- source/libs/sync/src/syncUtil.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index 056a597777..cf796c3862 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -202,21 +202,22 @@ void syncPrintNodeLog(const char* flags, ELogLevel level, int32_t dflag, SSyncNo // restore error code terrno = errCode; - + SyncIndex appliedIndex = pNode->pFsm->FpAppliedIndexCb(pNode->pFsm); + if (pNode != NULL) { taosPrintLog(flags, level, dflag, - "vgId:%d, %s, sync:%s, term:%" PRIu64 ", commit-index:%" PRId64 ", first-ver:%" PRId64 - ", last-ver:%" PRId64 ", min:%" PRId64 ", snap:%" PRId64 ", snap-term:%" PRIu64 + "vgId:%d, %s, sync:%s, term:%" PRIu64 ", commit-index:%" PRId64 ", applied-index:%" PRId64 + ", first-ver:%" PRId64 ", last-ver:%" PRId64 ", min:%" PRId64 ", snap:%" PRId64 ", snap-term:%" PRIu64 ", elect-times:%d, as-leader-times:%d, cfg-ch-times:%d, hb-slow:%d, hbr-slow:%d, " "aq-items:%d, snaping:%" PRId64 ", replicas:%d, last-cfg:%" PRId64 ", chging:%d, restore:%d, quorum:%d, elect-lc-timer:%" PRId64 ", hb:%" PRId64 ", buffer:%s, repl-mgrs:%s, members:%s, hb:%s, hb-reply:%s", - pNode->vgId, eventLog, syncStr(pNode->state), currentTerm, pNode->commitIndex, logBeginIndex, - logLastIndex, pNode->minMatchIndex, snapshot.lastApplyIndex, snapshot.lastApplyTerm, pNode->electNum, - pNode->becomeLeaderNum, pNode->configChangeNum, pNode->hbSlowNum, pNode->hbrSlowNum, aqItems, - pNode->snapshottingIndex, pNode->replicaNum, pNode->raftCfg.lastConfigIndex, pNode->changing, - pNode->restoreFinish, syncNodeDynamicQuorum(pNode), pNode->electTimerLogicClock, pNode->heartbeatTimerLogicClockUser, - bufferStatesStr, replMgrStatesStr, cfgStr, hbTimeStr, hbrTimeStr); + pNode->vgId, eventLog, syncStr(pNode->state), currentTerm, pNode->commitIndex, appliedIndex, + logBeginIndex, logLastIndex, pNode->minMatchIndex, snapshot.lastApplyIndex, snapshot.lastApplyTerm, + pNode->electNum, pNode->becomeLeaderNum, pNode->configChangeNum, pNode->hbSlowNum, pNode->hbrSlowNum, + aqItems, pNode->snapshottingIndex, pNode->replicaNum, pNode->raftCfg.lastConfigIndex, pNode->changing, + pNode->restoreFinish, syncNodeDynamicQuorum(pNode), pNode->electTimerLogicClock, + pNode->heartbeatTimerLogicClockUser, bufferStatesStr, replMgrStatesStr, cfgStr, hbTimeStr, hbrTimeStr); } } From a21e0fe75e9a15e7423e2e51e137123b61455bc6 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 20 Apr 2023 10:21:50 +0800 Subject: [PATCH 27/35] enh: propose vnode commit synchronously --- include/common/tmsg.h | 2 +- source/dnode/vnode/src/inc/vnd.h | 1 - source/dnode/vnode/src/inc/vnodeInt.h | 4 ---- source/dnode/vnode/src/vnd/vnodeCommit.c | 14 +------------- source/dnode/vnode/src/vnd/vnodeOpen.c | 2 -- source/dnode/vnode/src/vnd/vnodeSync.c | 5 ----- 6 files changed, 2 insertions(+), 26 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index bb2450e8f7..6d1d3ebce6 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -77,7 +77,7 @@ static inline bool tmsgIsValid(tmsg_t type) { } static inline bool vnodeIsMsgBlock(tmsg_t type) { return (type == TDMT_VND_CREATE_TABLE) || (type == TDMT_VND_ALTER_TABLE) || (type == TDMT_VND_DROP_TABLE) || - (type == TDMT_VND_UPDATE_TAG_VAL) || (type == TDMT_VND_ALTER_CONFIRM); + (type == TDMT_VND_UPDATE_TAG_VAL) || (type == TDMT_VND_ALTER_CONFIRM) || (type == TDMT_VND_COMMIT); } static inline bool syncUtilUserCommit(tmsg_t msgType) { diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h index ae65e2ba3f..a67f246e73 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnd.h @@ -97,7 +97,6 @@ int32_t vnodeGetBatchMeta(SVnode* pVnode, SRpcMsg* pMsg); // vnodeCommit.c int32_t vnodeBegin(SVnode* pVnode); int32_t vnodeShouldCommit(SVnode* pVnode, bool atExit); -void vnodeUpdCommitSched(SVnode* pVnode); void vnodeRollback(SVnode* pVnode); int32_t vnodeSaveInfo(const char* dir, const SVnodeInfo* pCfg); int32_t vnodeCommitInfo(const char* dir); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 81f7c3d52a..2576fce998 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -378,7 +378,6 @@ struct SVnode { STQ* pTq; SSink* pSink; tsem_t canCommit; - SVCommitSched commitSched; int64_t sync; TdThreadMutex lock; bool blocked; @@ -387,9 +386,6 @@ struct SVnode { int32_t blockSec; int64_t blockSeq; SQHandle* pQuery; -#if 0 - SRpcHandleInfo blockInfo; -#endif }; #define TD_VID(PVNODE) ((PVNODE)->config.vgId) diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 3eb813f394..847125018c 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -143,23 +143,13 @@ _exit: return code; } -void vnodeUpdCommitSched(SVnode *pVnode) { - int64_t randNum = taosRand(); - pVnode->commitSched.commitMs = taosGetMonoTimestampMs(); - pVnode->commitSched.maxWaitMs = tsVndCommitMaxIntervalMs + (randNum % tsVndCommitMaxIntervalMs); -} - int vnodeShouldCommit(SVnode *pVnode, bool atExit) { - SVCommitSched *pSched = &pVnode->commitSched; - int64_t nowMs = taosGetMonoTimestampMs(); bool diskAvail = osDataSpaceAvailable(); bool needCommit = false; taosThreadMutexLock(&pVnode->mutex); if (pVnode->inUse && diskAvail) { - needCommit = - ((pVnode->inUse->size > pVnode->inUse->node.size) && (pSched->commitMs + SYNC_VND_COMMIT_MIN_MS < nowMs)) || - ((pVnode->inUse->size > 0) && atExit); + needCommit = (pVnode->inUse->size > pVnode->inUse->node.size) || (pVnode->inUse->size > 0 && atExit); } taosThreadMutexUnlock(&pVnode->mutex); return needCommit; @@ -431,8 +421,6 @@ static int vnodeCommitImpl(SCommitInfo *pInfo) { vInfo("vgId:%d, start to commit, commitId:%" PRId64 " version:%" PRId64 " term: %" PRId64, TD_VID(pVnode), pInfo->info.state.commitID, pInfo->info.state.committed, pInfo->info.state.commitTerm); - vnodeUpdCommitSched(pVnode); - // persist wal before starting if (walPersist(pVnode->pWal) < 0) { vError("vgId:%d, failed to persist wal since %s", TD_VID(pVnode), terrstr()); diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index c7d155be0d..e8f34d27ac 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -286,8 +286,6 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { taosThreadMutexInit(&pVnode->mutex, NULL); taosThreadCondInit(&pVnode->poolNotEmpty, NULL); - vnodeUpdCommitSched(pVnode); - int8_t rollback = vnodeShouldRollback(pVnode); // open buffer pool diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index d4a394b584..fb7f2901a1 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -112,9 +112,6 @@ static int32_t inline vnodeProposeMsg(SVnode *pVnode, SRpcMsg *pMsg, bool isWeak pVnode->blocked = true; pVnode->blockSec = taosGetTimestampSec(); pVnode->blockSeq = seq; -#if 0 - pVnode->blockInfo = pMsg->info; -#endif } taosThreadMutexUnlock(&pVnode->lock); @@ -157,8 +154,6 @@ void vnodeProposeCommitOnNeed(SVnode *pVnode, bool atExit) { } else { tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &rpcMsg); } - - vnodeUpdCommitSched(pVnode); } #if BATCH_ENABLE From a1ca9466cdba97d6d04a994e5bd548e34e7df30e Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Thu, 20 Apr 2023 10:23:51 +0800 Subject: [PATCH 28/35] enh: adjust size limit of appyQ and negotiationWin --- include/util/taoserror.h | 1 + include/util/tdef.h | 5 ++++- source/libs/sync/src/syncPipeline.c | 10 +++++++++- source/util/src/terror.c | 1 + 4 files changed, 15 insertions(+), 2 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index ab89466a19..0d9292cc6b 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -547,6 +547,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_SYN_INVALID_SNAPSHOT_MSG TAOS_DEF_ERROR_CODE(0, 0x0915) // internal #define TSDB_CODE_SYN_BUFFER_FULL TAOS_DEF_ERROR_CODE(0, 0x0916) #define TSDB_CODE_SYN_WRITE_STALL TAOS_DEF_ERROR_CODE(0, 0x0917) +#define TSDB_CODE_SYN_NEGO_WIN_EXCEEDED TAOS_DEF_ERROR_CODE(0, 0X0918) #define TSDB_CODE_SYN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x09FF) // tq diff --git a/include/util/tdef.h b/include/util/tdef.h index 2f86395dad..9dd454bb68 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -285,8 +285,11 @@ typedef enum ELogicConditionType { #define TSDB_DNODE_ROLE_VNODE 2 #define TSDB_MAX_REPLICA 5 + #define TSDB_SYNC_LOG_BUFFER_SIZE 4096 -#define TSDB_SYNC_LOG_BUFFER_RETENTION (TSDB_SYNC_LOG_BUFFER_SIZE >> 4) +#define TSDB_SYNC_LOG_BUFFER_RETENTION 256 +#define TSDB_SYNC_APPLYQ_SIZE_LIMIT 512 +#define TSDB_SYNC_NEGOTIATION_WIN 512 #define TSDB_TBNAME_COLUMN_INDEX (-1) #define TSDB_MULTI_TABLEMETA_MAX_NUM 100000 // maximum batch size allowed to load table meta diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c index 6bebef77dc..5e6058ef03 100644 --- a/source/libs/sync/src/syncPipeline.c +++ b/source/libs/sync/src/syncPipeline.c @@ -53,8 +53,15 @@ int32_t syncLogBufferAppend(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEnt goto _err; } + if (pNode->restoreFinish && index - pBuf->commitIndex >= TSDB_SYNC_NEGOTIATION_WIN) { + terrno = TSDB_CODE_SYN_NEGO_WIN_EXCEEDED; + sError("vgId:%d, failed to append since %s, index:%" PRId64 ", commit-index:%" PRId64, pNode->vgId, terrstr(), + index, pBuf->commitIndex); + goto _err; + } + SyncIndex appliedIndex = pNode->pFsm->FpAppliedIndexCb(pNode->pFsm); - if (pNode->restoreFinish && pBuf->commitIndex - appliedIndex >= pBuf->size) { + if (pNode->restoreFinish && pBuf->commitIndex - appliedIndex >= TSDB_SYNC_APPLYQ_SIZE_LIMIT) { terrno = TSDB_CODE_SYN_WRITE_STALL; sError("vgId:%d, failed to append since %s. index:%" PRId64 ", commit-index:%" PRId64 ", applied-index:%" PRId64, pNode->vgId, terrstr(), index, pBuf->commitIndex, appliedIndex); @@ -83,6 +90,7 @@ int32_t syncLogBufferAppend(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEnt _err: syncLogBufferValidate(pBuf); taosThreadMutexUnlock(&pBuf->mutex); + taosMsleep(1); return -1; } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 002d605793..34b09761c8 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -426,6 +426,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SYN_RESTORING, "Sync leader is restor TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_SNAPSHOT_MSG, "Sync invalid snapshot msg") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_BUFFER_FULL, "Sync buffer is full") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_WRITE_STALL, "Sync write stall") +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_NEGO_WIN_EXCEEDED, "Sync negotiation win exceeded") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INTERNAL_ERROR, "Sync internal error") //tq From 1af9b52d9b3250fd9841b17f66c8395f76db4533 Mon Sep 17 00:00:00 2001 From: wade zhang <95411902+gccgdb1234@users.noreply.github.com> Date: Thu, 20 Apr 2023 16:40:10 +0800 Subject: [PATCH 29/35] Update 02-database.md --- docs/zh/12-taos-sql/02-database.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/zh/12-taos-sql/02-database.md b/docs/zh/12-taos-sql/02-database.md index b54998e08d..dd30635660 100644 --- a/docs/zh/12-taos-sql/02-database.md +++ b/docs/zh/12-taos-sql/02-database.md @@ -36,7 +36,6 @@ database_option: { | TSDB_PAGESIZE value | WAL_RETENTION_PERIOD value | WAL_RETENTION_SIZE value - | WAL_ROLL_PERIOD value | WAL_SEGMENT_SIZE value } ``` From bbb467725f96d5f952561aeecb4c29b72e7b2ffd Mon Sep 17 00:00:00 2001 From: cadem Date: Thu, 20 Apr 2023 21:50:55 +0800 Subject: [PATCH 30/35] message compatible --- source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index e2e2323bdb..d7197832e5 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -201,6 +201,11 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return -1; } + if(req.learnerReplica == 0) + { + req.learnerSelfIndex = -1; + } + dInfo("vgId:%d, vnode management handle msgType:%s, start to create vnode, page:%d pageSize:%d buffer:%d szPage:%d szBuf:%" PRIu64 ", cacheLast:%d cacheLastSize:%d sstTrigger:%d tsdbPageSize:%d %d dbname:%s dbId:%" PRId64 ", days:%d keep0:%d keep1:%d keep2:%d tsma:%d precision:%d compression:%d minRows:%d maxRows:%d" @@ -322,6 +327,10 @@ int32_t vmProcessAlterVnodeTypeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return -1; } + if(req.learnerReplicas == 0){ + req.learnerSelfIndex = -1; + } + dInfo("vgId:%d, vnode management handle msgType:%s, start to process alter-node-type-request", req.vgId, TMSG_INFO(pMsg->msgType)); @@ -509,6 +518,10 @@ int32_t vmProcessAlterVnodeReplicaReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return -1; } + if(alterReq.learnerReplica == 0){ + alterReq.learnerSelfIndex = -1; + } + int32_t vgId = alterReq.vgId; dInfo("vgId:%d,vnode management handle msgType:%s, start to alter vnode replica:%d selfIndex:%d leanerReplica:%d " "learnerSelfIndex:%d strict:%d", From bed2273cece5021130d5316c5ae77bbc22ec5787 Mon Sep 17 00:00:00 2001 From: xiaolei li <85657333+xleili@users.noreply.github.com> Date: Fri, 21 Apr 2023 09:03:20 +0800 Subject: [PATCH 31/35] enhance: package jdbc-driver for enterprise package (#20999) * enhance: package jdbc-driver for enterprise package * enhance: enterprise package include jdbc driver --- packaging/tools/makeclient.sh | 3 ++- packaging/tools/makepkg.sh | 15 ++++++++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/packaging/tools/makeclient.sh b/packaging/tools/makeclient.sh index b473f3b527..1dbfc897bc 100755 --- a/packaging/tools/makeclient.sh +++ b/packaging/tools/makeclient.sh @@ -197,7 +197,8 @@ if [[ $productName == "TDengine" ]]; then mkdir -p ${install_dir}/connector if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then if [ "$osType" != "Darwin" ]; then - [ -f ${build_dir}/lib/*.jar ] && cp ${build_dir}/lib/*.jar ${install_dir}/connector || : + jars=$(ls ${build_dir}/lib/*.jar 2>/dev/null|wc -l) + [ "${jars}" != "0" ] && cp ${build_dir}/lib/*.jar ${install_dir}/connector || : fi git clone --depth 1 https://github.com/taosdata/driver-go ${install_dir}/connector/go rm -rf ${install_dir}/connector/go/.git ||: diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index e4df233d67..a590835257 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -338,7 +338,20 @@ if [ "$verMode" == "cluster" ] || [ "$verMode" == "cloud" ]; then connector_dir="${code_dir}/connector" mkdir -p ${install_dir}/connector if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then - [ -f ${build_dir}/lib/*.jar ] && cp ${build_dir}/lib/*.jar ${install_dir}/connector || : + tmp_pwd=`pwd` + cd ${install_dir}/connector + if [ ! -d taos-connector-jdbc ];then + git clone -b main --depth=1 https://github.com/taosdata/taos-connector-jdbc.git ||: + fi + cd taos-connector-jdbc + mvn clean package -Dmaven.test.skip=true + echo ${build_dir}/lib/ + cp target/*.jar ${build_dir}/lib/ + cd ${install_dir}/connector + rm -rf taos-connector-jdbc + cd ${tmp_pwd} + jars=$(ls ${build_dir}/lib/*.jar 2>/dev/null|wc -l) + [ "${jars}" != "0" ] && cp ${build_dir}/lib/*.jar ${install_dir}/connector || : git clone --depth 1 https://github.com/taosdata/driver-go ${install_dir}/connector/go rm -rf ${install_dir}/connector/go/.git ||: From 6d948d8345cb9b837123ed51f6c8ce9d6d3de17c Mon Sep 17 00:00:00 2001 From: cadem Date: Fri, 21 Apr 2023 09:50:54 +0800 Subject: [PATCH 32/35] message compatible --- source/common/src/tmsg.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 8c73a45832..85c150066d 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -4213,7 +4213,7 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq * for (int32_t i = 0; i < 8; ++i) { if (tDecodeI64(&decoder, &pReq->reserved[i]) < 0) return -1; } - if (tDecodeIsEnd(&decoder)) { + if (!tDecodeIsEnd(&decoder)) { if (tDecodeI8(&decoder, &pReq->learnerReplica) < 0) return -1; if (tDecodeI8(&decoder, &pReq->learnerSelfIndex) < 0) return -1; for (int32_t i = 0; i < TSDB_MAX_LEARNER_REPLICA; ++i) { @@ -4476,7 +4476,7 @@ int32_t tDeserializeSAlterVnodeReplicaReq(void *buf, int32_t bufLen, SAlterVnode for (int32_t i = 0; i < 8; ++i) { if (tDecodeI64(&decoder, &pReq->reserved[i]) < 0) return -1; } - if (tDecodeIsEnd(&decoder)) { + if (!tDecodeIsEnd(&decoder)) { if (tDecodeI8(&decoder, &pReq->learnerSelfIndex) < 0) return -1; if (tDecodeI8(&decoder, &pReq->learnerReplica) < 0) return -1; for (int32_t i = 0; i < TSDB_MAX_LEARNER_REPLICA; ++i) { @@ -4818,7 +4818,7 @@ int32_t tDeserializeSDCreateMnodeReq(void *buf, int32_t bufLen, SDCreateMnodeReq SReplica *pReplica = &pReq->replicas[i]; if (tDecodeSReplica(&decoder, pReplica) < 0) return -1; } - if (tDecodeIsEnd(&decoder)) { + if (!tDecodeIsEnd(&decoder)) { if (tDecodeI8(&decoder, &pReq->learnerReplica) < 0) return -1; for (int32_t i = 0; i < TSDB_MAX_LEARNER_REPLICA; ++i) { SReplica *pReplica = &pReq->learnerReplicas[i]; From c7e930088f4aac46134830fde8774dff075fb819 Mon Sep 17 00:00:00 2001 From: sunpeng Date: Fri, 21 Apr 2023 11:12:11 +0800 Subject: [PATCH 33/35] docs: update taoskeeper and monitor docs (#21004) * docs: update taoskeeper and monitor docs * fix for doc --- docs/en/13-operation/10-monitor.md | 301 ++++++++++++++++++++++++++ docs/en/14-reference/14-taosKeeper.md | 8 +- docs/zh/14-reference/14-taosKeeper.md | 8 +- docs/zh/17-operation/10-monitor.md | 14 +- 4 files changed, 320 insertions(+), 11 deletions(-) diff --git a/docs/en/13-operation/10-monitor.md b/docs/en/13-operation/10-monitor.md index 346b874059..ea401c1e1c 100644 --- a/docs/en/13-operation/10-monitor.md +++ b/docs/en/13-operation/10-monitor.md @@ -42,3 +42,304 @@ An existing Grafana Notification Channel can be specified with parameter `-E`, t Launch `TDinsight.sh` with the command above and restart Grafana, then open Dashboard `http://localhost:3000/d/tdinsight`. For more use cases and restrictions please refer to [TDinsight](/reference/tdinsight/). + +## log database + +The data of tdinsight dashboard is stored in `log` database (default. You can change it in taoskeeper's config file. For more infrmation, please reference to [taoskeeper document](/reference/taosKeeper)). The taoskeeper will create log database on taoskeeper startup. + +### cluster\_info table + +`cluster_info` table contains cluster information records. + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|first\_ep|VARCHAR||first ep of cluster| +|first\_ep\_dnode\_id|INT||dnode id or first\_ep| +|version|VARCHAR||tdengine version. such as: 3.0.4.0| +|master\_uptime|FLOAT||days of master's uptime| +|monitor\_interval|INT||monitor interval in second| +|dbs\_total|INT||total number of databases in cluster| +|tbs\_total|BIGINT||total number of tables in cluster| +|stbs\_total|INT||total number of stables in cluster| +|dnodes\_total|INT||total number of dnodes in cluster| +|dnodes\_alive|INT||total number of dnodes in ready state| +|mnodes\_total|INT||total number of mnodes in cluster| +|mnodes\_alive|INT||total number of mnodes in ready state| +|vgroups\_total|INT||total number of vgroups in cluster| +|vgroups\_alive|INT||total number of vgroups in ready state| +|vnodes\_total|INT||total number of vnode in cluster| +|vnodes\_alive|INT||total number of vnode in ready state| +|connections\_total|INT||total number of connections to cluster| +|topics\_total|INT||total number of topics in cluster| +|streams\_total|INT||total number of streams in cluster| +|protocol|INT||protocol version| +|cluster\_id|NCHAR|TAG|cluster id| + +### d\_info table + +`d_info` table contains dnodes information records. + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|status|VARCHAR||dnode status| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### m\_info table + +`m_info` table contains mnode information records. + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|role|VARCHAR||the role of mnode. leader or follower| +|mnode\_id|INT|TAG|master node id| +|mnode\_ep|NCHAR|TAG|master node endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### dnodes\_info table + +`dnodes_info` table contains dnodes information records. + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|uptime|FLOAT||dnode uptime| +|cpu\_engine|FLOAT||cpu usage of tdengine. read from `/proc//stat`| +|cpu\_system|FLOAT||cpu usage of server. read from `/proc/stat`| +|cpu\_cores|FLOAT||cpu cores of server| +|mem\_engine|INT||memory usage of tdengine. read from `/proc//status`| +|mem\_system|INT||memory usage of server| +|mem\_total|INT||total memory of server in `KB`| +|disk\_engine|INT||| +|disk\_used|BIGINT||usage of data dir in `bytes`| +|disk\_total|BIGINT||the capacity of data dir in `bytes`| +|net\_in|FLOAT||network throughput rate in kb/s. read from `/proc/net/dev`| +|net\_out|FLOAT||network throughput rate in kb/s. read from `/proc/net/dev`| +|io\_read|FLOAT||io throughput rate in kb/s. read from `/proc//io`| +|io\_write|FLOAT||io throughput rate in kb/s. read from `/proc//io`| +|io\_read\_disk|FLOAT||io throughput rate of disk in kb/s. read from `/proc//io`| +|io\_write\_disk|FLOAT||io throughput rate of disk in kb/s. read from `/proc//io`| +|req\_select|INT||number of select queries received per dnode| +|req\_select\_rate|FLOAT||number of select queries received per dnode divided by monitor interval.| +|req\_insert|INT||number of insert queries received per dnode| +|req\_insert\_success|INT||number of successfully insert queries received per dnode| +|req\_insert\_rate|FLOAT||number of insert queries received per dnode divided by monitor interval| +|req\_insert\_batch|INT||number of batch insertions| +|req\_insert\_batch\_success|INT||number of successful batch insertions| +|req\_insert\_batch\_rate|FLOAT||number of batch insertions divided by monitor interval| +|errors|INT||dnode errors| +|vnodes\_num|INT||number of vnodes per dnode| +|masters|INT||number of master vnodes| +|has\_mnode|INT||if the dnode has mnode| +|has\_qnode|INT||if the dnode has qnode| +|has\_snode|INT||if the dnode has snode| +|has\_bnode|INT||if the dnode has bnode| +|dnode\_id|INT|TAG|dnode id| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### data\_dir table + +`data_dir` table contains data directory information records. + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|name|NCHAR||data directory. default is `/var/lib/taos`| +|level|INT||level for multi-level storage| +|avail|BIGINT||available space for data directory| +|used|BIGINT||used space for data directory| +|total|BIGINT||total space for data directory| +|dnode\_id|INT|TAG|dnode id| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### log\_dir table + +`log_dir` table contains log directory information records. + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|name|NCHAR||log directory. default is `/var/log/taos/`| +|avail|BIGINT||available space for log directory| +|used|BIGINT||used space for data directory| +|total|BIGINT||total space for data directory| +|dnode\_id|INT|TAG|dnode id| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### temp\_dir table + +`temp_dir` table contains temp dir information records. + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|name|NCHAR||temp directory. default is `/tmp/`| +|avail|BIGINT||available space for temp directory| +|used|BIGINT||used space for temp directory| +|total|BIGINT||total space for temp directory| +|dnode\_id|INT|TAG|dnode id| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### vgroups\_info table + +`vgroups_info` table contains vgroups information records. + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|vgroup\_id|INT||vgroup id| +|database\_name|VARCHAR||database for the vgroup| +|tables\_num|BIGINT||number of tables per vgroup| +|status|VARCHAR||status| +|dnode\_id|INT|TAG|dnode id| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### vnodes\_role table + +`vnodes_role` table contains vnode role information records. + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|vnode\_role|VARCHAR||role. leader or follower| +|dnode\_id|INT|TAG|dnode id| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### logs table + +`logs` table contains login information records. + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|level|VARCHAR||log level| +|content|NCHAR||log content| +|dnode\_id|INT|TAG|dnode id| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### log\_summary table + +`log_summary` table contains log summary information records. + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|error|INT||error count| +|info|INT||info count| +|debug|INT||debug count| +|trace|INT||trace count| +|dnode\_id|INT|TAG|dnode id| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### grants\_info table + +`grants_info` table contains grants information records. + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|expire\_time|BIGINT||time until grants expire in seconds| +|timeseries\_used|BIGINT||timeseries used| +|timeseries\_total|BIGINT||total timeseries| +|dnode\_id|INT|TAG|dnode id| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### keeper\_monitor table + +`keeper_monitor` table contains keeper monitor information records. + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|cpu|FLOAT||cpu usage| +|mem|FLOAT||memory usage| +|identify|NCHAR|TAG|| + +### taosadapter\_restful\_http\_request\_total table + +`taosadapter_restful_http_request_total` table contains taosadapter rest request information record. The timestamp column of this table is `_ts`. + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|\_ts|TIMESTAMP||timestamp| +|guage|DOUBLE||metric value| +|client\_ip|NCHAR|TAG|client ip| +|endpoint|NCHAR|TAG|taosadpater endpoint| +|request\_method|NCHAR|TAG|request method| +|request\_uri|NCHAR|TAG|request uri| +|status\_code|NCHAR|TAG|status code| + +### taosadapter\_restful\_http\_request\_fail table + +`taosadapter_restful_http_request_fail` table contains taosadapter failed rest request information record. The timestamp column of this table is `_ts`. + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|\_ts|TIMESTAMP||timestamp| +|guage|DOUBLE||metric value| +|client\_ip|NCHAR|TAG|client ip| +|endpoint|NCHAR|TAG|taosadpater endpoint| +|request\_method|NCHAR|TAG|request method| +|request\_uri|NCHAR|TAG|request uri| +|status\_code|NCHAR|TAG|status code| + +### taosadapter\_restful\_http\_request\_in\_flight table + +`taosadapter_restful_http_request_in_flight` table contains taosadapter rest request information record in real time. The timestamp column of this table is `_ts`. + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|\_ts|TIMESTAMP||timestamp| +|guage|DOUBLE||metric value| +|endpoint|NCHAR|TAG|taosadpater endpoint| + +### taosadapter\_restful\_http\_request\_summary\_milliseconds table + +`taosadapter_restful_http_request_summary_milliseconds` table contains the summary or rest information record. The timestamp column of this table is `_ts`. + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|\_ts|TIMESTAMP||timestamp| +|count|DOUBLE||| +|sum|DOUBLE||| +|0.5|DOUBLE||| +|0.9|DOUBLE||| +|0.99|DOUBLE||| +|0.1|DOUBLE||| +|0.2|DOUBLE||| +|endpoint|NCHAR|TAG|taosadpater endpoint| +|request\_method|NCHAR|TAG|request method| +|request\_uri|NCHAR|TAG|request uri| + +### taosadapter\_system\_mem\_percent table + +`taosadapter_system_mem_percent` table contains taosadapter memory usage information. The timestamp of this table is `_ts`. + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|\_ts|TIMESTAMP||timestamp| +|guage|DOUBLE||metric value| +|endpoint|NCHAR|TAG|taosadpater endpoint| + +### taosadapter\_system\_cpu\_percent table + +`taosadapter_system_cpu_percent` table contains taosadapter cup usage information. The timestamp of this table is `_ts`. + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|\_ts|TIMESTAMP||timestamp| +|guage|DOUBLE||mertic value| +|endpoint|NCHAR|TAG|taosadpater endpoint| + diff --git a/docs/en/14-reference/14-taosKeeper.md b/docs/en/14-reference/14-taosKeeper.md index 895bd82e19..9c4a2da921 100644 --- a/docs/en/14-reference/14-taosKeeper.md +++ b/docs/en/14-reference/14-taosKeeper.md @@ -108,7 +108,7 @@ The following `launchctl` commands can help you manage taoskeeper service: #### Launch With Configuration File -You can quickly launch taosKeeper with the following commands. If you do not specify a configuration file, `/etc/taos/keeper.toml` is used by default. If this file does not specify configurations, the default values are used. +You can quickly launch taosKeeper with the following commands. If you do not specify a configuration file, `/etc/taos/taoskeeper.toml` is used by default. If this file does not specify configurations, the default values are used. ```shell $ taoskeeper -c @@ -153,6 +153,10 @@ database = "log" # standard tables to monitor tables = ["normal_table"] + +# database options for db storing metrics data +[metrics.databaseoptions] +cachemodel = "none" ``` ### Obtain Monitoring Metrics @@ -203,7 +207,7 @@ taos_cluster_info_dnodes_total{cluster_id="5981392874047724755"} 1 taos_cluster_info_first_ep{cluster_id="5981392874047724755",value="hlb:6030"} 1 ``` -### check_health +### check\_health ``` $ curl -i http://127.0.0.1:6043/check_health diff --git a/docs/zh/14-reference/14-taosKeeper.md b/docs/zh/14-reference/14-taosKeeper.md index 12b609584a..03ca30781f 100644 --- a/docs/zh/14-reference/14-taosKeeper.md +++ b/docs/zh/14-reference/14-taosKeeper.md @@ -111,7 +111,7 @@ Active: inactive (dead) #### 配置文件启动 -执行以下命令即可快速体验 taosKeeper。当不指定 taosKeeper 配置文件时，优先使用 `/etc/taos/keeper.toml` 配置，否则将使用默认配置。 +执行以下命令即可快速体验 taosKeeper。当不指定 taosKeeper 配置文件时，优先使用 `/etc/taos/taoskeeper.toml` 配置，否则将使用默认配置。 ```shell $ taoskeeper -c @@ -156,6 +156,10 @@ database = "log" # 指定需要监控的普通表 tables = [] + +# database options for db storing metrics data +[metrics.databaseoptions] +cachemodel = "none" ``` ### 获取监控指标 @@ -206,7 +210,7 @@ taos_cluster_info_dnodes_total{cluster_id="5981392874047724755"} 1 taos_cluster_info_first_ep{cluster_id="5981392874047724755",value="hlb:6030"} 1 ``` -### check_health +### check\_health ``` $ curl -i http://127.0.0.1:6043/check_health diff --git a/docs/zh/17-operation/10-monitor.md b/docs/zh/17-operation/10-monitor.md index 01a2257286..f7b3ed8b64 100644 --- a/docs/zh/17-operation/10-monitor.md +++ b/docs/zh/17-operation/10-monitor.md @@ -54,7 +54,7 @@ TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db， |first\_ep\_dnode\_id|INT||集群 first ep 的 dnode id| |version|VARCHAR||tdengine version。例如：3.0.4.0| |master\_uptime|FLOAT||当前 master 节点的uptime。单位：天| -|monitor_interval|INT||monitor interval。单位：秒| +|monitor\_interval|INT||monitor interval。单位：秒| |dbs\_total|INT||database 总数| |tbs\_total|BIGINT||当前集群 table 总数| |stbs\_total|INT||当前集群 stable 总数| @@ -112,12 +112,12 @@ TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db， |disk\_engine|INT||| |disk\_used|BIGINT||data dir 挂载的磁盘使用量，单位 bytes| |disk\_total|BIGINT||data dir 挂载的磁盘总容量，单位 bytes| -|net\_in|FLOAT||网络吞吐率，从 `/proc/net/dev` 中读取的 received bytes。单位 bytes per second| -|net\_out|FLOAT||网络吞吐率，从 `/proc/net/dev` 中读取的 transmit bytes。单位 bytes per second| -|io\_read|FLOAT||io 吞吐率，从 `/proc//io` 中读取的 rchar 与上次数值计算之后，计算得到速度。单位 bytes per second| -|io\_write|FLOAT||io 吞吐率，从 `/proc//io` 中读取的 wchar 与上次数值计算之后，计算得到速度。单位 bytes per second| -|io\_read\_disk|FLOAT||磁盘 io 吞吐率，从 `/proc//io` 中读取的 read_bytes。单位 bytes per second| -|io\_write\_disk|FLOAT||磁盘 io 吞吐率，从 `/proc//io` 中读取的 write_bytes。单位 bytes per second| +|net\_in|FLOAT||网络吞吐率，从 `/proc/net/dev` 中读取的 received bytes。单位 kb/s| +|net\_out|FLOAT||网络吞吐率，从 `/proc/net/dev` 中读取的 transmit bytes。单位 kb/s| +|io\_read|FLOAT||io 吞吐率，从 `/proc//io` 中读取的 rchar 与上次数值计算之后，计算得到速度。单位 kb/s| +|io\_write|FLOAT||io 吞吐率，从 `/proc//io` 中读取的 wchar 与上次数值计算之后，计算得到速度。单位 kb/s| +|io\_read\_disk|FLOAT||磁盘 io 吞吐率，从 `/proc//io` 中读取的 read_bytes。单位 kb/s| +|io\_write\_disk|FLOAT||磁盘 io 吞吐率，从 `/proc//io` 中读取的 write_bytes。单位 kb/s| |req\_select|INT||两个间隔内发生的查询请求数目| |req\_select\_rate|FLOAT||两个间隔内的查询请求速度 = `req_select / monitorInterval`| |req\_insert|INT||两个间隔内发生的写入请求，包含的单条数据数目| From f32f90f79a63d9d91ba06b56f390213c8c7a05b9 Mon Sep 17 00:00:00 2001 From: wade zhang <95411902+gccgdb1234@users.noreply.github.com> Date: Fri, 21 Apr 2023 14:39:48 +0800 Subject: [PATCH 34/35] Update 10-monitor.md --- docs/zh/17-operation/10-monitor.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/17-operation/10-monitor.md b/docs/zh/17-operation/10-monitor.md index f7b3ed8b64..7def90916c 100644 --- a/docs/zh/17-operation/10-monitor.md +++ b/docs/zh/17-operation/10-monitor.md @@ -107,7 +107,7 @@ TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db， |cpu\_system|FLOAT||服务器 cpu 使用率，从 `/proc/stat` 读取| |cpu\_cores|FLOAT||服务器 cpu 核数| |mem\_engine|INT||taosd 内存使用率，从 `/proc//status` 读取| -|mem\_system|INT||服务器内存使用率| +|mem\_system|INT||服务器可用内存| |mem\_total|INT||服务器内存总量，单位 KB| |disk\_engine|INT||| |disk\_used|BIGINT||data dir 挂载的磁盘使用量，单位 bytes| From beb082059ad42d734412c7b8ae9d071e71ebb224 Mon Sep 17 00:00:00 2001 From: wade zhang <95411902+gccgdb1234@users.noreply.github.com> Date: Fri, 21 Apr 2023 14:40:26 +0800 Subject: [PATCH 35/35] Update 10-monitor.md --- docs/en/13-operation/10-monitor.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/13-operation/10-monitor.md b/docs/en/13-operation/10-monitor.md index ea401c1e1c..19107240bf 100644 --- a/docs/en/13-operation/10-monitor.md +++ b/docs/en/13-operation/10-monitor.md @@ -111,7 +111,7 @@ The data of tdinsight dashboard is stored in `log` database (default. You can ch |cpu\_system|FLOAT||cpu usage of server. read from `/proc/stat`| |cpu\_cores|FLOAT||cpu cores of server| |mem\_engine|INT||memory usage of tdengine. read from `/proc//status`| -|mem\_system|INT||memory usage of server| +|mem\_system|INT||available memory on the server| |mem\_total|INT||total memory of server in `KB`| |disk\_engine|INT||| |disk\_used|BIGINT||usage of data dir in `bytes`|