From 59b25a96d7f7db01cb5dd77165a8f3e10fde9a59 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 11 Nov 2020 20:21:37 +0800 Subject: [PATCH 1/6] TD-1896 --- src/vnode/src/vnodeMain.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index dab07d189b..138455e1b5 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -317,6 +317,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { pVnode->version = walGetVersion(pVnode->wal); } + tsdbSyncCommit(pVnode->tsdb); walRenew(pVnode->wal); SSyncInfo syncInfo; From 40728d1b316b78eab469686782d190aaca3d72e5 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 11 Nov 2020 22:38:46 +0800 Subject: [PATCH 2/6] TD-1949 --- src/inc/twal.h | 21 +++++++++++---------- src/vnode/src/vnodeMain.c | 3 ++- src/wal/src/walMgmt.c | 11 +---------- src/wal/src/walWrite.c | 18 +++++++++++++++++- 4 files changed, 31 insertions(+), 22 deletions(-) diff --git a/src/inc/twal.h b/src/inc/twal.h index b85377d8d4..8dd3a8a912 100644 --- a/src/inc/twal.h +++ b/src/inc/twal.h @@ -54,16 +54,17 @@ typedef int32_t FWalWrite(void *ahandle, void *pHead, int32_t qtype, void *pMsg) int32_t walInit(); void walCleanUp(); -twalh walOpen(char *path, SWalCfg *pCfg); -int32_t walAlter(twalh pWal, SWalCfg *pCfg); -void walStop(twalh); -void walClose(twalh); -int32_t walRenew(twalh); -void walRemoveOldFiles(twalh); -int32_t walWrite(twalh, SWalHead *); -void walFsync(twalh, bool forceFsync); -int32_t walRestore(twalh, void *pVnode, FWalWrite writeFp); -int32_t walGetWalFile(twalh, char *fileName, int64_t *fileId); +twalh walOpen(char *path, SWalCfg *pCfg); +int32_t walAlter(twalh pWal, SWalCfg *pCfg); +void walStop(twalh); +void walClose(twalh); +int32_t walRenew(twalh); +void walRemoveOneOldFile(twalh); +void walRemoveAllOldFiles(twalh); +int32_t walWrite(twalh, SWalHead *); +void walFsync(twalh, bool forceFsync); +int32_t walRestore(twalh, void *pVnode, FWalWrite writeFp); +int32_t walGetWalFile(twalh, char *fileName, int64_t *fileId); uint64_t walGetVersion(twalh); #ifdef __cplusplus diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 138455e1b5..e1ea89a76a 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -318,6 +318,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { } tsdbSyncCommit(pVnode->tsdb); + walRemoveAllOldFiles(pVnode->tsdb); walRenew(pVnode->wal); SSyncInfo syncInfo; @@ -593,7 +594,7 @@ static int vnodeProcessTsdbStatus(void *arg, int status) { if (status == TSDB_STATUS_COMMIT_OVER) { vDebug("vgId:%d, commit over, fver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId, pVnode->fversion, pVnode->version); - walRemoveOldFiles(pVnode->wal); + walRemoveOneOldFile(pVnode->wal); return vnodeSaveVersion(pVnode); } diff --git a/src/wal/src/walMgmt.c b/src/wal/src/walMgmt.c index 9ba0dfd124..fb49f38217 100644 --- a/src/wal/src/walMgmt.c +++ b/src/wal/src/walMgmt.c @@ -128,16 +128,7 @@ void walClose(void *handle) { taosClose(pWal->fd); if (pWal->keep != TAOS_WAL_KEEP) { - int64_t fileId = -1; - while (walGetNextFile(pWal, &fileId) >= 0) { - snprintf(pWal->name, sizeof(pWal->name), "%s/%s%" PRId64, pWal->path, WAL_PREFIX, fileId); - - if (remove(pWal->name) < 0) { - wError("vgId:%d, wal:%p file:%s, failed to remove", pWal->vgId, pWal, pWal->name); - } else { - wInfo("vgId:%d, wal:%p file:%s, it is removed", pWal->vgId, pWal, pWal->name); - } - } + walRemoveAllOldFiles(pWal); } else { wDebug("vgId:%d, wal:%p file:%s, it is closed and kept", pWal->vgId, pWal, pWal->name); } diff --git a/src/wal/src/walWrite.c b/src/wal/src/walWrite.c index d3a41ec6b2..48021eecfc 100644 --- a/src/wal/src/walWrite.c +++ b/src/wal/src/walWrite.c @@ -63,7 +63,7 @@ int32_t walRenew(void *handle) { return code; } -void walRemoveOldFiles(void *handle) { +void walRemoveOneOldFile(void *handle) { SWal *pWal = handle; if (pWal == NULL) return; if (pWal->keep == TAOS_WAL_KEEP) return; @@ -86,6 +86,22 @@ void walRemoveOldFiles(void *handle) { pthread_mutex_unlock(&pWal->mutex); } +void walRemoveAllOldFiles(void *handle) { + if (handle == NULL) return; + + SWal * pWal = handle; + int64_t fileId = -1; + while (walGetNextFile(pWal, &fileId) >= 0) { + snprintf(pWal->name, sizeof(pWal->name), "%s/%s%" PRId64, pWal->path, WAL_PREFIX, fileId); + + if (remove(pWal->name) < 0) { + wError("vgId:%d, wal:%p file:%s, failed to remove", pWal->vgId, pWal, pWal->name); + } else { + wInfo("vgId:%d, wal:%p file:%s, it is removed", pWal->vgId, pWal, pWal->name); + } + } +} + int32_t walWrite(void *handle, SWalHead *pHead) { if (handle == NULL) return -1; From a4cb007d6913d10aecccb12b96ba6bdfc05ba476 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 11 Nov 2020 23:26:20 +0800 Subject: [PATCH 3/6] TD-1896 --- src/vnode/src/vnodeMain.c | 2 +- tests/script/general/wal/maxtables.sim | 46 ++++++++++++++++++++++++++ tests/script/jenkins/basic.txt | 1 + 3 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 tests/script/general/wal/maxtables.sim diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index e1ea89a76a..7d45516ecc 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -318,7 +318,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { } tsdbSyncCommit(pVnode->tsdb); - walRemoveAllOldFiles(pVnode->tsdb); + walRemoveAllOldFiles(pVnode->wal); walRenew(pVnode->wal); SSyncInfo syncInfo; diff --git a/tests/script/general/wal/maxtables.sim b/tests/script/general/wal/maxtables.sim new file mode 100644 index 0000000000..f95997f1e4 --- /dev/null +++ b/tests/script/general/wal/maxtables.sim @@ -0,0 +1,46 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 100 +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1 +system sh/cfg.sh -n dnode1 -c tableIncStepPerVnode -v 2 + + +print ============== deploy +system sh/exec.sh -n dnode1 -s start +sleep 3001 +sql connect + +sql create database d1 +sql use d1 +sql create table st (ts timestamp, tbcol int) TAGS(tgcol int) + +$i = 0 +while $i < 100 + $tb = t . $i + sql create table $tb using st tags( $i ) + sql insert into $tb values (now , $i ) + $i = $i + 1 +endw + +sql_error sql create table tt (ts timestamp, i int) + +print =============== step3 +sql select * from st; +if $rows != 100 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 4 +sleep 3000 + +print =============== step4 +system sh/exec.sh -n dnode1 -s start -t +sleep 10000 + +sql select * from st; +if $rows != 100 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 2a84172da9..daf92679bd 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -238,6 +238,7 @@ cd ../../../debug; make ./test.sh -f general/wal/sync.sim ./test.sh -f general/wal/kill.sim +./test.sh -f general/wal/maxtables.sim ./test.sh -f unique/account/account_create.sim ./test.sh -f unique/account/account_delete.sim From 391e95bc5867134c6e29edf995a3d5bb668a970b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 11 Nov 2020 23:27:19 +0800 Subject: [PATCH 4/6] script --- tests/script/general/wal/maxtables.sim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/script/general/wal/maxtables.sim b/tests/script/general/wal/maxtables.sim index f95997f1e4..e504c7e92e 100644 --- a/tests/script/general/wal/maxtables.sim +++ b/tests/script/general/wal/maxtables.sim @@ -35,8 +35,8 @@ system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 4 sleep 3000 print =============== step4 -system sh/exec.sh -n dnode1 -s start -t -sleep 10000 +system sh/exec.sh -n dnode1 -s start +sleep 3000 sql select * from st; if $rows != 100 then From ab6387cae2bc04179fe326923d80c5adaef0835f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 11 Nov 2020 23:34:21 +0800 Subject: [PATCH 5/6] log error while client version not matched with server version --- src/util/src/tutil.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/util/src/tutil.c b/src/util/src/tutil.c index 099b9d9530..451976f563 100644 --- a/src/util/src/tutil.c +++ b/src/util/src/tutil.c @@ -377,7 +377,8 @@ int taosCheckVersion(char *input_client_version, char *input_server_version, int for(int32_t i = 0; i < comparedSegments; ++i) { if (clientVersionNumber[i] != serverVersionNumber[i]) { - uError("the %d-th number of server version:%s not matched with client version:%s", i, server_version, version); + uError("the %d-th number of server version:%s not matched with client version:%s", i, server_version, + client_version); return TSDB_CODE_TSC_INVALID_VERSION; } } From 028e62c974f9695b134dc5014ad2fa3e37a707ec Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 11 Nov 2020 23:45:10 +0800 Subject: [PATCH 6/6] TD-2067 --- src/common/inc/tglobal.h | 1 + src/common/src/tglobal.c | 11 +++++++++++ src/vnode/src/vnodeMain.c | 4 +--- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index dbcf50ba77..4087f638a9 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -44,6 +44,7 @@ extern int32_t tsMaxShellConns; extern int32_t tsShellActivityTimer; extern uint32_t tsMaxTmrCtrl; extern float tsNumOfThreadsPerCore; +extern int32_t tsNumOfCommitThreads; extern float tsRatioOfQueryThreads; // todo remove it extern int8_t tsDaylight; extern char tsTimezone[]; diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 3bc1e4d0cc..4495c3d928 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -51,6 +51,7 @@ int32_t tsMaxShellConns = 5000; int32_t tsMaxConnections = 5000; int32_t tsShellActivityTimer = 3; // second float tsNumOfThreadsPerCore = 1.0f; +int32_t tsNumOfCommitThreads = 1; float tsRatioOfQueryThreads = 0.5f; int8_t tsDaylight = 0; char tsTimezone[TSDB_TIMEZONE_LEN] = {0}; @@ -426,6 +427,16 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); + cfg.option = "numOfCommitThreads"; + cfg.ptr = &tsNumOfCommitThreads; + cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; + cfg.minValue = 1; + cfg.maxValue = 100; + cfg.ptrLength = 0; + cfg.unitType = TAOS_CFG_UTYPE_NONE; + taosInitConfigOption(cfg); + cfg.option = "ratioOfQueryThreads"; cfg.ptr = &tsRatioOfQueryThreads; cfg.valType = TAOS_CFG_VTYPE_FLOAT; diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 7d45516ecc..199619e851 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -28,8 +28,6 @@ #include "vnodeCfg.h" #include "vnodeVersion.h" -#define DEFAULT_COMMIT_THREADS 1 - static SHashObj*tsVnodesHash; static void vnodeCleanUp(SVnodeObj *pVnode); static int vnodeProcessTsdbStatus(void *arg, int status); @@ -69,7 +67,7 @@ int32_t vnodeInitResources() { return TSDB_CODE_VND_OUT_OF_MEMORY; } - if (tsdbInitCommitQueue(DEFAULT_COMMIT_THREADS) < 0) { + if (tsdbInitCommitQueue(tsNumOfCommitThreads) < 0) { vError("failed to init vnode commit queue"); return terrno; }