From f403fc0c6ee1c2a9cc02f3037ecaa54c55f90c2d Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 28 Oct 2020 11:11:25 +0800 Subject: [PATCH 1/4] fix TD-1835 --- src/tsdb/src/tsdbFile.c | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index 626ad77da2..559fb0c233 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -79,7 +79,7 @@ int tsdbOpenFileH(STsdbRepo *pRepo) { DIR * dir = NULL; int fid = 0; int vid = 0; - regex_t regex1, regex2; + regex_t regex1 = {0}, regex2 = {0}; int code = 0; char fname[TSDB_FILENAME_LEN] = "\0"; @@ -95,9 +95,24 @@ int tsdbOpenFileH(STsdbRepo *pRepo) { dir = opendir(tDataDir); if (dir == NULL) { - tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), tDataDir, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; + if (errno == ENOENT) { + if (taosMkDir(tDataDir, 0755) < 0) { + tsdbError("vgId:%d failed to create directory %s since %s", REPO_ID(pRepo), tDataDir, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + dir = opendir(tDataDir); + if (dir == NULL) { + tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), tDataDir, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + } else { + tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), tDataDir, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } } code = regcomp(®ex1, "^v[0-9]+f[0-9]+\\.(head|data|last|stat)$", REG_EXTENDED); From 11c94027794dc183abba0d00238bbd21058d3ede Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 28 Oct 2020 05:40:03 +0000 Subject: [PATCH 2/4] TD-1780 --- src/mnode/src/mnodeVgroup.c | 4 +- src/sync/src/syncRestore.c | 4 +- src/tsdb/src/tsdbFile.c | 3 + src/vnode/src/vnodeMain.c | 10 +- tests/script/unique/dnode/data1.sim | 137 ++++++++++++++++++++++++++++ 5 files changed, 152 insertions(+), 6 deletions(-) create mode 100644 tests/script/unique/dnode/data1.sim diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index 5084f1276a..df703241c2 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -663,13 +663,13 @@ static int32_t mnodeGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *p for (int32_t i = 0; i < pShow->maxReplica; ++i) { pShow->bytes[cols] = 2; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; - snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "dnode%d", i + 1); + snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%dDnode", i + 1); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; pShow->bytes[cols] = 9 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%dstatus", i + 1); + snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%dStatus", i + 1); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; } diff --git a/src/sync/src/syncRestore.c b/src/sync/src/syncRestore.c index ebb6c3a0a9..19a5d3ba41 100644 --- a/src/sync/src/syncRestore.c +++ b/src/sync/src/syncRestore.c @@ -56,6 +56,7 @@ static int syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { int code = -1; char name[TSDB_FILENAME_LEN * 2] = {0}; uint32_t pindex = 0; // index in last restore + bool fileChanged = false; *fversion = 0; sinfo.index = 0; @@ -114,10 +115,11 @@ static int syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { close(dfd); if (ret < 0) break; + fileChanged = true; sDebug("%s, %s is received, size:%" PRId64, pPeer->id, minfo.name, minfo.size); } - if (code == 0 && (minfo.fversion != sinfo.fversion)) { + if (code == 0 && fileChanged) { // data file is changed, code shall be set to 1 *fversion = minfo.fversion; code = 1; diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index 559fb0c233..5633e97cdf 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -96,6 +96,9 @@ int tsdbOpenFileH(STsdbRepo *pRepo) { dir = opendir(tDataDir); if (dir == NULL) { if (errno == ENOENT) { + tsdbError("vgId:%d directory %s not exist", REPO_ID(pRepo), tDataDir); + terrno = TAOS_SYSTEM_ERROR(errno); + if (taosMkDir(tDataDir, 0755) < 0) { tsdbError("vgId:%d failed to create directory %s since %s", REPO_ID(pRepo), tDataDir, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index e529f27f55..9c283dd9c4 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -291,11 +291,15 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { if (pVnode->tsdb == NULL) { vnodeCleanUp(pVnode); return terrno; - } else if (terrno != TSDB_CODE_SUCCESS && pVnode->syncCfg.replica <= 1) { + } else if (terrno != TSDB_CODE_SUCCESS) { vError("vgId:%d, failed to open tsdb, replica:%d reason:%s", pVnode->vgId, pVnode->syncCfg.replica, tstrerror(terrno)); - vnodeCleanUp(pVnode); - return terrno; + if (pVnode->syncCfg.replica <= 1) { + vnodeCleanUp(pVnode); + return terrno; + } else { + pVnode->version = 0; + } } sprintf(temp, "%s/wal", rootDir); diff --git a/tests/script/unique/dnode/data1.sim b/tests/script/unique/dnode/data1.sim new file mode 100644 index 0000000000..61a991148b --- /dev/null +++ b/tests/script/unique/dnode/data1.sim @@ -0,0 +1,137 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 + +system sh/cfg.sh -n dnode1 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode4 -c balanceInterval -v 10 + +system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 + +system sh/cfg.sh -n dnode1 -c wallevel -v 2 +system sh/cfg.sh -n dnode2 -c wallevel -v 2 +system sh/cfg.sh -n dnode3 -c wallevel -v 2 +system sh/cfg.sh -n dnode4 -c wallevel -v 2 + +print ========== step1 +system sh/exec.sh -n dnode1 -s start +sql connect +sleep 3000 + +print ========== step2 +sql create dnode $hostname2 +system sh/exec.sh -n dnode2 -s start +sql create dnode $hostname3 +system sh/exec.sh -n dnode3 -s start +sql create dnode $hostname4 +system sh/exec.sh -n dnode4 -s start + +$x = 0 +show2: + $x = $x + 1 + sleep 3000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 openVnodes $data2_1 +print dnode2 openVnodes $data2_2 +print dnode3 openVnodes $data2_3 +print dnode4 openVnodes $data2_4 +if $data2_1 != 0 then + goto show2 +endi +if $data2_2 != 0 then + goto show2 +endi +if $data2_3 != 0 then + goto show2 +endi +if $data2_4 != 0 then + goto show2 +endi + +print ========== step3 +sql create database d1 replica 3 +sql create table d1.t1 (t timestamp, i int) +sql insert into d1.t1 values(now+1s, 35) +sql insert into d1.t1 values(now+2s, 34) +sql insert into d1.t1 values(now+3s, 33) +sql insert into d1.t1 values(now+4s, 32) +sql insert into d1.t1 values(now+5s, 31) + +$x = 0 +show3: + $x = $x + 1 + sleep 3000 + if $x == 10 then + return -1 + endi + +sql show dnodes +print dnode1 openVnodes $data2_1 +print dnode2 openVnodes $data2_2 +print dnode3 openVnodes $data2_3 +print dnode4 openVnodes $data2_4 +if $data2_1 != 0 then + goto show3 +endi +if $data2_2 != 1 then + goto show3 +endi +if $data2_3 != 1 then + goto show3 +endi +if $data2_4 != 1 then + goto show3 +endi + +print ========== step4 +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT + +print ========== step5 +system_content rm -rf ../../../sim/dnode4/data/vnode/vnode2/tsdb/data + +print ========== step6 +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode4 -s start +sleep 10000 + +print ========== step7 +sql select * from d1.t1 order by t desc +print $data01 $data11 $data21 $data31 $data41 +if $data01 != 31 then + return -1 +endi +if $data11 != 32 then + return -1 +endi +if $data21 != 33 then + return -1 +endi +if $data31 != 34 then + return -1 +endi +if $data41 != 35 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT +system sh/exec.sh -n dnode4 -s stop -x SIGINT +system sh/exec.sh -n dnode5 -s stop -x SIGINT +system sh/exec.sh -n dnode6 -s stop -x SIGINT +system sh/exec.sh -n dnode7 -s stop -x SIGINT +system sh/exec.sh -n dnode8 -s stop -x SIGINT \ No newline at end of file From 64b9658d0dad8daedebe5ac77a9467e62e595a57 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 28 Oct 2020 05:42:17 +0000 Subject: [PATCH 3/4] TD-1780 --- tests/script/jenkins/basic.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 4e68d1566a..05bf2b915a 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -277,6 +277,7 @@ cd ../../../debug; make ./test.sh -f unique/dnode/balance2.sim ./test.sh -f unique/dnode/balance3.sim ./test.sh -f unique/dnode/balancex.sim +./test.sh -f unique/dnode/data1.sim ./test.sh -f unique/dnode/offline1.sim ./test.sh -f unique/dnode/offline2.sim ./test.sh -f unique/dnode/reason.sim From 13047d02e4a92ee1d280a35a29e162a4242bc466 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 28 Oct 2020 06:29:13 +0000 Subject: [PATCH 4/4] scripts --- tests/script/general/db/nosuchfile.sim | 66 ++++++++++++++++++++++++++ tests/script/jenkins/basic.txt | 1 + 2 files changed, 67 insertions(+) create mode 100644 tests/script/general/db/nosuchfile.sim diff --git a/tests/script/general/db/nosuchfile.sim b/tests/script/general/db/nosuchfile.sim new file mode 100644 index 0000000000..98ac4ec012 --- /dev/null +++ b/tests/script/general/db/nosuchfile.sim @@ -0,0 +1,66 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c wallevel -v 2 + +print ========== step1 +system sh/exec.sh -n dnode1 -s start +sql connect +sleep 3000 + +print ========== step3 +sql create database d1 +sql create table d1.t1 (t timestamp, i int) +sql insert into d1.t1 values(now+1s, 35) +sql insert into d1.t1 values(now+2s, 34) +sql insert into d1.t1 values(now+3s, 33) +sql insert into d1.t1 values(now+4s, 32) +sql insert into d1.t1 values(now+5s, 31) + +print ========== step4 +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode1 -s start +sleep 3000 + +print ========== step5 +sql select * from d1.t1 order by t desc +print $data01 $data11 $data21 $data31 $data41 +if $data01 != 31 then + return -1 +endi +if $data11 != 32 then + return -1 +endi +if $data21 != 33 then + return -1 +endi +if $data31 != 34 then + return -1 +endi +if $data41 != 35 then + return -1 +endi + +print ========== step6 +system_content rm -rf ../../../sim/dnode1/data/vnode/vnode2/tsdb/data + +print ========== step7 +sql select * from d1.t1 order by t desc +print $data01 $data11 $data21 $data31 $data41 +if $data01 != null then + return -1 +endi +if $data11 != null then + return -1 +endi +if $data21 != null then + return -1 +endi +if $data31 != null then + return -1 +endi +if $data41 != null then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 05bf2b915a..1b2fe37c71 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -64,6 +64,7 @@ cd ../../../debug; make ./test.sh -f general/db/repeat.sim ./test.sh -f general/db/tables.sim ./test.sh -f general/db/vnodes.sim +./test.sh -f general/db/nosuchfile.sim ./test.sh -f general/field/2.sim ./test.sh -f general/field/3.sim