From a4fd2def98108357e1097afb955cd7177ac151d0 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Mon, 17 Oct 2022 13:30:23 +0800 Subject: [PATCH 1/9] fix: resolve a typo in errormsg printing in walScanLogGetLastVer --- source/libs/wal/src/walMeta.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 279f4dc656..85791fa682 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -123,8 +123,8 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) { } SWalCkHead* logContent = (SWalCkHead*)candidate; if (walValidHeadCksum(logContent) != 0) { - wError("vgId:%d, failed to validate checksum of wal entry header. offset:% %" PRId64 ", file:%s", - ((char*)(logContent)-buf), fnameStr); + wWarn("vgId:%d, failed to validate checksum of wal entry header. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, + offset + ((char*)(logContent)-buf), fnameStr); haystack = candidate + 1; if (firstTrial) { break; @@ -162,8 +162,8 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) { } if (walValidBodyCksum(logContent) != 0) { terrno = TSDB_CODE_WAL_CHKSUM_MISMATCH; - wError("vgId:%d, failed to validate checksum of wal entry body. offset:% %" PRId64 ", file:%s", - ((char*)(logContent)-buf), fnameStr); + wWarn("vgId:%d, failed to validate checksum of wal entry body. offset:%" PRId64 ", file:%s", pWal->cfg.vgId, + offset + ((char*)(logContent)-buf), fnameStr); haystack = candidate + 1; if (firstTrial) { break; From 23a4a3767779d2c95b01787a5db25e88a9ca770e Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Mon, 17 Oct 2022 14:17:34 +0800 Subject: [PATCH 2/9] enh: not allowed to start when disk space unavailable --- source/dnode/mgmt/node_mgmt/src/dmEnv.c | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c index 076826ebc2..becefcd6a0 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c +++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c @@ -51,6 +51,7 @@ static int32_t dmInitMonitor() { static bool dmCheckDiskSpace() { osUpdate(); + // sufficiency if (!osDataSpaceSufficient()) { dWarn("free data disk size: %f GB, not sufficient, expected %f GB at least", (double)tsDataSpace.size.avail / 1024.0 / 1024.0 / 1024.0, (double)tsDataSpace.reserved / 1024.0 / 1024.0 / 1024.0); } @@ -60,7 +61,21 @@ static bool dmCheckDiskSpace() { if (!osTempSpaceSufficient()) { dWarn("free temp disk size: %f GB, not sufficient, expected %f GB at least", (double)tsTempSpace.size.avail / 1024.0 / 1024.0 / 1024.0, (double)tsTempSpace.reserved / 1024.0 / 1024.0 / 1024.0); } - return true; + // availability + bool ret = true; + if (!osDataSpaceAvailable()) { + dError("data disk space unavailable, i.e. %s", tsDataDir); + ret = false; + } + if (!osLogSpaceAvailable()) { + dError("log disk space unavailable, i.e. %s", tsLogDir); + ret = false; + } + if (!osTempSpaceAvailable()) { + dError("temp disk space unavailable, i.e. %s", tsTempDir); + ret = false; + } + return ret; } static bool dmCheckDataDirVersion() { From 3190ef65a3c43521e62cd015bd1fae6f1de9f3e5 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Mon, 17 Oct 2022 21:13:52 +0800 Subject: [PATCH 3/9] fix: deal with misaligned idx entries in walCheckAndRepairIdxFile --- source/libs/wal/src/walMeta.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 85791fa682..fa22805df2 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -481,6 +481,10 @@ int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) { continue; } + if (offset != (idxEntry.ver - pFileInfo->firstVer) * sizeof(SWalIdxEntry)) { + continue; + } + if (walReadLogHead(pLogFile, idxEntry.offset, &ckHead) < 0) { wWarn("vgId:%d, failed to read log file since %s. file:%s, offset:%" PRId64 ", idx entry ver:%" PRId64 "", pWal->cfg.vgId, terrstr(), fLogNameStr, idxEntry.offset, idxEntry.ver); @@ -493,6 +497,8 @@ int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) { } offset += sizeof(SWalIdxEntry); + ASSERT(offset == (idxEntry.ver - pFileInfo->firstVer + 1) * sizeof(SWalIdxEntry)); + // ftruncate idx file if (offset < fileSize) { if (taosFtruncateFile(pIdxFile, offset) < 0) { From 23e18bff76781825aed80efc4cfe90450c6ab74b Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 18 Oct 2022 01:31:19 +0800 Subject: [PATCH 4/9] fix: fsync fatal log messages --- source/util/src/tlog.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 2e2300ba14..9d97cf7ab2 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -443,10 +443,13 @@ static inline int32_t taosBuildLogHead(char *buffer, const char *flags) { static inline void taosPrintLogImp(ELogLevel level, int32_t dflag, const char *buffer, int32_t len) { if ((dflag & DEBUG_FILE) && tsLogObj.logHandle && tsLogObj.logHandle->pFile != NULL && osLogSpaceAvailable()) { taosUpdateLogNums(level); - if (tsAsyncLog) { + if (tsAsyncLog && level != DEBUG_FATAL) { taosPushLogBuffer(tsLogObj.logHandle, buffer, len); } else { taosWriteFile(tsLogObj.logHandle->pFile, buffer, len); + if (level == DEBUG_FATAL) { + taosFsyncFile(tsLogObj.logHandle->pFile); + } } if (tsLogObj.maxLines > 0) { From f0427c93223193fc03ea8261726b42a8159e863c Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 18 Oct 2022 01:37:55 +0800 Subject: [PATCH 5/9] fix: ftrunc on appending WAL log failure for recovery in walWriteImpl --- source/libs/wal/src/walWrite.c | 41 ++++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 1e5c63fa8d..7da88ec95d 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -408,25 +408,33 @@ END: static int32_t walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) { SWalIdxEntry entry = {.ver = ver, .offset = offset}; - int64_t idxOffset = taosLSeekFile(pWal->pIdxFile, 0, SEEK_END); + SWalFileInfo *pFileInfo = walGetCurFileInfo(pWal); + ASSERT(pFileInfo != NULL); + ASSERT(pFileInfo->firstVer >= 0); + int64_t idxOffset = (entry.ver - pFileInfo->firstVer) * sizeof(SWalIdxEntry); wDebug("vgId:%d, write index, index:%" PRId64 ", offset:%" PRId64 ", at %" PRId64, pWal->cfg.vgId, ver, offset, idxOffset); + int64_t size = taosWriteFile(pWal->pIdxFile, &entry, sizeof(SWalIdxEntry)); if (size != sizeof(SWalIdxEntry)) { + wError("vgId:%d, failed to write idx entry due to %s. ver:%lld", pWal->cfg.vgId, strerror(errno), ver); terrno = TAOS_SYSTEM_ERROR(errno); - // TODO truncate return -1; } return 0; } -// TODO gurantee atomicity by truncate failed writing static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgType, SWalSyncInfo syncMeta, const void *body, int32_t bodyLen) { int64_t code = 0; int64_t offset = walGetCurFileOffset(pWal); + SWalFileInfo *pFileInfo = walGetCurFileInfo(pWal); + ASSERT(pFileInfo != NULL); + if (pFileInfo->firstVer == -1) { + pFileInfo->firstVer = index; + } pWal->writeHead.head.version = index; pWal->writeHead.head.bodyLen = bodyLen; pWal->writeHead.head.msgType = msgType; @@ -437,11 +445,9 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy pWal->writeHead.cksumHead = walCalcHeadCksum(&pWal->writeHead); pWal->writeHead.cksumBody = walCalcBodyCksum(body, bodyLen); - wDebug("vgId:%d, wal write log %ld, msgType: %s", pWal->cfg.vgId, index, TMSG_INFO(msgType)); if (taosWriteFile(pWal->pLogFile, &pWal->writeHead, sizeof(SWalCkHead)) != sizeof(SWalCkHead)) { - // TODO ftruncate terrno = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(errno)); @@ -450,7 +456,6 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy } if (taosWriteFile(pWal->pLogFile, (char *)body, bodyLen) != bodyLen) { - // TODO ftruncate terrno = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(errno)); @@ -460,7 +465,6 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy code = walWriteIndex(pWal, index, offset); if (code < 0) { - // TODO ftruncate goto END; } @@ -468,14 +472,27 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy if (pWal->vers.firstVer == -1) pWal->vers.firstVer = index; pWal->vers.lastVer = index; pWal->totSize += sizeof(SWalCkHead) + bodyLen; - if (walGetCurFileInfo(pWal)->firstVer == -1) { - walGetCurFileInfo(pWal)->firstVer = index; - } - walGetCurFileInfo(pWal)->lastVer = index; - walGetCurFileInfo(pWal)->fileSize += sizeof(SWalCkHead) + bodyLen; + pFileInfo->lastVer = index; + pFileInfo->fileSize += sizeof(SWalCkHead) + bodyLen; return 0; + END: + // recover in a reverse order + if (taosFtruncateFile(pWal->pLogFile, offset) < 0) { + wFatal("vgId:%d, failed to ftruncate logfile to offset:%lld during recovery due to %s", pWal->cfg.vgId, offset, + strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + ASSERT(0 && "failed to recover from error"); + } + + int64_t idxOffset = (index - pFileInfo->firstVer) * sizeof(SWalIdxEntry); + if (taosFtruncateFile(pWal->pIdxFile, idxOffset) < 0) { + wFatal("vgId:%d, failed to ftruncate idxfile to offset:%lld during recovery due to %s", pWal->cfg.vgId, idxOffset, + strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + ASSERT(0 && "failed to recover from error"); + } return -1; } From af9aa9d78331061da00d37fdd95ee776d860d1a9 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 18 Oct 2022 02:12:05 +0800 Subject: [PATCH 6/9] enh: assert on alignment of idx entries in walWriteIndex --- source/libs/wal/src/walWrite.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 7da88ec95d..c088145ff6 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -421,6 +421,8 @@ static int32_t walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } + + ASSERT(taosLSeekFile(pWal->pIdxFile, 0, SEEK_END) == idxOffset + sizeof(SWalIdxEntry) && "Offset of idx entries misaligned"); return 0; } From cf2b6ccbfb5544070f379040a3f01a3a782d7e1d Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 18 Oct 2022 09:58:02 +0800 Subject: [PATCH 7/9] enh: write idx ahead of writing its log entry in WAL --- source/libs/wal/src/walWrite.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index c088145ff6..aa6d52174d 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -449,6 +449,11 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy pWal->writeHead.cksumBody = walCalcBodyCksum(body, bodyLen); wDebug("vgId:%d, wal write log %ld, msgType: %s", pWal->cfg.vgId, index, TMSG_INFO(msgType)); + code = walWriteIndex(pWal, index, offset); + if (code < 0) { + goto END; + } + if (taosWriteFile(pWal->pLogFile, &pWal->writeHead, sizeof(SWalCkHead)) != sizeof(SWalCkHead)) { terrno = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), @@ -465,11 +470,6 @@ static FORCE_INLINE int32_t walWriteImpl(SWal *pWal, int64_t index, tmsg_t msgTy goto END; } - code = walWriteIndex(pWal, index, offset); - if (code < 0) { - goto END; - } - // set status if (pWal->vers.firstVer == -1) pWal->vers.firstVer = index; pWal->vers.lastVer = index; From 0cbaa03fcc4cc6cc3ac836f89215d3ad4b929db0 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 18 Oct 2022 14:51:19 +0800 Subject: [PATCH 8/9] fix: osDefaultInit on all platform in testing Init at sut.cpp --- source/dnode/mgmt/test/sut/src/sut.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/source/dnode/mgmt/test/sut/src/sut.cpp b/source/dnode/mgmt/test/sut/src/sut.cpp index 699203e8c1..a4d2e46881 100644 --- a/source/dnode/mgmt/test/sut/src/sut.cpp +++ b/source/dnode/mgmt/test/sut/src/sut.cpp @@ -43,9 +43,7 @@ void Testbase::InitLog(const char* path) { } void Testbase::Init(const char* path, int16_t port) { -#ifdef _TD_DARWIN_64 osDefaultInit(); -#endif tsServerPort = port; strcpy(tsLocalFqdn, "localhost"); snprintf(tsLocalEp, TSDB_EP_LEN, "%s:%u", tsLocalFqdn, tsServerPort); From 2525a5cf1fb6567f5508a18f0bb7bc7d488fbe42 Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Tue, 18 Oct 2022 16:06:52 +0800 Subject: [PATCH 9/9] enh: report proper errorcode on disk unavailability --- source/dnode/mgmt/node_mgmt/src/dmEnv.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c index becefcd6a0..5a7f149bc6 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c +++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c @@ -65,14 +65,17 @@ static bool dmCheckDiskSpace() { bool ret = true; if (!osDataSpaceAvailable()) { dError("data disk space unavailable, i.e. %s", tsDataDir); + terrno = TSDB_CODE_VND_NO_DISKSPACE; ret = false; } if (!osLogSpaceAvailable()) { dError("log disk space unavailable, i.e. %s", tsLogDir); + terrno = TSDB_CODE_VND_NO_DISKSPACE; ret = false; } if (!osTempSpaceAvailable()) { dError("temp disk space unavailable, i.e. %s", tsTempDir); + terrno = TSDB_CODE_VND_NO_DISKSPACE; ret = false; } return ret;