xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix: ftruncate WAL log after the last valid WAL record properly in walScanLogGetLastVer

This commit is contained in:
Benguang Zhao 2022-09-30 18:54:49 +08:00
parent 76865258cf
commit 9055f63742
1 changed files with 42 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

79
source/libs/wal/src/walMeta.c
View File

@ -45,12 +45,11 @@ static FORCE_INLINE int walBuildTmpMetaName(SWal* pWal, char* buf) {
static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal) { static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal) {
int32_t sz = taosArrayGetSize(pWal->fileInfoSet); int32_t sz = taosArrayGetSize(pWal->fileInfoSet);
ASSERT(sz > 0); if (sz <= 0) {
#if 0 wError("No WAL log file found.");
for (int i = 0; i < sz; i++) { terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, i); return -1;
} }
#endif
SWalFileInfo* pLastFileInfo = taosArrayGet(pWal->fileInfoSet, sz - 1); SWalFileInfo* pLastFileInfo = taosArrayGet(pWal->fileInfoSet, sz - 1);
char fnameStr[WAL_FILE_LEN]; char fnameStr[WAL_FILE_LEN];
@ -61,7 +60,7 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal) {
int32_t readSize = TMIN(WAL_SCAN_BUF_SIZE, fileSize); int32_t readSize = TMIN(WAL_SCAN_BUF_SIZE, fileSize);
pLastFileInfo->fileSize = fileSize; pLastFileInfo->fileSize = fileSize;
TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ); TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ | TD_FILE_WRITE);
if (pFile == NULL) { if (pFile == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
return -1; return -1;
@ -69,28 +68,30 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal) {
uint64_t magic = WAL_MAGIC; uint64_t magic = WAL_MAGIC;
char* buf = taosMemoryMalloc(readSize + 5); char* buf = taosMemoryMalloc(readSize + sizeof(uint64_t));
if (buf == NULL) { if (buf == NULL) {
taosCloseFile(&pFile);
terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; terrno = TSDB_CODE_WAL_OUT_OF_MEMORY;
return -1; goto _err;
} }
int64_t offset; int64_t offset;
offset = taosLSeekFile(pFile, -readSize, SEEK_END); offset = taosLSeekFile(pFile, -readSize, SEEK_END);
if (readSize != taosReadFile(pFile, buf, readSize)) { if (readSize != taosReadFile(pFile, buf, readSize)) {
taosMemoryFree(buf);
taosCloseFile(&pFile);
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
return -1; goto _err;
} }
int64_t walCkHeadSz = sizeof(SWalCkHead);
char* found = NULL; char* found = NULL;
while (1) { while (1) {
char* haystack = buf; char* haystack = buf;
char* candidate; char* candidate = NULL;
while ((candidate = tmemmem(haystack, readSize - (haystack - buf), (char*)&magic, sizeof(uint64_t))) != NULL) { while ((candidate = tmemmem(haystack, readSize - (haystack - buf), (char*)&magic, sizeof(uint64_t))) != NULL) {
// read and validate // read and validate
int64_t len = readSize - (candidate - buf);
if (len < walCkHeadSz) {
break;
}
SWalCkHead* logContent = (SWalCkHead*)candidate; SWalCkHead* logContent = (SWalCkHead*)candidate;
if (walValidHeadCksum(logContent) == 0 && walValidBodyCksum(logContent) == 0) { if (walValidHeadCksum(logContent) == 0 && walValidBodyCksum(logContent) == 0) {
found = candidate; found = candidate;
@ -98,35 +99,26 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal) {
haystack = candidate + 1; haystack = candidate + 1;
} }
if (found || offset == 0) break; if (found || offset == 0) break;
offset = TMIN(0, offset - readSize + sizeof(uint64_t));
int64_t offset2 = taosLSeekFile(pFile, offset, SEEK_SET); // go backwards, i.e. by at most one WAL scan buf size
ASSERT(offset == offset2); offset = TMAX(0, offset - readSize + walCkHeadSz);
if (readSize != taosReadFile(pFile, buf, readSize)) { int64_t ret = taosLSeekFile(pFile, offset, SEEK_SET);
taosMemoryFree(buf); if (ret < 0) {
taosCloseFile(&pFile); wError("failed to lseek file due to %s. offset:%lld", strerror(errno), offset);
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
return -1; goto _err;
} }
#if 0 ASSERT(offset == ret);
if (found == buf) { if (readSize != taosReadFile(pFile, buf, readSize)) {
SWalCkHead* logContent = (SWalCkHead*)found; terrno = TAOS_SYSTEM_ERROR(errno);
if (walValidHeadCksum(logContent) != 0 || walValidBodyCksum(logContent) != 0) { goto _err;
// file has to be deleted
taosMemoryFree(buf);
taosCloseFile(&pFile);
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
return -1;
} }
} }
#endif
}
if (found == NULL) { if (found == NULL) {
// file corrupted, no complete log wError("WAL log file corrupted: no valid WAL record found. file: %s", fnameStr);
// TODO delete and search in previous files
/*ASSERT(0);*/
terrno = TSDB_CODE_WAL_FILE_CORRUPTED; terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
return -1; goto _err;
} }
// truncate file // truncate file
@ -137,15 +129,28 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal) {
if (lastEntryEndOffset != fileSize) { if (lastEntryEndOffset != fileSize) {
wWarn("vgId:%d repair meta truncate file %s to %ld, orig size %ld", pWal->cfg.vgId, fnameStr, lastEntryEndOffset, wWarn("vgId:%d repair meta truncate file %s to %ld, orig size %ld", pWal->cfg.vgId, fnameStr, lastEntryEndOffset,
fileSize); fileSize);
taosFtruncateFile(pFile, lastEntryEndOffset); if (taosFtruncateFile(pFile, lastEntryEndOffset) < 0) {
wError("failed to truncate file due to %s. file:%s", strerror(errno), fnameStr);
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
if (taosFsyncFile(pFile) < 0) {
wError("failed to fsync file due to %s. file:%s", strerror(errno), fnameStr);
terrno = TAOS_SYSTEM_ERROR(errno);
goto _err;
}
((SWalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->fileSize = lastEntryEndOffset; ((SWalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->fileSize = lastEntryEndOffset;
pWal->totSize -= (fileSize - lastEntryEndOffset); pWal->totSize -= (fileSize - lastEntryEndOffset);
} }
taosCloseFile(&pFile); taosCloseFile(&pFile);
taosMemoryFree(buf); taosMemoryFree(buf);
return retVer; return retVer;
_err:
taosCloseFile(&pFile);
taosMemoryFree(buf);
return -1;
} }
int walCheckAndRepairMeta(SWal* pWal) { int walCheckAndRepairMeta(SWal* pWal) {