xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

refactor wal's meta & reader

This commit is contained in:
Minglei Jin 2024-07-18 09:51:31 +08:00
parent 5e4b7b35a5
commit d2be5e0464
2 changed files with 45 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
source/libs/wal/src/walMeta.c
View File

@ -274,7 +274,7 @@ static void walRebuildFileInfoSet(SArray* metaLogList, SArray* actualLogList) {
} }
} }
void walAlignVersions(SWal* pWal) { static void walAlignVersions(SWal* pWal) {
if (pWal->vers.firstVer > pWal->vers.snapshotVer + 1) { if (pWal->vers.firstVer > pWal->vers.snapshotVer + 1) {
wWarn("vgId:%d, firstVer:%" PRId64 " is larger than snapshotVer:%" PRId64 " + 1. align with it.", pWal->cfg.vgId, wWarn("vgId:%d, firstVer:%" PRId64 " is larger than snapshotVer:%" PRId64 " + 1. align with it.", pWal->cfg.vgId,
pWal->vers.firstVer, pWal->vers.snapshotVer); pWal->vers.firstVer, pWal->vers.snapshotVer);
@ -294,7 +294,7 @@ void walAlignVersions(SWal* pWal) {
wInfo("vgId:%d, reset commitVer to %" PRId64, pWal->cfg.vgId, pWal->vers.commitVer); wInfo("vgId:%d, reset commitVer to %" PRId64, pWal->cfg.vgId, pWal->vers.commitVer);
} }
int walRepairLogFileTs(SWal* pWal, bool* updateMeta) { static int walRepairLogFileTs(SWal* pWal, bool* updateMeta) {
int32_t sz = taosArrayGetSize(pWal->fileInfoSet); int32_t sz = taosArrayGetSize(pWal->fileInfoSet);
int32_t fileIdx = -1; int32_t fileIdx = -1;
int32_t lastCloseTs = 0; int32_t lastCloseTs = 0;
@ -324,7 +324,7 @@ int walRepairLogFileTs(SWal* pWal, bool* updateMeta) {
return 0; return 0;
} }
bool walLogEntriesComplete(const SWal* pWal) { static bool walLogEntriesComplete(const SWal* pWal) {
int32_t sz = taosArrayGetSize(pWal->fileInfoSet); int32_t sz = taosArrayGetSize(pWal->fileInfoSet);
bool complete = true; bool complete = true;
int32_t fileIdx = -1; int32_t fileIdx = -1;
@ -352,7 +352,7 @@ bool walLogEntriesComplete(const SWal* pWal) {
return complete; return complete;
} }
int walTrimIdxFile(SWal* pWal, int32_t fileIdx) { static int walTrimIdxFile(SWal* pWal, int32_t fileIdx) {
SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx); SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx);
ASSERT(pFileInfo != NULL); ASSERT(pFileInfo != NULL);
char fnameStr[WAL_FILE_LEN]; char fnameStr[WAL_FILE_LEN];
@ -481,7 +481,7 @@ int walCheckAndRepairMeta(SWal* pWal) {
pWal->vers.firstVer = ((SWalFileInfo*)taosArrayGet(pWal->fileInfoSet, 0))->firstVer; pWal->vers.firstVer = ((SWalFileInfo*)taosArrayGet(pWal->fileInfoSet, 0))->firstVer;
pWal->vers.lastVer = ((SWalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->lastVer; pWal->vers.lastVer = ((SWalFileInfo*)taosArrayGetLast(pWal->fileInfoSet))->lastVer;
} }
(void)walAlignVersions(pWal); walAlignVersions(pWal);
// repair ts of files // repair ts of files
if (walRepairLogFileTs(pWal, &updateMeta) < 0) { if (walRepairLogFileTs(pWal, &updateMeta) < 0) {
@ -500,7 +500,7 @@ int walCheckAndRepairMeta(SWal* pWal) {
return 0; return 0;
} }
int walReadLogHead(TdFilePtr pLogFile, int64_t offset, SWalCkHead* pCkHead) { static int walReadLogHead(TdFilePtr pLogFile, int64_t offset, SWalCkHead* pCkHead) {
if (taosLSeekFile(pLogFile, offset, SEEK_SET) < 0) { if (taosLSeekFile(pLogFile, offset, SEEK_SET) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
return -1; return -1;
@ -519,7 +519,7 @@ int walReadLogHead(TdFilePtr pLogFile, int64_t offset, SWalCkHead* pCkHead) {
return 0; return 0;
} }
int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) { static int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) {
int32_t sz = taosArrayGetSize(pWal->fileInfoSet); int32_t sz = taosArrayGetSize(pWal->fileInfoSet);
SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx); SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx);
char fnameStr[WAL_FILE_LEN]; char fnameStr[WAL_FILE_LEN];
@ -866,7 +866,7 @@ static int walFindCurMetaVer(SWal* pWal) {
return metaVer; return metaVer;
} }
void walUpdateSyncedOffset(SWal* pWal) { static void walUpdateSyncedOffset(SWal* pWal) {
SWalFileInfo* pFileInfo = walGetCurFileInfo(pWal); SWalFileInfo* pFileInfo = walGetCurFileInfo(pWal);
if (pFileInfo == NULL) return; if (pFileInfo == NULL) return;
pFileInfo->syncedOffset = pFileInfo->fileSize; pFileInfo->syncedOffset = pFileInfo->fileSize;
@ -892,7 +892,7 @@ int walSaveMeta(SWal* pWal) {
} }
// update synced offset // update synced offset
(void)walUpdateSyncedOffset(pWal); walUpdateSyncedOffset(pWal);
// flush to a tmpfile // flush to a tmpfile
n = walBuildTmpMetaName(pWal, tmpFnameStr); n = walBuildTmpMetaName(pWal, tmpFnameStr);

30
source/libs/wal/src/walRead.c
View File

@ -199,7 +199,7 @@ static int32_t walReadChangeFile(SWalReader *pReader, int64_t fileFirstVer) {
return 0; return 0;
} }
int32_t walReadSeekVerImpl(SWalReader *pReader, int64_t ver) { static int32_t walReadSeekVerImpl(SWalReader *pReader, int64_t ver) {
SWal *pWal = pReader->pWal; SWal *pWal = pReader->pWal;
// bsearch in fileSet // bsearch in fileSet
@ -224,8 +224,7 @@ int32_t walReadSeekVerImpl(SWalReader *pReader, int64_t ver) {
return -1; return -1;
} }
wDebug("vgId:%d, wal version reset from %" PRId64 " to %" PRId64, pReader->pWal->cfg.vgId, wDebug("vgId:%d, wal version reset from %" PRId64 " to %" PRId64, pReader->pWal->cfg.vgId, pReader->curVersion, ver);
pReader->curVersion, ver);
pReader->curVersion = ver; pReader->curVersion = ver;
return 0; return 0;
@ -252,7 +251,6 @@ int32_t walReaderSeekVer(SWalReader *pReader, int64_t ver) {
return 0; return 0;
} }
int32_t walFetchHead(SWalReader *pRead, int64_t ver) { int32_t walFetchHead(SWalReader *pRead, int64_t ver) {
int64_t code; int64_t code;
int64_t contLen; int64_t contLen;
@ -294,8 +292,8 @@ int32_t walFetchHead(SWalReader *pRead, int64_t ver) {
code = walValidHeadCksum(pRead->pHead); code = walValidHeadCksum(pRead->pHead);
if (code != 0) { if (code != 0) {
wError("vgId:%d, unexpected wal log index:%" PRId64 ", since head checksum not passed, 0x%"PRIx64, pRead->pWal->cfg.vgId, ver, wError("vgId:%d, unexpected wal log index:%" PRId64 ", since head checksum not passed, 0x%" PRIx64,
pRead->readerId); pRead->pWal->cfg.vgId, ver, pRead->readerId);
terrno = TSDB_CODE_WAL_FILE_CORRUPTED; terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
return -1; return -1;
} }
@ -304,8 +302,8 @@ int32_t walFetchHead(SWalReader *pRead, int64_t ver) {
} }
int32_t walSkipFetchBody(SWalReader *pRead) { int32_t walSkipFetchBody(SWalReader *pRead) {
wDebug("vgId:%d, skip:%" PRId64 ", first:%" PRId64 ", commit:%" PRId64 ", last:%" PRId64 wDebug("vgId:%d, skip:%" PRId64 ", first:%" PRId64 ", commit:%" PRId64 ", last:%" PRId64 ", applied:%" PRId64
", applied:%" PRId64 ", 0x%" PRIx64, ", 0x%" PRIx64,
pRead->pWal->cfg.vgId, pRead->pHead->head.version, pRead->pWal->vers.firstVer, pRead->pWal->vers.commitVer, pRead->pWal->cfg.vgId, pRead->pHead->head.version, pRead->pWal->vers.firstVer, pRead->pWal->vers.commitVer,
pRead->pWal->vers.lastVer, pRead->pWal->vers.appliedVer, pRead->readerId); pRead->pWal->vers.lastVer, pRead->pWal->vers.appliedVer, pRead->readerId);
@ -358,10 +356,11 @@ int32_t walFetchBody(SWalReader *pRead) {
if (cryptedBodyLen != taosReadFile(pRead->pLogFile, pReadHead->body, cryptedBodyLen)) { if (cryptedBodyLen != taosReadFile(pRead->pLogFile, pReadHead->body, cryptedBodyLen)) {
if (plainBodyLen < 0) { if (plainBodyLen < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, wal fetch body error:%" PRId64 ", read request index:%" PRId64 ", since %s, 0x%"PRIx64, wError("vgId:%d, wal fetch body error:%" PRId64 ", read request index:%" PRId64 ", since %s, 0x%" PRIx64, vgId,
vgId, pReadHead->version, ver, tstrerror(terrno), id); pReadHead->version, ver, tstrerror(terrno), id);
} else { } else {
wError("vgId:%d, wal fetch body error:%" PRId64 ", read request index:%" PRId64 ", since file corrupted, 0x%"PRIx64, wError("vgId:%d, wal fetch body error:%" PRId64 ", read request index:%" PRId64
", since file corrupted, 0x%" PRIx64,
vgId, pReadHead->version, ver, id); vgId, pReadHead->version, ver, id);
terrno = TSDB_CODE_WAL_FILE_CORRUPTED; terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
} }
@ -378,7 +377,8 @@ int32_t walFetchBody(SWalReader *pRead) {
decryptBody(&pRead->pWal->cfg, pRead->pHead, plainBodyLen, __FUNCTION__); decryptBody(&pRead->pWal->cfg, pRead->pHead, plainBodyLen, __FUNCTION__);
if (walValidBodyCksum(pRead->pHead) != 0) { if (walValidBodyCksum(pRead->pHead) != 0) {
wError("vgId:%d, wal fetch body error, index:%" PRId64 ", since body checksum not passed, 0x%" PRIx64, vgId, ver, id); wError("vgId:%d, wal fetch body error, index:%" PRId64 ", since body checksum not passed, 0x%" PRIx64, vgId, ver,
id);
terrno = TSDB_CODE_WAL_FILE_CORRUPTED; terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
return -1; return -1;
} }
@ -458,8 +458,7 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) {
} }
if (pReader->capacity < cryptedBodyLen) { if (pReader->capacity < cryptedBodyLen) {
SWalCkHead *ptr = SWalCkHead *ptr = (SWalCkHead *)taosMemoryRealloc(pReader->pHead, sizeof(SWalCkHead) + cryptedBodyLen);
(SWalCkHead *)taosMemoryRealloc(pReader->pHead, sizeof(SWalCkHead) + cryptedBodyLen);
if (ptr == NULL) { if (ptr == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
taosThreadMutexUnlock(&pReader->mutex); taosThreadMutexUnlock(&pReader->mutex);
@ -469,8 +468,7 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) {
pReader->capacity = cryptedBodyLen; pReader->capacity = cryptedBodyLen;
} }
if ((contLen = taosReadFile(pReader->pLogFile, pReader->pHead->head.body, cryptedBodyLen)) != if ((contLen = taosReadFile(pReader->pLogFile, pReader->pHead->head.body, cryptedBodyLen)) != cryptedBodyLen) {
cryptedBodyLen) {
if (contLen < 0) if (contLen < 0)
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
else { else {