xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix stream state transfer

This commit is contained in:
yihaoDeng 2023-07-20 09:51:18 +00:00
parent 6e5be0d30e
commit 519d3b74d1
3 changed files with 40 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
source/dnode/vnode/src/tq/tqStreamStateSnap.c
View File

@ -51,15 +51,7 @@ int32_t streamStateSnapReaderOpen(STQ* pTq, int64_t sver, int64_t ever, SStreamS
SStreamSnapReader* pSnapReader = NULL; SStreamSnapReader* pSnapReader = NULL;
// restore from checkpoint if checkpointid != 0 if (streamSnapReaderOpen(pTq, sver, checkpointId, pTq->path, &pSnapReader) == 0) {
if (checkpointId != 0) {
sprintf(tdir, "%s%s%s%s%s%scheckpoint%" PRId64 "", pTq->path, TD_DIRSEP, VNODE_TQ_STREAM, TD_DIRSEP, "checkpoints",
TD_DIRSEP, checkpointId);
} else {
sprintf(tdir, "%s%s%s%s%s", pTq->path, TD_DIRSEP, VNODE_TQ_STREAM, TD_DIRSEP, "state");
}
if (streamSnapReaderOpen(pTq, sver, checkpointId, tdir, &pSnapReader) == 0) {
pReader->complete = 1; pReader->complete = 1;
} else { } else {
code = -1; code = -1;

59
source/libs/stream/src/streamSnapshot.c
View File

@ -79,31 +79,42 @@ const char* ROCKSDB_CURRENT = "CURRENT";
const char* ROCKSDB_CHECKPOINT_META = "CHECKPOINT"; const char* ROCKSDB_CHECKPOINT_META = "CHECKPOINT";
static int64_t kBlockSize = 64 * 1024; static int64_t kBlockSize = 64 * 1024;
int32_t streamSnapHandleInit(SStreamSnapHandle* handle, char* path); int32_t streamSnapHandleInit(SStreamSnapHandle* handle, char* path, int64_t chkpId);
void streamSnapHandleDestroy(SStreamSnapHandle* handle); void streamSnapHandleDestroy(SStreamSnapHandle* handle);
// static void streamBuildFname(char* path, char* file, char* fullname) // static void streamBuildFname(char* path, char* file, char* fullname)
#define STREAM_ROCKSDB_BUILD_FULLNAME(path, file, fullname) \ #define STREAM_ROCKSDB_BUILD_FULLNAME(path, file, fullname) \
do { \ do { \
sprintf(fullname, "%s/%s", path, file); \ sprintf(fullname, "%s%s%s", path, TD_DIRSEP, file); \
} while (0) } while (0)
int32_t streamSnapHandleInit(SStreamSnapHandle* pHandle, char* path) { int32_t streamSnapHandleInit(SStreamSnapHandle* pHandle, char* path, int64_t chkpId) {
// impl later // impl later
int len = strlen(path);
char* tdir = taosMemoryCalloc(1, len + 128);
memcpy(tdir, path, len);
if (chkpId != 0) {
sprintf(tdir, "%s%s%s%s%s%scheckpoint%" PRId64 "", path, TD_DIRSEP, "stream", TD_DIRSEP, "checkpoints", TD_DIRSEP,
chkpId);
} else {
sprintf(tdir, "%s%s%s%s%s", path, TD_DIRSEP, "stream", TD_DIRSEP, "state");
}
int32_t code = 0; int32_t code = 0;
TdDirPtr pDir = taosOpenDir(path); TdDirPtr pDir = taosOpenDir(tdir);
if (NULL == pDir) { if (NULL == pDir) {
qError("stream-state failed to open %s", tdir);
goto _err; goto _err;
} }
SBanckendFile* pFile = taosMemoryCalloc(1, sizeof(SBanckendFile)); SBanckendFile* pFile = taosMemoryCalloc(1, sizeof(SBanckendFile));
pHandle->pBackendFile = pFile; pHandle->pBackendFile = pFile;
pHandle->checkpointId = 0; pHandle->checkpointId = chkpId;
pHandle->seraial = 0; pHandle->seraial = 0;
pFile->path = taosStrdup(path); pFile->path = tdir;
pFile->pSst = taosArrayInit(16, sizeof(void*)); pFile->pSst = taosArrayInit(16, sizeof(void*));
TdDirEntryPtr pDirEntry; TdDirEntryPtr pDirEntry;
@ -135,7 +146,9 @@ int32_t streamSnapHandleInit(SStreamSnapHandle* pHandle, char* path) {
taosCloseDir(&pDir); taosCloseDir(&pDir);
if (pFile->pCurrent == NULL) { if (pFile->pCurrent == NULL) {
qError("stream-state failed to open %s, reason: no valid file", tdir);
code = -1; code = -1;
tdir = NULL;
goto _err; goto _err;
} }
SArray* list = taosArrayInit(64, sizeof(SBackendFileItem)); SArray* list = taosArrayInit(64, sizeof(SBackendFileItem));
@ -176,11 +189,13 @@ int32_t streamSnapHandleInit(SStreamSnapHandle* pHandle, char* path) {
pHandle->pFileList = list; pHandle->pFileList = list;
char fullname[256] = {0}; char fullname[256] = {0};
char* file = taosArrayGet(pHandle->pFileList, pHandle->currFileIdx); char* file = ((SBackendFileItem*)taosArrayGet(pHandle->pFileList, pHandle->currFileIdx))->name;
STREAM_ROCKSDB_BUILD_FULLNAME(pFile->path, file, fullname); STREAM_ROCKSDB_BUILD_FULLNAME(pFile->path, file, fullname);
pHandle->fd = taosOpenFile(fullname, TD_FILE_READ); pHandle->fd = taosOpenFile(fullname, TD_FILE_READ);
if (pHandle->fd == NULL) { if (pHandle->fd == NULL) {
qError("stream-state failed to open %s, reason: %s", tdir, tstrerror(errno));
tdir = NULL;
goto _err; goto _err;
} }
pHandle->seraial = 0; pHandle->seraial = 0;
@ -188,6 +203,7 @@ int32_t streamSnapHandleInit(SStreamSnapHandle* pHandle, char* path) {
return 0; return 0;
_err: _err:
streamSnapHandleDestroy(pHandle); streamSnapHandleDestroy(pHandle);
taosMemoryFreeClear(tdir);
code = -1; code = -1;
return code; return code;
@ -195,32 +211,33 @@ _err:
void streamSnapHandleDestroy(SStreamSnapHandle* handle) { void streamSnapHandleDestroy(SStreamSnapHandle* handle) {
SBanckendFile* pFile = handle->pBackendFile; SBanckendFile* pFile = handle->pBackendFile;
taosMemoryFree(pFile->pCheckpointMeta); if (pFile) {
taosMemoryFree(pFile->pCurrent); taosMemoryFree(pFile->pCheckpointMeta);
taosMemoryFree(pFile->pMainfest); taosMemoryFree(pFile->pCurrent);
taosMemoryFree(pFile->pOptions); taosMemoryFree(pFile->pMainfest);
taosMemoryFree(pFile->path); taosMemoryFree(pFile->pOptions);
for (int i = 0; i < taosArrayGetSize(pFile->pSst); i++) { taosMemoryFree(pFile->path);
char* sst = taosArrayGetP(pFile->pSst, i); for (int i = 0; i < taosArrayGetSize(pFile->pSst); i++) {
taosMemoryFree(sst); char* sst = taosArrayGetP(pFile->pSst, i);
taosMemoryFree(sst);
}
taosArrayDestroy(pFile->pSst);
taosMemoryFree(pFile);
} }
taosArrayDestroy(pFile->pSst);
taosMemoryFree(pFile);
taosArrayDestroy(handle->pFileList); taosArrayDestroy(handle->pFileList);
taosCloseFile(&handle->fd); taosCloseFile(&handle->fd);
return; return;
} }
int32_t streamSnapReaderOpen(void* pMeta, int64_t sver, int64_t ever, char* path, SStreamSnapReader** ppReader) { int32_t streamSnapReaderOpen(void* pMeta, int64_t sver, int64_t chkpId, char* path, SStreamSnapReader** ppReader) {
// impl later // impl later
SStreamSnapReader* pReader = taosMemoryCalloc(1, sizeof(SStreamSnapReader)); SStreamSnapReader* pReader = taosMemoryCalloc(1, sizeof(SStreamSnapReader));
if (pReader == NULL) { if (pReader == NULL) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
pReader->handle.checkpointId = ever;
// const char* path = NULL; if (streamSnapHandleInit(&pReader->handle, (char*)path, chkpId) < 0) {
if (streamSnapHandleInit(&pReader->handle, (char*)path) < 0) {
taosMemoryFree(pReader); taosMemoryFree(pReader);
return -1; return -1;
} }

2
source/libs/tdb/test/tdbPageRecycleTest.cpp
View File

@ -804,7 +804,7 @@ TEST(TdbPageRecycleTest, recycly_delete_interior_ofp_nocommit) {
// sprintf(&key[count - 2], "%c", i); // sprintf(&key[count - 2], "%c", i);
key[count - 2] = '0' + i; key[count - 2] = '0' + i;
ret = tdbTbInsert(pDb, key, count, NULL, NULL, txn); ret = tdbTbInsert(pDb, key, count, NULL, 0, txn);
GTEST_ASSERT_EQ(ret, 0); GTEST_ASSERT_EQ(ret, 0);
} }
} }