xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

support reopen stream state

This commit is contained in:
yihaoDeng 2023-08-10 01:49:26 +00:00
parent a20b299f9b
commit ef247cdb1d
4 changed files with 58 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
include/libs/stream/tstream.h
View File

@ -369,11 +369,11 @@ typedef struct SStreamMeta {
int32_t chkptNotReadyTasks; int32_t chkptNotReadyTasks;
int64_t checkpointId; int64_t chkpId;
SArray* checkpointSaved; SArray* chkpSaved;
SArray* checkpointInUse; SArray* chkpInUse;
int32_t checkpointCap; int32_t chkpCap;
SRWLatch checkpointDirLock; SRWLatch chkpDirLock;
} SStreamMeta; } SStreamMeta;
int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo); int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo);

4
source/dnode/vnode/src/tq/tqStreamStateSnap.c
View File

@ -47,11 +47,11 @@ int32_t streamStateSnapReaderOpen(STQ* pTq, int64_t sver, int64_t ever, SStreamS
pReader->sver = sver; pReader->sver = sver;
pReader->ever = ever; pReader->ever = ever;
int64_t checkpointId = meta ? meta->checkpointId : 0; int64_t chkpId = meta ? meta->chkpId : 0;
SStreamSnapReader* pSnapReader = NULL; SStreamSnapReader* pSnapReader = NULL;
if (streamSnapReaderOpen(pTq, sver, checkpointId, pTq->path, &pSnapReader) == 0) { if (streamSnapReaderOpen(pTq, sver, chkpId, pTq->path, &pSnapReader) == 0) {
pReader->complete = 1; pReader->complete = 1;
} else { } else {
code = -1; code = -1;

84
source/libs/stream/src/streamBackendRocksdb.c
View File

@ -372,8 +372,8 @@ int32_t copyFiles(const char* src, const char* dst) {
// opt later, just hard link // opt later, just hard link
int32_t sLen = strlen(src); int32_t sLen = strlen(src);
int32_t dLen = strlen(dst); int32_t dLen = strlen(dst);
char* absSrcPath = taosMemoryCalloc(1, sLen + 64); char* srcName = taosMemoryCalloc(1, sLen + 64);
char* absDstPath = taosMemoryCalloc(1, dLen + 64); char* dstName = taosMemoryCalloc(1, dLen + 64);
TdDirPtr pDir = taosOpenDir(src); TdDirPtr pDir = taosOpenDir(src);
if (pDir == NULL) return 0; if (pDir == NULL) return 0;
@ -383,22 +383,22 @@ int32_t copyFiles(const char* src, const char* dst) {
char* name = taosGetDirEntryName(de); char* name = taosGetDirEntryName(de);
if (strcmp(name, ".") == 0 || strcmp(name, "..") == 0) continue; if (strcmp(name, ".") == 0 || strcmp(name, "..") == 0) continue;
sprintf(absSrcPath, "%s%s%s", src, TD_DIRSEP, name); sprintf(srcName, "%s%s%s", src, TD_DIRSEP, name);
sprintf(absDstPath, "%s%s%s", dst, TD_DIRSEP, name); sprintf(dstName, "%s%s%s", dst, TD_DIRSEP, name);
if (!taosDirEntryIsDir(de)) { if (!taosDirEntryIsDir(de)) {
code = taosCopyFile(absSrcPath, absDstPath); code = taosCopyFile(srcName, dstName);
if (code == -1) { if (code == -1) {
goto _err; goto _err;
} }
} }
memset(absSrcPath, 0, sLen + 64); memset(srcName, 0, sLen + 64);
memset(absDstPath, 0, dLen + 64); memset(dstName, 0, dLen + 64);
} }
_err: _err:
taosMemoryFreeClear(absSrcPath); taosMemoryFreeClear(srcName);
taosMemoryFreeClear(absDstPath); taosMemoryFreeClear(dstName);
taosCloseDir(&pDir); taosCloseDir(&pDir);
return code >= 0 ? 0 : -1; return code >= 0 ? 0 : -1;
} }
@ -626,75 +626,75 @@ void streamBackendHandleCleanup(void* arg) {
int32_t getLatestCheckpoint(void* arg, int64_t* checkpoint) { int32_t getLatestCheckpoint(void* arg, int64_t* checkpoint) {
SStreamMeta* pMeta = arg; SStreamMeta* pMeta = arg;
taosWLockLatch(&pMeta->checkpointDirLock); taosWLockLatch(&pMeta->chkpDirLock);
int64_t tc = 0; int64_t tc = 0;
int32_t sz = taosArrayGetSize(pMeta->checkpointSaved); int32_t sz = taosArrayGetSize(pMeta->chkpSaved);
if (sz <= 0) { if (sz <= 0) {
taosWUnLockLatch(&pMeta->checkpointDirLock); taosWUnLockLatch(&pMeta->chkpDirLock);
return -1; return -1;
} else { } else {
tc = *(int64_t*)taosArrayGetLast(pMeta->checkpointSaved); tc = *(int64_t*)taosArrayGetLast(pMeta->chkpSaved);
} }
taosArrayPush(pMeta->checkpointInUse, &tc); taosArrayPush(pMeta->chkpInUse, &tc);
*checkpoint = tc; *checkpoint = tc;
taosWUnLockLatch(&pMeta->checkpointDirLock); taosWUnLockLatch(&pMeta->chkpDirLock);
return 0; return 0;
} }
/* /*
* checkpointSave |--cp1--|--cp2--|--cp3--|--cp4--|--cp5--| * checkpointSave |--cp1--|--cp2--|--cp3--|--cp4--|--cp5--|
* checkpointInUse: |--cp2--|--cp4--| * chkpInUse: |--cp2--|--cp4--|
* checkpointInUse is doing translation, cannot del until * chkpInUse is doing translation, cannot del until
* replication is finished * replication is finished
*/ */
int32_t delObsoleteCheckpoint(void* arg, const char* path) { int32_t delObsoleteCheckpoint(void* arg, const char* path) {
SStreamMeta* pMeta = arg; SStreamMeta* pMeta = arg;
taosWLockLatch(&pMeta->checkpointDirLock); taosWLockLatch(&pMeta->chkpDirLock);
SArray* checkpointDel = taosArrayInit(10, sizeof(int64_t)); SArray* chkpDel = taosArrayInit(10, sizeof(int64_t));
SArray* checkpointDup = taosArrayInit(10, sizeof(int64_t)); SArray* chkpDup = taosArrayInit(10, sizeof(int64_t));
int64_t minId = 0; int64_t minId = 0;
if (taosArrayGetSize(pMeta->checkpointInUse) >= 1) { if (taosArrayGetSize(pMeta->chkpInUse) >= 1) {
minId = *(int64_t*)taosArrayGet(pMeta->checkpointInUse, 0); minId = *(int64_t*)taosArrayGet(pMeta->chkpInUse, 0);
for (int i = 0; i < taosArrayGetSize(pMeta->checkpointSaved); i++) { for (int i = 0; i < taosArrayGetSize(pMeta->chkpSaved); i++) {
int64_t id = *(int64_t*)taosArrayGet(pMeta->checkpointSaved, i); int64_t id = *(int64_t*)taosArrayGet(pMeta->chkpSaved, i);
if (id >= minId) { if (id >= minId) {
taosArrayPush(checkpointDup, &id); taosArrayPush(chkpDup, &id);
} else { } else {
taosArrayPush(checkpointDel, &id); taosArrayPush(chkpDel, &id);
} }
} }
} else { } else {
int32_t sz = taosArrayGetSize(pMeta->checkpointSaved); int32_t sz = taosArrayGetSize(pMeta->chkpSaved);
int32_t dsz = sz - pMeta->checkpointCap; // del size int32_t dsz = sz - pMeta->chkpCap; // del size
for (int i = 0; i < dsz; i++) { for (int i = 0; i < dsz; i++) {
int64_t id = *(int64_t*)taosArrayGet(pMeta->checkpointSaved, i); int64_t id = *(int64_t*)taosArrayGet(pMeta->chkpSaved, i);
taosArrayPush(checkpointDel, &id); taosArrayPush(chkpDel, &id);
} }
for (int i = dsz < 0 ? 0 : dsz; i < sz; i++) { for (int i = dsz < 0 ? 0 : dsz; i < sz; i++) {
int64_t id = *(int64_t*)taosArrayGet(pMeta->checkpointSaved, i); int64_t id = *(int64_t*)taosArrayGet(pMeta->chkpSaved, i);
taosArrayPush(checkpointDup, &id); taosArrayPush(chkpDup, &id);
} }
} }
taosArrayDestroy(pMeta->checkpointSaved); taosArrayDestroy(pMeta->chkpSaved);
pMeta->checkpointSaved = checkpointDup; pMeta->chkpSaved = chkpDup;
taosWUnLockLatch(&pMeta->checkpointDirLock); taosWUnLockLatch(&pMeta->chkpDirLock);
for (int i = 0; i < taosArrayGetSize(checkpointDel); i++) { for (int i = 0; i < taosArrayGetSize(chkpDel); i++) {
int64_t id = *(int64_t*)taosArrayGet(checkpointDel, i); int64_t id = *(int64_t*)taosArrayGet(chkpDel, i);
char tbuf[256] = {0}; char tbuf[256] = {0};
sprintf(tbuf, "%s%scheckpoint%" PRId64 "", path, TD_DIRSEP, id); sprintf(tbuf, "%s%scheckpoint%" PRId64 "", path, TD_DIRSEP, id);
if (taosIsDir(tbuf)) { if (taosIsDir(tbuf)) {
taosRemoveDir(tbuf); taosRemoveDir(tbuf);
} }
} }
taosArrayDestroy(checkpointDel); taosArrayDestroy(chkpDel);
return 0; return 0;
} }
@ -742,7 +742,7 @@ int32_t streamBackendLoadCheckpointInfo(void* arg) {
for (int i = 0; i < taosArrayGetSize(suffix); i++) { for (int i = 0; i < taosArrayGetSize(suffix); i++) {
int64_t id = *(int64_t*)taosArrayGet(suffix, i); int64_t id = *(int64_t*)taosArrayGet(suffix, i);
taosArrayPush(pMeta->checkpointSaved, &id); taosArrayPush(pMeta->chkpSaved, &id);
} }
taosArrayDestroy(suffix); taosArrayDestroy(suffix);
@ -794,9 +794,9 @@ int32_t streamBackendDoCheckpoint(void* arg, uint64_t checkpointId) {
} }
rocksdb_checkpoint_object_destroy(cp); rocksdb_checkpoint_object_destroy(cp);
} }
taosWLockLatch(&pMeta->checkpointDirLock); taosWLockLatch(&pMeta->chkpDirLock);
taosArrayPush(pMeta->checkpointSaved, &checkpointId); taosArrayPush(pMeta->chkpSaved, &checkpointId);
taosWUnLockLatch(&pMeta->checkpointDirLock); taosWUnLockLatch(&pMeta->chkpDirLock);
delObsoleteCheckpoint(arg, path); delObsoleteCheckpoint(arg, path);

18
source/libs/stream/src/streamMeta.c
View File

@ -91,12 +91,13 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF
pMeta->pTaskBackendUnique = pMeta->pTaskBackendUnique =
taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK);
pMeta->checkpointSaved = taosArrayInit(4, sizeof(int64_t)); pMeta->chkpSaved = taosArrayInit(4, sizeof(int64_t));
pMeta->checkpointInUse = taosArrayInit(4, sizeof(int64_t)); pMeta->chkpInUse = taosArrayInit(4, sizeof(int64_t));
pMeta->checkpointCap = 8; pMeta->chkpCap = 8;
taosInitRWLatch(&pMeta->checkpointDirLock); taosInitRWLatch(&pMeta->chkpDirLock);
int64_t chkpId = streamGetLatestCheckpointId(pMeta); int64_t chkpId = streamGetLatestCheckpointId(pMeta);
pMeta->chkpId = chkpId;
pMeta->streamBackend = streamBackendInit(pMeta->path, chkpId); pMeta->streamBackend = streamBackendInit(pMeta->path, chkpId);
if (pMeta->streamBackend == NULL) { if (pMeta->streamBackend == NULL) {
@ -109,7 +110,6 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF
terrno = TAOS_SYSTEM_ERROR(code); terrno = TAOS_SYSTEM_ERROR(code);
goto _err; goto _err;
} }
taosInitRWLatch(&pMeta->lock); taosInitRWLatch(&pMeta->lock);
taosThreadMutexInit(&pMeta->backendMutex, NULL); taosThreadMutexInit(&pMeta->backendMutex, NULL);
@ -182,9 +182,9 @@ int32_t streamMetaReopen(SStreamMeta* pMeta, int64_t chkpId) {
taosHashClear(pMeta->pTaskBackendUnique); taosHashClear(pMeta->pTaskBackendUnique);
taosArrayClear(pMeta->checkpointSaved); taosArrayClear(pMeta->chkpSaved);
taosArrayClear(pMeta->checkpointInUse); taosArrayClear(pMeta->chkpInUse);
return 0; return 0;
} }
@ -222,8 +222,8 @@ void streamMetaClose(SStreamMeta* pMeta) {
taosThreadMutexDestroy(&pMeta->backendMutex); taosThreadMutexDestroy(&pMeta->backendMutex);
taosHashCleanup(pMeta->pTaskBackendUnique); taosHashCleanup(pMeta->pTaskBackendUnique);
taosArrayDestroy(pMeta->checkpointSaved); taosArrayDestroy(pMeta->chkpSaved);
taosArrayDestroy(pMeta->checkpointInUse); taosArrayDestroy(pMeta->chkpInUse);
taosMemoryFree(pMeta); taosMemoryFree(pMeta);
} }