xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix stream transfer error

This commit is contained in:
yihaoDeng 2023-11-08 17:10:54 +08:00
parent 7b0891981e
commit 76fcc49c6d
3 changed files with 66 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
source/libs/stream/inc/streamInt.h
View File

@ -162,6 +162,9 @@ int uploadCheckpoint(char* id, char* path);
int downloadCheckpoint(char* id, char* path); int downloadCheckpoint(char* id, char* path);
int deleteCheckpoint(char* id); int deleteCheckpoint(char* id);
typedef int32_t (*__stream_async_exec_fn_t)(void* param);
int32_t streamMetaAsyncExec(SStreamMeta* pMeta, __stream_async_exec_fn_t fn, void* param, int32_t* code);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

20
source/libs/stream/src/streamCheckpoint.c
View File

@ -13,8 +13,8 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include "streamInt.h"
#include "rsync.h" #include "rsync.h"
#include "streamInt.h"
int32_t tEncodeStreamCheckpointSourceReq(SEncoder* pEncoder, const SStreamCheckpointSourceReq* pReq) { int32_t tEncodeStreamCheckpointSourceReq(SEncoder* pEncoder, const SStreamCheckpointSourceReq* pReq) {
if (tStartEncode(pEncoder) < 0) return -1; if (tStartEncode(pEncoder) < 0) return -1;
@ -331,9 +331,9 @@ int32_t streamSaveAllTaskStatus(SStreamMeta* pMeta, SStreamTask* p, int64_t chec
return code; return code;
} }
int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) { int32_t streamTaskBuildCheckpointImpl(void* arg) {
int32_t code = 0; int32_t code = 0;
SStreamTask* pTask = arg;
// check for all tasks, and do generate the vnode-wide checkpoint data. // check for all tasks, and do generate the vnode-wide checkpoint data.
SStreamMeta* pMeta = pTask->pMeta; SStreamMeta* pMeta = pTask->pMeta;
// int32_t remain = atomic_sub_fetch_32(&pMeta->chkptNotReadyTasks, 1); // int32_t remain = atomic_sub_fetch_32(&pMeta->chkptNotReadyTasks, 1);
@ -350,9 +350,9 @@ int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) {
pMeta->vgId, pTask->id.idStr, pTask->info.taskLevel, el, pTask->checkpointingId); pMeta->vgId, pTask->id.idStr, pTask->info.taskLevel, el, pTask->checkpointingId);
// } else { // } else {
// stInfo( // stInfo(
// "vgId:%d vnode wide tasks not reach checkpoint ready status, ready s-task:%s, level:%d elapsed time:%.2f Sec " // "vgId:%d vnode wide tasks not reach checkpoint ready status, ready s-task:%s, level:%d elapsed time:%.2f Sec
// "not ready:%d/%d", // " "not ready:%d/%d", pMeta->vgId, pTask->id.idStr, pTask->info.taskLevel, el, remain,
// pMeta->vgId, pTask->id.idStr, pTask->info.taskLevel, el, remain, pMeta->numOfStreamTasks); // pMeta->numOfStreamTasks);
// } // }
// send check point response to upstream task // send check point response to upstream task
@ -373,7 +373,10 @@ int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) {
return code; return code;
} }
int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) {
int32_t code = 0;
return streamMetaAsyncExec(pTask->pMeta, streamTaskBuildCheckpointImpl, pTask, NULL);
}
// static int64_t kBlockSize = 64 * 1024; // static int64_t kBlockSize = 64 * 1024;
// static int sendCheckpointToS3(char* id, SArray* fileList){ // static int sendCheckpointToS3(char* id, SArray* fileList){
@ -474,7 +477,6 @@ int uploadCheckpoint(char* id, char* path){
if (strlen(tsSnodeIp) != 0) { if (strlen(tsSnodeIp) != 0) {
uploadRsync(id, path); uploadRsync(id, path);
// }else if(tsS3StreamEnabled){ // }else if(tsS3StreamEnabled){
} }
return 0; return 0;
} }
@ -487,7 +489,6 @@ int downloadCheckpoint(char* id, char* path){
if (strlen(tsSnodeIp) != 0) { if (strlen(tsSnodeIp) != 0) {
downloadRsync(id, path); downloadRsync(id, path);
// }else if(tsS3StreamEnabled){ // }else if(tsS3StreamEnabled){
} }
return 0; return 0;
} }
@ -500,7 +501,6 @@ int deleteCheckpoint(char* id){
if (strlen(tsSnodeIp) != 0) { if (strlen(tsSnodeIp) != 0) {
deleteRsync(id); deleteRsync(id);
// }else if(tsS3StreamEnabled){ // }else if(tsS3StreamEnabled){
} }
return 0; return 0;
} }

17
source/libs/stream/src/streamMeta.c
View File

@ -490,6 +490,7 @@ void streamMetaCloseImpl(void* arg) {
taosMemoryFree(pMeta->path); taosMemoryFree(pMeta->path);
taosThreadMutexDestroy(&pMeta->backendMutex); taosThreadMutexDestroy(&pMeta->backendMutex);
taosCleanUpScheduler(pMeta->qHandle);
pMeta->role = NODE_ROLE_UNINIT; pMeta->role = NODE_ROLE_UNINIT;
taosMemoryFree(pMeta); taosMemoryFree(pMeta);
stDebug("end to close stream meta"); stDebug("end to close stream meta");
@ -1261,3 +1262,19 @@ void streamMetaWUnLock(SStreamMeta* pMeta) {
stTrace("vgId:%d meta-wunlock", pMeta->vgId); stTrace("vgId:%d meta-wunlock", pMeta->vgId);
taosWUnLockLatch(&pMeta->lock); taosWUnLockLatch(&pMeta->lock);
} }
static void execHelper(struct SSchedMsg* pSchedMsg) {
__async_exec_fn_t execFn = (__async_exec_fn_t)pSchedMsg->ahandle;
int32_t code = execFn(pSchedMsg->thandle);
if (code != 0 && pSchedMsg->msg != NULL) {
*(int32_t*)pSchedMsg->msg = code;
}
}
int32_t streamMetaAsyncExec(SStreamMeta* pMeta, __stream_async_exec_fn_t fn, void* param, int32_t* code) {
SSchedMsg schedMsg = {0};
schedMsg.fp = execHelper;
schedMsg.ahandle = fn;
schedMsg.thandle = param;
schedMsg.msg = code;
return taosScheduleTask(pMeta->qHandle, &schedMsg);
}