xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

tcs/stream: use tcs interface instead of s3

This commit is contained in:
Minglei Jin 2024-10-11 15:57:47 +08:00
parent 6d6568c18f
commit 3f403569eb
2 changed files with 32 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
source/libs/stream/CMakeLists.txt
View File

@ -3,6 +3,7 @@ add_library(stream STATIC ${STREAM_SRC})
target_include_directories(
stream
PUBLIC "${TD_SOURCE_DIR}/include/libs/stream"
PUBLIC "${TD_SOURCE_DIR}/include/libs/tcs"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
)

32
source/libs/stream/src/streamCheckpoint.c
View File

@ -13,10 +13,10 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "cos.h"
#include "rsync.h"
#include "streamBackendRocksdb.h"
#include "streamInt.h"
#include "tcs.h"
static int32_t downloadCheckpointDataByName(const char* id, const char* fname, const char* dstName);
static int32_t deleteCheckpointFile(const char* id, const char* name);
@ -374,8 +374,8 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
stDebug("s-task:%s set childIdx:%d, and add checkpoint-trigger block into outputQ", id, pTask->info.selfChildId);
code = continueDispatchCheckpointTriggerBlock(pBlock, pTask); // todo handle this failure
} else { // only one task exists, no need to dispatch downstream info
code = appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT, pActiveInfo->activeId, pActiveInfo->transId,
-1);
code =
appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT, pActiveInfo->activeId, pActiveInfo->transId, -1);
streamFreeQitem((SStreamQueueItem*)pBlock);
}
} else if (taskLevel == TASK_LEVEL__SINK || taskLevel == TASK_LEVEL__AGG) {
@ -559,7 +559,7 @@ void streamTaskClearCheckInfo(SStreamTask* pTask, bool clearChkpReadyMsg) {
}
streamMutexUnlock(&pInfo->lock);
stDebug("s-task:%s clear active checkpointInfo, failed checkpointId:%"PRId64", current checkpointId:%"PRId64,
stDebug("s-task:%s clear active checkpointInfo, failed checkpointId:%" PRId64 ", current checkpointId:%" PRId64,
pTask->id.idStr, pInfo->failedId, pTask->chkInfo.checkpointId);
}
@ -574,8 +574,7 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SV
if (pReq->checkpointId <= pInfo->checkpointId) {
stDebug("s-task:%s vgId:%d latest checkpointId:%" PRId64 " Ver:%" PRId64
" no need to update checkpoint info, updated checkpointId:%" PRId64 " Ver:%" PRId64
" transId:%d ignored",
" no need to update checkpoint info, updated checkpointId:%" PRId64 " Ver:%" PRId64 " transId:%d ignored",
id, vgId, pInfo->checkpointId, pInfo->checkpointVer, pReq->checkpointId, pReq->checkpointVer,
pReq->transId);
streamMutexUnlock(&pTask->lock);
@ -1022,7 +1021,8 @@ void checkpointTriggerMonitorFn(void* param, void* tmrId) {
}
if (++pTmrInfo->activeCounter < 50) {
streamTmrStart(checkpointTriggerMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId, "trigger-recv-monitor");
streamTmrStart(checkpointTriggerMonitorFn, 200, pTask, streamTimer, &pTmrInfo->tmrHandle, vgId,
"trigger-recv-monitor");
return;
}
@ -1287,11 +1287,11 @@ void streamTaskSetTriggerDispatchConfirmed(SStreamTask* pTask, int32_t vgId) {
static int32_t uploadCheckpointToS3(const char* id, const char* path) {
int32_t code = 0;
int32_t nBytes = 0;
/*
if (s3Init() != 0) {
return TSDB_CODE_THIRDPARTY_ERROR;
}
*/
TdDirPtr pDir = taosOpenDir(path);
if (pDir == NULL) {
return terrno;
@ -1324,11 +1324,11 @@ static int32_t uploadCheckpointToS3(const char* id, const char* path) {
break;
}
code = s3PutObjectFromFile2(filename, object, 0);
code = tcsPutObjectFromFile2(filename, object, 0);
if (code != 0) {
stError("[s3] failed to upload checkpoint:%s, reason:%s", filename, tstrerror(code));
stError("[tcs] failed to upload checkpoint:%s, reason:%s", filename, tstrerror(code));
} else {
stDebug("[s3] upload checkpoint:%s", filename);
stDebug("[tcs] upload checkpoint:%s", filename);
}
}
@ -1354,7 +1354,7 @@ int32_t downloadCheckpointByNameS3(const char* id, const char* fname, const char
taosMemoryFree(buf);
return TSDB_CODE_OUT_OF_RANGE;
}
int32_t code = s3GetObjectToFile(buf, dstName);
int32_t code = tcsGetObjectToFile(buf, dstName);
if (code != 0) {
taosMemoryFree(buf);
return TAOS_SYSTEM_ERROR(errno);
@ -1417,7 +1417,7 @@ int32_t streamTaskDownloadCheckpointData(const char* id, char* path, int64_t che
if (strlen(tsSnodeAddress) != 0) {
return downloadByRsync(id, path, checkpointId);
} else if (tsS3StreamEnabled) {
return s3GetObjectsByPrefix(id, path);
return tcsGetObjectsByPrefix(id, path);
}
return 0;
@ -1431,7 +1431,7 @@ int32_t deleteCheckpoint(const char* id) {
if (strlen(tsSnodeAddress) != 0) {
return deleteRsync(id);
} else if (tsS3StreamEnabled) {
s3DeleteObjectsByPrefix(id);
tcsDeleteObjectsByPrefix(id);
}
return 0;
}
@ -1445,7 +1445,7 @@ int32_t deleteCheckpointFile(const char* id, const char* name) {
}
char* tmp = object;
int32_t code = s3DeleteObjects((const char**)&tmp, 1);
int32_t code = tcsDeleteObjects((const char**)&tmp, 1);
if (code != 0) {
return TSDB_CODE_THIRDPARTY_ERROR;
}