xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

refactor: do some internall refactor.

This commit is contained in:
Haojun Liao 2023-04-11 15:42:24 +08:00
parent 15cceb5a5a
commit 18479d8115
10 changed files with 40 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
include/libs/stream/tstream.h
View File

@ -50,7 +50,7 @@ enum {
TASK_STATUS__RECOVER_PREPARE, TASK_STATUS__RECOVER_PREPARE,
TASK_STATUS__RECOVER1, TASK_STATUS__RECOVER1,
TASK_STATUS__RECOVER2, TASK_STATUS__RECOVER2,
TASK_STATUS_RESTORE, // only available for source task to replay WAL from the checkpoint TASK_STATUS__RESTORE, // only available for source task to replay WAL from the checkpoint
}; };
enum { enum {
@ -353,9 +353,9 @@ struct SStreamTask {
int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo); int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo);
int32_t tDecodeStreamEpInfo(SDecoder* pDecoder, SStreamChildEpInfo* pInfo); int32_t tDecodeStreamEpInfo(SDecoder* pDecoder, SStreamChildEpInfo* pInfo);
SStreamTask* tNewSStreamTask(int64_t streamId); SStreamTask* tNewStreamTask(int64_t streamId);
int32_t tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask); int32_t tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask);
int32_t tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask); int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask);
void tFreeStreamTask(SStreamTask* pTask); void tFreeStreamTask(SStreamTask* pTask);
int32_t tAppendDataForStream(SStreamTask* pTask, SStreamQueueItem* pItem); int32_t tAppendDataForStream(SStreamTask* pTask, SStreamQueueItem* pItem);
bool tInputQueueIsFull(const SStreamTask* pTask); bool tInputQueueIsFull(const SStreamTask* pTask);

4
source/dnode/mnode/impl/src/mndDef.c
View File

@ -70,7 +70,7 @@ int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) {
if (tEncodeI32(pEncoder, innerSz) < 0) return -1; if (tEncodeI32(pEncoder, innerSz) < 0) return -1;
for (int32_t j = 0; j < innerSz; j++) { for (int32_t j = 0; j < innerSz; j++) {
SStreamTask *pTask = taosArrayGetP(pArray, j); SStreamTask *pTask = taosArrayGetP(pArray, j);
if (tEncodeSStreamTask(pEncoder, pTask) < 0) return -1; if (tEncodeStreamTask(pEncoder, pTask) < 0) return -1;
} }
} }
@ -130,7 +130,7 @@ int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj, int32_t sver) {
taosArrayDestroy(pArray); taosArrayDestroy(pArray);
return -1; return -1;
} }
if (tDecodeSStreamTask(pDecoder, pTask) < 0) { if (tDecodeStreamTask(pDecoder, pTask) < 0) {
taosMemoryFree(pTask); taosMemoryFree(pTask);
taosArrayDestroy(pArray); taosArrayDestroy(pArray);
return -1; return -1;

10
source/dnode/mnode/impl/src/mndScheduler.c
View File

@ -224,7 +224,7 @@ int32_t mndAddShuffleSinkTasksToStream(SMnode* pMnode, SStreamObj* pStream) {
continue; continue;
} }
SStreamTask* pTask = tNewSStreamTask(pStream->uid); SStreamTask* pTask = tNewStreamTask(pStream->uid);
if (pTask == NULL) { if (pTask == NULL) {
sdbRelease(pSdb, pVgroup); sdbRelease(pSdb, pVgroup);
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -260,7 +260,7 @@ int32_t mndAddShuffleSinkTasksToStream(SMnode* pMnode, SStreamObj* pStream) {
int32_t mndAddFixedSinkTaskToStream(SMnode* pMnode, SStreamObj* pStream) { int32_t mndAddFixedSinkTaskToStream(SMnode* pMnode, SStreamObj* pStream) {
SArray* tasks = taosArrayGetP(pStream->tasks, 0); SArray* tasks = taosArrayGetP(pStream->tasks, 0);
SStreamTask* pTask = tNewSStreamTask(pStream->uid); SStreamTask* pTask = tNewStreamTask(pStream->uid);
if (pTask == NULL) { if (pTask == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1; return -1;
@ -350,7 +350,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) {
return -1; return -1;
} }
pInnerTask = tNewSStreamTask(pStream->uid); pInnerTask = tNewStreamTask(pStream->uid);
if (pInnerTask == NULL) { if (pInnerTask == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
qDestroyQueryPlan(pPlan); qDestroyQueryPlan(pPlan);
@ -421,7 +421,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) {
continue; continue;
} }
SStreamTask* pTask = tNewSStreamTask(pStream->uid); SStreamTask* pTask = tNewStreamTask(pStream->uid);
if (pTask == NULL) { if (pTask == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
sdbRelease(pSdb, pVgroup); sdbRelease(pSdb, pVgroup);
@ -491,7 +491,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) {
continue; continue;
} }
SStreamTask* pTask = tNewSStreamTask(pStream->uid); SStreamTask* pTask = tNewStreamTask(pStream->uid);
if (pTask == NULL) { if (pTask == NULL) {
sdbRelease(pSdb, pVgroup); sdbRelease(pSdb, pVgroup);
qDestroyQueryPlan(pPlan); qDestroyQueryPlan(pPlan);

8
source/dnode/mnode/impl/src/mndStream.c
View File

@ -39,8 +39,8 @@ static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pOldStream, SStream
static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq); static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq);
static int32_t mndProcessDropStreamReq(SRpcMsg *pReq); static int32_t mndProcessDropStreamReq(SRpcMsg *pReq);
static int32_t mndProcessStreamCheckpointTmr(SRpcMsg *pReq); static int32_t mndProcessStreamCheckpointTmr(SRpcMsg *pReq);
// static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq); static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq);
/*static int32_t mndProcessRecoverStreamReq(SRpcMsg *pReq);*/ static int32_t mndProcessRecoverStreamReq(SRpcMsg *pReq);
static int32_t mndProcessStreamMetaReq(SRpcMsg *pReq); static int32_t mndProcessStreamMetaReq(SRpcMsg *pReq);
static int32_t mndGetStreamMeta(SRpcMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); static int32_t mndGetStreamMeta(SRpcMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta);
static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows);
@ -418,7 +418,7 @@ FAIL:
int32_t mndPersistTaskDeployReq(STrans *pTrans, const SStreamTask *pTask) { int32_t mndPersistTaskDeployReq(STrans *pTrans, const SStreamTask *pTask) {
SEncoder encoder; SEncoder encoder;
tEncoderInit(&encoder, NULL, 0); tEncoderInit(&encoder, NULL, 0);
tEncodeSStreamTask(&encoder, pTask); tEncodeStreamTask(&encoder, pTask);
int32_t size = encoder.pos; int32_t size = encoder.pos;
int32_t tlen = sizeof(SMsgHead) + size; int32_t tlen = sizeof(SMsgHead) + size;
tEncoderClear(&encoder); tEncoderClear(&encoder);
@ -430,7 +430,7 @@ int32_t mndPersistTaskDeployReq(STrans *pTrans, const SStreamTask *pTask) {
((SMsgHead *)buf)->vgId = htonl(pTask->nodeId); ((SMsgHead *)buf)->vgId = htonl(pTask->nodeId);
void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
tEncoderInit(&encoder, abuf, size); tEncoderInit(&encoder, abuf, size);
tEncodeSStreamTask(&encoder, pTask); tEncodeStreamTask(&encoder, pTask);
tEncoderClear(&encoder); tEncoderClear(&encoder);
STransAction action = {0}; STransAction action = {0};

2
source/dnode/snode/src/snode.c
View File

@ -139,7 +139,7 @@ int32_t sndProcessTaskDeployReq(SSnode *pSnode, char *msg, int32_t msgLen) {
} }
SDecoder decoder; SDecoder decoder;
tDecoderInit(&decoder, (uint8_t *)msg, msgLen); tDecoderInit(&decoder, (uint8_t *)msg, msgLen);
code = tDecodeSStreamTask(&decoder, pTask); code = tDecodeStreamTask(&decoder, pTask);
if (code < 0) { if (code < 0) {
tDecoderClear(&decoder); tDecoderClear(&decoder);
taosMemoryFree(pTask); taosMemoryFree(pTask);

10
source/dnode/vnode/src/tq/tq.c
View File

@ -906,14 +906,13 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) {
pTask->inputStatus = TASK_INPUT_STATUS__NORMAL; pTask->inputStatus = TASK_INPUT_STATUS__NORMAL;
pTask->outputStatus = TASK_OUTPUT_STATUS__NORMAL; pTask->outputStatus = TASK_OUTPUT_STATUS__NORMAL;
pTask->pMsgCb = &pTq->pVnode->msgCb; pTask->pMsgCb = &pTq->pVnode->msgCb;
pTask->chkInfo.version = ver;
pTask->pMeta = pTq->pStreamMeta; pTask->pMeta = pTq->pStreamMeta;
// expand executor // expand executor
if (pTask->fillHistory) { if (pTask->fillHistory) {
pTask->taskStatus = TASK_STATUS__WAIT_DOWNSTREAM; pTask->taskStatus = TASK_STATUS__WAIT_DOWNSTREAM;
} else { } else {
pTask->taskStatus = TASK_STATUS_RESTORE; pTask->taskStatus = TASK_STATUS__RESTORE;
} }
if (pTask->taskLevel == TASK_LEVEL__SOURCE) { if (pTask->taskLevel == TASK_LEVEL__SOURCE) {
@ -1089,7 +1088,7 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t sversion, char* msg, int32_t ms
SDecoder decoder; SDecoder decoder;
tDecoderInit(&decoder, (uint8_t*)msg, msgLen); tDecoderInit(&decoder, (uint8_t*)msg, msgLen);
code = tDecodeSStreamTask(&decoder, pTask); code = tDecodeStreamTask(&decoder, pTask);
if (code < 0) { if (code < 0) {
tDecoderClear(&decoder); tDecoderClear(&decoder);
taosMemoryFree(pTask); taosMemoryFree(pTask);
@ -1485,8 +1484,9 @@ int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) {
if (pTask->taskStatus == TASK_STATUS__NORMAL) { if (pTask->taskStatus == TASK_STATUS__NORMAL) {
tqDebug("vgId:%d s-task:%s start to process run req", vgId, pTask->id.idStr); tqDebug("vgId:%d s-task:%s start to process run req", vgId, pTask->id.idStr);
streamProcessRunReq(pTask); streamProcessRunReq(pTask);
} else if (pTask->taskStatus == TASK_STATUS_RESTORE) { } else if (pTask->taskStatus == TASK_STATUS__RESTORE) {
tqDebug("vgId:%d s-task:%s start to restore from last ck", vgId, pTask->id.idStr); tqDebug("vgId:%d s-task:%s start to process in restore procedure from last chk point:%" PRId64, vgId,
pTask->id.idStr, pTask->chkInfo.version);
streamProcessRunReq(pTask); streamProcessRunReq(pTask);
} else { } else {
tqDebug("vgId:%d s-task:%s ignore run req since not in ready state", vgId, pTask->id.idStr); tqDebug("vgId:%d s-task:%s ignore run req since not in ready state", vgId, pTask->id.idStr);

5
source/dnode/vnode/src/tq/tqRestore.c
View File

@ -49,9 +49,10 @@ int32_t transferToNormalTask(SStreamMeta* pStreamMeta, SArray* pTaskList) {
} }
// todo: add lock // todo: add lock
for(int32_t i = 0; i < numOfTask; ++i){ for (int32_t i = 0; i < numOfTask; ++i) {
SStreamTask* pTask = taosArrayGetP(pTaskList, i); SStreamTask* pTask = taosArrayGetP(pTaskList, i);
tqDebug("vgId:%d transfer s-task:%s state restore -> ready", pStreamMeta->vgId, pTask->id.idStr); tqDebug("vgId:%d transfer s-task:%s state restore -> ready, checkpoint:%" PRId64 " checkpoint id:%" PRId64,
pStreamMeta->vgId, pTask->id.idStr, pTask->chkInfo.version, pTask->chkInfo.id);
taosHashRemove(pStreamMeta->pRestoreTasks, &pTask->id.taskId, sizeof(pTask->id.taskId)); taosHashRemove(pStreamMeta->pRestoreTasks, &pTask->id.taskId, sizeof(pTask->id.taskId));
// NOTE: do not change the following order // NOTE: do not change the following order

14
source/libs/stream/src/streamExec.c
View File

@ -21,11 +21,15 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray*
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
void* pExecutor = pTask->exec.pExecutor; void* pExecutor = pTask->exec.pExecutor;
while (pTask->taskLevel == TASK_LEVEL__SOURCE && atomic_load_8(&pTask->taskStatus) != TASK_STATUS__NORMAL) { while (pTask->taskLevel == TASK_LEVEL__SOURCE) {
qError("stream task wait for the end of fill history, s-task:%s, status:%d", pTask->id.idStr, int8_t status = atomic_load_8(&pTask->taskStatus);
atomic_load_8(&pTask->taskStatus)); if (status != TASK_STATUS__NORMAL && status != TASK_STATUS__RESTORE) {
taosMsleep(2); qError("stream task wait for the end of fill history, s-task:%s, status:%d", pTask->id.idStr,
continue; atomic_load_8(&pTask->taskStatus));
taosMsleep(2);
} else {
break;
}
} }
// set input // set input

8
source/libs/stream/src/streamMeta.c
View File

@ -109,7 +109,7 @@ int32_t streamMetaAddSerializedTask(SStreamMeta* pMeta, int64_t ver, char* msg,
} }
SDecoder decoder; SDecoder decoder;
tDecoderInit(&decoder, (uint8_t*)msg, msgLen); tDecoderInit(&decoder, (uint8_t*)msg, msgLen);
if (tDecodeSStreamTask(&decoder, pTask) < 0) { if (tDecodeStreamTask(&decoder, pTask) < 0) {
tDecoderClear(&decoder); tDecoderClear(&decoder);
goto FAIL; goto FAIL;
} }
@ -142,7 +142,7 @@ int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask) {
void* buf = NULL; void* buf = NULL;
int32_t len; int32_t len;
int32_t code; int32_t code;
tEncodeSize(tEncodeSStreamTask, pTask, len, code); tEncodeSize(tEncodeStreamTask, pTask, len, code);
if (code < 0) { if (code < 0) {
return -1; return -1;
} }
@ -153,7 +153,7 @@ int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask) {
SEncoder encoder = {0}; SEncoder encoder = {0};
tEncoderInit(&encoder, buf, len); tEncoderInit(&encoder, buf, len);
tEncodeSStreamTask(&encoder, pTask); tEncodeStreamTask(&encoder, pTask);
tEncoderClear(&encoder); tEncoderClear(&encoder);
if (tdbTbUpsert(pMeta->pTaskDb, &pTask->id.taskId, sizeof(int32_t), buf, len, pMeta->txn) < 0) { if (tdbTbUpsert(pMeta->pTaskDb, &pTask->id.taskId, sizeof(int32_t), buf, len, pMeta->txn) < 0) {
@ -321,7 +321,7 @@ int32_t streamLoadTasks(SStreamMeta* pMeta, int64_t ver) {
return -1; return -1;
} }
tDecoderInit(&decoder, (uint8_t*)pVal, vLen); tDecoderInit(&decoder, (uint8_t*)pVal, vLen);
tDecodeSStreamTask(&decoder, pTask); tDecodeStreamTask(&decoder, pTask);
tDecoderClear(&decoder); tDecoderClear(&decoder);
if (pMeta->expandFunc(pMeta->ahandle, pTask, -1) < 0) { if (pMeta->expandFunc(pMeta->ahandle, pTask, -1) < 0) {

6
source/libs/stream/src/streamTask.c
View File

@ -16,7 +16,7 @@
#include "executor.h" #include "executor.h"
#include "tstream.h" #include "tstream.h"
SStreamTask* tNewSStreamTask(int64_t streamId) { SStreamTask* tNewStreamTask(int64_t streamId) {
SStreamTask* pTask = (SStreamTask*)taosMemoryCalloc(1, sizeof(SStreamTask)); SStreamTask* pTask = (SStreamTask*)taosMemoryCalloc(1, sizeof(SStreamTask));
if (pTask == NULL) { if (pTask == NULL) {
return NULL; return NULL;
@ -54,7 +54,7 @@ int32_t tDecodeStreamEpInfo(SDecoder* pDecoder, SStreamChildEpInfo* pInfo) {
return 0; return 0;
} }
int32_t tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) { int32_t tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) {
if (tStartEncode(pEncoder) < 0) return -1; if (tStartEncode(pEncoder) < 0) return -1;
if (tEncodeI64(pEncoder, pTask->id.streamId) < 0) return -1; if (tEncodeI64(pEncoder, pTask->id.streamId) < 0) return -1;
if (tEncodeI32(pEncoder, pTask->id.taskId) < 0) return -1; if (tEncodeI32(pEncoder, pTask->id.taskId) < 0) return -1;
@ -107,7 +107,7 @@ int32_t tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) {
return pEncoder->pos; return pEncoder->pos;
} }
int32_t tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask) { int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask) {
if (tStartDecode(pDecoder) < 0) return -1; if (tStartDecode(pDecoder) < 0) return -1;
if (tDecodeI64(pDecoder, &pTask->id.streamId) < 0) return -1; if (tDecodeI64(pDecoder, &pTask->id.streamId) < 0) return -1;
if (tDecodeI32(pDecoder, &pTask->id.taskId) < 0) return -1; if (tDecodeI32(pDecoder, &pTask->id.taskId) < 0) return -1;