diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index fff58feeb1..969edae30d 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1064,7 +1064,7 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t sversion, char* msg, int32_t ms return code; } -static void doStartStep2(SStreamTask* pTask, SStreamTask* pStreamTask, STQ* pTq) { +static void doStartFillhistoryStep2(SStreamTask* pTask, SStreamTask* pStreamTask, STQ* pTq) { const char* id = pTask->id.idStr; int64_t nextProcessedVer = pStreamTask->hTaskInfo.haltVer; @@ -1105,7 +1105,7 @@ static void doStartStep2(SStreamTask* pTask, SStreamTask* pStreamTask, STQ* pTq) } } -// this function should be executed by only one thread +// this function should be executed by only one thread, so we set an sentinel to protect this function int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) { SStreamScanHistoryReq* pReq = (SStreamScanHistoryReq*)pMsg->pCont; SStreamMeta* pMeta = pTq->pStreamMeta; @@ -1134,6 +1134,7 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) { } } + // let's decide which step should be executed now if (pTask->execInfo.step1Start == 0) { ASSERT(pTask->status.pauseAllowed == false); int64_t ts = taosGetTimestampMs(); @@ -1167,14 +1168,28 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) { return 0; } - streamScanHistoryData(pTask); + EScanHistoryRet ret = streamScanHistoryData(pTask); + // todo update the step1 exec elapsed time double el = (taosGetTimestampMs() - pTask->execInfo.step1Start) / 1000.0; - if (streamTaskGetStatus(pTask, NULL) == TASK_STATUS__PAUSE) { - int8_t status = streamTaskSetSchedStatusInactive(pTask); - tqDebug("s-task:%s is paused in the step1, elapsed time:%.2fs, sched-status:%d", pTask->id.idStr, el, status); + if (ret == TASK_SCANHISTORY_QUIT || ret == TASK_SCANHISTORY_REXEC) { + int8_t status = streamTaskSetSchedStatusInactive(pTask); atomic_store_32(&pTask->status.inScanHistorySentinel, 0); + + if (ret == TASK_SCANHISTORY_REXEC) { + streamStartScanHistoryAsync(pTask, 0); + } else { + char* p = NULL; + ETaskStatus s = streamTaskGetStatus(pTask, &p); + + if (s == TASK_STATUS__PAUSE) { + tqDebug("s-task:%s is paused in the step1, elapsed time:%.2fs, sched-status:%d", pTask->id.idStr, el, status); + } else if (s == TASK_STATUS__STOP || s == TASK_STATUS__DROPPING) { + tqDebug("s-task:%s status:%p not continue scan-history data", pTask->id.idStr, p); + } + } + streamMetaReleaseTask(pMeta, pTask); return 0; } @@ -1203,23 +1218,20 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) { code = streamTaskHandleEvent(pStreamTask->status.pSM, TASK_EVENT_HALT); if (code == TSDB_CODE_SUCCESS) { - doStartStep2(pTask, pStreamTask, pTq); + doStartFillhistoryStep2(pTask, pStreamTask, pTq); } else { tqError("s-task:%s failed to halt s-task:%s, not launch step2", id, pStreamTask->id.idStr); } streamMetaReleaseTask(pMeta, pStreamTask); - } else { STimeWindow* pWindow = &pTask->dataRange.window; ASSERT(HAS_RELATED_FILLHISTORY_TASK(pTask)); - // Not update the fill-history time window until the state transfer is completed if the related fill-history task - // exists. - tqDebug( - "s-task:%s scan-history in stream time window completed, now start to handle data from WAL, startVer:%" PRId64 - ", window:%" PRId64 " - %" PRId64, - id, pTask->chkInfo.nextProcessVer, pWindow->skey, pWindow->ekey); + // Not update the fill-history time window until the state transfer is completed. + tqDebug("s-task:%s scan-history in stream time window completed, start to handle data from WAL, startVer:%" PRId64 + ", window:%" PRId64 " - %" PRId64, + id, pTask->chkInfo.nextProcessVer, pWindow->skey, pWindow->ekey); code = streamTaskScanHistoryDataComplete(pTask); } diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 4452eff8c7..9ed7d6e033 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -1046,7 +1046,7 @@ int32_t qRestoreStreamOperatorOption(qTaskInfo_t tinfo) { } } -bool qStreamRecoverScanFinished(qTaskInfo_t tinfo) { +bool qStreamScanhistoryFinished(qTaskInfo_t tinfo) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; return pTaskInfo->streamInfo.recoverScanFinished; } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 76945f17a9..7dfe88e6de 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -1104,7 +1104,6 @@ void streamMetaRLock(SStreamMeta* pMeta) { void streamMetaRUnLock(SStreamMeta* pMeta) { stTrace("vgId:%d meta-runlock", pMeta->vgId); taosRUnLockLatch(&pMeta->lock); - } void streamMetaWLock(SStreamMeta* pMeta) { stTrace("vgId:%d meta-wlock", pMeta->vgId); diff --git a/source/libs/stream/src/streamQueue.c b/source/libs/stream/src/streamQueue.c index eae4605dbc..90e862005b 100644 --- a/source/libs/stream/src/streamQueue.c +++ b/source/libs/stream/src/streamQueue.c @@ -340,10 +340,11 @@ int32_t streamTaskPutDataIntoInputQ(SStreamTask* pTask, SStreamQueueItem* pItem) return 0; } -// the result should be put into the outputQ in any cases, otherwise, the result may be lost +// the result should be put into the outputQ in any cases, the result may be lost otherwise. int32_t streamTaskPutDataIntoOutputQ(SStreamTask* pTask, SStreamDataBlock* pBlock) { STaosQueue* pQueue = pTask->outputq.queue->pQueue; + // wait for the output queue is available for new data to dispatch while (streamQueueIsFull(pTask->outputq.queue)) { if (streamTaskShouldStop(pTask)) { stInfo("s-task:%s discard result block due to task stop", pTask->id.idStr); diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index f949d46315..a1245ba32c 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -400,8 +400,7 @@ void tFreeStreamTask(SStreamTask* pTask) { taosMemoryFree(pTask->outputInfo.pTokenBucket); taosThreadMutexDestroy(&pTask->lock); - taosArrayDestroy(pTask->outputInfo.pDownstreamUpdateList); - pTask->outputInfo.pDownstreamUpdateList = NULL; + pTask->outputInfo.pDownstreamUpdateList = taosArrayDestroy(pTask->outputInfo.pDownstreamUpdateList); taosMemoryFree(pTask); stDebug("s-task:0x%x free task completed", taskId); diff --git a/source/libs/stream/src/streamTaskSm.c b/source/libs/stream/src/streamTaskSm.c index 09656cbe97..04b449aaaf 100644 --- a/source/libs/stream/src/streamTaskSm.c +++ b/source/libs/stream/src/streamTaskSm.c @@ -315,7 +315,6 @@ int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent even GET_EVT_NAME(event), pSM->current.name, GET_EVT_NAME(pSM->prev.evt)); taosThreadMutexUnlock(&pTask->lock); - stDebug("s-task:%s unlockx", pTask->id.idStr); return TSDB_CODE_STREAM_INVALID_STATETRANS; }