diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index b2ea08d50d..f58bbe0d68 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -129,7 +129,7 @@ typedef struct { static STqMgmt tqMgmt = {0}; // tqRead -int64_t tqScanLog(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, STqOffsetVal* offset); +int64_t tqScan(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, STqOffsetVal* offset); int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHead** pHeadWithCkSum); // tqExec diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 0c72d35027..2aef9e7175 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -112,7 +112,8 @@ int32_t tqSendMetaPollRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, }; tmsgSendRsp(&resp); - tqDebug("vgId:%d from consumer:%" PRId64 ", (epoch %d) send rsp, res msg type %d, reqOffset:%" PRId64 ", rspOffset:%" PRId64, + tqDebug("vgId:%d from consumer:%" PRId64 ", (epoch %d) send rsp, res msg type %d, reqOffset:%" PRId64 + ", rspOffset:%" PRId64, TD_VID(pTq->pVnode), pReq->consumerId, pReq->epoch, pRsp->resMsgType, pRsp->reqOffset, pRsp->rspOffset); return 0; @@ -179,8 +180,8 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, char* msg, int32_t msgLen) { tDecoderClear(&decoder); if (offset.val.type == TMQ_OFFSET__SNAPSHOT_DATA) { - tqDebug("receive offset commit msg to %s on vgId:%d, offset(type:snapshot) uid:%" PRId64 ", ts:%" PRId64, offset.subKey, - TD_VID(pTq->pVnode), offset.val.uid, offset.val.ts); + tqDebug("receive offset commit msg to %s on vgId:%d, offset(type:snapshot) uid:%" PRId64 ", ts:%" PRId64, + offset.subKey, TD_VID(pTq->pVnode), offset.val.uid, offset.val.ts); } else if (offset.val.type == TMQ_OFFSET__LOG) { tqDebug("receive offset commit msg to %s on vgId:%d, offset(type:log) version:%" PRId64, offset.subKey, TD_VID(pTq->pVnode), offset.val.version); @@ -316,9 +317,9 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { } // 3.query - if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN && fetchOffsetNew.type == TMQ_OFFSET__LOG) { + if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { fetchOffsetNew.version++; - if (tqScanLog(pTq, &pHandle->execHandle, &dataRsp, &fetchOffsetNew) < 0) { + if (tqScan(pTq, &pHandle->execHandle, &dataRsp, &fetchOffsetNew) < 0) { ASSERT(0); code = -1; goto OVER; @@ -333,7 +334,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { goto OVER; } - if (pHandle->execHandle.subType != TOPIC_SUB_TYPE__COLUMN && fetchOffsetNew.type == TMQ_OFFSET__LOG) { + if (pHandle->execHandle.subType != TOPIC_SUB_TYPE__COLUMN) { int64_t fetchVer = fetchOffsetNew.version + 1; SWalCkHead* pCkHead = taosMemoryMalloc(sizeof(SWalCkHead) + 2048); if (pCkHead == NULL) { @@ -411,6 +412,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { } taosMemoryFree(pCkHead); +#if 0 } else if (fetchOffsetNew.type == TMQ_OFFSET__SNAPSHOT_DATA) { tqInfo("retrieve using snapshot actual offset: uid %" PRId64 " ts %" PRId64, fetchOffsetNew.uid, fetchOffsetNew.ts); if (tqScanSnapshot(pTq, &pHandle->execHandle, &dataRsp, fetchOffsetNew, workerId) < 0) { @@ -421,6 +423,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) { code = -1; } +#endif } else if (fetchOffsetNew.type == TMQ_OFFSET__SNAPSHOT_META) { ASSERT(0); } diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index 4ecd88b1fd..b2fd5a5ddc 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -59,15 +59,17 @@ static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, SMqDataRsp* pRsp) { return 0; } -int64_t tqScanLog(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, STqOffsetVal* pOffset) { +int64_t tqScan(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, STqOffsetVal* pOffset) { qTaskInfo_t task = pExec->execCol.task[0]; if (qStreamPrepareScan(task, pOffset) < 0) { + ASSERT(pOffset->type == TMQ_OFFSET__LOG); pRsp->rspOffset = *pOffset; pRsp->rspOffset.version--; return 0; } + int32_t rowCnt = 0; while (1) { SSDataBlock* pDataBlock = NULL; uint64_t ts = 0; @@ -77,11 +79,19 @@ int64_t tqScanLog(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, STqOff if (pDataBlock != NULL) { tqAddBlockDataToRsp(pDataBlock, pRsp); - if (pRsp->withTbName) { - int64_t uid = pExec->pExecReader[0]->msgIter.uid; - tqAddTbNameToRsp(pTq, uid, pRsp); - } pRsp->blockNum++; + if (pRsp->withTbName) { + if (pOffset->type == TMQ_OFFSET__LOG) { + int64_t uid = pExec->pExecReader[0]->msgIter.uid; + tqAddTbNameToRsp(pTq, uid, pRsp); + } else { + pRsp->withTbName = 0; + } + } + if (pOffset->type == TMQ_OFFSET__LOG) { + rowCnt += pDataBlock->info.rows; + if (rowCnt >= 4096) break; + } continue; } @@ -94,12 +104,12 @@ int64_t tqScanLog(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, STqOff ASSERT(0); } + ASSERT(pRsp->rspOffset.type != 0); + if (pRsp->rspOffset.type == TMQ_OFFSET__LOG) { ASSERT(pRsp->rspOffset.version + 1 >= pRsp->reqOffset.version); } - ASSERT(pRsp->rspOffset.type != 0); - break; } diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 82387119b6..21eb7680d5 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -144,6 +144,7 @@ typedef struct { void* metaBlk; // for tmq fetching meta SSDataBlock* pullOverBlk; // for streaming SWalFilterCond cond; + int64_t lastScanUid; } SStreamTaskInfo; typedef struct SExecTaskInfo { diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index ca1d8b2b10..42ad173c74 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -336,8 +336,6 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, const STqOffsetVal* pOffset) { qDebug("tsdb reader offset seek to uid %ld ts %ld, table cur set to %d , all table num %d", uid, ts, pTableScanInfo->currentTable, tableSz); - } else { - // switch to log } } else { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index d2f63849d0..57e74415a7 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1273,6 +1273,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { /*pTaskInfo->streamInfo.lastStatus.version = pTaskInfo->streamInfo.prepareStatus.version - 1;*/ /*} else {*/ pTaskInfo->streamInfo.lastStatus = ret.offset; + /*pTaskInfo->streamInfo.lastScanUid */ ASSERT(pTaskInfo->streamInfo.lastStatus.version + 1 >= pTaskInfo->streamInfo.prepareStatus.version); /*}*/ return NULL; @@ -1490,7 +1491,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { return (pBlockInfo->rows == 0) ? NULL : pInfo->pRes; } else if (pInfo->blockType == STREAM_INPUT__TABLE_SCAN) { - /*ASSERT(0);*/ + ASSERT(0); // check reader last status // if not match, reset status SSDataBlock* pResult = doTableScan(pInfo->pTableScanOp);