xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

tq handle multiple topic

This commit is contained in:
Liu Jicong 2022-01-20 14:53:09 +08:00
parent f7726ce8a4
commit 51f02913f4
4 changed files with 95 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/common/tmsg.h
View File

@ -1561,7 +1561,7 @@ typedef struct SMqSetCVgRsp {
typedef struct SMqCVConsumeReq { typedef struct SMqCVConsumeReq {
int64_t reqId; int64_t reqId;
int64_t offset; int64_t offset;
int64_t clientId; int64_t consumerId;
int64_t blockingTime; int64_t blockingTime;
char topicName[TSDB_TOPIC_FNAME_LEN]; char topicName[TSDB_TOPIC_FNAME_LEN];
char cgroup[TSDB_CONSUMER_GROUP_LEN]; char cgroup[TSDB_CONSUMER_GROUP_LEN];

2
source/dnode/mnode/impl/src/mndSubscribe.c
View File

@ -55,8 +55,6 @@ int32_t mndInitSubscribe(SMnode *pMnode) {
.deleteFp = (SdbDeleteFp)mndSubActionDelete}; .deleteFp = (SdbDeleteFp)mndSubActionDelete};
mndSetMsgHandle(pMnode, TDMT_MND_SUBSCRIBE, mndProcessSubscribeReq); mndSetMsgHandle(pMnode, TDMT_MND_SUBSCRIBE, mndProcessSubscribeReq);
/*mndSetMsgHandle(pMnode, TDMT_MND_SUBSCRIBE_RSP, mndProcessSubscribeRsp);*/
/*mndSetMsgHandle(pMnode, TDMT_VND_SUBSCRIBE, mndProcessSubscribeInternalReq);*/
mndSetMsgHandle(pMnode, TDMT_VND_SUBSCRIBE_RSP, mndProcessSubscribeInternalRsp); mndSetMsgHandle(pMnode, TDMT_VND_SUBSCRIBE_RSP, mndProcessSubscribeInternalRsp);
mndSetMsgHandle(pMnode, TDMT_MND_MQ_TIMER, mndProcessMqTimerMsg); mndSetMsgHandle(pMnode, TDMT_MND_MQ_TIMER, mndProcessMqTimerMsg);
return sdbSetTable(pMnode->pSdb, table); return sdbSetTable(pMnode->pSdb, table);

10
source/dnode/vnode/inc/tq.h
View File

@ -175,8 +175,7 @@ typedef struct STqBuffer {
STqTaskItem output[TQ_BUFFER_SIZE]; STqTaskItem output[TQ_BUFFER_SIZE];
} STqBuffer; } STqBuffer;
typedef struct STqClientHandle { typedef struct STqTopicHandle {
int64_t clientId;
char topicName[TSDB_TOPIC_FNAME_LEN]; char topicName[TSDB_TOPIC_FNAME_LEN];
char cGroup[TSDB_TOPIC_FNAME_LEN]; char cGroup[TSDB_TOPIC_FNAME_LEN];
char* sql; char* sql;
@ -186,7 +185,12 @@ typedef struct STqClientHandle {
int64_t currentOffset; int64_t currentOffset;
STqBuffer buffer; STqBuffer buffer;
SWalReadHandle* pReadhandle; SWalReadHandle* pReadhandle;
} STqClientHandle; } STqTopicHandle;
typedef struct STqConsumerHandle {
int64_t consumerId;
SArray* topics; // SArray<STqClientTopic>
} STqConsumerHandle;
typedef struct STqQueryMsg { typedef struct STqQueryMsg {
STqMsgItem* item; STqMsgItem* item;

19
source/dnode/vnode/src/tq/tq.c
View File

@ -343,7 +343,6 @@ int tqConsume(STQ* pTq, SRpcMsg* pReq, SRpcMsg** pRsp) {
int numOfMsgs = 0; int numOfMsgs = 0;
int sizeLimit = 4096; int sizeLimit = 4096;
STqConsumeRsp* pCsmRsp = (*pRsp)->pCont; STqConsumeRsp* pCsmRsp = (*pRsp)->pCont;
void* ptr = realloc((*pRsp)->pCont, sizeof(STqConsumeRsp) + sizeLimit); void* ptr = realloc((*pRsp)->pCont, sizeof(STqConsumeRsp) + sizeLimit);
if (ptr == NULL) { if (ptr == NULL) {
@ -611,11 +610,16 @@ int tqItemSSize() {
int32_t tqProcessConsume(STQ* pTq, SRpcMsg* pMsg, SRpcMsg** ppRsp) { int32_t tqProcessConsume(STQ* pTq, SRpcMsg* pMsg, SRpcMsg** ppRsp) {
SMqCVConsumeReq* pReq = pMsg->pCont; SMqCVConsumeReq* pReq = pMsg->pCont;
int64_t reqId = pReq->reqId; int64_t reqId = pReq->reqId;
int64_t clientId = pReq->clientId; int64_t consumerId = pReq->consumerId;
int64_t offset = pReq->offset; int64_t offset = pReq->offset;
int64_t blockingTime = pReq->blockingTime; int64_t blockingTime = pReq->blockingTime;
STqClientHandle* pHandle = tqHandleGet(pTq->tqMeta, clientId); STqConsumerHandle* pConsumer = tqHandleGet(pTq->tqMeta, consumerId);
int sz = taosArrayGetSize(pConsumer->topics);
for (int i = 0 ; i < sz; i++) {
STqTopicHandle *pHandle = taosArrayGet(pConsumer->topics, i);
int8_t pos = offset % TQ_BUFFER_SIZE; int8_t pos = offset % TQ_BUFFER_SIZE;
int8_t old = atomic_val_compare_exchange_8(&pHandle->buffer.output[pos].status, 0, 1); int8_t old = atomic_val_compare_exchange_8(&pHandle->buffer.output[pos].status, 0, 1);
if (old == 1) { if (old == 1) {
@ -633,11 +637,13 @@ int32_t tqProcessConsume(STQ* pTq, SRpcMsg* pMsg, SRpcMsg **ppRsp) {
SSubQueryMsg* pQueryMsg = pHandle->buffer.output[pos].pMsg; SSubQueryMsg* pQueryMsg = pHandle->buffer.output[pos].pMsg;
void* outputData; void* outputData;
atomic_store_8(&pHandle->buffer.output[pos].status, 1); atomic_store_8(&pHandle->buffer.output[pos].status, 1);
// put output into rsp
}
// launch query // launch query
// get result // get result
// put into
SMqCvConsumeRsp* pRsp; SMqCvConsumeRsp* pRsp;
return 0; return 0;
} }
@ -705,6 +711,7 @@ SArray *tqRetrieveDataBlock(STqReadHandle* pHandle, SArray* pColumnIdList) {
taosArrayPush(pArray, &colInfo); taosArrayPush(pArray, &colInfo);
return pArray; return pArray;
} }
/*int tqLoadDataBlock(SExecTaskInfo* pTaskInfo, SSubmitBlkScanInfo* pSubmitBlkScanInfo, SSDataBlock* pBlock, uint32_t status) {*/ /*int tqLoadDataBlock(SExecTaskInfo* pTaskInfo, SSubmitBlkScanInfo* pSubmitBlkScanInfo, SSDataBlock* pBlock, uint32_t
* status) {*/
/*return 0;*/ /*return 0;*/
/*}*/ /*}*/