xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

enh(tmq): support seek in tmq.

This commit is contained in:
Haojun Liao 2023-04-19 13:56:42 +08:00
parent 9627e8105b
commit 3f12156c53
2 changed files with 90 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

183
source/client/src/clientTmq.c
View File

@ -560,6 +560,21 @@ static int32_t doSendCommitMsg(tmq_t* tmq, SMqClientVg* pVg, const char* pTopicN
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static SMqClientTopic* getTopicByName(tmq_t* tmq, const char* pTopicName) {
int32_t numOfTopics = taosArrayGetSize(tmq->clientTopics);
for (int32_t i = 0; i < numOfTopics; ++i) {
SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
if (strcmp(pTopic->topicName, pTopicName) != 0) {
continue;
}
return pTopic;
}
tscError("consumer:0x%" PRIx64 ", failed to find topic:%s", tmq->consumerId, pTopicName);
return NULL;
}
static void asyncCommitOffset(tmq_t* tmq, const TAOS_RES* pRes, tmq_commit_cb* pCommitFp, void* userParam) { static void asyncCommitOffset(tmq_t* tmq, const TAOS_RES* pRes, tmq_commit_cb* pCommitFp, void* userParam) {
char* pTopicName = NULL; char* pTopicName = NULL;
int32_t vgId = 0; int32_t vgId = 0;
@ -602,15 +617,8 @@ static void asyncCommitOffset(tmq_t* tmq, const TAOS_RES* pRes, tmq_commit_cb* p
tscDebug("consumer:0x%" PRIx64 " do manual commit offset for %s, vgId:%d", tmq->consumerId, pTopicName, vgId); tscDebug("consumer:0x%" PRIx64 " do manual commit offset for %s, vgId:%d", tmq->consumerId, pTopicName, vgId);
int32_t i = 0; SMqClientTopic* pTopic = getTopicByName(tmq, pTopicName);
for (; i < numOfTopics; i++) { if (pTopic == NULL) {
SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
if (strcmp(pTopic->topicName, pTopicName) == 0) {
break;
}
}
if (i == numOfTopics) {
tscWarn("consumer:0x%" PRIx64 " failed to find the specified topic:%s, total topics:%d", tmq->consumerId, tscWarn("consumer:0x%" PRIx64 " failed to find the specified topic:%s, total topics:%d", tmq->consumerId,
pTopicName, numOfTopics); pTopicName, numOfTopics);
taosMemoryFree(pParamSet); taosMemoryFree(pParamSet);
@ -618,8 +626,6 @@ static void asyncCommitOffset(tmq_t* tmq, const TAOS_RES* pRes, tmq_commit_cb* p
return; return;
} }
SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
int32_t j = 0; int32_t j = 0;
int32_t numOfVgroups = taosArrayGetSize(pTopic->vgs); int32_t numOfVgroups = taosArrayGetSize(pTopic->vgs);
for (j = 0; j < numOfVgroups; j++) { for (j = 0; j < numOfVgroups; j++) {
@ -2334,20 +2340,7 @@ SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4) {
return NULL; return NULL;
} }
static SMqClientTopic* getTopicByName(tmq_t* tmq, const char* pTopicName) {
int32_t numOfTopics = taosArrayGetSize(tmq->clientTopics);
for (int32_t i = 0; i < numOfTopics; ++i) {
SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
if (strcmp(pTopic->topicName, pTopicName) != 0) {
continue;
}
return pTopic;
}
tscError("consumer:0x%" PRIx64 ", failed to find topic:%s", tmq->consumerId, pTopicName);
return NULL;
}
int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_assignment** assignment, int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_assignment** assignment,
int32_t* numOfAssignment) { int32_t* numOfAssignment) {
@ -2408,74 +2401,78 @@ int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgroupHandle
return TSDB_CODE_INVALID_PARA; return TSDB_CODE_INVALID_PARA;
} }
return 0; // return 0;
#if 0 //#if 0
// tmq_commit_sync(tmq, ); SMqRspObj rspObj = {.resType = RES_TYPE__TMQ, .vgId = pVg->vgId};
{ tstrncpy(rspObj.topic, pTopicName, tListLen(rspObj.topic));
SMqCommitCbParamSet* pParamSet = taosMemoryCalloc(1, sizeof(SMqCommitCbParamSet)); tmq_commit_sync(tmq, &rspObj);
if (pParamSet == NULL) { // {
// pCommitFp(tmq, TSDB_CODE_OUT_OF_MEMORY, userParam); // SMqCommitCbParamSet* pParamSet = taosMemoryCalloc(1, sizeof(SMqCommitCbParamSet));
return -1; // if (pParamSet == NULL) {
} //// pCommitFp(tmq, TSDB_CODE_OUT_OF_MEMORY, userParam);
// return -1;
pParamSet->refId = tmq->refId; // }
pParamSet->epoch = tmq->epoch; //
pParamSet->callbackFn = pCommitFp; // pParamSet->refId = tmq->refId;
pParamSet->userParam = userParam; // pParamSet->epoch = tmq->epoch;
// pParamSet->callbackFn = pCommitFp;
int32_t numOfTopics = taosArrayGetSize(tmq->clientTopics); // pParamSet->userParam = userParam;
//
tscDebug("consumer:0x%" PRIx64 " do manual commit offset for %s, vgId:%d", tmq->consumerId, pTopicName, vgId); // int32_t numOfTopics = taosArrayGetSize(tmq->clientTopics);
//
int32_t i = 0; // tscDebug("consumer:0x%" PRIx64 " do manual commit offset for %s, vgId:%d", tmq->consumerId, pTopicName, vgId);
for (; i < numOfTopics; i++) { //
SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); // int32_t i = 0;
if (strcmp(pTopic->topicName, pTopicName) == 0) { // for (; i < numOfTopics; i++) {
break; // SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
} // if (strcmp(pTopic->topicName, pTopicName) == 0) {
} // break;
// }
if (i == numOfTopics) { // }
tscWarn("consumer:0x%" PRIx64 " failed to find the specified topic:%s, total topics:%d", tmq->consumerId, //
pTopicName, numOfTopics); // if (i == numOfTopics) {
taosMemoryFree(pParamSet); // tscWarn("consumer:0x%" PRIx64 " failed to find the specified topic:%s, total topics:%d", tmq->consumerId,
pCommitFp(tmq, TSDB_CODE_SUCCESS, userParam); // pTopicName, numOfTopics);
return; // taosMemoryFree(pParamSet);
} // pCommitFp(tmq, TSDB_CODE_SUCCESS, userParam);
// return;
SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); // }
//
int32_t j = 0; // SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i);
int32_t numOfVgroups = taosArrayGetSize(pTopic->vgs); //
for (j = 0; j < numOfVgroups; j++) { // int32_t j = 0;
SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); // int32_t numOfVgroups = taosArrayGetSize(pTopic->vgs);
if (pVg->vgId == vgId) { // for (j = 0; j < numOfVgroups; j++) {
break; // SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j);
} // if (pVg->vgId == vgId) {
} // break;
// }
if (j == numOfVgroups) { // }
tscWarn("consumer:0x%" PRIx64 " failed to find the specified vgId:%d, total Vgs:%d, topic:%s", tmq->consumerId, //
vgId, numOfVgroups, pTopicName); // if (j == numOfVgroups) {
taosMemoryFree(pParamSet); // tscWarn("consumer:0x%" PRIx64 " failed to find the specified vgId:%d, total Vgs:%d, topic:%s",
pCommitFp(tmq, TSDB_CODE_SUCCESS, userParam); // tmq->consumerId,
return; // vgId, numOfVgroups, pTopicName);
} // taosMemoryFree(pParamSet);
// pCommitFp(tmq, TSDB_CODE_SUCCESS, userParam);
SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); // return;
if (pVg->offsetInfo.currentOffset.type > 0 && !tOffsetEqual(&pVg->offsetInfo.currentOffset, &pVg->offsetInfo.committedOffset)) { // }
code = doSendCommitMsg(tmq, pVg, pTopic->topicName, pParamSet, j, numOfVgroups); //
// SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j);
// failed to commit, callback user function directly. // if (pVg->offsetInfo.currentOffset.type > 0 && !tOffsetEqual(&pVg->offsetInfo.currentOffset,
if (code != TSDB_CODE_SUCCESS) { // &pVg->offsetInfo.committedOffset)) {
taosMemoryFree(pParamSet); // code = doSendCommitMsg(tmq, pVg, pTopic->topicName, pParamSet, j, numOfVgroups);
pCommitFp(tmq, code, userParam); //
} // // failed to commit, callback user function directly.
} else { // do not perform commit, callback user function directly. // if (code != TSDB_CODE_SUCCESS) {
taosMemoryFree(pParamSet); // taosMemoryFree(pParamSet);
pCommitFp(tmq, code, userParam); // pCommitFp(tmq, code, userParam);
} // }
} // } else { // do not perform commit, callback user function directly.
#endif // taosMemoryFree(pParamSet);
// pCommitFp(tmq, code, userParam);
// }
// }
//#endif
} }

21
source/dnode/vnode/src/tq/tqRestore.c
View File

@ -55,27 +55,6 @@ int tqStreamTasksScanWal(STQ* pTq) {
return 0; return 0;
} }
//int32_t transferToNormalTask(SStreamMeta* pStreamMeta, SArray* pTaskList) {
// int32_t numOfTask = taosArrayGetSize(pTaskList);
// if (numOfTask <= 0) {
// return TSDB_CODE_SUCCESS;
// }
//
// // todo: add lock
// for (int32_t i = 0; i < numOfTask; ++i) {
// SStreamTask* pTask = taosArrayGetP(pTaskList, i);
// tqDebug("vgId:%d transfer s-task:%s state restore -> ready, checkpoint:%" PRId64 " checkpoint id:%" PRId64,
// pStreamMeta->vgId, pTask->id.idStr, pTask->chkInfo.version, pTask->chkInfo.id);
// taosHashRemove(pStreamMeta->pWalReadTasks, &pTask->id.taskId, sizeof(pTask->id.taskId));
//
// // NOTE: do not change the following order
// atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__NORMAL);
// taosHashPut(pStreamMeta->pTasks, &pTask->id.taskId, sizeof(pTask->id.taskId), &pTask, POINTER_BYTES);
// }
//
// return TSDB_CODE_SUCCESS;
//}
int32_t streamTaskReplayWal(SStreamMeta* pStreamMeta, STqOffsetStore* pOffsetStore, bool* pScanIdle) { int32_t streamTaskReplayWal(SStreamMeta* pStreamMeta, STqOffsetStore* pOffsetStore, bool* pScanIdle) {
void* pIter = NULL; void* pIter = NULL;
int32_t vgId = pStreamMeta->vgId; int32_t vgId = pStreamMeta->vgId;