xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix:[TS-4592]remove lost status

This commit is contained in:
wangmm0220 2024-07-11 14:56:25 +08:00
parent b28437aaba
commit 4ef2c39649
7 changed files with 93 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/common/tmsg.h
View File

@ -2773,7 +2773,7 @@ enum {
}; };
#define DEFAULT_MAX_POLL_INTERVAL 3000000 #define DEFAULT_MAX_POLL_INTERVAL 3000000
#define DEFAULT_SESSION_TIMEOUT 10000 #define DEFAULT_SESSION_TIMEOUT 12000
typedef struct { typedef struct {
char name[TSDB_TOPIC_FNAME_LEN]; // accout.topic char name[TSDB_TOPIC_FNAME_LEN]; // accout.topic

2
source/client/src/clientEnv.c
View File

@ -127,7 +127,7 @@ static void generateWriteSlowLog(STscObj *pTscObj, SRequestObj *pRequest, int32_
cJSON_AddItemToObject(json, "error_info", cJSON_CreateString(tstrerror(pRequest->code))); cJSON_AddItemToObject(json, "error_info", cJSON_CreateString(tstrerror(pRequest->code)));
cJSON_AddItemToObject(json, "type", cJSON_CreateNumber(reqType)); cJSON_AddItemToObject(json, "type", cJSON_CreateNumber(reqType));
cJSON_AddItemToObject(json, "rows_num", cJSON_CreateNumber(pRequest->body.resInfo.numOfRows + pRequest->body.resInfo.totalRows)); cJSON_AddItemToObject(json, "rows_num", cJSON_CreateNumber(pRequest->body.resInfo.numOfRows + pRequest->body.resInfo.totalRows));
if(strlen(pRequest->sqlstr) > pTscObj->pAppInfo->monitorParas.tsSlowLogMaxLen){ if(pRequest->sqlstr != NULL && strlen(pRequest->sqlstr) > pTscObj->pAppInfo->monitorParas.tsSlowLogMaxLen){
char tmp = pRequest->sqlstr[pTscObj->pAppInfo->monitorParas.tsSlowLogMaxLen]; char tmp = pRequest->sqlstr[pTscObj->pAppInfo->monitorParas.tsSlowLogMaxLen];
pRequest->sqlstr[pTscObj->pAppInfo->monitorParas.tsSlowLogMaxLen] = '\0'; pRequest->sqlstr[pTscObj->pAppInfo->monitorParas.tsSlowLogMaxLen] = '\0';
cJSON_AddItemToObject(json, "sql", cJSON_CreateString(pRequest->sqlstr)); cJSON_AddItemToObject(json, "sql", cJSON_CreateString(pRequest->sqlstr));

2
source/dnode/mnode/impl/inc/mndConsumer.h
View File

@ -25,7 +25,7 @@ extern "C" {
enum { enum {
MQ_CONSUMER_STATUS_REBALANCE = 1, MQ_CONSUMER_STATUS_REBALANCE = 1,
MQ_CONSUMER_STATUS_READY, MQ_CONSUMER_STATUS_READY,
MQ_CONSUMER_STATUS_LOST, // MQ_CONSUMER_STATUS_LOST,
}; };
int32_t mndInitConsumer(SMnode *pMnode); int32_t mndInitConsumer(SMnode *pMnode);

82
source/dnode/mnode/impl/src/mndSubscribe.c
View File

@ -749,11 +749,13 @@ static void mndCheckConsumer(SRpcMsg *pMsg, SHashObj *rebSubHash) {
} else { } else {
checkForVgroupSplit(pMnode, pConsumer, rebSubHash); checkForVgroupSplit(pMnode, pConsumer, rebSubHash);
} }
} else { } else if (status == MQ_CONSUMER_STATUS_REBALANCE) {
taosRLockLatch(&pConsumer->lock); taosRLockLatch(&pConsumer->lock);
buildRebInfo(rebSubHash, pConsumer->rebNewTopics, 1, pConsumer->cgroup, pConsumer->consumerId); buildRebInfo(rebSubHash, pConsumer->rebNewTopics, 1, pConsumer->cgroup, pConsumer->consumerId);
buildRebInfo(rebSubHash, pConsumer->rebRemovedTopics, 0, pConsumer->cgroup, pConsumer->consumerId); buildRebInfo(rebSubHash, pConsumer->rebRemovedTopics, 0, pConsumer->cgroup, pConsumer->consumerId);
taosRUnLockLatch(&pConsumer->lock); taosRUnLockLatch(&pConsumer->lock);
} else {
mndSendConsumerMsg(pMnode, pConsumer->consumerId, TDMT_MND_TMQ_LOST_CONSUMER_CLEAR, &pMsg->info);
} }
mndReleaseConsumer(pMnode, pConsumer); mndReleaseConsumer(pMnode, pConsumer);
@ -974,41 +976,41 @@ END:
return ret; return ret;
} }
static int32_t mndDropConsumerByGroup(SMnode *pMnode, STrans *pTrans, char *cgroup, char *topic) { //static int32_t mndDropConsumerByGroup(SMnode *pMnode, STrans *pTrans, char *cgroup, char *topic) {
void *pIter = NULL; // void *pIter = NULL;
SMqConsumerObj *pConsumer = NULL; // SMqConsumerObj *pConsumer = NULL;
int ret = 0; // int ret = 0;
while (1) { // while (1) {
pIter = sdbFetch(pMnode->pSdb, SDB_CONSUMER, pIter, (void **)&pConsumer); // pIter = sdbFetch(pMnode->pSdb, SDB_CONSUMER, pIter, (void **)&pConsumer);
if (pIter == NULL) { // if (pIter == NULL) {
break; // break;
} // }
//
// drop consumer in lost status, other consumers not in lost status already deleted by rebalance // // drop consumer in lost status, other consumers not in lost status already deleted by rebalance
if (pConsumer->status != MQ_CONSUMER_STATUS_LOST || strcmp(cgroup, pConsumer->cgroup) != 0) { // if (pConsumer->status != MQ_CONSUMER_STATUS_LOST || strcmp(cgroup, pConsumer->cgroup) != 0) {
sdbRelease(pMnode->pSdb, pConsumer); // sdbRelease(pMnode->pSdb, pConsumer);
continue; // continue;
} // }
int32_t sz = taosArrayGetSize(pConsumer->assignedTopics); // int32_t sz = taosArrayGetSize(pConsumer->assignedTopics);
for (int32_t i = 0; i < sz; i++) { // for (int32_t i = 0; i < sz; i++) {
char *name = taosArrayGetP(pConsumer->assignedTopics, i); // char *name = taosArrayGetP(pConsumer->assignedTopics, i);
if (strcmp(topic, name) == 0) { // if (strcmp(topic, name) == 0) {
int32_t code = mndSetConsumerDropLogs(pTrans, pConsumer); // int32_t code = mndSetConsumerDropLogs(pTrans, pConsumer);
if (code != 0) { // if (code != 0) {
ret = code; // ret = code;
goto END; // goto END;
} // }
} // }
} // }
//
sdbRelease(pMnode->pSdb, pConsumer); // sdbRelease(pMnode->pSdb, pConsumer);
} // }
//
END: //END:
sdbRelease(pMnode->pSdb, pConsumer); // sdbRelease(pMnode->pSdb, pConsumer);
sdbCancelFetch(pMnode->pSdb, pIter); // sdbCancelFetch(pMnode->pSdb, pIter);
return ret; // return ret;
} //}
static int32_t mndProcessDropCgroupReq(SRpcMsg *pMsg) { static int32_t mndProcessDropCgroupReq(SRpcMsg *pMsg) {
SMnode *pMnode = pMsg->info.node; SMnode *pMnode = pMsg->info.node;
@ -1055,10 +1057,10 @@ static int32_t mndProcessDropCgroupReq(SRpcMsg *pMsg) {
goto end; goto end;
} }
code = mndDropConsumerByGroup(pMnode, pTrans, dropReq.cgroup, dropReq.topic); // code = mndDropConsumerByGroup(pMnode, pTrans, dropReq.cgroup, dropReq.topic);
if (code != 0) { // if (code != 0) {
goto end; // goto end;
} // }
code = sendDeleteSubToVnode(pMnode, pSub, pTrans); code = sendDeleteSubToVnode(pMnode, pSub, pTrans);
if (code != 0) { if (code != 0) {

90
source/dnode/mnode/impl/src/mndTopic.c
View File

@ -668,47 +668,47 @@ static bool checkTopic(SArray *topics, char *topicName){
return false; return false;
} }
static int32_t mndDropConsumerByTopic(SMnode *pMnode, STrans *pTrans, char *topicName){ //static int32_t mndDropConsumerByTopic(SMnode *pMnode, STrans *pTrans, char *topicName){
int32_t code = 0; // int32_t code = 0;
SSdb *pSdb = pMnode->pSdb; // SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL; // void *pIter = NULL;
SMqConsumerObj *pConsumer = NULL; // SMqConsumerObj *pConsumer = NULL;
while (1) { // while (1) {
pIter = sdbFetch(pSdb, SDB_CONSUMER, pIter, (void **)&pConsumer); // pIter = sdbFetch(pSdb, SDB_CONSUMER, pIter, (void **)&pConsumer);
if (pIter == NULL) { // if (pIter == NULL) {
break; // break;
} // }
//
bool found = checkTopic(pConsumer->assignedTopics, topicName); // bool found = checkTopic(pConsumer->assignedTopics, topicName);
if (found){ // if (found){
if (pConsumer->status == MQ_CONSUMER_STATUS_LOST) { // if (pConsumer->status == MQ_CONSUMER_STATUS_LOST) {
code = mndSetConsumerDropLogs(pTrans, pConsumer); // code = mndSetConsumerDropLogs(pTrans, pConsumer);
if (code != 0) { // if (code != 0) {
goto end; // goto end;
} // }
sdbRelease(pSdb, pConsumer); // sdbRelease(pSdb, pConsumer);
continue; // continue;
} // }
mError("topic:%s, failed to drop since subscribed by consumer:0x%" PRIx64 ", in consumer group %s", // mError("topic:%s, failed to drop since subscribed by consumer:0x%" PRIx64 ", in consumer group %s",
topicName, pConsumer->consumerId, pConsumer->cgroup); // topicName, pConsumer->consumerId, pConsumer->cgroup);
code = TSDB_CODE_MND_TOPIC_SUBSCRIBED; // code = TSDB_CODE_MND_TOPIC_SUBSCRIBED;
goto end; // goto end;
} // }
//
if (checkTopic(pConsumer->rebNewTopics, topicName) || checkTopic(pConsumer->rebRemovedTopics, topicName)) { // if (checkTopic(pConsumer->rebNewTopics, topicName) || checkTopic(pConsumer->rebRemovedTopics, topicName)) {
code = TSDB_CODE_MND_TOPIC_SUBSCRIBED; // code = TSDB_CODE_MND_TOPIC_SUBSCRIBED;
mError("topic:%s, failed to drop since subscribed by consumer:%" PRId64 ", in consumer group %s (reb new)", // mError("topic:%s, failed to drop since subscribed by consumer:%" PRId64 ", in consumer group %s (reb new)",
topicName, pConsumer->consumerId, pConsumer->cgroup); // topicName, pConsumer->consumerId, pConsumer->cgroup);
goto end; // goto end;
} // }
sdbRelease(pSdb, pConsumer); // sdbRelease(pSdb, pConsumer);
} // }
//
end: //end:
sdbRelease(pSdb, pConsumer); // sdbRelease(pSdb, pConsumer);
sdbCancelFetch(pSdb, pIter); // sdbCancelFetch(pSdb, pIter);
return code; // return code;
} //}
static int32_t mndDropCheckInfoByTopic(SMnode *pMnode, STrans *pTrans, SMqTopicObj *pTopic){ static int32_t mndDropCheckInfoByTopic(SMnode *pMnode, STrans *pTrans, SMqTopicObj *pTopic){
// broadcast to all vnode // broadcast to all vnode
@ -804,10 +804,10 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) {
goto end; goto end;
} }
code = mndDropConsumerByTopic(pMnode, pTrans, dropReq.name); // code = mndDropConsumerByTopic(pMnode, pTrans, dropReq.name);
if (code != 0) { // if (code != 0) {
goto end; // goto end;
} // }
code = mndDropSubByTopic(pMnode, pTrans, dropReq.name); code = mndDropSubByTopic(pMnode, pTrans, dropReq.name);
if (code < 0) { if (code < 0) {

2
source/dnode/mnode/sdb/src/sdbFile.c
View File

@ -339,6 +339,8 @@ static int32_t sdbReadFileImp(SSdb *pSdb) {
code = sdbWriteWithoutFree(pSdb, pRaw); code = sdbWriteWithoutFree(pSdb, pRaw);
if (code != 0) { if (code != 0) {
mError("failed to read sdb file:%s since %s", file, terrstr()); mError("failed to read sdb file:%s since %s", file, terrstr());
code = sdbWriteWithoutFree(pSdb, pRaw);
goto _OVER; goto _OVER;
} }
} }

1
utils/test/c/tmq_taosx_ci.c
View File

@ -596,6 +596,7 @@ tmq_t* build_consumer() {
tmq_conf_set(conf, "enable.auto.commit", "true"); tmq_conf_set(conf, "enable.auto.commit", "true");
tmq_conf_set(conf, "auto.offset.reset", "earliest"); tmq_conf_set(conf, "auto.offset.reset", "earliest");
tmq_conf_set(conf, "msg.consume.excluded", "1"); tmq_conf_set(conf, "msg.consume.excluded", "1");
// tmq_conf_set(conf, "max.poll.interval.ms", "20000");
if (g_conf.snapShot) { if (g_conf.snapShot) {
tmq_conf_set(conf, "experimental.snapshot.enable", "true"); tmq_conf_set(conf, "experimental.snapshot.enable", "true");