diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index 1ea3eaf219..448243cc3d 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -21,8 +21,8 @@ #include "tglobal.h" #include "tmsgtype.h" -#define LOG_ID_TAG "connId:0x%"PRIx64",reqId:0x%"PRIx64 -#define LOG_ID_VALUE *(int64_t*)taos,pRequest->requestId +#define LOG_ID_TAG "connId:0x%" PRIx64 ",reqId:0x%" PRIx64 +#define LOG_ID_VALUE *(int64_t*)taos, pRequest->requestId static tb_uid_t processSuid(tb_uid_t suid, char* db) { return suid + MurmurHash3_32(db, strlen(db)); } @@ -31,8 +31,7 @@ static char* buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* sch char* string = NULL; cJSON* json = cJSON_CreateObject(); if (json == NULL) { - uError("create json object failed") - return NULL; + uError("create json object failed") return NULL; } cJSON* type = cJSON_CreateString("create"); cJSON_AddItemToObject(json, "type", type); @@ -56,7 +55,7 @@ static char* buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* sch cJSON_AddItemToObject(column, "name", cname); cJSON* ctype = cJSON_CreateNumber(s->type); cJSON_AddItemToObject(column, "type", ctype); - if (s->type == TSDB_DATA_TYPE_BINARY || s->type == TSDB_DATA_TYPE_VARBINARY|| s->type == TSDB_DATA_TYPE_GEOMETRY) { + if (s->type == TSDB_DATA_TYPE_BINARY || s->type == TSDB_DATA_TYPE_VARBINARY || s->type == TSDB_DATA_TYPE_GEOMETRY) { int32_t length = s->bytes - VARSTR_HEADER_SIZE; cJSON* cbytes = cJSON_CreateNumber(length); cJSON_AddItemToObject(column, "length", cbytes); @@ -131,7 +130,8 @@ static char* buildAlterSTableJson(void* alterData, int32_t alterDataLen) { cJSON* colType = cJSON_CreateNumber(field->type); cJSON_AddItemToObject(json, "colType", colType); - if (field->type == TSDB_DATA_TYPE_BINARY || field->type == TSDB_DATA_TYPE_VARBINARY || field->type == TSDB_DATA_TYPE_GEOMETRY) { + if (field->type == TSDB_DATA_TYPE_BINARY || field->type == TSDB_DATA_TYPE_VARBINARY || + field->type == TSDB_DATA_TYPE_GEOMETRY) { int32_t length = field->bytes - VARSTR_HEADER_SIZE; cJSON* cbytes = cJSON_CreateNumber(length); cJSON_AddItemToObject(json, "colLength", cbytes); @@ -156,7 +156,8 @@ static char* buildAlterSTableJson(void* alterData, int32_t alterDataLen) { cJSON_AddItemToObject(json, "colName", colName); cJSON* colType = cJSON_CreateNumber(field->type); cJSON_AddItemToObject(json, "colType", colType); - if (field->type == TSDB_DATA_TYPE_BINARY || field->type == TSDB_DATA_TYPE_VARBINARY || field->type == TSDB_DATA_TYPE_GEOMETRY) { + if (field->type == TSDB_DATA_TYPE_BINARY || field->type == TSDB_DATA_TYPE_VARBINARY || + field->type == TSDB_DATA_TYPE_GEOMETRY) { int32_t length = field->bytes - VARSTR_HEADER_SIZE; cJSON* cbytes = cJSON_CreateNumber(length); cJSON_AddItemToObject(json, "colLength", cbytes); @@ -182,7 +183,7 @@ static char* buildAlterSTableJson(void* alterData, int32_t alterDataLen) { } string = cJSON_PrintUnformatted(json); - end: +end: cJSON_Delete(json); tFreeSMAltertbReq(&req); return string; @@ -295,9 +296,9 @@ static void buildChildElement(cJSON* json, SVCreateTbReq* pCreateReq) { cJSON* tvalue = NULL; if (IS_VAR_DATA_TYPE(pTagVal->type)) { char* buf = NULL; - if(pTagVal->type == TSDB_DATA_TYPE_VARBINARY){ - buf = taosMemoryCalloc(pTagVal->nData*2 + 2 + 3, 1); - }else{ + if (pTagVal->type == TSDB_DATA_TYPE_VARBINARY) { + buf = taosMemoryCalloc(pTagVal->nData * 2 + 2 + 3, 1); + } else { buf = taosMemoryCalloc(pTagVal->nData + 3, 1); } @@ -315,7 +316,7 @@ static void buildChildElement(cJSON* json, SVCreateTbReq* pCreateReq) { cJSON_AddItemToArray(tags, tag); } - end: +end: cJSON_AddItemToObject(json, "tags", tags); taosArrayDestroy(pTagVals); } @@ -469,7 +470,8 @@ static char* processAlterTable(SMqMetaRsp* metaRsp) { cJSON* colType = cJSON_CreateNumber(vAlterTbReq.type); cJSON_AddItemToObject(json, "colType", colType); - if (vAlterTbReq.type == TSDB_DATA_TYPE_BINARY || vAlterTbReq.type == TSDB_DATA_TYPE_VARBINARY || vAlterTbReq.type == TSDB_DATA_TYPE_GEOMETRY) { + if (vAlterTbReq.type == TSDB_DATA_TYPE_BINARY || vAlterTbReq.type == TSDB_DATA_TYPE_VARBINARY || + vAlterTbReq.type == TSDB_DATA_TYPE_GEOMETRY) { int32_t length = vAlterTbReq.bytes - VARSTR_HEADER_SIZE; cJSON* cbytes = cJSON_CreateNumber(length); cJSON_AddItemToObject(json, "colLength", cbytes); @@ -490,7 +492,8 @@ static char* processAlterTable(SMqMetaRsp* metaRsp) { cJSON_AddItemToObject(json, "colName", colName); cJSON* colType = cJSON_CreateNumber(vAlterTbReq.colModType); cJSON_AddItemToObject(json, "colType", colType); - if (vAlterTbReq.colModType == TSDB_DATA_TYPE_BINARY || vAlterTbReq.colModType == TSDB_DATA_TYPE_VARBINARY || vAlterTbReq.colModType == TSDB_DATA_TYPE_GEOMETRY) { + if (vAlterTbReq.colModType == TSDB_DATA_TYPE_BINARY || vAlterTbReq.colModType == TSDB_DATA_TYPE_VARBINARY || + vAlterTbReq.colModType == TSDB_DATA_TYPE_GEOMETRY) { int32_t length = vAlterTbReq.colModBytes - VARSTR_HEADER_SIZE; cJSON* cbytes = cJSON_CreateNumber(length); cJSON_AddItemToObject(json, "colLength", cbytes); @@ -527,9 +530,9 @@ static char* processAlterTable(SMqMetaRsp* metaRsp) { } buf = parseTagDatatoJson(vAlterTbReq.pTagVal); } else { - if(vAlterTbReq.tagType == TSDB_DATA_TYPE_VARBINARY){ - buf = taosMemoryCalloc(vAlterTbReq.nTagVal*2 + 2 + 3, 1); - }else{ + if (vAlterTbReq.tagType == TSDB_DATA_TYPE_VARBINARY) { + buf = taosMemoryCalloc(vAlterTbReq.nTagVal * 2 + 2 + 3, 1); + } else { buf = taosMemoryCalloc(vAlterTbReq.nTagVal + 3, 1); } dataConverToStr(buf, vAlterTbReq.tagType, vAlterTbReq.pTagVal, vAlterTbReq.nTagVal, NULL); @@ -677,7 +680,7 @@ _exit: } static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) { - if(taos == NULL || meta == NULL) { + if (taos == NULL || meta == NULL) { terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -692,7 +695,7 @@ static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) { terrno = code; return code; } - uDebug(LOG_ID_TAG" create stable, meta:%p, metaLen:%d", LOG_ID_VALUE, meta, metaLen); + uDebug(LOG_ID_TAG " create stable, meta:%p, metaLen:%d", LOG_ID_VALUE, meta, metaLen); pRequest->syncQuery = true; if (!pRequest->pDb) { code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; @@ -731,8 +734,8 @@ static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) { pReq.source = TD_REQ_FROM_TAOX; pReq.igExists = true; - uDebug(LOG_ID_TAG" create stable name:%s suid:%" PRId64 " processSuid:%" PRId64, - LOG_ID_VALUE, req.name, req.suid, pReq.suid); + uDebug(LOG_ID_TAG " create stable name:%s suid:%" PRId64 " processSuid:%" PRId64, LOG_ID_VALUE, req.name, req.suid, + pReq.suid); STscObj* pTscObj = pRequest->pTscObj; SName tableName; tNameExtractFullName(toName(pTscObj->acctId, pRequest->pDb, req.name, &tableName), pReq.name); @@ -766,7 +769,7 @@ static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) { taosMemoryFree(pCmdMsg.pMsg); end: - uDebug(LOG_ID_TAG" create stable return, msg:%s", LOG_ID_VALUE, tstrerror(code)); + uDebug(LOG_ID_TAG " create stable return, msg:%s", LOG_ID_VALUE, tstrerror(code)); destroyRequest(pRequest); tFreeSMCreateStbReq(&pReq); tDecoderClear(&coder); @@ -775,7 +778,7 @@ end: } static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) { - if(taos == NULL || meta == NULL) { + if (taos == NULL || meta == NULL) { terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -791,7 +794,7 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) { return code; } - uDebug(LOG_ID_TAG" drop stable, meta:%p, metaLen:%d", LOG_ID_VALUE, meta, metaLen); + uDebug(LOG_ID_TAG " drop stable, meta:%p, metaLen:%d", LOG_ID_VALUE, meta, metaLen); pRequest->syncQuery = true; if (!pRequest->pDb) { code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; @@ -812,9 +815,9 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) { goto end; } SRequestConnInfo conn = {.pTrans = pRequest->pTscObj->pAppInfo->pTransporter, - .requestId = pRequest->requestId, - .requestObjRefId = pRequest->self, - .mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp)}; + .requestId = pRequest->requestId, + .requestObjRefId = pRequest->self, + .mgmtEps = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp)}; SName pName = {0}; toName(pRequest->pTscObj->acctId, pRequest->pDb, req.name, &pName); STableMeta* pTableMeta = NULL; @@ -835,8 +838,8 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) { pReq.source = TD_REQ_FROM_TAOX; // pReq.suid = processSuid(req.suid, pRequest->pDb); - uDebug(LOG_ID_TAG" drop stable name:%s suid:%" PRId64 " new suid:%" PRId64, - LOG_ID_VALUE, req.name, req.suid, pReq.suid); + uDebug(LOG_ID_TAG " drop stable name:%s suid:%" PRId64 " new suid:%" PRId64, LOG_ID_VALUE, req.name, req.suid, + pReq.suid); STscObj* pTscObj = pRequest->pTscObj; SName tableName = {0}; tNameExtractFullName(toName(pTscObj->acctId, pRequest->pDb, req.name, &tableName), pReq.name); @@ -870,7 +873,7 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) { taosMemoryFree(pCmdMsg.pMsg); end: - uDebug(LOG_ID_TAG" drop stable return, msg:%s", LOG_ID_VALUE, tstrerror(code)); + uDebug(LOG_ID_TAG " drop stable return, msg:%s", LOG_ID_VALUE, tstrerror(code)); destroyRequest(pRequest); tDecoderClear(&coder); terrno = code; @@ -889,7 +892,7 @@ static void destroyCreateTbReqBatch(void* data) { } static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) { - if(taos == NULL || meta == NULL) { + if (taos == NULL || meta == NULL) { terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -939,9 +942,9 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) { taosHashSetFreeFp(pVgroupHashmap, destroyCreateTbReqBatch); SRequestConnInfo conn = {.pTrans = pTscObj->pAppInfo->pTransporter, - .requestId = pRequest->requestId, - .requestObjRefId = pRequest->self, - .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)}; + .requestId = pRequest->requestId, + .requestObjRefId = pRequest->self, + .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)}; pRequest->tableList = taosArrayInit(req.nReqs, sizeof(SName)); // loop to create table @@ -1033,7 +1036,7 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) { code = pRequest->code; end: - uDebug(LOG_ID_TAG" create table return, msg:%s", LOG_ID_VALUE, tstrerror(code)); + uDebug(LOG_ID_TAG " create table return, msg:%s", LOG_ID_VALUE, tstrerror(code)); for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { pCreateReq = req.pReqs + iReq; taosMemoryFreeClear(pCreateReq->comment); @@ -1062,7 +1065,7 @@ static void destroyDropTbReqBatch(void* data) { } static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) { - if(taos == NULL || meta == NULL) { + if (taos == NULL || meta == NULL) { terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -1111,9 +1114,9 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) { taosHashSetFreeFp(pVgroupHashmap, destroyDropTbReqBatch); SRequestConnInfo conn = {.pTrans = pTscObj->pAppInfo->pTransporter, - .requestId = pRequest->requestId, - .requestObjRefId = pRequest->self, - .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)}; + .requestId = pRequest->requestId, + .requestObjRefId = pRequest->self, + .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)}; pRequest->tableList = taosArrayInit(req.nReqs, sizeof(SName)); // loop to create table for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { @@ -1142,7 +1145,8 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) { tb_uid_t oldSuid = pDropReq->suid; pDropReq->suid = pTableMeta->suid; taosMemoryFreeClear(pTableMeta); - uDebug(LOG_ID_TAG" drop table name:%s suid:%" PRId64 " new suid:%" PRId64, LOG_ID_VALUE, pDropReq->name, oldSuid, pDropReq->suid); + uDebug(LOG_ID_TAG " drop table name:%s suid:%" PRId64 " new suid:%" PRId64, LOG_ID_VALUE, pDropReq->name, oldSuid, + pDropReq->suid); taosArrayPush(pRequest->tableList, &pName); SVgroupDropTableBatch* pTableBatch = taosHashGet(pVgroupHashmap, &pInfo.vgId, sizeof(pInfo.vgId)); @@ -1185,7 +1189,7 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) { code = pRequest->code; end: - uDebug(LOG_ID_TAG" drop table return, msg:%s", LOG_ID_VALUE, tstrerror(code)); + uDebug(LOG_ID_TAG " drop table return, msg:%s", LOG_ID_VALUE, tstrerror(code)); taosHashCleanup(pVgroupHashmap); destroyRequest(pRequest); tDecoderClear(&coder); @@ -1227,16 +1231,16 @@ end: //} static int32_t taosDeleteData(TAOS* taos, void* meta, int32_t metaLen) { - if(taos == NULL || meta == NULL) { + if (taos == NULL || meta == NULL) { terrno = TSDB_CODE_INVALID_PARA; return terrno; } - SDeleteRes req = {0}; - SDecoder coder = {0}; + SDeleteRes req = {0}; + SDecoder coder = {0}; char sql[256] = {0}; int32_t code = TSDB_CODE_SUCCESS; - uDebug("connId:0x%"PRIx64" delete data, meta:%p, len:%d", *(int64_t*)taos, meta, metaLen); + uDebug("connId:0x%" PRIx64 " delete data, meta:%p, len:%d", *(int64_t*)taos, meta, metaLen); // decode and process req void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); @@ -1260,14 +1264,14 @@ static int32_t taosDeleteData(TAOS* taos, void* meta, int32_t metaLen) { taos_free_result(res); end: - uDebug("connId:0x%"PRIx64" delete data sql:%s, code:%s", *(int64_t*)taos, sql, tstrerror(code)); + uDebug("connId:0x%" PRIx64 " delete data sql:%s, code:%s", *(int64_t*)taos, sql, tstrerror(code)); tDecoderClear(&coder); terrno = code; return code; } static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) { - if(taos == NULL || meta == NULL) { + if (taos == NULL || meta == NULL) { terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -1312,9 +1316,9 @@ static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) { } SRequestConnInfo conn = {.pTrans = pTscObj->pAppInfo->pTransporter, - .requestId = pRequest->requestId, - .requestObjRefId = pRequest->self, - .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)}; + .requestId = pRequest->requestId, + .requestObjRefId = pRequest->self, + .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)}; SVgroupInfo pInfo = {0}; SName pName = {0}; @@ -1394,8 +1398,8 @@ int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const ch return taos_write_raw_block_with_fields_with_reqid(taos, rows, pData, tbname, fields, numFields, 0); } -int taos_write_raw_block_with_fields_with_reqid(TAOS *taos, int rows, char *pData, const char *tbname, - TAOS_FIELD *fields, int numFields, int64_t reqid){ +int taos_write_raw_block_with_fields_with_reqid(TAOS* taos, int rows, char* pData, const char* tbname, + TAOS_FIELD* fields, int numFields, int64_t reqid) { if (!taos || !pData || !tbname) { terrno = TSDB_CODE_INVALID_PARA; return terrno; @@ -1410,8 +1414,8 @@ int taos_write_raw_block_with_fields_with_reqid(TAOS *taos, int rows, char *pDat return terrno; } - uDebug(LOG_ID_TAG " write raw block with field, rows:%d, pData:%p, tbname:%s, fields:%p, numFields:%d", - LOG_ID_VALUE, rows, pData, tbname, fields, numFields); + uDebug(LOG_ID_TAG " write raw block with field, rows:%d, pData:%p, tbname:%s, fields:%p, numFields:%d", LOG_ID_VALUE, + rows, pData, tbname, fields, numFields); pRequest->syncQuery = true; if (!pRequest->pDb) { @@ -1562,12 +1566,12 @@ end: return code; } -static void* getRawDataFromRes(void *pRetrieve){ +static void* getRawDataFromRes(void* pRetrieve) { void* rawData = NULL; // deal with compatibility - if(*(int64_t*)pRetrieve == 0){ + if (*(int64_t*)pRetrieve == 0) { rawData = ((SRetrieveTableRsp*)pRetrieve)->data; - }else if(*(int64_t*)pRetrieve == 1){ + } else if (*(int64_t*)pRetrieve == 1) { rawData = ((SRetrieveTableRspForTmq*)pRetrieve)->data; } ASSERT(rawData != NULL); @@ -1575,7 +1579,7 @@ static void* getRawDataFromRes(void *pRetrieve){ } static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { - if(taos == NULL || data == NULL){ + if (taos == NULL || data == NULL) { terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -1644,11 +1648,11 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { strcpy(pName.tname, tbName); code = catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta); -// if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) { -// uError("WriteRaw:catalogGetTableMeta table not exist. table name: %s", tbName); -// code = TSDB_CODE_SUCCESS; -// continue; -// } + // if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) { + // uError("WriteRaw:catalogGetTableMeta table not exist. table name: %s", tbName); + // code = TSDB_CODE_SUCCESS; + // continue; + // } if (code != TSDB_CODE_SUCCESS) { goto end; } @@ -1704,7 +1708,7 @@ end: } static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) { - if(taos == NULL || data == NULL){ + if (taos == NULL || data == NULL) { terrno = TSDB_CODE_INVALID_PARA; return terrno; } @@ -1757,7 +1761,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) } pVgHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); - uDebug(LOG_ID_TAG" write raw metadata block num:%d", LOG_ID_VALUE, rspObj.rsp.blockNum); + uDebug(LOG_ID_TAG " write raw metadata block num:%d", LOG_ID_VALUE, rspObj.rsp.blockNum); while (++rspObj.resIter < rspObj.rsp.blockNum) { void* pRetrieve = taosArrayGetP(rspObj.rsp.blockData, rspObj.resIter); if (!rspObj.rsp.withSchema) { @@ -1770,7 +1774,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) goto end; } - uDebug(LOG_ID_TAG" write raw metadata block tbname:%s", LOG_ID_VALUE, tbName); + uDebug(LOG_ID_TAG " write raw metadata block tbname:%s", LOG_ID_VALUE, tbName); SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}}; strcpy(pName.dbname, pRequest->pDb); strcpy(pName.tname, tbName); @@ -1817,11 +1821,11 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) strcpy(pName.tname, pCreateReqDst->ctb.stbName); } code = catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta); -// if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) { -// uError("WriteRaw:catalogGetTableMeta table not exist. table name: %s", tbName); -// code = TSDB_CODE_SUCCESS; -// continue; -// } + // if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) { + // uError("WriteRaw:catalogGetTableMeta table not exist. table name: %s", tbName); + // code = TSDB_CODE_SUCCESS; + // continue; + // } if (code != TSDB_CODE_SUCCESS) { goto end; } diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 3c3aee3032..1904874a0b 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -26,7 +26,7 @@ #define EMPTY_BLOCK_POLL_IDLE_DURATION 10 #define DEFAULT_AUTO_COMMIT_INTERVAL 5000 -#define DEFAULT_HEARTBEAT_INTERVAL 3000 +#define DEFAULT_HEARTBEAT_INTERVAL 3000 struct SMqMgmt { int8_t inited; @@ -63,7 +63,7 @@ struct tmq_conf_t { int8_t withTbName; int8_t snapEnable; int8_t replayEnable; - int8_t sourceExcluded; // do not consume, bit + int8_t sourceExcluded; // do not consume, bit uint16_t port; int32_t autoCommitInterval; char* ip; @@ -83,15 +83,15 @@ struct tmq_t { int32_t autoCommitInterval; int8_t resetOffsetCfg; int8_t replayEnable; - int8_t sourceExcluded; // do not consume, bit + int8_t sourceExcluded; // do not consume, bit uint64_t consumerId; tmq_commit_cb* commitCb; void* commitCbUserParam; // status - SRWLatch lock; - int8_t status; - int32_t epoch; + SRWLatch lock; + int8_t status; + int32_t epoch; // poll info int64_t pollCnt; int64_t totalRows; @@ -132,8 +132,8 @@ enum { typedef struct SVgOffsetInfo { STqOffsetVal committedOffset; - STqOffsetVal endOffset; // the last version in TAOS_RES + 1 - STqOffsetVal beginOffset; // the first version in TAOS_RES + STqOffsetVal endOffset; // the last version in TAOS_RES + 1 + STqOffsetVal beginOffset; // the first version in TAOS_RES int64_t walVerBegin; int64_t walVerEnd; } SVgOffsetInfo; @@ -144,11 +144,11 @@ typedef struct { SVgOffsetInfo offsetInfo; int32_t vgId; int32_t vgStatus; - int32_t vgSkipCnt; // here used to mark the slow vgroups - int64_t emptyBlockReceiveTs; // once empty block is received, idle for ignoreCnt then start to poll data - int64_t blockReceiveTs; // once empty block is received, idle for ignoreCnt then start to poll data - int64_t blockSleepForReplay; // once empty block is received, idle for ignoreCnt then start to poll data - bool seekUpdated; // offset is updated by seek operator, therefore, not update by vnode rsp. + int32_t vgSkipCnt; // here used to mark the slow vgroups + int64_t emptyBlockReceiveTs; // once empty block is received, idle for ignoreCnt then start to poll data + int64_t blockReceiveTs; // once empty block is received, idle for ignoreCnt then start to poll data + int64_t blockSleepForReplay; // once empty block is received, idle for ignoreCnt then start to poll data + bool seekUpdated; // offset is updated by seek operator, therefore, not update by vnode rsp. SEpSet epSet; } SMqClientVg; @@ -178,23 +178,23 @@ typedef struct { } SMqPollRspWrapper; typedef struct { -// int64_t refId; -// int32_t epoch; + // int64_t refId; + // int32_t epoch; tsem_t rspSem; int32_t rspErr; } SMqSubscribeCbParam; typedef struct { - int64_t refId; - bool sync; - void* pParam; + int64_t refId; + bool sync; + void* pParam; } SMqAskEpCbParam; typedef struct { - int64_t refId; - char topicName[TSDB_TOPIC_FNAME_LEN]; - int32_t vgId; - uint64_t requestId; // request id for debug purpose + int64_t refId; + char topicName[TSDB_TOPIC_FNAME_LEN]; + int32_t vgId; + uint64_t requestId; // request id for debug purpose } SMqPollCbParam; typedef struct SMqVgCommon { @@ -208,14 +208,14 @@ typedef struct SMqVgCommon { } SMqVgCommon; typedef struct SMqSeekParam { - tsem_t sem; - int32_t code; + tsem_t sem; + int32_t code; } SMqSeekParam; typedef struct SMqCommittedParam { - tsem_t sem; - int32_t code; - SMqVgOffset vgOffset; + tsem_t sem; + int32_t code; + SMqVgOffset vgOffset; } SMqCommittedParam; typedef struct SMqVgWalInfoParam { @@ -238,10 +238,10 @@ typedef struct { typedef struct { SMqCommitCbParamSet* params; -// SMqVgOffset* pOffset; - char topicName[TSDB_TOPIC_FNAME_LEN]; - int32_t vgId; - tmq_t* pTmq; + // SMqVgOffset* pOffset; + char topicName[TSDB_TOPIC_FNAME_LEN]; + int32_t vgId; + tmq_t* pTmq; } SMqCommitCbParam; typedef struct SSyncCommitInfo { @@ -252,7 +252,8 @@ typedef struct SSyncCommitInfo { static int32_t syncAskEp(tmq_t* tmq); static int32_t makeTopicVgroupKey(char* dst, const char* topicName, int32_t vg); static int32_t tmqCommitDone(SMqCommitCbParamSet* pParamSet); -static int32_t doSendCommitMsg(tmq_t* tmq, int32_t vgId, SEpSet* epSet, STqOffsetVal* offset, const char* pTopicName, SMqCommitCbParamSet* pParamSet); +static int32_t doSendCommitMsg(tmq_t* tmq, int32_t vgId, SEpSet* epSet, STqOffsetVal* offset, const char* pTopicName, + SMqCommitCbParamSet* pParamSet); static void commitRspCountDown(SMqCommitCbParamSet* pParamSet, int64_t consumerId, const char* pTopic, int32_t vgId); static void askEp(tmq_t* pTmq, void* param, bool sync, bool updateEpset); @@ -287,7 +288,7 @@ void tmq_conf_destroy(tmq_conf_t* conf) { } tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value) { - if (conf == NULL || key == NULL || value == NULL){ + if (conf == NULL || key == NULL || value == NULL) { return TMQ_CONF_INVALID; } if (strcasecmp(key, "group.id") == 0) { @@ -402,7 +403,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value tmq_list_t* tmq_list_new() { return (tmq_list_t*)taosArrayInit(0, sizeof(void*)); } int32_t tmq_list_append(tmq_list_t* list, const char* src) { - if(list == NULL) return -1; + if (list == NULL) return -1; SArray* container = &list->container; if (src == NULL || src[0] == 0) return -1; char* topic = taosStrdup(src); @@ -411,19 +412,19 @@ int32_t tmq_list_append(tmq_list_t* list, const char* src) { } void tmq_list_destroy(tmq_list_t* list) { - if(list == NULL) return; + if (list == NULL) return; SArray* container = &list->container; taosArrayDestroyP(container, taosMemoryFree); } int32_t tmq_list_get_size(const tmq_list_t* list) { - if(list == NULL) return -1; + if (list == NULL) return -1; const SArray* container = &list->container; return taosArrayGetSize(container); } char** tmq_list_to_c_array(const tmq_list_t* list) { - if(list == NULL) return NULL; + if (list == NULL) return NULL; const SArray* container = &list->container; return container->pData; } @@ -439,7 +440,8 @@ static int32_t tmqCommitCb(void* param, SDataBuf* pBuf, int32_t code) { return 0; } -static int32_t doSendCommitMsg(tmq_t* tmq, int32_t vgId, SEpSet* epSet, STqOffsetVal* offset, const char* pTopicName, SMqCommitCbParamSet* pParamSet) { +static int32_t doSendCommitMsg(tmq_t* tmq, int32_t vgId, SEpSet* epSet, STqOffsetVal* offset, const char* pTopicName, + SMqCommitCbParamSet* pParamSet) { SMqVgOffset pOffset = {0}; pOffset.consumerId = tmq->consumerId; @@ -479,7 +481,7 @@ static int32_t doSendCommitMsg(tmq_t* tmq, int32_t vgId, SEpSet* epSet, STqOffse } pParam->params = pParamSet; -// pParam->pOffset = pOffset; + // pParam->pOffset = pOffset; pParam->vgId = vgId; pParam->pTmq = tmq; @@ -493,7 +495,7 @@ static int32_t doSendCommitMsg(tmq_t* tmq, int32_t vgId, SEpSet* epSet, STqOffse return TSDB_CODE_OUT_OF_MEMORY; } - pMsgSendInfo->msgInfo = (SDataBuf) { .pData = buf, .len = sizeof(SMsgHead) + len, .handle = NULL }; + pMsgSendInfo->msgInfo = (SDataBuf){.pData = buf, .len = sizeof(SMsgHead) + len, .handle = NULL}; pMsgSendInfo->requestId = generateRequestId(); pMsgSendInfo->requestObjRefId = 0; @@ -505,7 +507,7 @@ static int32_t doSendCommitMsg(tmq_t* tmq, int32_t vgId, SEpSet* epSet, STqOffse int64_t transporterId = 0; atomic_add_fetch_32(&pParamSet->waitingRspNum, 1); code = asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, epSet, &transporterId, pMsgSendInfo); - if(code != 0){ + if (code != 0) { atomic_sub_fetch_32(&pParamSet->waitingRspNum, 1); return code; } @@ -527,7 +529,8 @@ static SMqClientTopic* getTopicByName(tmq_t* tmq, const char* pTopicName) { return NULL; } -static SMqCommitCbParamSet* prepareCommitCbParamSet(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* userParam, int32_t rspNum){ +static SMqCommitCbParamSet* prepareCommitCbParamSet(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* userParam, + int32_t rspNum) { SMqCommitCbParamSet* pParamSet = taosMemoryCalloc(1, sizeof(SMqCommitCbParamSet)); if (pParamSet == NULL) { return NULL; @@ -542,14 +545,14 @@ static SMqCommitCbParamSet* prepareCommitCbParamSet(tmq_t* tmq, tmq_commit_cb* p return pParamSet; } -static int32_t getClientVg(tmq_t* tmq, char* pTopicName, int32_t vgId, SMqClientVg** pVg){ +static int32_t getClientVg(tmq_t* tmq, char* pTopicName, int32_t vgId, SMqClientVg** pVg) { SMqClientTopic* pTopic = getTopicByName(tmq, pTopicName); if (pTopic == NULL) { tscError("consumer:0x%" PRIx64 " invalid topic name:%s", tmq->consumerId, pTopicName); return TSDB_CODE_TMQ_INVALID_TOPIC; } - int32_t numOfVgs = taosArrayGetSize(pTopic->vgs); + int32_t numOfVgs = taosArrayGetSize(pTopic->vgs); for (int32_t i = 0; i < numOfVgs; ++i) { SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, i); if (pClientVg->vgId == vgId) { @@ -561,19 +564,20 @@ static int32_t getClientVg(tmq_t* tmq, char* pTopicName, int32_t vgId, SMqClient return *pVg == NULL ? TSDB_CODE_TMQ_INVALID_VGID : TSDB_CODE_SUCCESS; } -static int32_t asyncCommitOffset(tmq_t* tmq, char* pTopicName, int32_t vgId, STqOffsetVal* offsetVal, tmq_commit_cb* pCommitFp, void* userParam) { +static int32_t asyncCommitOffset(tmq_t* tmq, char* pTopicName, int32_t vgId, STqOffsetVal* offsetVal, + tmq_commit_cb* pCommitFp, void* userParam) { tscInfo("consumer:0x%" PRIx64 " do manual commit offset for %s, vgId:%d", tmq->consumerId, pTopicName, vgId); taosRLockLatch(&tmq->lock); SMqClientVg* pVg = NULL; - int32_t code = getClientVg(tmq, pTopicName, vgId, &pVg); - if(code != 0){ + int32_t code = getClientVg(tmq, pTopicName, vgId, &pVg); + if (code != 0) { goto end; } if (offsetVal->type <= 0) { code = TSDB_CODE_TMQ_INVALID_MSG; goto end; } - if (tOffsetEqual(offsetVal, &pVg->offsetInfo.committedOffset)){ + if (tOffsetEqual(offsetVal, &pVg->offsetInfo.committedOffset)) { code = TSDB_CODE_TMQ_SAME_COMMITTED_VALUE; goto end; } @@ -605,11 +609,11 @@ end: return code; } -static void asyncCommitFromResult(tmq_t* tmq, const TAOS_RES* pRes, tmq_commit_cb* pCommitFp, void* userParam){ - char* pTopicName = NULL; - int32_t vgId = 0; +static void asyncCommitFromResult(tmq_t* tmq, const TAOS_RES* pRes, tmq_commit_cb* pCommitFp, void* userParam) { + char* pTopicName = NULL; + int32_t vgId = 0; STqOffsetVal offsetVal = {0}; - int32_t code = 0; + int32_t code = 0; if (pRes == NULL || tmq == NULL) { code = TSDB_CODE_INVALID_PARA; @@ -639,8 +643,8 @@ static void asyncCommitFromResult(tmq_t* tmq, const TAOS_RES* pRes, tmq_commit_c code = asyncCommitOffset(tmq, pTopicName, vgId, &offsetVal, pCommitFp, userParam); end: - if(code != TSDB_CODE_SUCCESS && pCommitFp != NULL){ - if(code == TSDB_CODE_TMQ_SAME_COMMITTED_VALUE) code = TSDB_CODE_SUCCESS; + if (code != TSDB_CODE_SUCCESS && pCommitFp != NULL) { + if (code == TSDB_CODE_TMQ_SAME_COMMITTED_VALUE) code = TSDB_CODE_SUCCESS; pCommitFp(tmq, code, userParam); } } @@ -662,11 +666,13 @@ static void asyncCommitAllOffsets(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* us SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); int32_t numOfVgroups = taosArrayGetSize(pTopic->vgs); - tscInfo("consumer:0x%" PRIx64 " commit offset for topics:%s, numOfVgs:%d", tmq->consumerId, pTopic->topicName, numOfVgroups); + tscInfo("consumer:0x%" PRIx64 " commit offset for topics:%s, numOfVgs:%d", tmq->consumerId, pTopic->topicName, + numOfVgroups); for (int32_t j = 0; j < numOfVgroups; j++) { SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); - if (pVg->offsetInfo.endOffset.type > 0 && !tOffsetEqual(&pVg->offsetInfo.endOffset, &pVg->offsetInfo.committedOffset)) { + if (pVg->offsetInfo.endOffset.type > 0 && + !tOffsetEqual(&pVg->offsetInfo.endOffset, &pVg->offsetInfo.committedOffset)) { char offsetBuf[TSDB_OFFSET_LEN] = {0}; tFormatOffset(offsetBuf, tListLen(offsetBuf), &pVg->offsetInfo.endOffset); @@ -675,23 +681,27 @@ static void asyncCommitAllOffsets(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* us code = doSendCommitMsg(tmq, pVg->vgId, &pVg->epSet, &pVg->offsetInfo.endOffset, pTopic->topicName, pParamSet); if (code != TSDB_CODE_SUCCESS) { - tscError("consumer:0x%" PRIx64 " topic:%s on vgId:%d end commit msg failed, send offset:%s committed:%s, code:%s ordinal:%d/%d", - tmq->consumerId, pTopic->topicName, pVg->vgId, offsetBuf, commitBuf, tstrerror(terrno), j + 1, numOfVgroups); + tscError("consumer:0x%" PRIx64 + " topic:%s on vgId:%d end commit msg failed, send offset:%s committed:%s, code:%s ordinal:%d/%d", + tmq->consumerId, pTopic->topicName, pVg->vgId, offsetBuf, commitBuf, tstrerror(terrno), j + 1, + numOfVgroups); continue; } - tscInfo("consumer:0x%" PRIx64 " topic:%s on vgId:%d send commit msg success, send offset:%s committed:%s, ordinal:%d/%d", + tscInfo("consumer:0x%" PRIx64 + " topic:%s on vgId:%d send commit msg success, send offset:%s committed:%s, ordinal:%d/%d", tmq->consumerId, pTopic->topicName, pVg->vgId, offsetBuf, commitBuf, j + 1, numOfVgroups); pVg->offsetInfo.committedOffset = pVg->offsetInfo.endOffset; } else { tscInfo("consumer:0x%" PRIx64 " topic:%s vgId:%d, no commit, current:%" PRId64 ", ordinal:%d/%d", - tmq->consumerId, pTopic->topicName, pVg->vgId, pVg->offsetInfo.endOffset.version, j + 1, numOfVgroups); + tmq->consumerId, pTopic->topicName, pVg->vgId, pVg->offsetInfo.endOffset.version, j + 1, numOfVgroups); } } } taosRUnLockLatch(&tmq->lock); - tscInfo("consumer:0x%" PRIx64 " total commit:%d for %d topics", tmq->consumerId, pParamSet->waitingRspNum - 1, numOfTopics); + tscInfo("consumer:0x%" PRIx64 " total commit:%d for %d topics", tmq->consumerId, pParamSet->waitingRspNum - 1, + numOfTopics); // request is sent if (pParamSet->waitingRspNum != 1) { @@ -702,7 +712,7 @@ static void asyncCommitAllOffsets(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* us end: taosMemoryFree(pParamSet); - if(pCommitFp != NULL) { + if (pCommitFp != NULL) { pCommitFp(tmq, code, userParam); } return; @@ -710,10 +720,10 @@ end: static void generateTimedTask(int64_t refId, int32_t type) { tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId); - if(tmq == NULL) return; + if (tmq == NULL) return; int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t), DEF_QITEM, 0); - if(pTaskType == NULL) return; + if (pTaskType == NULL) return; *pTaskType = type; taosWriteQitem(tmq->delayedTask, pTaskType); @@ -729,8 +739,8 @@ void tmqAssignAskEpTask(void* param, void* tmrId) { void tmqReplayTask(void* param, void* tmrId) { int64_t refId = *(int64_t*)param; - tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId); - if(tmq == NULL) goto END; + tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId); + if (tmq == NULL) goto END; tsem_post(&tmq->rspSem); taosReleaseRef(tmqMgmt.rsetId, refId); @@ -753,13 +763,13 @@ int32_t tmqHbCb(void* param, SDataBuf* pMsg, int32_t code) { tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId); if (tmq != NULL) { taosWLockLatch(&tmq->lock); - for(int32_t i = 0; i < taosArrayGetSize(rsp.topicPrivileges); i++){ + for (int32_t i = 0; i < taosArrayGetSize(rsp.topicPrivileges); i++) { STopicPrivilege* privilege = taosArrayGet(rsp.topicPrivileges, i); - if(privilege->noPrivilege == 1){ + if (privilege->noPrivilege == 1) { int32_t topicNumCur = taosArrayGetSize(tmq->clientTopics); for (int32_t j = 0; j < topicNumCur; j++) { SMqClientTopic* pTopicCur = taosArrayGet(tmq->clientTopics, j); - if(strcmp(pTopicCur->topicName, privilege->topic) == 0){ + if (strcmp(pTopicCur->topicName, privilege->topic) == 0) { tscInfo("consumer:0x%" PRIx64 ", has no privilege, topic:%s", tmq->consumerId, privilege->topic); pTopicCur->noPrivilege = 1; } @@ -790,22 +800,22 @@ void tmqSendHbReq(void* param, void* tmrId) { req.epoch = tmq->epoch; taosRLockLatch(&tmq->lock); req.topics = taosArrayInit(taosArrayGetSize(tmq->clientTopics), sizeof(TopicOffsetRows)); - for(int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++){ - SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); - int32_t numOfVgroups = taosArrayGetSize(pTopic->vgs); + for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { + SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); + int32_t numOfVgroups = taosArrayGetSize(pTopic->vgs); TopicOffsetRows* data = taosArrayReserve(req.topics, 1); strcpy(data->topicName, pTopic->topicName); data->offsetRows = taosArrayInit(numOfVgroups, sizeof(OffsetRows)); - for(int j = 0; j < numOfVgroups; j++){ + for (int j = 0; j < numOfVgroups; j++) { SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); - OffsetRows* offRows = taosArrayReserve(data->offsetRows, 1); + OffsetRows* offRows = taosArrayReserve(data->offsetRows, 1); offRows->vgId = pVg->vgId; offRows->rows = pVg->numOfRows; offRows->offset = pVg->offsetInfo.endOffset; - offRows->ever = pVg->offsetInfo.walVerEnd; + offRows->ever = pVg->offsetInfo.walVerEnd; char buf[TSDB_OFFSET_LEN] = {0}; tFormatOffset(buf, TSDB_OFFSET_LEN, &offRows->offset); - tscInfo("consumer:0x%" PRIx64 ",report offset, group:%s vgId:%d, offset:%s/%"PRId64", rows:%"PRId64, + tscInfo("consumer:0x%" PRIx64 ",report offset, group:%s vgId:%d, offset:%s/%" PRId64 ", rows:%" PRId64, tmq->consumerId, tmq->groupId, offRows->vgId, buf, offRows->ever, offRows->rows); } } @@ -835,13 +845,13 @@ void tmqSendHbReq(void* param, void* tmrId) { goto OVER; } - sendInfo->msgInfo = (SDataBuf){ .pData = pReq, .len = tlen, .handle = NULL }; + sendInfo->msgInfo = (SDataBuf){.pData = pReq, .len = tlen, .handle = NULL}; sendInfo->requestId = generateRequestId(); sendInfo->requestObjRefId = 0; sendInfo->paramFreeFp = taosMemoryFree; sendInfo->param = taosMemoryMalloc(sizeof(int64_t)); - *(int64_t *)sendInfo->param = refId; + *(int64_t*)sendInfo->param = refId; sendInfo->fp = tmqHbCb; sendInfo->msgType = TDMT_MND_TMQ_HB; @@ -972,7 +982,7 @@ int32_t tmqSubscribeCb(void* param, SDataBuf* pMsg, int32_t code) { } int32_t tmq_subscription(tmq_t* tmq, tmq_list_t** topics) { - if(tmq == NULL) return TSDB_CODE_INVALID_PARA; + if (tmq == NULL) return TSDB_CODE_INVALID_PARA; if (*topics == NULL) { *topics = tmq_list_new(); } @@ -986,7 +996,7 @@ int32_t tmq_subscription(tmq_t* tmq, tmq_list_t** topics) { } int32_t tmq_unsubscribe(tmq_t* tmq) { - if(tmq == NULL) return TSDB_CODE_INVALID_PARA; + if (tmq == NULL) return TSDB_CODE_INVALID_PARA; if (tmq->status != TMQ_CONSUMER_STATUS__READY) { tscInfo("consumer:0x%" PRIx64 " not in ready state, unsubscribe it directly", tmq->consumerId); return 0; @@ -1060,9 +1070,10 @@ static void tmqMgmtInit(void) { } } -#define SET_ERROR_MSG(MSG) if(errstr!=NULL)snprintf(errstr,errstrLen,MSG); +#define SET_ERROR_MSG(MSG) \ + if (errstr != NULL) snprintf(errstr, errstrLen, MSG); tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { - if(conf == NULL) { + if (conf == NULL) { SET_ERROR_MSG("configure is null") return NULL; } @@ -1114,7 +1125,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { pTmq->resetOffsetCfg = conf->resetOffset; pTmq->replayEnable = conf->replayEnable; pTmq->sourceExcluded = conf->sourceExcluded; - if(conf->replayEnable){ + if (conf->replayEnable) { pTmq->autoCommit = false; } taosInitRWLatch(&pTmq->lock); @@ -1154,7 +1165,8 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { tFormatOffset(buf, tListLen(buf), &offset); tscInfo("consumer:0x%" PRIx64 " is setup, refId:%" PRId64 ", groupId:%s, snapshot:%d, autoCommit:%d, commitInterval:%dms, offset:%s", - pTmq->consumerId, pTmq->refId, pTmq->groupId, pTmq->useSnapshot, pTmq->autoCommit, pTmq->autoCommitInterval, buf); + pTmq->consumerId, pTmq->refId, pTmq->groupId, pTmq->useSnapshot, pTmq->autoCommit, pTmq->autoCommitInterval, + buf); return pTmq; @@ -1164,7 +1176,7 @@ _failed: } int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { - if(tmq == NULL || topic_list == NULL) return TSDB_CODE_INVALID_PARA; + if (tmq == NULL || topic_list == NULL) return TSDB_CODE_INVALID_PARA; const int32_t MAX_RETRY_COUNT = 120 * 2; // let's wait for 2 mins at most const SArray* container = &topic_list->container; int32_t sz = taosArrayGetSize(container); @@ -1224,7 +1236,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { goto FAIL; } - SMqSubscribeCbParam param = { .rspErr = 0}; + SMqSubscribeCbParam param = {.rspErr = 0}; if (tsem_init(¶m.rspSem, 0, 0) != 0) { code = TSDB_CODE_TSC_INTERNAL_ERROR; goto FAIL; @@ -1242,7 +1254,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { int64_t transporterId = 0; code = asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); - if(code != 0){ + if (code != 0) { goto FAIL; } @@ -1293,20 +1305,20 @@ FAIL: } void tmq_conf_set_auto_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb, void* param) { - if(conf == NULL) return; + if (conf == NULL) return; conf->commitCb = cb; conf->commitCbUserParam = param; } -static SMqClientVg* getVgInfo(tmq_t* tmq, char* topicName, int32_t vgId){ +static SMqClientVg* getVgInfo(tmq_t* tmq, char* topicName, int32_t vgId) { int32_t topicNumCur = taosArrayGetSize(tmq->clientTopics); - for(int i = 0; i < topicNumCur; i++){ + for (int i = 0; i < topicNumCur; i++) { SMqClientTopic* pTopicCur = taosArrayGet(tmq->clientTopics, i); - if(strcmp(pTopicCur->topicName, topicName) == 0){ + if (strcmp(pTopicCur->topicName, topicName) == 0) { int32_t vgNumCur = taosArrayGetSize(pTopicCur->vgs); for (int32_t j = 0; j < vgNumCur; j++) { SMqClientVg* pVgCur = taosArrayGet(pTopicCur->vgs, j); - if(pVgCur->vgId == vgId){ + if (pVgCur->vgId == vgId) { return pVgCur; } } @@ -1315,21 +1327,21 @@ static SMqClientVg* getVgInfo(tmq_t* tmq, char* topicName, int32_t vgId){ return NULL; } -static SMqClientTopic* getTopicInfo(tmq_t* tmq, char* topicName){ +static SMqClientTopic* getTopicInfo(tmq_t* tmq, char* topicName) { int32_t topicNumCur = taosArrayGetSize(tmq->clientTopics); - for(int i = 0; i < topicNumCur; i++){ + for (int i = 0; i < topicNumCur; i++) { SMqClientTopic* pTopicCur = taosArrayGet(tmq->clientTopics, i); - if(strcmp(pTopicCur->topicName, topicName) == 0){ + if (strcmp(pTopicCur->topicName, topicName) == 0) { return pTopicCur; } } return NULL; } -static void setVgIdle(tmq_t* tmq, char* topicName, int32_t vgId){ +static void setVgIdle(tmq_t* tmq, char* topicName, int32_t vgId) { taosWLockLatch(&tmq->lock); SMqClientVg* pVg = getVgInfo(tmq, topicName, vgId); - if(pVg){ + if (pVg) { atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); } taosWUnLockLatch(&tmq->lock); @@ -1337,10 +1349,10 @@ static void setVgIdle(tmq_t* tmq, char* topicName, int32_t vgId){ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { SMqPollCbParam* pParam = (SMqPollCbParam*)param; - int64_t refId = pParam->refId; - int32_t vgId = pParam->vgId; - uint64_t requestId = pParam->requestId; - tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId); + int64_t refId = pParam->refId; + int32_t vgId = pParam->vgId; + uint64_t requestId = pParam->requestId; + tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId); if (tmq == NULL) { code = TSDB_CODE_TMQ_CONSUMER_CLOSED; goto FAIL; @@ -1354,7 +1366,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { goto FAIL; } - if(code != 0){ + if (code != 0) { goto END; } @@ -1363,7 +1375,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { if (msgEpoch < clientEpoch) { // do not write into queue since updating epoch reset tscWarn("consumer:0x%" PRIx64 - " msg discard from vgId:%d since from earlier epoch, rsp epoch %d, current epoch %d, reqId:0x%" PRIx64, + " msg discard from vgId:%d since from earlier epoch, rsp epoch %d, current epoch %d, reqId:0x%" PRIx64, tmq->consumerId, vgId, msgEpoch, clientEpoch, requestId); code = -1; goto END; @@ -1416,10 +1428,10 @@ END: taosReleaseRef(tmqMgmt.rsetId, refId); FAIL: - if(tmq) tsem_post(&tmq->rspSem); + if (tmq) tsem_post(&tmq->rspSem); taosMemoryFree(pParam); - if(pMsg) taosMemoryFreeClear(pMsg->pData); - if(pMsg) taosMemoryFreeClear(pMsg->pEpSet); + if (pMsg) taosMemoryFreeClear(pMsg->pData); + if (pMsg) taosMemoryFreeClear(pMsg->pEpSet); return code; } @@ -1456,7 +1468,8 @@ static void initClientTopicFromRsp(SMqClientTopic* pTopic, SMqSubTopicEp* pTopic STqOffsetVal offsetNew = {0}; offsetNew.type = tmq->resetOffsetCfg; - tscInfo("consumer:0x%" PRIx64 ", update topic:%s, new numOfVgs:%d, num:%d, port:%d", tmq->consumerId, pTopic->topicName, vgNumGet, pVgEp->epSet.numOfEps,pVgEp->epSet.eps[pVgEp->epSet.inUse].port); + tscInfo("consumer:0x%" PRIx64 ", update topic:%s, new numOfVgs:%d, num:%d, port:%d", tmq->consumerId, + pTopic->topicName, vgNumGet, pVgEp->epSet.numOfEps, pVgEp->epSet.eps[pVgEp->epSet.inUse].port); SMqClientVg clientVg = { .pollCnt = 0, @@ -1495,9 +1508,9 @@ static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) int32_t topicNumGet = taosArrayGetSize(pRsp->topics); if (epoch < tmq->epoch || (epoch == tmq->epoch && topicNumGet == 0)) { - tscInfo("consumer:0x%" PRIx64 " no update ep epoch from %d to epoch %d, incoming topics:%d", - tmq->consumerId, tmq->epoch, epoch, topicNumGet); - if(atomic_load_8(&tmq->status) == TMQ_CONSUMER_STATUS__RECOVER){ + tscInfo("consumer:0x%" PRIx64 " no update ep epoch from %d to epoch %d, incoming topics:%d", tmq->consumerId, + tmq->epoch, epoch, topicNumGet); + if (atomic_load_8(&tmq->status) == TMQ_CONSUMER_STATUS__RECOVER) { atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__READY); } return false; @@ -1533,10 +1546,12 @@ static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) char buf[TSDB_OFFSET_LEN] = {0}; tFormatOffset(buf, TSDB_OFFSET_LEN, &pVgCur->offsetInfo.endOffset); tscInfo("consumer:0x%" PRIx64 ", epoch:%d vgId:%d vgKey:%s, offset:%s", tmq->consumerId, epoch, pVgCur->vgId, - vgKey, buf); + vgKey, buf); - SVgroupSaveInfo info = {.currentOffset = pVgCur->offsetInfo.endOffset, .seekOffset = pVgCur->offsetInfo.beginOffset, - .commitOffset = pVgCur->offsetInfo.committedOffset, .numOfRows = pVgCur->numOfRows, + SVgroupSaveInfo info = {.currentOffset = pVgCur->offsetInfo.endOffset, + .seekOffset = pVgCur->offsetInfo.beginOffset, + .commitOffset = pVgCur->offsetInfo.committedOffset, + .numOfRows = pVgCur->numOfRows, .vgStatus = pVgCur->vgStatus}; taosHashPut(pVgOffsetHashMap, vgKey, strlen(vgKey), &info, sizeof(SVgroupSaveInfo)); } @@ -1616,16 +1631,16 @@ SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, pRspObj->rsp.withSchema = true; pRspObj->rsp.blockSchema = taosArrayInit(pRspObj->rsp.blockNum, sizeof(void*)); } - // extract the rows in this data packet + // extract the rows in this data packet for (int32_t i = 0; i < pRspObj->rsp.blockNum; ++i) { SRetrieveTableRspForTmq* pRetrieve = (SRetrieveTableRspForTmq*)taosArrayGetP(pRspObj->rsp.blockData, i); - int64_t rows = htobe64(pRetrieve->numOfRows); + int64_t rows = htobe64(pRetrieve->numOfRows); pVg->numOfRows += rows; (*numOfRows) += rows; - if (needTransformSchema) { //withSchema is false if subscribe subquery, true if subscribe db or stable - SSchemaWrapper *schema = tCloneSSchemaWrapper(&pWrapper->topicHandle->schema); - if(schema){ + if (needTransformSchema) { // withSchema is false if subscribe subquery, true if subscribe db or stable + SSchemaWrapper* schema = tCloneSSchemaWrapper(&pWrapper->topicHandle->schema); + if (schema) { taosArrayPush(pRspObj->rsp.blockSchema, &schema); } } @@ -1649,7 +1664,7 @@ SMqTaosxRspObj* tmqBuildTaosxRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClie // extract the rows in this data packet for (int32_t i = 0; i < pRspObj->rsp.blockNum; ++i) { SRetrieveTableRspForTmq* pRetrieve = (SRetrieveTableRspForTmq*)taosArrayGetP(pRspObj->rsp.blockData, i); - int64_t rows = htobe64(pRetrieve->numOfRows); + int64_t rows = htobe64(pRetrieve->numOfRows); pVg->numOfRows += rows; (*numOfRows) += rows; } @@ -1657,15 +1672,15 @@ SMqTaosxRspObj* tmqBuildTaosxRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClie } static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* pVg, int64_t timeout) { - SMqPollReq req = {0}; - char* msg = NULL; + SMqPollReq req = {0}; + char* msg = NULL; SMqPollCbParam* pParam = NULL; - SMsgSendInfo* sendInfo = NULL; - int code = 0; + SMsgSendInfo* sendInfo = NULL; + int code = 0; tmqBuildConsumeReqImpl(&req, pTmq, timeout, pTopic, pVg); int32_t msgSize = tSerializeSMqPollReq(NULL, 0, &req); - if (msgSize < 0){ + if (msgSize < 0) { code = TSDB_CODE_INVALID_MSG; goto FAIL; } @@ -1709,14 +1724,14 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p char offsetFormatBuf[TSDB_OFFSET_LEN] = {0}; tFormatOffset(offsetFormatBuf, tListLen(offsetFormatBuf), &pVg->offsetInfo.endOffset); code = asyncSendMsgToServer(pTmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo); - tscDebug("consumer:0x%" PRIx64 " send poll to %s vgId:%d, code:%d, epoch %d, req:%s, reqId:0x%" PRIx64, pTmq->consumerId, - pTopic->topicName, pVg->vgId, code, pTmq->epoch, offsetFormatBuf, req.reqId); - if(code != 0){ + tscDebug("consumer:0x%" PRIx64 " send poll to %s vgId:%d, code:%d, epoch %d, req:%s, reqId:0x%" PRIx64, + pTmq->consumerId, pTopic->topicName, pVg->vgId, code, pTmq->epoch, offsetFormatBuf, req.reqId); + if (code != 0) { goto FAIL; } pVg->pollCnt++; - pVg->seekUpdated = false; // reset this flag. + pVg->seekUpdated = false; // reset this flag. pTmq->pollCnt++; return 0; @@ -1727,7 +1742,7 @@ FAIL: // broadcast the poll request to all related vnodes static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { - if(atomic_load_8(&tmq->status) == TMQ_CONSUMER_STATUS__RECOVER){ + if (atomic_load_8(&tmq->status) == TMQ_CONSUMER_STATUS__RECOVER) { return 0; } int32_t code = 0; @@ -1739,7 +1754,7 @@ static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { for (int i = 0; i < numOfTopics; i++) { SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); int32_t numOfVg = taosArrayGetSize(pTopic->vgs); - if(pTopic->noPrivilege){ + if (pTopic->noPrivilege) { tscDebug("consumer:0x%" PRIx64 " has no privilegr for topic:%s", tmq->consumerId, pTopic->topicName); continue; } @@ -1751,9 +1766,10 @@ static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { continue; } - if (tmq->replayEnable && taosGetTimestampMs() - pVg->blockReceiveTs < pVg->blockSleepForReplay) { // less than 10ms - tscTrace("consumer:0x%" PRIx64 " epoch %d, vgId:%d idle for %" PRId64 "ms before start next poll when replay", tmq->consumerId, - tmq->epoch, pVg->vgId, pVg->blockSleepForReplay); + if (tmq->replayEnable && + taosGetTimestampMs() - pVg->blockReceiveTs < pVg->blockSleepForReplay) { // less than 10ms + tscTrace("consumer:0x%" PRIx64 " epoch %d, vgId:%d idle for %" PRId64 "ms before start next poll when replay", + tmq->consumerId, tmq->epoch, pVg->vgId, pVg->blockSleepForReplay); continue; } @@ -1779,13 +1795,14 @@ end: return code; } -static void updateVgInfo(SMqClientVg* pVg, STqOffsetVal* reqOffset, STqOffsetVal* rspOffset, int64_t sver, int64_t ever, int64_t consumerId, bool hasData){ +static void updateVgInfo(SMqClientVg* pVg, STqOffsetVal* reqOffset, STqOffsetVal* rspOffset, int64_t sver, int64_t ever, + int64_t consumerId, bool hasData) { if (!pVg->seekUpdated) { - tscDebug("consumer:0x%" PRIx64" local offset is update, since seekupdate not set", consumerId); - if(hasData) pVg->offsetInfo.beginOffset = *reqOffset; + tscDebug("consumer:0x%" PRIx64 " local offset is update, since seekupdate not set", consumerId); + if (hasData) pVg->offsetInfo.beginOffset = *reqOffset; pVg->offsetInfo.endOffset = *rspOffset; } else { - tscDebug("consumer:0x%" PRIx64" local offset is NOT update, since seekupdate is set", consumerId); + tscDebug("consumer:0x%" PRIx64 " local offset is NOT update, since seekupdate is set", consumerId); } // update the status @@ -1820,17 +1837,19 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { tscDebug("consumer:0x%" PRIx64 " wait for the re-balance, set status to be RECOVER", tmq->consumerId); } else if (pRspWrapper->code == TSDB_CODE_TQ_NO_COMMITTED_OFFSET) { terrno = pRspWrapper->code; - tscError("consumer:0x%" PRIx64 " unexpected rsp from poll, code:%s", tmq->consumerId, tstrerror(pRspWrapper->code)); + tscError("consumer:0x%" PRIx64 " unexpected rsp from poll, code:%s", tmq->consumerId, + tstrerror(pRspWrapper->code)); taosFreeQitem(pRspWrapper); return NULL; - } else{ - if(pRspWrapper->code == TSDB_CODE_VND_INVALID_VGROUP_ID){ // for vnode transform + } else { + if (pRspWrapper->code == TSDB_CODE_VND_INVALID_VGROUP_ID) { // for vnode transform askEp(tmq, NULL, false, true); } - tscError("consumer:0x%" PRIx64 " msg from vgId:%d discarded, since %s", tmq->consumerId, pollRspWrapper->vgId, tstrerror(pRspWrapper->code)); + tscError("consumer:0x%" PRIx64 " msg from vgId:%d discarded, since %s", tmq->consumerId, pollRspWrapper->vgId, + tstrerror(pRspWrapper->code)); taosWLockLatch(&tmq->lock); SMqClientVg* pVg = getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId); - if(pVg) pVg->emptyBlockReceiveTs = taosGetTimestampMs(); + if (pVg) pVg->emptyBlockReceiveTs = taosGetTimestampMs(); taosWUnLockLatch(&tmq->lock); } setVgIdle(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId); @@ -1846,7 +1865,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { SMqClientVg* pVg = getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId); pollRspWrapper->vgHandle = pVg; pollRspWrapper->topicHandle = getTopicInfo(tmq, pollRspWrapper->topicName); - if(pollRspWrapper->vgHandle == NULL || pollRspWrapper->topicHandle == NULL){ + if (pollRspWrapper->vgHandle == NULL || pollRspWrapper->topicHandle == NULL) { tscError("consumer:0x%" PRIx64 " get vg or topic error, topic:%s vgId:%d", tmq->consumerId, pollRspWrapper->topicName, pollRspWrapper->vgId); taosWUnLockLatch(&tmq->lock); @@ -1861,7 +1880,8 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { pVg->epSet = *pollRspWrapper->pEpset; } - updateVgInfo(pVg, &pDataRsp->reqOffset, &pDataRsp->rspOffset, pDataRsp->head.walsver, pDataRsp->head.walever, tmq->consumerId, pDataRsp->blockNum != 0); + updateVgInfo(pVg, &pDataRsp->reqOffset, &pDataRsp->rspOffset, pDataRsp->head.walsver, pDataRsp->head.walever, + tmq->consumerId, pDataRsp->blockNum != 0); char buf[TSDB_OFFSET_LEN] = {0}; tFormatOffset(buf, TSDB_OFFSET_LEN, &pDataRsp->rspOffset); @@ -1878,10 +1898,10 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { SMqRspObj* pRsp = tmqBuildRspFromWrapper(pollRspWrapper, pVg, &numOfRows); tmq->totalRows += numOfRows; pVg->emptyBlockReceiveTs = 0; - if(tmq->replayEnable){ + if (tmq->replayEnable) { pVg->blockReceiveTs = taosGetTimestampMs(); pVg->blockSleepForReplay = pRsp->rsp.sleepTime; - if(pVg->blockSleepForReplay > 0){ + if (pVg->blockSleepForReplay > 0) { int64_t* pRefId1 = taosMemoryMalloc(sizeof(int64_t)); *pRefId1 = tmq->refId; taosTmrStart(tmqReplayTask, pVg->blockSleepForReplay, pRefId1, tmqMgmt.timer); @@ -1897,7 +1917,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { } } else { tscInfo("consumer:0x%" PRIx64 " vgId:%d msg discard since epoch mismatch: msg epoch %d, consumer epoch %d", - tmq->consumerId, pollRspWrapper->vgId, pDataRsp->head.epoch, consumerEpoch); + tmq->consumerId, pollRspWrapper->vgId, pDataRsp->head.epoch, consumerEpoch); setVgIdle(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId); tmqFreeRspWrapper(pRspWrapper); taosFreeQitem(pRspWrapper); @@ -1913,14 +1933,15 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { SMqClientVg* pVg = getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId); pollRspWrapper->vgHandle = pVg; pollRspWrapper->topicHandle = getTopicInfo(tmq, pollRspWrapper->topicName); - if(pollRspWrapper->vgHandle == NULL || pollRspWrapper->topicHandle == NULL){ + if (pollRspWrapper->vgHandle == NULL || pollRspWrapper->topicHandle == NULL) { tscError("consumer:0x%" PRIx64 " get vg or topic error, topic:%s vgId:%d", tmq->consumerId, pollRspWrapper->topicName, pollRspWrapper->vgId); taosWUnLockLatch(&tmq->lock); return NULL; } - updateVgInfo(pVg, &pollRspWrapper->metaRsp.rspOffset, &pollRspWrapper->metaRsp.rspOffset, pollRspWrapper->metaRsp.head.walsver, pollRspWrapper->metaRsp.head.walever, tmq->consumerId, true); + updateVgInfo(pVg, &pollRspWrapper->metaRsp.rspOffset, &pollRspWrapper->metaRsp.rspOffset, + pollRspWrapper->metaRsp.head.walsver, pollRspWrapper->metaRsp.head.walever, tmq->consumerId, true); // build rsp SMqMetaRspObj* pRsp = tmqBuildMetaRspFromWrapper(pollRspWrapper); taosFreeQitem(pRspWrapper); @@ -1928,7 +1949,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { return pRsp; } else { tscInfo("consumer:0x%" PRIx64 " vgId:%d msg discard since epoch mismatch: msg epoch %d, consumer epoch %d", - tmq->consumerId, pollRspWrapper->vgId, pollRspWrapper->metaRsp.head.epoch, consumerEpoch); + tmq->consumerId, pollRspWrapper->vgId, pollRspWrapper->metaRsp.head.epoch, consumerEpoch); setVgIdle(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId); tmqFreeRspWrapper(pRspWrapper); taosFreeQitem(pRspWrapper); @@ -1942,14 +1963,16 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { SMqClientVg* pVg = getVgInfo(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId); pollRspWrapper->vgHandle = pVg; pollRspWrapper->topicHandle = getTopicInfo(tmq, pollRspWrapper->topicName); - if(pollRspWrapper->vgHandle == NULL || pollRspWrapper->topicHandle == NULL){ + if (pollRspWrapper->vgHandle == NULL || pollRspWrapper->topicHandle == NULL) { tscError("consumer:0x%" PRIx64 " get vg or topic error, topic:%s vgId:%d", tmq->consumerId, pollRspWrapper->topicName, pollRspWrapper->vgId); taosWUnLockLatch(&tmq->lock); return NULL; } - updateVgInfo(pVg, &pollRspWrapper->taosxRsp.reqOffset, &pollRspWrapper->taosxRsp.rspOffset, pollRspWrapper->taosxRsp.head.walsver, pollRspWrapper->taosxRsp.head.walever, tmq->consumerId, pollRspWrapper->taosxRsp.blockNum != 0); + updateVgInfo(pVg, &pollRspWrapper->taosxRsp.reqOffset, &pollRspWrapper->taosxRsp.rspOffset, + pollRspWrapper->taosxRsp.head.walsver, pollRspWrapper->taosxRsp.head.walever, tmq->consumerId, + pollRspWrapper->taosxRsp.blockNum != 0); if (pollRspWrapper->taosxRsp.blockNum == 0) { tscDebug("consumer:0x%" PRIx64 " taosx empty block received, vgId:%d, vg total:%" PRId64 ", reqId:0x%" PRIx64, @@ -1967,7 +1990,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { void* pRsp = NULL; int64_t numOfRows = 0; if (pollRspWrapper->taosxRsp.createTableNum == 0) { - tscError("consumer:0x%" PRIx64" createTableNum should > 0 if rsp type is data_meta", tmq->consumerId); + tscError("consumer:0x%" PRIx64 " createTableNum should > 0 if rsp type is data_meta", tmq->consumerId); } else { pRsp = tmqBuildTaosxRspFromWrapper(pollRspWrapper, pVg, &numOfRows); } @@ -1977,7 +2000,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { char buf[TSDB_OFFSET_LEN] = {0}; tFormatOffset(buf, TSDB_OFFSET_LEN, &pVg->offsetInfo.endOffset); tscDebug("consumer:0x%" PRIx64 " process taosx poll rsp, vgId:%d, offset:%s, blocks:%d, rows:%" PRId64 - ", vg total:%" PRId64 ", total:%" PRId64 ", reqId:0x%" PRIx64, + ", vg total:%" PRId64 ", total:%" PRId64 ", reqId:0x%" PRIx64, tmq->consumerId, pVg->vgId, buf, pollRspWrapper->dataRsp.blockNum, numOfRows, pVg->numOfRows, tmq->totalRows, pollRspWrapper->reqId); @@ -1986,12 +2009,12 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { return pRsp; } else { tscInfo("consumer:0x%" PRIx64 " vgId:%d msg discard since epoch mismatch: msg epoch %d, consumer epoch %d", - tmq->consumerId, pollRspWrapper->vgId, pollRspWrapper->taosxRsp.head.epoch, consumerEpoch); + tmq->consumerId, pollRspWrapper->vgId, pollRspWrapper->taosxRsp.head.epoch, consumerEpoch); setVgIdle(tmq, pollRspWrapper->topicName, pollRspWrapper->vgId); tmqFreeRspWrapper(pRspWrapper); taosFreeQitem(pRspWrapper); } - } else if(pRspWrapper->tmqRspType == TMQ_MSG_TYPE__EP_RSP){ + } else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__EP_RSP) { tscDebug("consumer:0x%" PRIx64 " ep msg received", tmq->consumerId); SMqAskEpRspWrapper* pEpRspWrapper = (SMqAskEpRspWrapper*)pRspWrapper; SMqAskEpRsp* rspMsg = &pEpRspWrapper->msg; @@ -2005,7 +2028,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { } TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { - if(tmq == NULL) return NULL; + if (tmq == NULL) return NULL; void* rspObj = NULL; int64_t startTime = taosGetTimestampMs(); @@ -2021,7 +2044,7 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { } while (1) { - if(atomic_load_8(&tmq->status) != TMQ_CONSUMER_STATUS__RECOVER){ + if (atomic_load_8(&tmq->status) != TMQ_CONSUMER_STATUS__RECOVER) { break; } tscInfo("consumer:0x%" PRIx64 " tmq status is recover", tmq->consumerId); @@ -2091,7 +2114,7 @@ static void displayConsumeStatistics(tmq_t* pTmq) { } int32_t tmq_consumer_close(tmq_t* tmq) { - if(tmq == NULL) return TSDB_CODE_INVALID_PARA; + if (tmq == NULL) return TSDB_CODE_INVALID_PARA; tscInfo("consumer:0x%" PRIx64 " start to close consumer, status:%d", tmq->consumerId, tmq->status); displayConsumeStatistics(tmq); @@ -2138,7 +2161,7 @@ const char* tmq_err2str(int32_t err) { } tmq_res_t tmq_get_res_type(TAOS_RES* res) { - if (res == NULL){ + if (res == NULL) { return TMQ_RES_INVALID; } if (TD_RES_TMQ(res)) { @@ -2153,7 +2176,7 @@ tmq_res_t tmq_get_res_type(TAOS_RES* res) { } const char* tmq_get_topic_name(TAOS_RES* res) { - if (res == NULL){ + if (res == NULL) { return NULL; } if (TD_RES_TMQ(res)) { @@ -2171,7 +2194,7 @@ const char* tmq_get_topic_name(TAOS_RES* res) { } const char* tmq_get_db_name(TAOS_RES* res) { - if (res == NULL){ + if (res == NULL) { return NULL; } @@ -2190,7 +2213,7 @@ const char* tmq_get_db_name(TAOS_RES* res) { } int32_t tmq_get_vgroup_id(TAOS_RES* res) { - if (res == NULL){ + if (res == NULL) { return -1; } if (TD_RES_TMQ(res)) { @@ -2208,15 +2231,15 @@ int32_t tmq_get_vgroup_id(TAOS_RES* res) { } int64_t tmq_get_vgroup_offset(TAOS_RES* res) { - if (res == NULL){ + if (res == NULL) { return TSDB_CODE_INVALID_PARA; } if (TD_RES_TMQ(res)) { - SMqRspObj* pRspObj = (SMqRspObj*) res; + SMqRspObj* pRspObj = (SMqRspObj*)res; STqOffsetVal* pOffset = &pRspObj->rsp.reqOffset; if (pOffset->type == TMQ_OFFSET__LOG) { return pRspObj->rsp.reqOffset.version; - }else{ + } else { tscError("invalid offset type:%d", pOffset->type); } } else if (TD_RES_TMQ_META(res)) { @@ -2225,11 +2248,11 @@ int64_t tmq_get_vgroup_offset(TAOS_RES* res) { return pRspObj->metaRsp.rspOffset.version; } } else if (TD_RES_TMQ_METADATA(res)) { - SMqTaosxRspObj* pRspObj = (SMqTaosxRspObj*) res; + SMqTaosxRspObj* pRspObj = (SMqTaosxRspObj*)res; if (pRspObj->rsp.reqOffset.type == TMQ_OFFSET__LOG) { return pRspObj->rsp.reqOffset.version; } - } else{ + } else { tscError("invalid tmq type:%d", *(int8_t*)res); } @@ -2238,7 +2261,7 @@ int64_t tmq_get_vgroup_offset(TAOS_RES* res) { } const char* tmq_get_table_name(TAOS_RES* res) { - if (res == NULL){ + if (res == NULL) { return NULL; } if (TD_RES_TMQ(res)) { @@ -2262,7 +2285,7 @@ const char* tmq_get_table_name(TAOS_RES* res) { void tmq_commit_async(tmq_t* tmq, const TAOS_RES* pRes, tmq_commit_cb* cb, void* param) { if (tmq == NULL) { tscError("invalid tmq handle, null"); - if(cb != NULL) { + if (cb != NULL) { cb(tmq, TSDB_CODE_INVALID_PARA, param); } return; @@ -2274,8 +2297,8 @@ void tmq_commit_async(tmq_t* tmq, const TAOS_RES* pRes, tmq_commit_cb* cb, void* } } -static void commitCallBackFn(tmq_t *UNUSED_PARAM(tmq), int32_t code, void* param) { - SSyncCommitInfo* pInfo = (SSyncCommitInfo*) param; +static void commitCallBackFn(tmq_t* UNUSED_PARAM(tmq), int32_t code, void* param) { + SSyncCommitInfo* pInfo = (SSyncCommitInfo*)param; pInfo->code = code; tsem_post(&pInfo->sem); } @@ -2309,34 +2332,35 @@ int32_t tmq_commit_sync(tmq_t* tmq, const TAOS_RES* pRes) { } // wal range will be ok after calling tmq_get_topic_assignment or poll interface -static int32_t checkWalRange(SVgOffsetInfo* offset, int64_t value){ +static int32_t checkWalRange(SVgOffsetInfo* offset, int64_t value) { if (offset->walVerBegin == -1 || offset->walVerEnd == -1) { tscError("Assignment or poll interface need to be called first"); return TSDB_CODE_TMQ_NEED_INITIALIZED; } if (value != -1 && (value < offset->walVerBegin || value > offset->walVerEnd)) { - tscError("invalid seek params, offset:%" PRId64 ", valid range:[%" PRId64 ", %" PRId64 "]", value, offset->walVerBegin, offset->walVerEnd); + tscError("invalid seek params, offset:%" PRId64 ", valid range:[%" PRId64 ", %" PRId64 "]", value, + offset->walVerBegin, offset->walVerEnd); return TSDB_CODE_TMQ_VERSION_OUT_OF_RANGE; } return 0; } -int32_t tmq_commit_offset_sync(tmq_t *tmq, const char *pTopicName, int32_t vgId, int64_t offset){ +int32_t tmq_commit_offset_sync(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_t offset) { if (tmq == NULL || pTopicName == NULL) { tscError("invalid tmq handle, null"); return TSDB_CODE_INVALID_PARA; } int32_t accId = tmq->pTscObj->acctId; - char tname[TSDB_TOPIC_FNAME_LEN] = {0}; + char tname[TSDB_TOPIC_FNAME_LEN] = {0}; sprintf(tname, "%d.%s", accId, pTopicName); taosWLockLatch(&tmq->lock); SMqClientVg* pVg = NULL; - int32_t code = getClientVg(tmq, tname, vgId, &pVg); - if(code != 0){ + int32_t code = getClientVg(tmq, tname, vgId, &pVg); + if (code != 0) { taosWUnLockLatch(&tmq->lock); return code; } @@ -2353,7 +2377,7 @@ int32_t tmq_commit_offset_sync(tmq_t *tmq, const char *pTopicName, int32_t vgId, SSyncCommitInfo* pInfo = taosMemoryMalloc(sizeof(SSyncCommitInfo)); if (pInfo == NULL) { - tscError("consumer:0x%"PRIx64" failed to prepare seek operation", tmq->consumerId); + tscError("consumer:0x%" PRIx64 " failed to prepare seek operation", tmq->consumerId); return TSDB_CODE_OUT_OF_MEMORY; } @@ -2361,36 +2385,38 @@ int32_t tmq_commit_offset_sync(tmq_t *tmq, const char *pTopicName, int32_t vgId, pInfo->code = 0; code = asyncCommitOffset(tmq, tname, vgId, &offsetVal, commitCallBackFn, pInfo); - if(code == 0){ + if (code == 0) { tsem_wait(&pInfo->sem); code = pInfo->code; } - if(code == TSDB_CODE_TMQ_SAME_COMMITTED_VALUE) code = TSDB_CODE_SUCCESS; + if (code == TSDB_CODE_TMQ_SAME_COMMITTED_VALUE) code = TSDB_CODE_SUCCESS; tsem_destroy(&pInfo->sem); taosMemoryFree(pInfo); - tscInfo("consumer:0x%" PRIx64 " sync send commit to vgId:%d, offset:%" PRId64" code:%s", tmq->consumerId, vgId, offset, tstrerror(code)); + tscInfo("consumer:0x%" PRIx64 " sync send commit to vgId:%d, offset:%" PRId64 " code:%s", tmq->consumerId, vgId, + offset, tstrerror(code)); return code; } -void tmq_commit_offset_async(tmq_t *tmq, const char *pTopicName, int32_t vgId, int64_t offset, tmq_commit_cb *cb, void *param){ +void tmq_commit_offset_async(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_t offset, tmq_commit_cb* cb, + void* param) { int32_t code = 0; if (tmq == NULL || pTopicName == NULL) { tscError("invalid tmq handle, null"); code = TSDB_CODE_INVALID_PARA; - goto end; + goto end; } int32_t accId = tmq->pTscObj->acctId; - char tname[TSDB_TOPIC_FNAME_LEN] = {0}; + char tname[TSDB_TOPIC_FNAME_LEN] = {0}; sprintf(tname, "%d.%s", accId, pTopicName); taosWLockLatch(&tmq->lock); SMqClientVg* pVg = NULL; code = getClientVg(tmq, tname, vgId, &pVg); - if(code != 0){ + if (code != 0) { taosWUnLockLatch(&tmq->lock); goto end; } @@ -2407,11 +2433,12 @@ void tmq_commit_offset_async(tmq_t *tmq, const char *pTopicName, int32_t vgId, i code = asyncCommitOffset(tmq, tname, vgId, &offsetVal, cb, param); - tscInfo("consumer:0x%" PRIx64 " async send commit to vgId:%d, offset:%" PRId64" code:%s", tmq->consumerId, vgId, offset, tstrerror(code)); + tscInfo("consumer:0x%" PRIx64 " async send commit to vgId:%d, offset:%" PRId64 " code:%s", tmq->consumerId, vgId, + offset, tstrerror(code)); end: - if(code != 0 && cb != NULL){ - if(code == TSDB_CODE_TMQ_SAME_COMMITTED_VALUE) code = TSDB_CODE_SUCCESS; + if (code != 0 && cb != NULL) { + if (code == TSDB_CODE_TMQ_SAME_COMMITTED_VALUE) code = TSDB_CODE_SUCCESS; cb(tmq, code, param); } } @@ -2431,14 +2458,13 @@ int32_t askEpCb(void* param, SDataBuf* pMsg, int32_t code) { SMqRspHead* head = pMsg->pData; int32_t epoch = atomic_load_32(&tmq->epoch); - tscInfo("consumer:0x%" PRIx64 ", recv ep, msg epoch %d, current epoch %d", tmq->consumerId, - head->epoch, epoch); - if(pParam->sync){ + tscInfo("consumer:0x%" PRIx64 ", recv ep, msg epoch %d, current epoch %d", tmq->consumerId, head->epoch, epoch); + if (pParam->sync) { SMqAskEpRsp rsp = {0}; tDecodeSMqAskEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &rsp); doUpdateLocalEp(tmq, head->epoch, &rsp); tDeleteSMqAskEpRsp(&rsp); - }else{ + } else { SMqAskEpRspWrapper* pWrapper = taosAllocateQitem(sizeof(SMqAskEpRspWrapper), DEF_QITEM, 0); if (pWrapper == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -2457,7 +2483,7 @@ END: taosReleaseRef(tmqMgmt.rsetId, pParam->refId); FAIL: - if(pParam->sync){ + if (pParam->sync) { SAskEpInfo* pInfo = pParam->pParam; pInfo->code = code; tsem_post(&pInfo->sem); @@ -2485,11 +2511,11 @@ int32_t syncAskEp(tmq_t* pTmq) { void askEp(tmq_t* pTmq, void* param, bool sync, bool updateEpSet) { SMqAskEpReq req = {0}; req.consumerId = pTmq->consumerId; - req.epoch = updateEpSet ? -1 :pTmq->epoch; + req.epoch = updateEpSet ? -1 : pTmq->epoch; strcpy(req.cgroup, pTmq->groupId); - int code = 0; + int code = 0; SMqAskEpCbParam* pParam = NULL; - void* pReq = NULL; + void* pReq = NULL; int32_t tlen = tSerializeSMqAskEpReq(NULL, 0, &req); if (tlen < 0) { @@ -2541,7 +2567,7 @@ void askEp(tmq_t* pTmq, void* param, bool sync, bool updateEpSet) { int64_t transporterId = 0; code = asyncSendMsgToServer(pTmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); - if(code == 0){ + if (code == 0) { return; } @@ -2566,7 +2592,7 @@ int32_t tmqCommitDone(SMqCommitCbParamSet* pParamSet) { } // if no more waiting rsp - if(pParamSet->callbackFn != NULL){ + if (pParamSet->callbackFn != NULL) { pParamSet->callbackFn(tmq, pParamSet->code, pParamSet->userParam); } @@ -2578,10 +2604,12 @@ int32_t tmqCommitDone(SMqCommitCbParamSet* pParamSet) { void commitRspCountDown(SMqCommitCbParamSet* pParamSet, int64_t consumerId, const char* pTopic, int32_t vgId) { int32_t waitingRspNum = atomic_sub_fetch_32(&pParamSet->waitingRspNum, 1); if (waitingRspNum == 0) { - tscInfo("consumer:0x%" PRIx64 " topic:%s vgId:%d all commit-rsp received, commit completed", consumerId, pTopic, vgId); + tscInfo("consumer:0x%" PRIx64 " topic:%s vgId:%d all commit-rsp received, commit completed", consumerId, pTopic, + vgId); tmqCommitDone(pParamSet); } else { - tscInfo("consumer:0x%" PRIx64 " topic:%s vgId:%d commit-rsp received, remain:%d", consumerId, pTopic, vgId, waitingRspNum); + tscInfo("consumer:0x%" PRIx64 " topic:%s vgId:%d commit-rsp received, remain:%d", consumerId, pTopic, vgId, + waitingRspNum); } } @@ -2590,7 +2618,8 @@ SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4) { pRspObj->resIter++; if (pRspObj->resIter < pRspObj->rsp.blockNum) { - SRetrieveTableRspForTmq* pRetrieveTmq = (SRetrieveTableRspForTmq*)taosArrayGetP(pRspObj->rsp.blockData, pRspObj->resIter); + SRetrieveTableRspForTmq* pRetrieveTmq = + (SRetrieveTableRspForTmq*)taosArrayGetP(pRspObj->rsp.blockData, pRspObj->resIter); if (pRspObj->rsp.withSchema) { SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(pRspObj->rsp.blockSchema, pRspObj->resIter); setResSchemaInfo(&pRspObj->resInfo, pSW->pSchema, pSW->nCols); @@ -2609,7 +2638,7 @@ SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4) { // TODO handle the compressed case pRspObj->resInfo.totalRows += pRspObj->resInfo.numOfRows; setResultDataPtr(&pRspObj->resInfo, pRspObj->resInfo.fields, pRspObj->resInfo.numOfCols, pRspObj->resInfo.numOfRows, - convertUcs4); + convertUcs4); return &pRspObj->resInfo; } @@ -2619,7 +2648,7 @@ SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4) { static int32_t tmqGetWalInfoCb(void* param, SDataBuf* pMsg, int32_t code) { SMqVgWalInfoParam* pParam = param; - SMqVgCommon* pCommon = pParam->pCommon; + SMqVgCommon* pCommon = pParam->pCommon; int32_t total = atomic_add_fetch_32(&pCommon->numOfRsp, 1); if (code != TSDB_CODE_SUCCESS) { @@ -2628,7 +2657,7 @@ static int32_t tmqGetWalInfoCb(void* param, SDataBuf* pMsg, int32_t code) { pCommon->code = code; } else { SMqDataRsp rsp; - SDecoder decoder; + SDecoder decoder; tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead)); tDecodeMqDataRsp(&decoder, &rsp); tDecoderClear(&decoder); @@ -2656,7 +2685,7 @@ static int32_t tmqGetWalInfoCb(void* param, SDataBuf* pMsg, int32_t code) { } static void destroyCommonInfo(SMqVgCommon* pCommon) { - if(pCommon == NULL){ + if (pCommon == NULL) { return; } taosArrayDestroy(pCommon->pList); @@ -2666,7 +2695,7 @@ static void destroyCommonInfo(SMqVgCommon* pCommon) { taosMemoryFree(pCommon); } -static bool isInSnapshotMode(int8_t type, bool useSnapshot){ +static bool isInSnapshotMode(int8_t type, bool useSnapshot) { if ((type < TMQ_OFFSET__LOG && useSnapshot) || type > TMQ_OFFSET__LOG) { return true; } @@ -2676,7 +2705,7 @@ static bool isInSnapshotMode(int8_t type, bool useSnapshot){ static int32_t tmCommittedCb(void* param, SDataBuf* pMsg, int32_t code) { SMqCommittedParam* pParam = param; - if (code != 0){ + if (code != 0) { goto end; } if (pMsg) { @@ -2689,8 +2718,8 @@ static int32_t tmCommittedCb(void* param, SDataBuf* pMsg, int32_t code) { tDecoderClear(&decoder); } - end: - if(pMsg){ +end: + if (pMsg) { taosMemoryFree(pMsg->pData); taosMemoryFree(pMsg->pEpSet); } @@ -2699,8 +2728,8 @@ static int32_t tmCommittedCb(void* param, SDataBuf* pMsg, int32_t code) { return 0; } -int64_t getCommittedFromServer(tmq_t *tmq, char* tname, int32_t vgId, SEpSet* epSet){ - int32_t code = 0; +int64_t getCommittedFromServer(tmq_t* tmq, char* tname, int32_t vgId, SEpSet* epSet) { + int32_t code = 0; SMqVgOffset pOffset = {0}; pOffset.consumerId = tmq->consumerId; @@ -2753,7 +2782,7 @@ int64_t getCommittedFromServer(tmq_t *tmq, char* tname, int32_t vgId, SEpSet* ep int64_t transporterId = 0; code = asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, epSet, &transporterId, sendInfo); - if(code != 0){ + if (code != 0) { taosMemoryFree(buf); taosMemoryFree(sendInfo); tsem_destroy(&pParam->sem); @@ -2763,10 +2792,10 @@ int64_t getCommittedFromServer(tmq_t *tmq, char* tname, int32_t vgId, SEpSet* ep tsem_wait(&pParam->sem); code = pParam->code; - if(code == TSDB_CODE_SUCCESS){ - if(pParam->vgOffset.offset.val.type == TMQ_OFFSET__LOG){ + if (code == TSDB_CODE_SUCCESS) { + if (pParam->vgOffset.offset.val.type == TMQ_OFFSET__LOG) { code = pParam->vgOffset.offset.val.version; - }else{ + } else { code = TSDB_CODE_TMQ_SNAPSHOT_ERROR; } } @@ -2776,27 +2805,27 @@ int64_t getCommittedFromServer(tmq_t *tmq, char* tname, int32_t vgId, SEpSet* ep return code; } -int64_t tmq_position(tmq_t *tmq, const char *pTopicName, int32_t vgId){ +int64_t tmq_position(tmq_t* tmq, const char* pTopicName, int32_t vgId) { if (tmq == NULL || pTopicName == NULL) { tscError("invalid tmq handle, null"); return TSDB_CODE_INVALID_PARA; } int32_t accId = tmq->pTscObj->acctId; - char tname[TSDB_TOPIC_FNAME_LEN] = {0}; + char tname[TSDB_TOPIC_FNAME_LEN] = {0}; sprintf(tname, "%d.%s", accId, pTopicName); taosWLockLatch(&tmq->lock); SMqClientVg* pVg = NULL; - int32_t code = getClientVg(tmq, tname, vgId, &pVg); - if(code != 0){ + int32_t code = getClientVg(tmq, tname, vgId, &pVg); + if (code != 0) { taosWUnLockLatch(&tmq->lock); return code; } SVgOffsetInfo* pOffsetInfo = &pVg->offsetInfo; - int32_t type = pOffsetInfo->endOffset.type; + int32_t type = pOffsetInfo->endOffset.type; if (isInSnapshotMode(type, tmq->useSnapshot)) { tscError("consumer:0x%" PRIx64 " offset type:%d not wal version, position error", tmq->consumerId, type); taosWUnLockLatch(&tmq->lock); @@ -2808,26 +2837,26 @@ int64_t tmq_position(tmq_t *tmq, const char *pTopicName, int32_t vgId){ taosWUnLockLatch(&tmq->lock); return code; } - SEpSet epSet = pVg->epSet; + SEpSet epSet = pVg->epSet; int64_t begin = pVg->offsetInfo.walVerBegin; int64_t end = pVg->offsetInfo.walVerEnd; taosWUnLockLatch(&tmq->lock); int64_t position = 0; - if(type == TMQ_OFFSET__LOG){ + if (type == TMQ_OFFSET__LOG) { position = pOffsetInfo->endOffset.version; - }else if(type == TMQ_OFFSET__RESET_EARLIEST || type == TMQ_OFFSET__RESET_LATEST){ + } else if (type == TMQ_OFFSET__RESET_EARLIEST || type == TMQ_OFFSET__RESET_LATEST) { code = getCommittedFromServer(tmq, tname, vgId, &epSet); - if(code == TSDB_CODE_TMQ_NO_COMMITTED){ - if(type == TMQ_OFFSET__RESET_EARLIEST){ + if (code == TSDB_CODE_TMQ_NO_COMMITTED) { + if (type == TMQ_OFFSET__RESET_EARLIEST) { position = begin; - } else if(type == TMQ_OFFSET__RESET_LATEST){ + } else if (type == TMQ_OFFSET__RESET_LATEST) { position = end; } - }else{ + } else { position = code; } - }else{ + } else { tscError("consumer:0x%" PRIx64 " offset type:%d can not be reach here", tmq->consumerId, type); } @@ -2835,40 +2864,42 @@ int64_t tmq_position(tmq_t *tmq, const char *pTopicName, int32_t vgId){ return position; } -int64_t tmq_committed(tmq_t *tmq, const char *pTopicName, int32_t vgId){ +int64_t tmq_committed(tmq_t* tmq, const char* pTopicName, int32_t vgId) { if (tmq == NULL || pTopicName == NULL) { tscError("invalid tmq handle, null"); return TSDB_CODE_INVALID_PARA; } int32_t accId = tmq->pTscObj->acctId; - char tname[TSDB_TOPIC_FNAME_LEN] = {0}; + char tname[TSDB_TOPIC_FNAME_LEN] = {0}; sprintf(tname, "%d.%s", accId, pTopicName); taosWLockLatch(&tmq->lock); SMqClientVg* pVg = NULL; - int32_t code = getClientVg(tmq, tname, vgId, &pVg); - if(code != 0){ + int32_t code = getClientVg(tmq, tname, vgId, &pVg); + if (code != 0) { taosWUnLockLatch(&tmq->lock); return code; } SVgOffsetInfo* pOffsetInfo = &pVg->offsetInfo; if (isInSnapshotMode(pOffsetInfo->endOffset.type, tmq->useSnapshot)) { - tscError("consumer:0x%" PRIx64 " offset type:%d not wal version, committed error", tmq->consumerId, pOffsetInfo->endOffset.type); + tscError("consumer:0x%" PRIx64 " offset type:%d not wal version, committed error", tmq->consumerId, + pOffsetInfo->endOffset.type); taosWUnLockLatch(&tmq->lock); return TSDB_CODE_TMQ_SNAPSHOT_ERROR; } if (isInSnapshotMode(pOffsetInfo->committedOffset.type, tmq->useSnapshot)) { - tscError("consumer:0x%" PRIx64 " offset type:%d not wal version, committed error", tmq->consumerId, pOffsetInfo->committedOffset.type); + tscError("consumer:0x%" PRIx64 " offset type:%d not wal version, committed error", tmq->consumerId, + pOffsetInfo->committedOffset.type); taosWUnLockLatch(&tmq->lock); return TSDB_CODE_TMQ_SNAPSHOT_ERROR; } int64_t committed = 0; - if(pOffsetInfo->committedOffset.type == TMQ_OFFSET__LOG){ + if (pOffsetInfo->committedOffset.type == TMQ_OFFSET__LOG) { committed = pOffsetInfo->committedOffset.version; taosWUnLockLatch(&tmq->lock); goto end; @@ -2885,7 +2916,7 @@ end: int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_assignment** assignment, int32_t* numOfAssignment) { - if(tmq == NULL || pTopicName == NULL || assignment == NULL || numOfAssignment == NULL){ + if (tmq == NULL || pTopicName == NULL || assignment == NULL || numOfAssignment == NULL) { tscError("invalid tmq handle, null"); return TSDB_CODE_INVALID_PARA; } @@ -2909,7 +2940,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a *numOfAssignment = taosArrayGetSize(pTopic->vgs); for (int32_t j = 0; j < (*numOfAssignment); ++j) { SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, j); - int32_t type = pClientVg->offsetInfo.beginOffset.type; + int32_t type = pClientVg->offsetInfo.beginOffset.type; if (isInSnapshotMode(type, tmq->useSnapshot)) { tscError("consumer:0x%" PRIx64 " offset type:%d not wal version, assignment not allowed", tmq->consumerId, type); code = TSDB_CODE_TMQ_SNAPSHOT_ERROR; @@ -2939,8 +2970,8 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a pAssignment->begin = pClientVg->offsetInfo.walVerBegin; pAssignment->end = pClientVg->offsetInfo.walVerEnd; pAssignment->vgId = pClientVg->vgId; - tscInfo("consumer:0x%" PRIx64 " get assignment from local:%d->%" PRId64, tmq->consumerId, - pAssignment->vgId, pAssignment->currentOffset); + tscInfo("consumer:0x%" PRIx64 " get assignment from local:%d->%" PRId64, tmq->consumerId, pAssignment->vgId, + pAssignment->currentOffset); } if (needFetch) { @@ -2951,7 +2982,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a goto end; } - pCommon->pList= taosArrayInit(4, sizeof(tmq_topic_assignment)); + pCommon->pList = taosArrayInit(4, sizeof(tmq_topic_assignment)); tsem_init(&pCommon->rsp, 0, 0); taosThreadMutexInit(&pCommon->mutex, 0); pCommon->pTopicName = taosStrdup(pTopic->topicName); @@ -3019,7 +3050,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a tscInfo("consumer:0x%" PRIx64 " %s retrieve wal info vgId:%d, epoch %d, req:%s, reqId:0x%" PRIx64, tmq->consumerId, pTopic->topicName, pClientVg->vgId, tmq->epoch, offsetFormatBuf, req.reqId); code = asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pClientVg->epSet, &transporterId, sendInfo); - if(code != 0){ + if (code != 0) { taosMemoryFree(pParam); taosMemoryFree(msg); goto end; @@ -3034,7 +3065,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a goto end; } int32_t num = taosArrayGetSize(pCommon->pList); - for(int32_t i = 0; i < num; ++i) { + for (int32_t i = 0; i < num; ++i) { (*assignment)[i] = *(tmq_topic_assignment*)taosArrayGet(pCommon->pList, i); } *numOfAssignment = num; @@ -3042,14 +3073,15 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a for (int32_t j = 0; j < (*numOfAssignment); ++j) { tmq_topic_assignment* p = &(*assignment)[j]; - for(int32_t i = 0; i < taosArrayGetSize(pTopic->vgs); ++i) { + for (int32_t i = 0; i < taosArrayGetSize(pTopic->vgs); ++i) { SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, i); if (pClientVg->vgId != p->vgId) { continue; } SVgOffsetInfo* pOffsetInfo = &pClientVg->offsetInfo; - tscInfo("consumer:0x%" PRIx64 " %s vgId:%d offset is update to:%"PRId64, tmq->consumerId, pTopic->topicName, p->vgId, p->currentOffset); + tscInfo("consumer:0x%" PRIx64 " %s vgId:%d offset is update to:%" PRId64, tmq->consumerId, pTopic->topicName, + p->vgId, p->currentOffset); pOffsetInfo->walVerBegin = p->begin; pOffsetInfo->walVerEnd = p->end; @@ -3058,7 +3090,7 @@ int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_a } end: - if(code != TSDB_CODE_SUCCESS){ + if (code != TSDB_CODE_SUCCESS) { taosMemoryFree(*assignment); *assignment = NULL; *numOfAssignment = 0; @@ -3069,11 +3101,11 @@ end: } void tmq_free_assignment(tmq_topic_assignment* pAssignment) { - if (pAssignment == NULL) { - return; - } + if (pAssignment == NULL) { + return; + } - taosMemoryFree(pAssignment); + taosMemoryFree(pAssignment); } static int32_t tmqSeekCb(void* param, SDataBuf* pMsg, int32_t code) { @@ -3087,7 +3119,8 @@ static int32_t tmqSeekCb(void* param, SDataBuf* pMsg, int32_t code) { return 0; } -// seek interface have to send msg to server to cancel push handle if needed, because consumer may be in wait status if there is no data to poll +// seek interface have to send msg to server to cancel push handle if needed, because consumer may be in wait status if +// there is no data to poll int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_t offset) { if (tmq == NULL || pTopicName == NULL) { tscError("invalid tmq handle, null"); @@ -3095,14 +3128,14 @@ int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_ } int32_t accId = tmq->pTscObj->acctId; - char tname[TSDB_TOPIC_FNAME_LEN] = {0}; + char tname[TSDB_TOPIC_FNAME_LEN] = {0}; sprintf(tname, "%d.%s", accId, pTopicName); taosWLockLatch(&tmq->lock); SMqClientVg* pVg = NULL; - int32_t code = getClientVg(tmq, tname, vgId, &pVg); - if(code != 0){ + int32_t code = getClientVg(tmq, tname, vgId, &pVg); + if (code != 0) { taosWUnLockLatch(&tmq->lock); return code; } @@ -3174,7 +3207,7 @@ int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_ int64_t transporterId = 0; code = asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); - if(code != 0){ + if (code != 0) { taosMemoryFree(msg); taosMemoryFree(sendInfo); tsem_destroy(&pParam->sem); @@ -3192,9 +3225,9 @@ int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_ return code; } -TAOS *tmq_get_connect(tmq_t *tmq){ +TAOS* tmq_get_connect(tmq_t* tmq) { if (tmq && tmq->pTscObj) { - return (TAOS *)(&(tmq->pTscObj->id)); + return (TAOS*)(&(tmq->pTscObj->id)); } return NULL; } diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index e8458a2fdc..f60d8035e3 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -15,20 +15,20 @@ #define _DEFAULT_SOURCE #include "mndConsumer.h" -#include "mndPrivilege.h" -#include "mndVgroup.h" -#include "mndShow.h" #include "mndDb.h" +#include "mndPrivilege.h" +#include "mndShow.h" #include "mndSubscribe.h" #include "mndTopic.h" #include "mndTrans.h" +#include "mndVgroup.h" #include "tcompare.h" #include "tname.h" #define MND_CONSUMER_VER_NUMBER 2 #define MND_CONSUMER_RESERVE_SIZE 64 -#define MND_MAX_GROUP_PER_TOPIC 100 +#define MND_MAX_GROUP_PER_TOPIC 100 static int32_t mndConsumerActionInsert(SSdb *pSdb, SMqConsumerObj *pConsumer); static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer); @@ -56,7 +56,7 @@ int32_t mndInitConsumer(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_TMQ_SUBSCRIBE, mndProcessSubscribeReq); mndSetMsgHandle(pMnode, TDMT_MND_TMQ_HB, mndProcessMqHbReq); mndSetMsgHandle(pMnode, TDMT_MND_TMQ_ASK_EP, mndProcessAskEpReq); -// mndSetMsgHandle(pMnode, TDMT_MND_TMQ_TIMER, mndProcessMqTimerMsg); + // mndSetMsgHandle(pMnode, TDMT_MND_TMQ_TIMER, mndProcessMqTimerMsg); mndSetMsgHandle(pMnode, TDMT_MND_TMQ_CONSUMER_RECOVER, mndProcessConsumerRecoverMsg); mndSetMsgHandle(pMnode, TDMT_MND_TMQ_LOST_CONSUMER_CLEAR, mndProcessConsumerClearMsg); @@ -68,10 +68,11 @@ int32_t mndInitConsumer(SMnode *pMnode) { void mndCleanupConsumer(SMnode *pMnode) {} -void mndDropConsumerFromSdb(SMnode *pMnode, int64_t consumerId, SRpcHandleInfo* info){ +void mndDropConsumerFromSdb(SMnode *pMnode, int64_t consumerId, SRpcHandleInfo *info) { SMqConsumerClearMsg *pClearMsg = rpcMallocCont(sizeof(SMqConsumerClearMsg)); if (pClearMsg == NULL) { - mError("consumer:0x%"PRIx64" failed to clear consumer due to out of memory. alloc size:%d", consumerId, (int32_t)sizeof(SMqConsumerClearMsg)); + mError("consumer:0x%" PRIx64 " failed to clear consumer due to out of memory. alloc size:%d", consumerId, + (int32_t)sizeof(SMqConsumerClearMsg)); return; } @@ -88,13 +89,14 @@ void mndDropConsumerFromSdb(SMnode *pMnode, int64_t consumerId, SRpcHandleInfo* return; } -static int32_t validateTopics(STrans *pTrans, const SArray *pTopicList, SMnode *pMnode, const char *pUser, bool enableReplay) { +static int32_t validateTopics(STrans *pTrans, const SArray *pTopicList, SMnode *pMnode, const char *pUser, + bool enableReplay) { SMqTopicObj *pTopic = NULL; - int32_t code = 0; + int32_t code = 0; int32_t numOfTopics = taosArrayGetSize(pTopicList); for (int32_t i = 0; i < numOfTopics; i++) { - char *pOneTopic = taosArrayGetP(pTopicList, i); + char *pOneTopic = taosArrayGetP(pTopicList, i); pTopic = mndAcquireTopic(pMnode, pOneTopic); if (pTopic == NULL) { // terrno has been set by callee function code = -1; @@ -112,11 +114,11 @@ static int32_t validateTopics(STrans *pTrans, const SArray *pTopicList, SMnode * goto FAILED; } - if(enableReplay){ - if(pTopic->subType != TOPIC_SUB_TYPE__COLUMN){ + if (enableReplay) { + if (pTopic->subType != TOPIC_SUB_TYPE__COLUMN) { code = TSDB_CODE_TMQ_REPLAY_NOT_SUPPORT; goto FAILED; - }else if(pTopic->ntbUid == 0 && pTopic->ctbStbUid == 0) { + } else if (pTopic->ntbUid == 0 && pTopic->ctbStbUid == 0) { SDbObj *pDb = mndAcquireDb(pMnode, pTopic->db); if (pDb == NULL) { code = -1; @@ -132,7 +134,7 @@ static int32_t validateTopics(STrans *pTrans, const SArray *pTopicList, SMnode * } mndTransSetDbName(pTrans, pOneTopic, NULL); - if(mndTransCheckConflict(pMnode, pTrans) != 0){ + if (mndTransCheckConflict(pMnode, pTrans) != 0) { code = -1; goto FAILED; } @@ -172,7 +174,7 @@ static int32_t mndProcessConsumerRecoverMsg(SRpcMsg *pMsg) { if (pTrans == NULL) { goto FAIL; } - if(validateTopics(pTrans, pConsumer->assignedTopics, pMnode, pMsg->info.conn.user, false) != 0){ + if (validateTopics(pTrans, pConsumer->assignedTopics, pMnode, pMsg->info.conn.user, false) != 0) { goto FAIL; } @@ -197,7 +199,7 @@ static int32_t mndProcessConsumerClearMsg(SRpcMsg *pMsg) { SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, pClearMsg->consumerId); if (pConsumer == NULL) { - mError("consumer:0x%"PRIx64" failed to be found to clear it", pClearMsg->consumerId); + mError("consumer:0x%" PRIx64 " failed to be found to clear it", pClearMsg->consumerId); return 0; } @@ -226,15 +228,15 @@ FAIL: return -1; } -static int32_t checkPrivilege(SMnode *pMnode, SMqConsumerObj *pConsumer, SMqHbRsp *rsp, char* user){ +static int32_t checkPrivilege(SMnode *pMnode, SMqConsumerObj *pConsumer, SMqHbRsp *rsp, char *user) { rsp->topicPrivileges = taosArrayInit(taosArrayGetSize(pConsumer->currentTopics), sizeof(STopicPrivilege)); - if(rsp->topicPrivileges == NULL){ + if (rsp->topicPrivileges == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno; } - for(int32_t i = 0; i < taosArrayGetSize(pConsumer->currentTopics); i++){ - char *topic = taosArrayGetP(pConsumer->currentTopics, i); - SMqTopicObj* pTopic = mndAcquireTopic(pMnode, topic); + for (int32_t i = 0; i < taosArrayGetSize(pConsumer->currentTopics); i++) { + char *topic = taosArrayGetP(pConsumer->currentTopics, i); + SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic); if (pTopic == NULL) { // terrno has been set by callee function continue; } @@ -252,10 +254,10 @@ static int32_t checkPrivilege(SMnode *pMnode, SMqConsumerObj *pConsumer, SMqHbR } static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) { - int32_t code = 0; - SMnode *pMnode = pMsg->info.node; - SMqHbReq req = {0}; - SMqHbRsp rsp = {0}; + int32_t code = 0; + SMnode *pMnode = pMsg->info.node; + SMqHbReq req = {0}; + SMqHbRsp rsp = {0}; SMqConsumerObj *pConsumer = NULL; if (tDeserializeSMqHbReq(pMsg->pCont, pMsg->contLen, &req) < 0) { @@ -264,7 +266,7 @@ static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) { goto end; } - int64_t consumerId = req.consumerId; + int64_t consumerId = req.consumerId; pConsumer = mndAcquireConsumer(pMnode, consumerId); if (pConsumer == NULL) { mError("consumer:0x%" PRIx64 " not exist", consumerId); @@ -273,7 +275,7 @@ static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) { goto end; } code = checkPrivilege(pMnode, pConsumer, &rsp, pMsg->info.conn.user); - if(code != 0){ + if (code != 0) { goto end; } @@ -295,12 +297,12 @@ static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) { tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &pRpcMsg); } - for(int i = 0; i < taosArrayGetSize(req.topics); i++){ - TopicOffsetRows* data = taosArrayGet(req.topics, i); + for (int i = 0; i < taosArrayGetSize(req.topics); i++) { + TopicOffsetRows *data = taosArrayGet(req.topics, i); mInfo("heartbeat report offset rows.%s:%s", pConsumer->cgroup, data->topicName); SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, pConsumer->cgroup, data->topicName); - if(pSub == NULL){ + if (pSub == NULL) { #ifdef TMQ_DEBUG ASSERT(0); #endif @@ -308,7 +310,7 @@ static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) { } taosWLockLatch(&pSub->lock); SMqConsumerEp *pConsumerEp = taosHashGet(pSub->consumerHash, &consumerId, sizeof(int64_t)); - if(pConsumerEp){ + if (pConsumerEp) { taosArrayDestroy(pConsumerEp->offsetRows); pConsumerEp->offsetRows = data->offsetRows; data->offsetRows = NULL; @@ -413,7 +415,7 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { char *topic = taosArrayGetP(pConsumer->currentTopics, i); SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, pConsumer->cgroup, topic); // txn guarantees pSub is created - if(pSub == NULL) { + if (pSub == NULL) { #ifdef TMQ_DEBUG ASSERT(0); #endif @@ -426,7 +428,7 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { // 2.1 fetch topic schema SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic); - if(pTopic == NULL) { + if (pTopic == NULL) { #ifdef TMQ_DEBUG ASSERT(0); #endif @@ -460,12 +462,12 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { for (int32_t j = 0; j < vgNum; j++) { SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, j); -// char offsetKey[TSDB_PARTITION_KEY_LEN]; -// mndMakePartitionKey(offsetKey, pConsumer->cgroup, topic, pVgEp->vgId); + // char offsetKey[TSDB_PARTITION_KEY_LEN]; + // mndMakePartitionKey(offsetKey, pConsumer->cgroup, topic, pVgEp->vgId); - if(epoch == -1){ + if (epoch == -1) { SVgObj *pVgroup = mndAcquireVgroup(pMnode, pVgEp->vgId); - if(pVgroup){ + if (pVgroup) { pVgEp->epSet = mndGetVgroupEpset(pMnode, pVgroup); mndReleaseVgroup(pMnode, pVgroup); } @@ -495,7 +497,7 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { goto FAIL; } - SMqRspHead* pHead = buf; + SMqRspHead *pHead = buf; pHead->mqMsgType = TMQ_MSG_TYPE__EP_RSP; pHead->epoch = serverEpoch; @@ -503,7 +505,6 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { pHead->walsver = 0; pHead->walever = 0; - void *abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); tEncodeSMqAskEpRsp(&abuf, &rsp); @@ -555,21 +556,20 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { SCMSubscribeReq subscribe = {0}; tDeserializeSCMSubscribeReq(msgStr, &subscribe); - int64_t consumerId = subscribe.consumerId; + int64_t consumerId = subscribe.consumerId; char *cgroup = subscribe.cgroup; SMqConsumerObj *pExistedConsumer = NULL; SMqConsumerObj *pConsumerNew = NULL; - STrans *pTrans = NULL; + STrans *pTrans = NULL; - SArray *pTopicList = subscribe.topicNames; taosArraySort(pTopicList, taosArrayCompareString); taosArrayRemoveDuplicate(pTopicList, taosArrayCompareString, freeItem); int32_t newTopicNum = taosArrayGetSize(pTopicList); - for(int i = 0; i < newTopicNum; i++){ - int32_t gNum = mndGetGroupNumByTopic(pMnode, (const char*)taosArrayGetP(pTopicList, i)); - if(gNum >= MND_MAX_GROUP_PER_TOPIC){ + for (int i = 0; i < newTopicNum; i++) { + int32_t gNum = mndGetGroupNumByTopic(pMnode, (const char *)taosArrayGetP(pTopicList, i)); + if (gNum >= MND_MAX_GROUP_PER_TOPIC) { terrno = TSDB_CODE_TMQ_GROUP_OUT_OF_RANGE; code = terrno; goto _over; @@ -600,7 +600,7 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { pConsumerNew->autoCommitInterval = subscribe.autoCommitInterval; pConsumerNew->resetOffsetCfg = subscribe.resetOffsetCfg; -// pConsumerNew->updateType = CONSUMER_UPDATE_SUB; // use insert logic + // pConsumerNew->updateType = CONSUMER_UPDATE_SUB; // use insert logic taosArrayDestroy(pConsumerNew->assignedTopics); pConsumerNew->assignedTopics = taosArrayDup(pTopicList, topicNameDup); @@ -787,16 +787,15 @@ CM_DECODE_OVER: } static int32_t mndConsumerActionInsert(SSdb *pSdb, SMqConsumerObj *pConsumer) { - mInfo("consumer:0x%" PRIx64 " sub insert, cgroup:%s status:%d(%s) epoch:%d", - pConsumer->consumerId, pConsumer->cgroup, pConsumer->status, mndConsumerStatusName(pConsumer->status), - pConsumer->epoch); + mInfo("consumer:0x%" PRIx64 " sub insert, cgroup:%s status:%d(%s) epoch:%d", pConsumer->consumerId, pConsumer->cgroup, + pConsumer->status, mndConsumerStatusName(pConsumer->status), pConsumer->epoch); pConsumer->subscribeTime = pConsumer->createTime; return 0; } static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer) { mInfo("consumer:0x%" PRIx64 " perform delete action, status:(%d)%s", pConsumer->consumerId, pConsumer->status, - mndConsumerStatusName(pConsumer->status)); + mndConsumerStatusName(pConsumer->status)); tDeleteSMqConsumerObj(pConsumer, false); return 0; } @@ -823,7 +822,7 @@ static void removeFromNewTopicList(SMqConsumerObj *pConsumer, const char *pTopic taosMemoryFree(p); mInfo("consumer:0x%" PRIx64 " remove new topic:%s in the topic list, remain newTopics:%d", pConsumer->consumerId, - pTopic, (int)taosArrayGetSize(pConsumer->rebNewTopics)); + pTopic, (int)taosArrayGetSize(pConsumer->rebNewTopics)); break; } } @@ -839,7 +838,7 @@ static void removeFromRemoveTopicList(SMqConsumerObj *pConsumer, const char *pTo taosMemoryFree(p); mInfo("consumer:0x%" PRIx64 " remove topic:%s in the removed topic list, remain removedTopics:%d", - pConsumer->consumerId, pTopic, (int)taosArrayGetSize(pConsumer->rebRemovedTopics)); + pConsumer->consumerId, pTopic, (int)taosArrayGetSize(pConsumer->rebRemovedTopics)); break; } } @@ -854,13 +853,13 @@ static void removeFromCurrentTopicList(SMqConsumerObj *pConsumer, const char *pT taosMemoryFree(topic); mInfo("consumer:0x%" PRIx64 " remove topic:%s in the current topic list, remain currentTopics:%d", - pConsumer->consumerId, pTopic, (int)taosArrayGetSize(pConsumer->currentTopics)); + pConsumer->consumerId, pTopic, (int)taosArrayGetSize(pConsumer->currentTopics)); break; } } } -static bool existInCurrentTopicList(const SMqConsumerObj* pConsumer, const char* pTopic) { +static bool existInCurrentTopicList(const SMqConsumerObj *pConsumer, const char *pTopic) { bool existing = false; int32_t size = taosArrayGetSize(pConsumer->currentTopics); for (int32_t i = 0; i < size; i++) { @@ -877,7 +876,7 @@ static bool existInCurrentTopicList(const SMqConsumerObj* pConsumer, const char* static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, SMqConsumerObj *pNewConsumer) { mInfo("consumer:0x%" PRIx64 " perform update action, update type:%d, subscribe-time:%" PRId64 ", createTime:%" PRId64, - pOldConsumer->consumerId, pNewConsumer->updateType, pOldConsumer->subscribeTime, pOldConsumer->createTime); + pOldConsumer->consumerId, pNewConsumer->updateType, pOldConsumer->subscribeTime, pOldConsumer->createTime); taosWLockLatch(&pOldConsumer->lock); @@ -888,19 +887,21 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, pOldConsumer->subscribeTime = taosGetTimestampMs(); pOldConsumer->status = MQ_CONSUMER_STATUS_REBALANCE; - mInfo("consumer:0x%" PRIx64 " sub update, modify existed consumer",pOldConsumer->consumerId); -// } else if (pNewConsumer->updateType == CONSUMER_UPDATE_TIMER_LOST) { -// int32_t sz = taosArrayGetSize(pOldConsumer->currentTopics); -// for (int32_t i = 0; i < sz; i++) { -// char *topic = taosStrdup(taosArrayGetP(pOldConsumer->currentTopics, i)); -// taosArrayPush(pOldConsumer->rebRemovedTopics, &topic); -// } -// -// int32_t prevStatus = pOldConsumer->status; -// pOldConsumer->status = MQ_CONSUMER_STATUS_LOST; -// mInfo("consumer:0x%" PRIx64 " timer update, timer lost. state %s -> %s, reb-time:%" PRId64 ", reb-removed-topics:%d", -// pOldConsumer->consumerId, mndConsumerStatusName(prevStatus), mndConsumerStatusName(pOldConsumer->status), -// pOldConsumer->rebalanceTime, (int)taosArrayGetSize(pOldConsumer->rebRemovedTopics)); + mInfo("consumer:0x%" PRIx64 " sub update, modify existed consumer", pOldConsumer->consumerId); + // } else if (pNewConsumer->updateType == CONSUMER_UPDATE_TIMER_LOST) { + // int32_t sz = taosArrayGetSize(pOldConsumer->currentTopics); + // for (int32_t i = 0; i < sz; i++) { + // char *topic = taosStrdup(taosArrayGetP(pOldConsumer->currentTopics, i)); + // taosArrayPush(pOldConsumer->rebRemovedTopics, &topic); + // } + // + // int32_t prevStatus = pOldConsumer->status; + // pOldConsumer->status = MQ_CONSUMER_STATUS_LOST; + // mInfo("consumer:0x%" PRIx64 " timer update, timer lost. state %s -> %s, reb-time:%" PRId64 ", + // reb-removed-topics:%d", + // pOldConsumer->consumerId, mndConsumerStatusName(prevStatus), + // mndConsumerStatusName(pOldConsumer->status), pOldConsumer->rebalanceTime, + // (int)taosArrayGetSize(pOldConsumer->rebRemovedTopics)); } else if (pNewConsumer->updateType == CONSUMER_UPDATE_REC) { int32_t sz = taosArrayGetSize(pOldConsumer->assignedTopics); for (int32_t i = 0; i < sz; i++) { @@ -909,7 +910,7 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, } pOldConsumer->status = MQ_CONSUMER_STATUS_REBALANCE; - mInfo("consumer:0x%" PRIx64 " timer update, timer recover",pOldConsumer->consumerId); + mInfo("consumer:0x%" PRIx64 " timer update, timer recover", pOldConsumer->consumerId); } else if (pNewConsumer->updateType == CONSUMER_UPDATE_REB) { atomic_add_fetch_32(&pOldConsumer->epoch, 1); @@ -940,11 +941,11 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, atomic_add_fetch_32(&pOldConsumer->epoch, 1); mInfo("consumer:0x%" PRIx64 " reb update add, state (%d)%s -> (%d)%s, new epoch:%d, reb-time:%" PRId64 - ", current topics:%d, newTopics:%d, removeTopics:%d", - pOldConsumer->consumerId, status, mndConsumerStatusName(status), pOldConsumer->status, - mndConsumerStatusName(pOldConsumer->status), pOldConsumer->epoch, pOldConsumer->rebalanceTime, - (int)taosArrayGetSize(pOldConsumer->currentTopics), (int)taosArrayGetSize(pOldConsumer->rebNewTopics), - (int)taosArrayGetSize(pOldConsumer->rebRemovedTopics)); + ", current topics:%d, newTopics:%d, removeTopics:%d", + pOldConsumer->consumerId, status, mndConsumerStatusName(status), pOldConsumer->status, + mndConsumerStatusName(pOldConsumer->status), pOldConsumer->epoch, pOldConsumer->rebalanceTime, + (int)taosArrayGetSize(pOldConsumer->currentTopics), (int)taosArrayGetSize(pOldConsumer->rebNewTopics), + (int)taosArrayGetSize(pOldConsumer->rebRemovedTopics)); } else if (pNewConsumer->updateType == CONSUMER_REMOVE_REB) { char *removedTopic = taosArrayGetP(pNewConsumer->rebRemovedTopics, 0); @@ -963,11 +964,11 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, atomic_add_fetch_32(&pOldConsumer->epoch, 1); mInfo("consumer:0x%" PRIx64 " reb update remove, state (%d)%s -> (%d)%s, new epoch:%d, reb-time:%" PRId64 - ", current topics:%d, newTopics:%d, removeTopics:%d", - pOldConsumer->consumerId, status, mndConsumerStatusName(status), pOldConsumer->status, - mndConsumerStatusName(pOldConsumer->status), pOldConsumer->epoch, pOldConsumer->rebalanceTime, - (int)taosArrayGetSize(pOldConsumer->currentTopics), (int)taosArrayGetSize(pOldConsumer->rebNewTopics), - (int)taosArrayGetSize(pOldConsumer->rebRemovedTopics)); + ", current topics:%d, newTopics:%d, removeTopics:%d", + pOldConsumer->consumerId, status, mndConsumerStatusName(status), pOldConsumer->status, + mndConsumerStatusName(pOldConsumer->status), pOldConsumer->epoch, pOldConsumer->rebalanceTime, + (int)taosArrayGetSize(pOldConsumer->currentTopics), (int)taosArrayGetSize(pOldConsumer->rebNewTopics), + (int)taosArrayGetSize(pOldConsumer->rebRemovedTopics)); } taosWUnLockLatch(&pOldConsumer->lock); @@ -1025,8 +1026,8 @@ static int32_t mndRetrieveConsumer(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock * int32_t cols = 0; // consumer id - char consumerIdHex[32] = {0}; - sprintf(varDataVal(consumerIdHex), "0x%"PRIx64, pConsumer->consumerId); + char consumerIdHex[32] = {0}; + sprintf(varDataVal(consumerIdHex), "0x%" PRIx64, pConsumer->consumerId); varDataSetLen(consumerIdHex, strlen(varDataVal(consumerIdHex))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); @@ -1081,12 +1082,13 @@ static int32_t mndRetrieveConsumer(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock * pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)&pConsumer->rebalanceTime, pConsumer->rebalanceTime == 0); - char buf[TSDB_OFFSET_LEN] = {0}; + char buf[TSDB_OFFSET_LEN] = {0}; STqOffsetVal pVal = {.type = pConsumer->resetOffsetCfg}; tFormatOffset(buf, TSDB_OFFSET_LEN, &pVal); char parasStr[64 + TSDB_OFFSET_LEN + VARSTR_HEADER_SIZE] = {0}; - sprintf(varDataVal(parasStr), "tbname:%d,commit:%d,interval:%dms,reset:%s", pConsumer->withTbName, pConsumer->autoCommit, pConsumer->autoCommitInterval, buf); + sprintf(varDataVal(parasStr), "tbname:%d,commit:%d,interval:%dms,reset:%s", pConsumer->withTbName, + pConsumer->autoCommit, pConsumer->autoCommitInterval, buf); varDataSetLen(parasStr, strlen(varDataVal(parasStr))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 67b9cf1ad6..6b7420b55f 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -14,8 +14,8 @@ */ #include "tq.h" -#include "vnd.h" #include "tqCommon.h" +#include "vnd.h" // 0: not init // 1: already inited @@ -865,9 +865,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t nextProcessVer) { return 0; } -int32_t tqProcessTaskCheckReq(STQ* pTq, SRpcMsg* pMsg) { - return tqStreamTaskProcessCheckReq(pTq->pStreamMeta, pMsg); -} +int32_t tqProcessTaskCheckReq(STQ* pTq, SRpcMsg* pMsg) { return tqStreamTaskProcessCheckReq(pTq->pStreamMeta, pMsg); } int32_t tqProcessTaskCheckRsp(STQ* pTq, SRpcMsg* pMsg) { return tqStreamTaskProcessCheckRsp(pTq->pStreamMeta, pMsg, vnodeIsRoleLeader(pTq->pVnode)); @@ -991,7 +989,7 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) { streamReExecScanHistoryFuture(pTask, retInfo.idleTime); } else { SStreamTaskState* p = streamTaskGetStatus(pTask); - ETaskStatus s = p->state; + ETaskStatus s = p->state; if (s == TASK_STATUS__PAUSE) { tqDebug("s-task:%s is paused in the step1, elapsed time:%.2fs total:%.2fs, sched-status:%d", pTask->id.idStr, @@ -1065,7 +1063,7 @@ int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) { } int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg) { - return tqStreamTaskProcessDispatchReq(pTq->pStreamMeta, pMsg); + return tqStreamTaskProcessDispatchReq(pTq->pStreamMeta, pMsg); } int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg) { @@ -1104,7 +1102,7 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) pRsp->info.handle = NULL; SStreamCheckpointSourceReq req = {0}; - SDecoder decoder; + SDecoder decoder; tDecoderInit(&decoder, (uint8_t*)msg, len); if (tDecodeStreamCheckpointSourceReq(&decoder, &req) < 0) { code = TSDB_CODE_MSG_DECODE_ERROR; @@ -1195,8 +1193,8 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) streamProcessCheckpointSourceReq(pTask, &req); taosThreadMutexUnlock(&pTask->lock); - qInfo("s-task:%s (vgId:%d) level:%d receive checkpoint-source msg chkpt:%" PRId64 ", transId:%d", - pTask->id.idStr, vgId, pTask->info.taskLevel, req.checkpointId, req.transId); + qInfo("s-task:%s (vgId:%d) level:%d receive checkpoint-source msg chkpt:%" PRId64 ", transId:%d", pTask->id.idStr, + vgId, pTask->info.taskLevel, req.checkpointId, req.transId); code = streamAddCheckpointSourceRspMsg(&req, &pMsg->info, pTask, 1); if (code != TSDB_CODE_SUCCESS) { diff --git a/source/dnode/vnode/src/tq/tqHandleSnapshot.c b/source/dnode/vnode/src/tq/tqHandleSnapshot.c index 3ce838ce8b..28fd315eb6 100644 --- a/source/dnode/vnode/src/tq/tqHandleSnapshot.c +++ b/source/dnode/vnode/src/tq/tqHandleSnapshot.c @@ -74,11 +74,11 @@ int32_t tqSnapReaderClose(STqSnapReader** ppReader) { } int32_t tqSnapRead(STqSnapReader* pReader, uint8_t** ppData) { - int32_t code = 0; - void* pKey = NULL; - void* pVal = NULL; - int32_t kLen = 0; - int32_t vLen = 0; + int32_t code = 0; + void* pKey = NULL; + void* pVal = NULL; + int32_t kLen = 0; + int32_t vLen = 0; if (tdbTbcNext(pReader->pCur, &pKey, &kLen, &pVal, &vLen)) { goto _exit; diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 3ae007ce34..727157a2f8 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -207,8 +207,8 @@ int32_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, uint64_t goto END; } - tqDebug("vgId:%d, consumer:0x%" PRIx64 " taosx get msg ver %" PRId64 ", type: %s, reqId:0x%" PRIx64" 0x%"PRIx64, vgId, - pHandle->consumerId, offset, TMSG_INFO(pHandle->pWalReader->pHead->head.msgType), reqId, id); + tqDebug("vgId:%d, consumer:0x%" PRIx64 " taosx get msg ver %" PRId64 ", type: %s, reqId:0x%" PRIx64 " 0x%" PRIx64, + vgId, pHandle->consumerId, offset, TMSG_INFO(pHandle->pWalReader->pHead->head.msgType), reqId, id); if (pHandle->pWalReader->pHead->head.msgType == TDMT_VND_SUBMIT) { code = walFetchBody(pHandle->pWalReader); @@ -303,7 +303,7 @@ int32_t tqReaderSeek(STqReader* pReader, int64_t ver, const char* id) { int32_t extractMsgFromWal(SWalReader* pReader, void** pItem, int64_t maxVer, const char* id) { int32_t code = 0; - while(1) { + while (1) { code = walNextValidMsg(pReader); if (code != TSDB_CODE_SUCCESS) { return code; @@ -322,7 +322,8 @@ int32_t extractMsgFromWal(SWalReader* pReader, void** pItem, int64_t maxVer, con void* data = taosMemoryMalloc(len); if (data == NULL) { - // todo: for all stream in this vnode, keep this offset in the offset files, and wait for a moment, and then retry + // todo: for all stream in this vnode, keep this offset in the offset files, and wait for a moment, and then + // retry code = TSDB_CODE_OUT_OF_MEMORY; terrno = code; @@ -369,7 +370,7 @@ int32_t extractMsgFromWal(SWalReader* pReader, void** pItem, int64_t maxVer, con // todo ignore the error in wal? bool tqNextBlockInWal(STqReader* pReader, const char* id, int sourceExcluded) { - SWalReader* pWalReader = pReader->pWalReader; + SWalReader* pWalReader = pReader->pWalReader; SSDataBlock* pDataBlock = NULL; uint64_t st = taosGetTimestampMs(); @@ -387,22 +388,22 @@ bool tqNextBlockInWal(STqReader* pReader, const char* id, int sourceExcluded) { pReader->nextBlk = 0; int32_t numOfBlocks = taosArrayGetSize(pReader->submit.aSubmitTbData); while (pReader->nextBlk < numOfBlocks) { - tqTrace("tq reader next data block %d/%d, len:%d %" PRId64, pReader->nextBlk, - numOfBlocks, pReader->msg.msgLen, pReader->msg.ver); + tqTrace("tq reader next data block %d/%d, len:%d %" PRId64, pReader->nextBlk, numOfBlocks, pReader->msg.msgLen, + pReader->msg.ver); SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); - if ((pSubmitTbData->source & sourceExcluded) != 0){ + if ((pSubmitTbData->source & sourceExcluded) != 0) { pReader->nextBlk += 1; continue; } if (pReader->tbIdHash == NULL || taosHashGet(pReader->tbIdHash, &pSubmitTbData->uid, sizeof(int64_t)) != NULL) { tqTrace("tq reader return submit block, uid:%" PRId64, pSubmitTbData->uid); SSDataBlock* pRes = NULL; - int32_t code = tqRetrieveDataBlock(pReader, &pRes, NULL); + int32_t code = tqRetrieveDataBlock(pReader, &pRes, NULL); if (code == TSDB_CODE_SUCCESS && pRes->info.rows > 0) { - if(pDataBlock == NULL){ + if (pDataBlock == NULL) { pDataBlock = createOneDataBlock(pRes, true); - }else{ + } else { blockDataMerge(pDataBlock, pRes); } } @@ -414,16 +415,16 @@ bool tqNextBlockInWal(STqReader* pReader, const char* id, int sourceExcluded) { tDestroySubmitReq(&pReader->submit, TSDB_MSG_FLG_DECODE); pReader->msg.msgStr = NULL; - if(pDataBlock != NULL){ + if (pDataBlock != NULL) { blockDataCleanup(pReader->pResBlock); copyDataBlock(pReader->pResBlock, pDataBlock); blockDataDestroy(pDataBlock); return true; - }else{ + } else { qTrace("stream scan return empty, all %d submit blocks consumed, %s", numOfBlocks, id); } - if(taosGetTimestampMs() - st > 1000){ + if (taosGetTimestampMs() - st > 1000) { return false; } } @@ -448,17 +449,11 @@ int32_t tqReaderSetSubmitMsg(STqReader* pReader, void* msgStr, int32_t msgLen, i return 0; } -SWalReader* tqGetWalReader(STqReader* pReader) { - return pReader->pWalReader; -} +SWalReader* tqGetWalReader(STqReader* pReader) { return pReader->pWalReader; } -SSDataBlock* tqGetResultBlock (STqReader* pReader) { - return pReader->pResBlock; -} +SSDataBlock* tqGetResultBlock(STqReader* pReader) { return pReader->pResBlock; } -int64_t tqGetResultBlockTime(STqReader *pReader){ - return pReader->lastTs; -} +int64_t tqGetResultBlockTime(STqReader* pReader) { return pReader->lastTs; } bool tqNextBlockImpl(STqReader* pReader, const char* idstr) { if (pReader->msg.msgStr == NULL) { @@ -599,7 +594,7 @@ static int32_t doSetVal(SColumnInfoData* pColumnInfoData, int32_t rowIndex, SCol if (IS_STR_DATA_TYPE(pColVal->type)) { char val[65535 + 2] = {0}; - if(COL_VAL_IS_VALUE(pColVal)){ + if (COL_VAL_IS_VALUE(pColVal)) { if (pColVal->value.pData != NULL) { memcpy(varDataVal(val), pColVal->value.pData, pColVal->value.nData); } @@ -874,7 +869,7 @@ int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas sourceIdx++; } else if (pCol->cid == pColData->info.colId) { tColDataGetValue(pCol, i, &colVal); - if(doSetVal(pColData, curRow - lastRow, &colVal) != TDB_CODE_SUCCESS){ + if (doSetVal(pColData, curRow - lastRow, &colVal) != TDB_CODE_SUCCESS) { goto FAIL; } sourceIdx++; @@ -958,10 +953,10 @@ int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas if (colVal.cid < pColData->info.colId) { sourceIdx++; } else if (colVal.cid == pColData->info.colId) { - if(doSetVal(pColData, curRow - lastRow, &colVal) != TDB_CODE_SUCCESS){ + if (doSetVal(pColData, curRow - lastRow, &colVal) != TDB_CODE_SUCCESS) { goto FAIL; } - sourceIdx++; + sourceIdx++; targetIdx++; } } @@ -1001,7 +996,7 @@ int tqReaderSetTbUidList(STqReader* pReader, const SArray* tbUidList, const char taosHashPut(pReader->tbIdHash, pKey, sizeof(int64_t), NULL, 0); } - tqDebug("s-task:%s %d tables are set to be queried target table", id, (int32_t) taosArrayGetSize(tbUidList)); + tqDebug("s-task:%s %d tables are set to be queried target table", id, (int32_t)taosArrayGetSize(tbUidList)); return 0; } @@ -1027,9 +1022,7 @@ bool tqReaderIsQueriedTable(STqReader* pReader, uint64_t uid) { return taosHashGet(pReader->tbIdHash, &uid, sizeof(uint64_t)); } -bool tqCurrentBlockConsumed(const STqReader* pReader) { - return pReader->msg.msgStr == NULL; -} +bool tqCurrentBlockConsumed(const STqReader* pReader) { return pReader->msg.msgStr == NULL; } int tqReaderRemoveTbUidList(STqReader* pReader, const SArray* tbUidList) { for (int32_t i = 0; i < taosArrayGetSize(tbUidList); i++) { @@ -1071,9 +1064,11 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { } else if (pTqHandle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) { if (isAdd) { SArray* list = NULL; - int ret = qGetTableList(pTqHandle->execHandle.execTb.suid, pTq->pVnode, pTqHandle->execHandle.execTb.node, &list, pTqHandle->execHandle.task); - if(ret != TDB_CODE_SUCCESS) { - tqError("qGetTableList in tqUpdateTbUidList error:%d handle %s consumer:0x%" PRIx64, ret, pTqHandle->subKey, pTqHandle->consumerId); + int ret = qGetTableList(pTqHandle->execHandle.execTb.suid, pTq->pVnode, pTqHandle->execHandle.execTb.node, + &list, pTqHandle->execHandle.task); + if (ret != TDB_CODE_SUCCESS) { + tqError("qGetTableList in tqUpdateTbUidList error:%d handle %s consumer:0x%" PRIx64, ret, pTqHandle->subKey, + pTqHandle->consumerId); taosArrayDestroy(list); taosHashCancelIterate(pTq->pHandle, pIter); taosWUnLockLatch(&pTq->lock); diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c index d86c0a0474..f108cedcf5 100644 --- a/source/dnode/vnode/src/tq/tqScan.c +++ b/source/dnode/vnode/src/tq/tqScan.c @@ -63,8 +63,8 @@ static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, STaosxRsp* pRsp, in return 0; } -int32_t getDataBlock(qTaskInfo_t task, const STqHandle* pHandle, int32_t vgId, SSDataBlock** res){ - uint64_t ts = 0; +int32_t getDataBlock(qTaskInfo_t task, const STqHandle* pHandle, int32_t vgId, SSDataBlock** res) { + uint64_t ts = 0; qStreamSetOpen(task); tqDebug("consumer:0x%" PRIx64 " vgId:%d, tmq one task start execute", pHandle->consumerId, vgId); @@ -97,27 +97,27 @@ int32_t tqScanData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* while (1) { SSDataBlock* pDataBlock = NULL; code = getDataBlock(task, pHandle, vgId, &pDataBlock); - if (code != 0){ + if (code != 0) { return code; } - if(pRequest->enableReplay){ - if(IS_OFFSET_RESET_TYPE(pRequest->reqOffset.type) && pHandle->block != NULL){ + if (pRequest->enableReplay) { + if (IS_OFFSET_RESET_TYPE(pRequest->reqOffset.type) && pHandle->block != NULL) { blockDataDestroy(pHandle->block); pHandle->block = NULL; } - if(pHandle->block == NULL){ + if (pHandle->block == NULL) { if (pDataBlock == NULL) { break; } STqOffsetVal offset = {0}; qStreamExtractOffset(task, &offset); pHandle->block = createOneDataBlock(pDataBlock, true); -// pHandle->block = createDataBlock(); -// copyDataBlock(pHandle->block, pDataBlock); + // pHandle->block = createDataBlock(); + // copyDataBlock(pHandle->block, pDataBlock); pHandle->blockTime = offset.ts; code = getDataBlock(task, pHandle, vgId, &pDataBlock); - if (code != 0){ + if (code != 0) { return code; } } @@ -132,7 +132,7 @@ int32_t tqScanData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* if (pDataBlock == NULL) { blockDataDestroy(pHandle->block); pHandle->block = NULL; - }else{ + } else { copyDataBlock(pHandle->block, pDataBlock); STqOffsetVal offset = {0}; @@ -141,7 +141,7 @@ int32_t tqScanData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* pHandle->blockTime = offset.ts; } break; - }else{ + } else { if (pDataBlock == NULL) { break; } @@ -250,7 +250,8 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta return 0; } -int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxRsp* pRsp, int32_t* totalRows, int8_t sourceExcluded) { +int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxRsp* pRsp, int32_t* totalRows, + int8_t sourceExcluded) { STqExecHandle* pExec = &pHandle->execHandle; SArray* pBlocks = taosArrayInit(0, sizeof(SSDataBlock)); SArray* pSchemas = taosArrayInit(0, sizeof(void*)); @@ -266,7 +267,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) goto loop_table; } - if ((pSubmitTbDataRet->source & sourceExcluded) != 0){ + if ((pSubmitTbDataRet->source & sourceExcluded) != 0) { goto loop_table; } if (pRsp->withTbName) { @@ -303,7 +304,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR tEncoderClear(&encoder); } - if (pHandle->fetchMeta == ONLY_META && pSubmitTbDataRet->pCreateTbReq == NULL){ + if (pHandle->fetchMeta == ONLY_META && pSubmitTbDataRet->pCreateTbReq == NULL) { goto loop_table; } for (int32_t i = 0; i < taosArrayGetSize(pBlocks); i++) { @@ -334,7 +335,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) goto loop_db; } - if ((pSubmitTbDataRet->source & sourceExcluded) != 0){ + if ((pSubmitTbDataRet->source & sourceExcluded) != 0) { goto loop_db; } if (pRsp->withTbName) { @@ -371,7 +372,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR tEncoderClear(&encoder); } - if (pHandle->fetchMeta == ONLY_META && pSubmitTbDataRet->pCreateTbReq == NULL){ + if (pHandle->fetchMeta == ONLY_META && pSubmitTbDataRet->pCreateTbReq == NULL) { goto loop_db; } for (int32_t i = 0; i < taosArrayGetSize(pBlocks); i++) { diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 72a73c0ff2..31267dbf52 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -72,10 +72,11 @@ static int32_t tqInitTaosxRsp(STaosxRsp* pRsp, STqOffsetVal pOffset) { return 0; } -static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg, bool* pBlockReturned) { - uint64_t consumerId = pRequest->consumerId; - STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, pRequest->subKey); - int32_t vgId = TD_VID(pTq->pVnode); +static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, + SRpcMsg* pMsg, bool* pBlockReturned) { + uint64_t consumerId = pRequest->consumerId; + STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, pRequest->subKey); + int32_t vgId = TD_VID(pTq->pVnode); *pBlockReturned = false; // In this vnode, data has been polled by consumer for this topic, so let's continue from the last offset value. @@ -132,7 +133,7 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, SRpcMsg* pMsg, STqOffsetVal* pOffset) { uint64_t consumerId = pRequest->consumerId; int32_t vgId = TD_VID(pTq->pVnode); - terrno = 0; + terrno = 0; SMqDataRsp dataRsp = {0}; tqInitDataRsp(&dataRsp, *pOffset); @@ -156,7 +157,7 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, taosWUnLockLatch(&pTq->lock); } - dataRsp.reqOffset = *pOffset; // reqOffset represents the current date offset, may be changed if wal not exists + dataRsp.reqOffset = *pOffset; // reqOffset represents the current date offset, may be changed if wal not exists code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&dataRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); end : { @@ -167,15 +168,15 @@ end : { consumerId, pHandle->subKey, vgId, dataRsp.blockNum, buf, pRequest->reqId, code); tDeleteMqDataRsp(&dataRsp); return code; - } +} } static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg, STqOffsetVal* offset) { - int code = 0; - int32_t vgId = TD_VID(pTq->pVnode); - SMqMetaRsp metaRsp = {0}; - STaosxRsp taosxRsp = {0}; + int code = 0; + int32_t vgId = TD_VID(pTq->pVnode); + SMqMetaRsp metaRsp = {0}; + STaosxRsp taosxRsp = {0}; tqInitTaosxRsp(&taosxRsp, *offset); if (offset->type != TMQ_OFFSET__LOG) { @@ -211,14 +212,16 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, int64_t fetchVer = offset->version; uint64_t st = taosGetTimestampMs(); - int totalRows = 0; + int totalRows = 0; while (1) { int32_t savedEpoch = atomic_load_32(&pHandle->epoch); ASSERT(savedEpoch <= pRequest->epoch); if (tqFetchLog(pTq, pHandle, &fetchVer, pRequest->reqId) < 0) { tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer); - code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); + code = tqSendDataRsp( + pHandle, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, + taosxRsp.createTableNum > 0 ? TMQ_MSG_TYPE__POLL_DATA_META_RSP : TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); goto end; } @@ -230,7 +233,9 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, if (pHead->msgType != TDMT_VND_SUBMIT) { if (totalRows > 0) { tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer); - code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); + code = tqSendDataRsp( + pHandle, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, + taosxRsp.createTableNum > 0 ? TMQ_MSG_TYPE__POLL_DATA_META_RSP : TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); goto end; } @@ -257,9 +262,11 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, goto end; } - if (totalRows >= 4096 || taosxRsp.createTableNum > 0 || (taosGetTimestampMs() - st > 1000)) { + if (totalRows >= 4096 || (taosGetTimestampMs() - st > 1000)) { tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer + 1); - code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, taosxRsp.createTableNum > 0 ? TMQ_MSG_TYPE__POLL_DATA_META_RSP : TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); + code = tqSendDataRsp( + pHandle, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, + taosxRsp.createTableNum > 0 ? TMQ_MSG_TYPE__POLL_DATA_META_RSP : TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); goto end; } else { fetchVer++; @@ -279,7 +286,7 @@ int32_t tqExtractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequ // 1. reset the offset if needed if (IS_OFFSET_RESET_TYPE(pRequest->reqOffset.type)) { // handle the reset offset cases, according to the consumer's choice. - bool blockReturned = false; + bool blockReturned = false; int32_t code = extractResetOffsetVal(&reqOffset, pTq, pHandle, pRequest, pMsg, &blockReturned); if (code != 0) { return code; @@ -289,7 +296,7 @@ int32_t tqExtractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequ if (blockReturned) { return 0; } - } else if(reqOffset.type == 0){ // use the consumer specified offset + } else if (reqOffset.type == 0) { // use the consumer specified offset uError("req offset type is 0"); return TSDB_CODE_TMQ_INVALID_MSG; } @@ -452,8 +459,8 @@ int32_t tqExtractDelDataBlock(const void* pData, int32_t len, int64_t ver, void* int32_t tqGetStreamExecInfo(SVnode* pVnode, int64_t streamId, int64_t* pDelay, bool* fhFinished) { SStreamMeta* pMeta = pVnode->pTq->pStreamMeta; - int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); - int32_t code = TSDB_CODE_SUCCESS; + int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); + int32_t code = TSDB_CODE_SUCCESS; if (pDelay != NULL) { *pDelay = 0; diff --git a/tests/system-test/7-tmq/tmq_taosx.py b/tests/system-test/7-tmq/tmq_taosx.py index 5bd70a5d60..ac8a6f377c 100644 --- a/tests/system-test/7-tmq/tmq_taosx.py +++ b/tests/system-test/7-tmq/tmq_taosx.py @@ -311,44 +311,57 @@ class TDTestCase: return - def consumeExcluded(self): - tdSql.execute(f'create topic topic_excluded as database db_taosx') + def consumeTest(self): + tdSql.execute(f'create database if not exists d1 vgroups 1') + tdSql.execute(f'use d1') + tdSql.execute(f'create table st(ts timestamp, i int) tags(t int)') + tdSql.execute(f'insert into t1 using st tags(1) values(now, 1) (now+1s, 2)') + tdSql.execute(f'insert into t2 using st tags(2) values(now, 1) (now+1s, 2)') + tdSql.execute(f'insert into t3 using st tags(3) values(now, 1) (now+1s, 2)') + tdSql.execute(f'insert into t1 using st tags(1) values(now+5s, 11) (now+10s, 12)') + + tdSql.query("select * from st") + tdSql.checkRows(8) + + tdSql.execute(f'create topic topic_excluded with meta as database d1') consumer_dict = { "group.id": "g1", "td.connect.user": "root", "td.connect.pass": "taosdata", "auto.offset.reset": "earliest", - "msg.consume.excluded": 1 } consumer = Consumer(consumer_dict) - tdLog.debug("test subscribe topic created by other user") - exceptOccured = False try: consumer.subscribe(["topic_excluded"]) except TmqError: - exceptOccured = True - - if exceptOccured: tdLog.exit(f"subscribe error") + index = 0 try: while True: res = consumer.poll(1) if not res: + if index != 1: + tdLog.exit("consume error") break - err = res.error() - if err is not None: - raise err val = res.value() - + if val is None: + continue + cnt = 0; for block in val: - print(block.fetchall()) + cnt += len(block.fetchall()) + if cnt != 8: + tdLog.exit("consume error") + + index += 1 finally: consumer.close() def run(self): + self.consumeTest() + tdSql.prepare() self.checkWal1VgroupOnlyMeta() @@ -362,7 +375,6 @@ class TDTestCase: self.checkSnapshotMultiVgroups() self.checkWalMultiVgroupsWithDropTable() - # self.consumeExcluded() self.checkSnapshotMultiVgroupsWithDropTable()