From 2eec17b20cbd3da2efb64b7da32312712e012962 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 17 Jul 2024 19:23:20 +0800 Subject: [PATCH] enh: add catalog return code processing --- include/os/osString.h | 3 +- source/libs/catalog/inc/catalogInt.h | 15 +- source/libs/catalog/src/catalog.c | 212 ++- source/libs/catalog/src/ctgAsync.c | 1652 ++++++++++++----- source/libs/catalog/src/ctgCache.c | 8 +- source/libs/catalog/src/ctgRemote.c | 4 +- source/libs/catalog/src/ctgUtil.c | 30 +- source/libs/scheduler/test/schedulerTests.cpp | 76 +- 8 files changed, 1408 insertions(+), 592 deletions(-) diff --git a/include/os/osString.h b/include/os/osString.h index 5fa5a25c6d..e53aceb83a 100644 --- a/include/os/osString.h +++ b/include/os/osString.h @@ -55,11 +55,12 @@ typedef enum { M2C = 0, C2M } ConvType; #define tstrncpy(dst, src, size) \ do { \ - strncpy((dst), (src), (size)); \ + (void)strncpy((dst), (src), (size)); \ (dst)[(size)-1] = 0; \ } while (0) #define TAOS_STRCPY(_dst, _src) ((void)strcpy(_dst, _src)) +#define TAOS_STRNCPY(_dst, _src, _size) ((void)strncpy(_dst, _src, _size)) char *tstrdup(const char *src); int32_t taosUcs4len(TdUcs4 *ucs4); diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 6aaa79bb31..4f5e1b069b 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -669,12 +669,12 @@ typedef struct SCtgCacheItemInfo { #define CTG_AUTH_READ(_t) ((_t) == AUTH_TYPE_READ || (_t) == AUTH_TYPE_READ_OR_WRITE) #define CTG_AUTH_WRITE(_t) ((_t) == AUTH_TYPE_WRITE || (_t) == AUTH_TYPE_READ_OR_WRITE) -#define CTG_QUEUE_INC() atomic_add_fetch_64(&gCtgMgmt.queue.qRemainNum, 1) -#define CTG_QUEUE_DEC() atomic_sub_fetch_64(&gCtgMgmt.queue.qRemainNum, 1) +#define CTG_QUEUE_INC() (void)atomic_add_fetch_64(&gCtgMgmt.queue.qRemainNum, 1) +#define CTG_QUEUE_DEC() (void)atomic_sub_fetch_64(&gCtgMgmt.queue.qRemainNum, 1) -#define CTG_STAT_INC(_item, _n) atomic_add_fetch_64(&(_item), _n) -#define CTG_STAT_DEC(_item, _n) atomic_sub_fetch_64(&(_item), _n) -#define CTG_STAT_GET(_item) atomic_load_64(&(_item)) +#define CTG_STAT_INC(_item, _n) (void)atomic_add_fetch_64(&(_item), _n) +#define CTG_STAT_DEC(_item, _n) (void)atomic_sub_fetch_64(&(_item), _n) +#define CTG_STAT_GET(_item) (void)atomic_load_64(&(_item)) #define CTG_STAT_API_INC(item, n) (CTG_STAT_INC(gCtgMgmt.statInfo.api.item, n)) #define CTG_STAT_RT_INC(item, n) (CTG_STAT_INC(gCtgMgmt.statInfo.runtime.item, n)) @@ -971,7 +971,7 @@ int32_t ctgRemoveTbMetaFromCache(SCatalog* pCtg, SName* pTableName, bool syncReq int32_t ctgGetTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta); int32_t ctgGetTbMetasFromCache(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetasCtx* ctx, int32_t dbIdx, int32_t* fetchIdx, int32_t baseResIdx, SArray* pList); -void* ctgCloneDbCfgInfo(void* pSrc); +int32_t ctgCloneDbCfgInfo(void* pSrc, SDbCfgInfo** ppDst); int32_t ctgOpUpdateVgroup(SCtgCacheOperation* action); int32_t ctgOpUpdateDbCfg(SCtgCacheOperation *operation); @@ -1112,7 +1112,7 @@ int32_t ctgRemoveTbMeta(SCatalog* pCtg, SName* pTableName); int32_t ctgRemoveCacheUser(SCatalog* pCtg, SCtgUserAuth* pUser, const char* user); int32_t ctgGetTbHashVgroup(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SVgroupInfo* pVgroup, bool* exists); -SName* ctgGetFetchName(SArray* pNames, SCtgFetch* pFetch); +int32_t ctgGetFetchName(SArray* pNames, SCtgFetch* pFetch, SName** ppName); int32_t ctgdGetOneHandle(SCatalog** pHandle); int ctgVgInfoComp(const void* lp, const void* rp); int32_t ctgMakeVgArray(SDBVgInfo* dbInfo); @@ -1165,6 +1165,7 @@ int32_t ctgGetStreamProgressFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, void* bInput); int32_t ctgAddTSMAFetch(SArray** pFetchs, int32_t dbIdx, int32_t tbIdx, int32_t* fetchIdx, int32_t resIdx, int32_t flag, CTG_TSMA_FETCH_TYPE fetchType, const SName* sourceTbName); +void ctgFreeTask(SCtgTask* pTask, bool freeRes); extern SCatalogMgmt gCtgMgmt; extern SCtgDebug gCTGDebug; diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 4048c8841b..07982afd5d 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -86,7 +86,7 @@ int32_t ctgRefreshDBVgInfo(SCatalog* pCtg, SRequestConnInfo* pConn, const char* if (code) { if (CTG_DB_NOT_EXIST(code) && (NULL != dbCache)) { ctgDebug("db no longer exist, dbFName:%s, dbId:0x%" PRIx64, input.db, input.dbId); - ctgDropDbCacheEnqueue(pCtg, input.db, input.dbId); + CTG_ERR_RET(ctgDropDbCacheEnqueue(pCtg, input.db, input.dbId)); } CTG_ERR_RET(code); @@ -116,8 +116,7 @@ int32_t ctgRefreshTbMeta(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetaCtx* if (CTG_FLAG_IS_SYS_DB(ctx->flag)) { ctgDebug("will refresh tbmeta, supposed in information_schema, tbName:%s", tNameGetTableName(ctx->pName)); - CTG_ERR_JRET( - ctgGetTbMetaFromMnodeImpl(pCtg, pConn, (char*)ctx->pName->dbname, (char*)ctx->pName->tname, output, NULL)); + CTG_ERR_JRET(ctgGetTbMetaFromMnodeImpl(pCtg, pConn, (char*)ctx->pName->dbname, (char*)ctx->pName->tname, output, NULL)); } else if (CTG_FLAG_IS_STB(ctx->flag)) { ctgDebug("will refresh tbmeta, supposed to be stb, tbName:%s", tNameGetTableName(ctx->pName)); @@ -128,8 +127,7 @@ int32_t ctgRefreshTbMeta(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetaCtx* CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, ctx->pName, &vgroupInfo, output, NULL)); } } else { - ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(ctx->pName), - ctx->flag); + ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(ctx->pName), ctx->flag); // if get from vnode failed or no table meta, will not try mnode CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, ctx->pName, &vgroupInfo, output, NULL)); @@ -166,7 +164,7 @@ int32_t ctgRefreshTbMeta(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetaCtx* if (CTG_IS_META_NULL(output->metaType)) { ctgError("no tbmeta got, tbName:%s", tNameGetTableName(ctx->pName)); - ctgRemoveTbMetaFromCache(pCtg, ctx->pName, false); + CTG_ERR_JRET(ctgRemoveTbMetaFromCache(pCtg, ctx->pName, false)); CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST); } @@ -216,7 +214,7 @@ int32_t ctgGetTbMeta(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetaCtx* ctx } if (CTG_IS_META_BOTH(output->metaType)) { - memcpy(output->tbMeta, &output->ctbMeta, sizeof(output->ctbMeta)); + TAOS_MEMCPY(output->tbMeta, &output->ctbMeta, sizeof(output->ctbMeta)); *pTableMeta = output->tbMeta; goto _return; @@ -233,7 +231,7 @@ int32_t ctgGetTbMeta(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetaCtx* ctx taosMemoryFreeClear(output->tbMeta); SName stbName = *ctx->pName; - strcpy(stbName.tname, output->tbName); + TAOS_STRCPY(stbName.tname, output->tbName); SCtgTbMetaCtx stbCtx = {0}; stbCtx.flag = ctx->flag; stbCtx.pName = &stbName; @@ -244,7 +242,7 @@ int32_t ctgGetTbMeta(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetaCtx* ctx continue; } - memcpy(*pTableMeta, &output->ctbMeta, sizeof(output->ctbMeta)); + TAOS_MEMCPY(*pTableMeta, &output->ctbMeta, sizeof(output->ctbMeta)); break; } @@ -254,15 +252,15 @@ _return: if (CTG_TABLE_NOT_EXIST(code) && ctx->tbInfo.inCache) { char dbFName[TSDB_DB_FNAME_LEN] = {0}; if (CTG_FLAG_IS_SYS_DB(ctx->flag)) { - strcpy(dbFName, ctx->pName->dbname); + TAOS_STRCPY(dbFName, ctx->pName->dbname); } else { - tNameGetFullDbName(ctx->pName, dbFName); + (void)tNameGetFullDbName(ctx->pName, dbFName); } if (TSDB_SUPER_TABLE == ctx->tbInfo.tbType) { - ctgDropStbMetaEnqueue(pCtg, dbFName, ctx->tbInfo.dbId, ctx->pName->tname, ctx->tbInfo.suid, false); + (void)ctgDropStbMetaEnqueue(pCtg, dbFName, ctx->tbInfo.dbId, ctx->pName->tname, ctx->tbInfo.suid, false); // already in error } else { - ctgDropTbMetaEnqueue(pCtg, dbFName, ctx->tbInfo.dbId, ctx->pName->tname, false); + (void)ctgDropTbMetaEnqueue(pCtg, dbFName, ctx->tbInfo.dbId, ctx->pName->tname, false); // already in error } } @@ -285,18 +283,18 @@ int32_t ctgUpdateTbMeta(SCatalog* pCtg, STableMetaRsp* rspMsg, bool syncOp) { int32_t code = 0; - strcpy(output->dbFName, rspMsg->dbFName); + TAOS_STRCPY(output->dbFName, rspMsg->dbFName); output->dbId = rspMsg->dbId; if (TSDB_CHILD_TABLE == rspMsg->tableType && NULL == rspMsg->pSchemas) { - strcpy(output->ctbName, rspMsg->tbName); + TAOS_STRCPY(output->ctbName, rspMsg->tbName); SET_META_TYPE_CTABLE(output->metaType); CTG_ERR_JRET(queryCreateCTableMetaFromMsg(rspMsg, &output->ctbMeta)); } else { - strcpy(output->tbName, rspMsg->tbName); + TAOS_STRCPY(output->tbName, rspMsg->tbName); SET_META_TYPE_TABLE(output->metaType); @@ -348,14 +346,14 @@ int32_t ctgChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, SUserAuthInfo *pReq, _return: - ctgUpdateUserEnqueue(pCtg, &req.authInfo, false); + (void)ctgUpdateUserEnqueue(pCtg, &req.authInfo, false); // cache update not fatal error CTG_RET(code); } int32_t ctgGetTbType(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTableName, int32_t* tbType) { char dbFName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(pTableName, dbFName); + (void)tNameGetFullDbName(pTableName, dbFName); CTG_ERR_RET(ctgReadTbTypeFromCache(pCtg, dbFName, pTableName->tname, tbType)); if (*tbType > 0) { return TSDB_CODE_SUCCESS; @@ -454,7 +452,11 @@ int32_t ctgGetTbTag(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTableName, tagVal.type = TSDB_DATA_TYPE_JSON; tagVal.pData = pJson; tagVal.nData = strlen(pJson); - taosArrayPush(pTagVals, &tagVal); + if (NULL == taosArrayPush(pTagVals, &tagVal)) { + taosMemoryFree(pJson); + taosArrayDestroy(pTagVals); + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } } else { CTG_ERR_JRET(tTagToValArray((const STag*)pCfg->pTags, &pTagVals)); } @@ -484,7 +486,7 @@ int32_t ctgGetTbDistVgInfo(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTabl CTG_ERR_JRET(ctgGetTbMeta(pCtg, pConn, &ctx, &tbMeta)); char db[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(pTableName, db); + (void)tNameGetFullDbName(pTableName, db); SHashObj* vgHash = NULL; CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pConn, db, &dbCache, &vgInfo, NULL)); @@ -555,7 +557,7 @@ int32_t ctgGetTbHashVgroup(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* SCtgDBCache* dbCache = NULL; int32_t code = 0; char db[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(pTableName, db); + (void)tNameGetFullDbName(pTableName, db); SDBVgInfo* vgInfo = NULL; CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pConn, db, &dbCache, &vgInfo, exists)); @@ -590,7 +592,7 @@ int32_t ctgGetTbsHashVgId(SCatalog* pCtg, SRequestConnInfo* pConn, int32_t acctI SCtgDBCache* dbCache = NULL; int32_t code = 0; char dbFName[TSDB_DB_FNAME_LEN] = {0}; - snprintf(dbFName, TSDB_DB_FNAME_LEN, "%d.%s", acctId, pDb); + (void)snprintf(dbFName, TSDB_DB_FNAME_LEN, "%d.%s", acctId, pDb); SDBVgInfo* vgInfo = NULL; CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pConn, dbFName, &dbCache, &vgInfo, NULL)); @@ -615,7 +617,7 @@ _return: int32_t ctgGetCachedTbVgMeta(SCatalog* pCtg, const SName* pTableName, SVgroupInfo* pVgroup, STableMeta** pTableMeta) { int32_t code = 0; char db[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(pTableName, db); + (void)tNameGetFullDbName(pTableName, db); SCtgDBCache *dbCache = NULL; SCtgTbCache *tbCache = NULL; @@ -688,8 +690,8 @@ _return: void ctgProcessTimerEvent(void *param, void *tmrId) { CTG_API_NENTER(); - ctgdShowCacheInfo(); - ctgdShowStatInfo(); + (void)ctgdShowCacheInfo(); + (void)ctgdShowStatInfo(); int32_t cacheMaxSize = atomic_load_32(&tsMetaCacheMaxSize); if (cacheMaxSize >= 0) { @@ -703,7 +705,7 @@ void ctgProcessTimerEvent(void *param, void *tmrId) { int32_t code = ctgClearCacheEnqueue(NULL, true, false, false, false); if (code) { qError("clear cache enqueue failed, error:%s", tstrerror(code)); - taosTmrReset(ctgProcessTimerEvent, CTG_DEFAULT_CACHE_MON_MSEC, NULL, gCtgMgmt.timer, &gCtgMgmt.cacheTimer); + (void)taosTmrReset(ctgProcessTimerEvent, CTG_DEFAULT_CACHE_MON_MSEC, NULL, gCtgMgmt.timer, &gCtgMgmt.cacheTimer); } goto _return; @@ -711,7 +713,7 @@ void ctgProcessTimerEvent(void *param, void *tmrId) { } qTrace("reset catalog timer"); - taosTmrReset(ctgProcessTimerEvent, CTG_DEFAULT_CACHE_MON_MSEC, NULL, gCtgMgmt.timer, &gCtgMgmt.cacheTimer); + (void)taosTmrReset(ctgProcessTimerEvent, CTG_DEFAULT_CACHE_MON_MSEC, NULL, gCtgMgmt.timer, &gCtgMgmt.cacheTimer); _return: @@ -723,10 +725,8 @@ int32_t ctgGetDBCfg(SCatalog* pCtg, SRequestConnInfo* pConn, const char* dbFName if (pDbCfg->cfgVersion < 0) { CTG_ERR_RET(ctgGetDBCfgFromMnode(pCtg, pConn, dbFName, pDbCfg, NULL)); - SDbCfgInfo *pCfg = ctgCloneDbCfgInfo(pDbCfg); - if (NULL == pCfg) { - return TSDB_CODE_OUT_OF_MEMORY; - } + SDbCfgInfo *pCfg = NULL; + CTG_ERR_RET(ctgCloneDbCfgInfo(pDbCfg, &pCfg)); CTG_ERR_RET(ctgUpdateDbCfgEnqueue(pCtg, dbFName, pDbCfg->dbId, pCfg, false)); } @@ -735,6 +735,59 @@ int32_t ctgGetDBCfg(SCatalog* pCtg, SRequestConnInfo* pConn, const char* dbFName } +int32_t ctgGetTbTsmas(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTableName, SArray** ppRes) { + STableTSMAInfoRsp tsmasRsp = {0}; + int32_t code = ctgGetTbTSMAFromMnode(pCtg, pConn, pTableName, &tsmasRsp, NULL, TDMT_MND_GET_TABLE_TSMA); + if (code == TSDB_CODE_MND_SMA_NOT_EXIST) { + code = 0; + goto _return; + } + + CTG_ERR_JRET(code); + + *ppRes = tsmasRsp.pTsmas; + tsmasRsp.pTsmas = NULL; + + for (int32_t i = 0; i < (*ppRes)->size; ++i) { + CTG_ERR_JRET(ctgUpdateTbTSMAEnqueue(pCtg, taosArrayGet((*ppRes), i), 0, false)); + } + + return TSDB_CODE_SUCCESS; + +_return: + + if (tsmasRsp.pTsmas) { + tFreeTableTSMAInfoRsp(&tsmasRsp); + } + CTG_RET(code); +} + + +int32_t ctgGetTsma(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTsmaName, STableTSMAInfo** pTsma) { + STableTSMAInfoRsp tsmaRsp = {0}; + int32_t code = ctgGetTbTSMAFromMnode(pCtg, pConn, pTsmaName, &tsmaRsp, NULL, TDMT_MND_GET_TSMA); + if (code == TSDB_CODE_MND_SMA_NOT_EXIST) { + code = 0; + goto _return; + } + + CTG_ERR_JRET(code); + + ASSERT(tsmaRsp.pTsmas && tsmaRsp.pTsmas->size == 1); + + *pTsma = taosArrayGetP(tsmaRsp.pTsmas, 0); + taosArrayDestroy(tsmaRsp.pTsmas); + tsmaRsp.pTsmas = NULL; + +_return: + + if (tsmaRsp.pTsmas) { + tFreeTableTSMAInfoRsp(&tsmaRsp); + } + CTG_RET(code); +} + + int32_t catalogInit(SCatalogCfg* cfg) { qDebug("catalogInit start"); if (gCtgMgmt.pCluster) { @@ -742,10 +795,10 @@ int32_t catalogInit(SCatalogCfg* cfg) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } - memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); + TAOS_MEMSET(&gCtgMgmt, 0, sizeof(gCtgMgmt)); if (cfg) { - memcpy(&gCtgMgmt.cfg, cfg, sizeof(*cfg)); + TAOS_MEMCPY(&gCtgMgmt.cfg, cfg, sizeof(*cfg)); if (gCtgMgmt.cfg.maxDBCacheNum == 0) { gCtgMgmt.cfg.maxDBCacheNum = CTG_DEFAULT_CACHE_DB_NUMBER; @@ -1129,7 +1182,7 @@ int32_t catalogUpdateTableIndex(SCatalog* pCtg, STableIndexRsp* pRsp) { CTG_API_LEAVE(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(pIndex, pRsp, sizeof(STableIndex)); + TAOS_MEMCPY(pIndex, pRsp, sizeof(STableIndex)); CTG_ERR_JRET(ctgUpdateTbIndexEnqueue(pCtg, &pIndex, false)); @@ -1247,17 +1300,26 @@ int32_t catalogChkTbMetaVersion(SCatalog* pCtg, SRequestConnInfo* pConn, SArray* CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } + int32_t code = TSDB_CODE_SUCCESS; SName name = {0}; int32_t sver = 0; int32_t tver = 0; int32_t tbNum = taosArrayGetSize(pTables); for (int32_t i = 0; i < tbNum; ++i) { STbSVersion* pTb = (STbSVersion*)taosArrayGet(pTables, i); + if (NULL == pTb) { + ctgError("fail to get the %dth table, tbNum:%d", i, tbNum); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } + if (NULL == pTb->tbFName || 0 == pTb->tbFName[0]) { continue; } - tNameFromString(&name, pTb->tbFName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + if (tNameFromString(&name, pTb->tbFName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE)) { + ctgError("invalid tbFName format, tbFName:%s, idx:%d", pTb->tbFName, i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } if (IS_SYS_DBNAME(name.dbname)) { continue; @@ -1266,18 +1328,18 @@ int32_t catalogChkTbMetaVersion(SCatalog* pCtg, SRequestConnInfo* pConn, SArray* int32_t tbType = 0; uint64_t suid = 0; char stbName[TSDB_TABLE_FNAME_LEN]; - ctgReadTbVerFromCache(pCtg, &name, &sver, &tver, &tbType, &suid, stbName); + CTG_ERR_JRET(ctgReadTbVerFromCache(pCtg, &name, &sver, &tver, &tbType, &suid, stbName)); if ((sver >= 0 && sver < pTb->sver) || (tver >= 0 && tver < pTb->tver)) { switch (tbType) { case TSDB_CHILD_TABLE: { SName stb = name; tstrncpy(stb.tname, stbName, sizeof(stb.tname)); - ctgRemoveTbMeta(pCtg, &stb); + CTG_ERR_JRET(ctgRemoveTbMeta(pCtg, &stb)); break; } case TSDB_SUPER_TABLE: case TSDB_NORMAL_TABLE: - ctgRemoveTbMeta(pCtg, &name); + CTG_ERR_JRET(ctgRemoveTbMeta(pCtg, &name)); break; default: ctgError("ignore table type %d", tbType); @@ -1286,7 +1348,9 @@ int32_t catalogChkTbMetaVersion(SCatalog* pCtg, SRequestConnInfo* pConn, SArray* } } - CTG_API_LEAVE(TSDB_CODE_SUCCESS); +_return: + + CTG_API_LEAVE(code); } int32_t catalogRefreshDBVgInfo(SCatalog* pCtg, SRequestConnInfo* pConn, const char* dbFName) { @@ -1454,10 +1518,10 @@ int32_t catalogAsyncGetAllMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SC _return: if (pJob) { - taosReleaseRef(gCtgMgmt.jobPool, pJob->refId); + (void)taosReleaseRef(gCtgMgmt.jobPool, pJob->refId); if (code) { - taosRemoveRef(gCtgMgmt.jobPool, pJob->refId); + (void)taosRemoveRef(gCtgMgmt.jobPool, pJob->refId); } } @@ -1558,7 +1622,7 @@ int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion** users, uint32_ while (pAuth != NULL) { size_t len = 0; void* key = taosHashGetKey(pAuth, &len); - strncpy((*users)[i].user, key, len); + TAOS_STRNCPY((*users)[i].user, key, len); (*users)[i].user[len] = 0; (*users)[i].version = pAuth->userAuth.version; ++i; @@ -1795,10 +1859,12 @@ int32_t catalogAsyncUpdateTSMA(SCatalog* pCtg, STableTSMAInfo** ppTsma, int32_t if (!pCtg || !ppTsma) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } + int32_t code = 0; CTG_ERR_JRET(ctgUpdateTbTSMAEnqueue(pCtg, ppTsma, tsmaVersion, false)); _return: + CTG_API_LEAVE(code); } @@ -1807,10 +1873,12 @@ int32_t catalogUpdateTSMA(SCatalog* pCtg, STableTSMAInfo** pTsma) { if (!pCtg || !pTsma) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } + int32_t code = 0; CTG_ERR_JRET(ctgUpdateTbTSMAEnqueue(pCtg, pTsma, 0, true)); _return: + CTG_API_LEAVE(code); } @@ -1823,36 +1891,14 @@ int32_t catalogRemoveTSMA(SCatalog* pCtg, const STableTSMAInfo* pTsma) { } if (!pCtg->dbCache) { - return TSDB_CODE_SUCCESS; - } - CTG_ERR_JRET(ctgDropTbTSMAEnqueue(pCtg, pTsma, true)); -_return: - CTG_API_LEAVE(code); -} - -int32_t ctgGetTbTsmas(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTableName, SArray** ppRes) { - STableTSMAInfoRsp tsmasRsp = {0}; - int32_t code = ctgGetTbTSMAFromMnode(pCtg, pConn, pTableName, &tsmasRsp, NULL, TDMT_MND_GET_TABLE_TSMA); - if (code == TSDB_CODE_MND_SMA_NOT_EXIST) { - code = 0; goto _return; } - CTG_ERR_JRET(code); - assert(tsmasRsp.pTsmas); - assert(tsmasRsp.pTsmas->size > 0); - *ppRes = tsmasRsp.pTsmas; - tsmasRsp.pTsmas = NULL; - - for (int32_t i = 0; i < (*ppRes)->size; ++i) { - CTG_ERR_JRET(ctgUpdateTbTSMAEnqueue(pCtg, taosArrayGet((*ppRes), i), 0, false)); - } - return TSDB_CODE_SUCCESS; - + + CTG_ERR_JRET(ctgDropTbTSMAEnqueue(pCtg, pTsma, true)); + _return: - if (tsmasRsp.pTsmas) { - tFreeTableTSMAInfoRsp(&tsmasRsp); - } - CTG_RET(code); + + CTG_API_LEAVE(code); } int32_t catalogGetTableTsmas(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SArray** pRes) { @@ -1870,27 +1916,6 @@ _return: CTG_API_LEAVE(code); } -int32_t ctgGetTsma(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTsmaName, STableTSMAInfo** pTsma) { - STableTSMAInfoRsp tsmaRsp = {0}; - int32_t code = ctgGetTbTSMAFromMnode(pCtg, pConn, pTsmaName, &tsmaRsp, NULL, TDMT_MND_GET_TSMA); - if (code == TSDB_CODE_MND_SMA_NOT_EXIST) { - code = 0; - goto _return; - } - - CTG_ERR_JRET(code); - - ASSERT(tsmaRsp.pTsmas && tsmaRsp.pTsmas->size == 1); - *pTsma = taosArrayGetP(tsmaRsp.pTsmas, 0); - taosArrayDestroy(tsmaRsp.pTsmas); - tsmaRsp.pTsmas = NULL; - -_return: - if (tsmaRsp.pTsmas) { - tFreeTableTSMAInfoRsp(&tsmaRsp); - } - CTG_RET(code); -} int32_t catalogGetTsma(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTsmaName, STableTSMAInfo** pTsma) { CTG_API_ENTER(); @@ -1903,6 +1928,7 @@ int32_t catalogGetTsma(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTs CTG_ERR_JRET(ctgGetTsma(pCtg, pConn, pTsmaName, pTsma)); _return: + CTG_API_LEAVE(code); } @@ -1930,7 +1956,7 @@ void catalogDestroy(void) { } if (gCtgMgmt.cacheTimer) { - taosTmrStop(gCtgMgmt.cacheTimer); + (void)taosTmrStop(gCtgMgmt.cacheTimer); gCtgMgmt.cacheTimer = NULL; taosTmrCleanUp(gCtgMgmt.timer); gCtgMgmt.timer = NULL; @@ -1939,8 +1965,8 @@ void catalogDestroy(void) { atomic_store_8((int8_t*)&gCtgMgmt.exit, true); if (!taosCheckCurrentInDll()) { - ctgClearCacheEnqueue(NULL, false, true, true, true); - taosThreadJoin(gCtgMgmt.updateThread, NULL); + (void)ctgClearCacheEnqueue(NULL, false, true, true, true); + (void)taosThreadJoin(gCtgMgmt.updateThread, NULL); } taosHashCleanup(gCtgMgmt.pCluster); diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 601e01f7e9..a32d029016 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -65,10 +65,13 @@ int32_t ctgInitGetTbMetaTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(ctx->pName, name, sizeof(*name)); + TAOS_MEMCPY(ctx->pName, name, sizeof(*name)); ctx->flag = pParam->flag | CTG_FLAG_UNKNOWN_STB; - taosArrayPush(pJob->pTasks, &task); + if (NULL != taosArrayPush(pJob->pTasks, &task)) { + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tbName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name->tname); @@ -91,8 +94,16 @@ int32_t ctgInitGetTbMetasTask(SCtgJob* pJob, int32_t taskIdx, void* param) { SCtgTbMetasCtx* ctx = task.taskCtx; ctx->pNames = param; ctx->pResList = taosArrayInit(pJob->tbMetaNum, sizeof(SMetaRes)); - - taosArrayPush(pJob->pTasks, &task); + if (NULL == ctx->pResList) { + qError("QID:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->tbMetaNum, (int32_t)sizeof(SMetaRes)); + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + if (NULL == taosArrayPush(pJob->pTasks, &task)) { + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbNum:%lu, tbNum:%d", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), taosArrayGetSize(ctx->pNames), pJob->tbMetaNum); @@ -115,9 +126,12 @@ int32_t ctgInitGetDbVgTask(SCtgJob* pJob, int32_t taskIdx, void* param) { SCtgDbVgCtx* ctx = task.taskCtx; - memcpy(ctx->dbFName, dbFName, sizeof(ctx->dbFName)); + TAOS_MEMCPY(ctx->dbFName, dbFName, sizeof(ctx->dbFName)); - taosArrayPush(pJob->pTasks, &task); + if (NULL == taosArrayPush(pJob->pTasks, &task)) { + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbFName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), dbFName); @@ -140,9 +154,12 @@ int32_t ctgInitGetDbCfgTask(SCtgJob* pJob, int32_t taskIdx, void* param) { SCtgDbCfgCtx* ctx = task.taskCtx; - memcpy(ctx->dbFName, dbFName, sizeof(ctx->dbFName)); + TAOS_MEMCPY(ctx->dbFName, dbFName, sizeof(ctx->dbFName)); - taosArrayPush(pJob->pTasks, &task); + if (NULL == taosArrayPush(pJob->pTasks, &task)) { + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbFName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), dbFName); @@ -165,9 +182,12 @@ int32_t ctgInitGetDbInfoTask(SCtgJob* pJob, int32_t taskIdx, void* param) { SCtgDbInfoCtx* ctx = task.taskCtx; - memcpy(ctx->dbFName, dbFName, sizeof(ctx->dbFName)); + TAOS_MEMCPY(ctx->dbFName, dbFName, sizeof(ctx->dbFName)); - taosArrayPush(pJob->pTasks, &task); + if (NULL == taosArrayPush(pJob->pTasks, &task)) { + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbFName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), dbFName); @@ -195,10 +215,13 @@ int32_t ctgInitGetTbHashTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(ctx->pName, name, sizeof(*name)); - tNameGetFullDbName(ctx->pName, ctx->dbFName); + TAOS_MEMCPY(ctx->pName, name, sizeof(*name)); + (void)tNameGetFullDbName(ctx->pName, ctx->dbFName); - taosArrayPush(pJob->pTasks, &task); + if (NULL == taosArrayPush(pJob->pTasks, &task)) { + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tableName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name->tname); @@ -221,8 +244,16 @@ int32_t ctgInitGetTbHashsTask(SCtgJob* pJob, int32_t taskIdx, void* param) { SCtgTbHashsCtx* ctx = task.taskCtx; ctx->pNames = param; ctx->pResList = taosArrayInit(pJob->tbHashNum, sizeof(SMetaRes)); - - taosArrayPush(pJob->pTasks, &task); + if (NULL == ctx->pResList) { + qError("QID:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->tbHashNum, (int32_t)sizeof(SMetaRes)); + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + if (NULL == taosArrayPush(pJob->pTasks, &task)) { + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbNum:%lu, tbNum:%d", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), taosArrayGetSize(ctx->pNames), pJob->tbHashNum); @@ -238,7 +269,10 @@ int32_t ctgInitGetQnodeTask(SCtgJob* pJob, int32_t taskIdx, void* param) { task.pJob = pJob; task.taskCtx = NULL; - taosArrayPush(pJob->pTasks, &task); + if (NULL == taosArrayPush(pJob->pTasks, &task)) { + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type)); @@ -253,7 +287,10 @@ int32_t ctgInitGetDnodeTask(SCtgJob* pJob, int32_t taskIdx, void* param) { task.pJob = pJob; task.taskCtx = NULL; - taosArrayPush(pJob->pTasks, &task); + if (NULL == taosArrayPush(pJob->pTasks, &task)) { + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type)); @@ -277,7 +314,10 @@ int32_t ctgInitGetIndexTask(SCtgJob* pJob, int32_t taskIdx, void* param) { tstrncpy(ctx->indexFName, name, sizeof(ctx->indexFName)); - taosArrayPush(pJob->pTasks, &task); + if (NULL == taosArrayPush(pJob->pTasks, &task)) { + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, indexFName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name); @@ -302,7 +342,10 @@ int32_t ctgInitGetUdfTask(SCtgJob* pJob, int32_t taskIdx, void* param) { tstrncpy(ctx->udfName, name, sizeof(ctx->udfName)); - taosArrayPush(pJob->pTasks, &task); + if (NULL == taosArrayPush(pJob->pTasks, &task)) { + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, udfName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name); @@ -325,9 +368,12 @@ int32_t ctgInitGetUserTask(SCtgJob* pJob, int32_t taskIdx, void* param) { SCtgUserCtx* ctx = task.taskCtx; - memcpy(&ctx->user, user, sizeof(*user)); + TAOS_MEMCPY(&ctx->user, user, sizeof(*user)); - taosArrayPush(pJob->pTasks, &task); + if (NULL == taosArrayPush(pJob->pTasks, &task)) { + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, user:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), user->user); @@ -342,7 +388,10 @@ int32_t ctgInitGetSvrVerTask(SCtgJob* pJob, int32_t taskIdx, void* param) { task.taskId = taskIdx; task.pJob = pJob; - taosArrayPush(pJob->pTasks, &task); + if (NULL == taosArrayPush(pJob->pTasks, &task)) { + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type)); @@ -369,9 +418,12 @@ int32_t ctgInitGetTbIndexTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(ctx->pName, name, sizeof(*name)); + TAOS_MEMCPY(ctx->pName, name, sizeof(*name)); - taosArrayPush(pJob->pTasks, &task); + if (NULL == taosArrayPush(pJob->pTasks, &task)) { + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tbName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name->tname); @@ -399,9 +451,12 @@ int32_t ctgInitGetTbCfgTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(ctx->pName, name, sizeof(*name)); + TAOS_MEMCPY(ctx->pName, name, sizeof(*name)); - taosArrayPush(pJob->pTasks, &task); + if (NULL == taosArrayPush(pJob->pTasks, &task)) { + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tbName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name->tname); @@ -429,9 +484,12 @@ int32_t ctgInitGetTbTagTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(ctx->pName, name, sizeof(*name)); + TAOS_MEMCPY(ctx->pName, name, sizeof(*name)); - taosArrayPush(pJob->pTasks, &task); + if (NULL == taosArrayPush(pJob->pTasks, &task)) { + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tbName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name->tname); @@ -454,8 +512,16 @@ int32_t ctgInitGetViewsTask(SCtgJob* pJob, int32_t taskIdx, void* param) { SCtgViewsCtx* ctx = task.taskCtx; ctx->pNames = param; ctx->pResList = taosArrayInit(pJob->viewNum, sizeof(SMetaRes)); + if (NULL == ctx->pResList) { + qError("QID:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->viewNum, (int32_t)sizeof(SMetaRes)); + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } - taosArrayPush(pJob->pTasks, &task); + if (NULL == taosArrayPush(pJob->pTasks, &task)) { + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbNum:%lu, viewNum:%d", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), taosArrayGetSize(ctx->pNames), pJob->viewNum); @@ -463,19 +529,6 @@ int32_t ctgInitGetViewsTask(SCtgJob* pJob, int32_t taskIdx, void* param) { return TSDB_CODE_SUCCESS; } -int32_t ctgHandleForceUpdateView(SCatalog* pCtg, const SCatalogReq* pReq) { - int32_t viewNum = taosArrayGetSize(pReq->pView); - for (int32_t i = 0; i < viewNum; ++i) { - STablesReq* p = taosArrayGet(pReq->pView, i); - int32_t viewNum = taosArrayGetSize(p->pTables); - for (int32_t m = 0; m < viewNum; ++m) { - SName* name = taosArrayGet(p->pTables, m); - ctgDropViewMetaEnqueue(pCtg, p->dbFName, 0, name->tname, 0, true); - } - } - return TSDB_CODE_SUCCESS; -} - int32_t ctgInitGetTbTSMATask(SCtgJob* pJob, int32_t taskId, void* param) { SCtgTask task = {0}; task.type = CTG_TASK_GET_TB_TSMA; @@ -483,12 +536,24 @@ int32_t ctgInitGetTbTSMATask(SCtgJob* pJob, int32_t taskId, void* param) { task.pJob = pJob; SCtgTbTSMACtx* pTaskCtx = taosMemoryCalloc(1, sizeof(SCtgTbTSMACtx)); - if (!pTaskCtx) CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + if (NULL == pTaskCtx) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + task.taskCtx = pTaskCtx; pTaskCtx->pNames = param; pTaskCtx->pResList = taosArrayInit(pJob->tbTsmaNum, sizeof(SMetaRes)); - - taosArrayPush(pJob->pTasks, &task); + if (NULL == pTaskCtx->pResList) { + qError("QID:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->tbTsmaNum, (int32_t)sizeof(SMetaRes)); + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + if (NULL == taosArrayPush(pJob->pTasks, &task)) { + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + return TSDB_CODE_SUCCESS; } @@ -499,16 +564,49 @@ int32_t ctgInitGetTSMATask(SCtgJob* pJob, int32_t taskId, void* param) { task.pJob = pJob; SCtgTbTSMACtx* pTaskCtx = taosMemoryCalloc(1, sizeof(SCtgTbTSMACtx)); - if (!pTaskCtx) CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + if (NULL == pTaskCtx) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } task.taskCtx = pTaskCtx; pTaskCtx->pNames = param; pTaskCtx->pResList = taosArrayInit(pJob->tsmaNum, sizeof(SMetaRes)); + if (NULL == pTaskCtx->pResList) { + qError("QID:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->tsmaNum, (int32_t)sizeof(SMetaRes)); + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } - taosArrayPush(pJob->pTasks, &task); - return 0; + if (NULL == taosArrayPush(pJob->pTasks, &task)) { + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + return TSDB_CODE_SUCCESS; } + +int32_t ctgHandleForceUpdateView(SCatalog* pCtg, const SCatalogReq* pReq) { + int32_t viewNum = taosArrayGetSize(pReq->pView); + for (int32_t i = 0; i < viewNum; ++i) { + STablesReq* p = taosArrayGet(pReq->pView, i); + if (NULL == p) { + qError("taosArrayGet the %dth view in req failed", i); + CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); + } + + int32_t viewNum = taosArrayGetSize(p->pTables); + for (int32_t m = 0; m < viewNum; ++m) { + SName* name = taosArrayGet(p->pTables, m); + CTG_ERR_RET(ctgDropViewMetaEnqueue(pCtg, p->dbFName, 0, name->tname, 0, true)); + } + } + + return TSDB_CODE_SUCCESS; +} + + int32_t ctgHandleForceUpdate(SCatalog* pCtg, int32_t taskNum, SCtgJob* pJob, const SCatalogReq* pReq) { + int32_t code = TSDB_CODE_SUCCESS; SHashObj* pDb = taosHashInit(taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); SHashObj* pTb = taosHashInit(taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); if (NULL == pDb || NULL == pTb) { @@ -519,101 +617,167 @@ int32_t ctgHandleForceUpdate(SCatalog* pCtg, int32_t taskNum, SCtgJob* pJob, con for (int32_t i = 0; i < pJob->dbVgNum; ++i) { char* dbFName = taosArrayGet(pReq->pDbVgroup, i); - taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN); + if (NULL == dbFName) { + qError("taosArrayGet the %dth db in req failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } + CTG_ERR_JRET(taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN)); } for (int32_t i = 0; i < pJob->dbCfgNum; ++i) { char* dbFName = taosArrayGet(pReq->pDbCfg, i); - taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN); + if (NULL == dbFName) { + qError("taosArrayGet the %dth db in req failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } + CTG_ERR_JRET(taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN)); } for (int32_t i = 0; i < pJob->dbInfoNum; ++i) { char* dbFName = taosArrayGet(pReq->pDbInfo, i); - taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN); + if (NULL == dbFName) { + qError("taosArrayGet the %dth db in req failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } + CTG_ERR_JRET(taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN)); } int32_t dbNum = taosArrayGetSize(pReq->pTableMeta); for (int32_t i = 0; i < dbNum; ++i) { STablesReq* p = taosArrayGet(pReq->pTableMeta, i); - taosHashPut(pDb, p->dbFName, strlen(p->dbFName), p->dbFName, TSDB_DB_FNAME_LEN); + if (NULL == p) { + qError("taosArrayGet the %dth tb in req failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } + CTG_ERR_JRET(taosHashPut(pDb, p->dbFName, strlen(p->dbFName), p->dbFName, TSDB_DB_FNAME_LEN)); + int32_t tbNum = taosArrayGetSize(p->pTables); for (int32_t m = 0; m < tbNum; ++m) { SName* name = taosArrayGet(p->pTables, m); - taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName)); + if (NULL == name) { + qError("taosArrayGet the %dth tb in req failed", m); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } + CTG_ERR_JRET(taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName))); } } dbNum = taosArrayGetSize(pReq->pTableHash); for (int32_t i = 0; i < dbNum; ++i) { STablesReq* p = taosArrayGet(pReq->pTableHash, i); - taosHashPut(pDb, p->dbFName, strlen(p->dbFName), p->dbFName, TSDB_DB_FNAME_LEN); + if (NULL == p) { + qError("taosArrayGet the %dth tb in req failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } + + CTG_ERR_JRET(taosHashPut(pDb, p->dbFName, strlen(p->dbFName), p->dbFName, TSDB_DB_FNAME_LEN)); + int32_t tbNum = taosArrayGetSize(p->pTables); for (int32_t m = 0; m < tbNum; ++m) { SName* name = taosArrayGet(p->pTables, m); - taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName)); + if (NULL == name) { + qError("taosArrayGet the %dth tb in req failed", m); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } + CTG_ERR_JRET(taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName))); } } dbNum = taosArrayGetSize(pReq->pView); for (int32_t i = 0; i < dbNum; ++i) { STablesReq* p = taosArrayGet(pReq->pView, i); - taosHashPut(pDb, p->dbFName, strlen(p->dbFName), p->dbFName, TSDB_DB_FNAME_LEN); + if (NULL == p) { + qError("taosArrayGet the %dth db in req failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } + CTG_ERR_JRET(taosHashPut(pDb, p->dbFName, strlen(p->dbFName), p->dbFName, TSDB_DB_FNAME_LEN)); } for (int32_t i = 0; i < pJob->tbCfgNum; ++i) { SName* name = taosArrayGet(pReq->pTableCfg, i); + if (NULL == name) { + qError("taosArrayGet the %dth tb in req failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } + char dbFName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(name, dbFName); - taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN); - taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName)); + (void)tNameGetFullDbName(name, dbFName); + CTG_ERR_JRET(taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN)); + CTG_ERR_JRET(taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName))); } for (int32_t i = 0; i < pJob->tbTagNum; ++i) { SName* name = taosArrayGet(pReq->pTableTag, i); + if (NULL == name) { + qError("taosArrayGet the %dth tb in req failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } + char dbFName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(name, dbFName); - taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN); - taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName)); + (void)tNameGetFullDbName(name, dbFName); + CTG_ERR_JRET(taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN)); + CTG_ERR_JRET(taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName))); } char* dbFName = taosHashIterate(pDb, NULL); while (dbFName) { - ctgDropDbVgroupEnqueue(pCtg, dbFName, true); + CTG_ERR_JRET(ctgDropDbVgroupEnqueue(pCtg, dbFName, true)); dbFName = taosHashIterate(pDb, dbFName); } taosHashCleanup(pDb); + pDb = NULL; // REFRESH TABLE META for (int32_t i = 0; i < pJob->tbCfgNum; ++i) { SName* name = taosArrayGet(pReq->pTableCfg, i); - taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName)); + if (NULL == name) { + qError("taosArrayGet the %dth tb in req failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } + CTG_ERR_JRET(taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName))); } SName* name = taosHashIterate(pTb, NULL); while (name) { - ctgRemoveTbMeta(pCtg, name); + CTG_ERR_JRET(ctgRemoveTbMeta(pCtg, name)); name = taosHashIterate(pTb, name); } taosHashCleanup(pTb); + pTb = NULL; for (int32_t i = 0; i < pJob->tbIndexNum; ++i) { SName* name = taosArrayGet(pReq->pTableIndex, i); - ctgDropTbIndexEnqueue(pCtg, name, true); + if (NULL == name) { + qError("taosArrayGet the %dth tb in req failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } + CTG_ERR_JRET(ctgDropTbIndexEnqueue(pCtg, name, true)); } for (int32_t i = 0; i < pJob->tbTsmaNum; ++i) { STablesReq* pTbReq = taosArrayGet(pReq->pTableTSMAs, i); + if (NULL == pTbReq) { + qError("taosArrayGet the %dth tb in req failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } for (int32_t j = 0; j < pTbReq->pTables->size; ++j) { SName* name = taosArrayGet(pTbReq->pTables, j); - ctgDropTSMAForTbEnqueue(pCtg, name, true); + CTG_ERR_JRET(ctgDropTSMAForTbEnqueue(pCtg, name, true)); } } // REFRESH VIEW META - return ctgHandleForceUpdateView(pCtg, pReq); + CTG_ERR_JRET(ctgHandleForceUpdateView(pCtg, pReq)); + +_return: + + taosHashCleanup(pDb); + taosHashCleanup(pTb); + + return code; } int32_t ctgInitTask(SCtgJob* pJob, CTG_TASK_TYPE type, void* param, int32_t* taskId) { @@ -628,6 +792,7 @@ int32_t ctgInitTask(SCtgJob* pJob, CTG_TASK_TYPE type, void* param, int32_t* tas } _return: + CTG_UNLOCK(CTG_WRITE, &pJob->taskLock); return code; @@ -714,16 +879,28 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const for (int32_t i = 0; i < dbVgNum; ++i) { char* dbFName = taosArrayGet(pReq->pDbVgroup, i); + if (NULL == dbFName) { + qError("taosArrayGet the %dth db in pDbVgroup failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_DB_VGROUP, dbFName, NULL)); } for (int32_t i = 0; i < dbCfgNum; ++i) { char* dbFName = taosArrayGet(pReq->pDbCfg, i); + if (NULL == dbFName) { + qError("taosArrayGet the %dth db in pDbCfg failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_DB_CFG, dbFName, NULL)); } for (int32_t i = 0; i < dbInfoNum; ++i) { char* dbFName = taosArrayGet(pReq->pDbInfo, i); + if (NULL == dbFName) { + qError("taosArrayGet the %dth db in pDbInfo failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_DB_INFO, dbFName, NULL)); } @@ -751,54 +928,74 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const for (int32_t i = 0; i < tbIndexNum; ++i) { SName* name = taosArrayGet(pReq->pTableIndex, i); + if (NULL == name) { + qError("taosArrayGet the %dth tb in pTableIndex failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_TB_SMA_INDEX, name, NULL)); } for (int32_t i = 0; i < tbCfgNum; ++i) { SName* name = taosArrayGet(pReq->pTableCfg, i); + if (NULL == name) { + qError("taosArrayGet the %dth tb in pTableCfg failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_TB_CFG, name, NULL)); } for (int32_t i = 0; i < tbTagNum; ++i) { SName* name = taosArrayGet(pReq->pTableTag, i); + if (NULL == name) { + qError("taosArrayGet the %dth tb in pTableTag failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_TB_TAG, name, NULL)); } for (int32_t i = 0; i < indexNum; ++i) { char* indexName = taosArrayGet(pReq->pIndex, i); + if (NULL == indexName) { + qError("taosArrayGet the %dth index in pIndex failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_INDEX_INFO, indexName, NULL)); } for (int32_t i = 0; i < udfNum; ++i) { char* udfName = taosArrayGet(pReq->pUdf, i); + if (NULL == udfName) { + qError("taosArrayGet the %dth udf in pUdf failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_UDF, udfName, NULL)); } for (int32_t i = 0; i < userNum; ++i) { SUserAuthInfo* user = taosArrayGet(pReq->pUser, i); + if (NULL == user) { + qError("taosArrayGet the %dth user in pUser failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_USER, user, NULL)); } if (viewNum > 0) { CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_VIEW, pReq->pView, NULL)); } - if (tbTsmaNum > 0) { CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_TB_TSMA, pReq->pTableTSMAs, NULL)); } if (tsmaNum > 0) { CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_TSMA, pReq->pTSMAs, NULL)); } - if (qnodeNum) { CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_QNODE, NULL, NULL)); } - if (dnodeNum) { CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_DNODE, NULL, NULL)); } - if (svrVerNum) { CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_SVR_VER, NULL, NULL)); } @@ -809,7 +1006,7 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const CTG_ERR_JRET(terrno); } - taosAcquireRef(gCtgMgmt.jobPool, pJob->refId); + (void)taosAcquireRef(gCtgMgmt.jobPool, pJob->refId); double el = (taosGetTimestampUs() - st) / 1000.0; qDebug("QID:0x%" PRIx64 ", jobId: 0x%" PRIx64 " initialized, task num %d, forceUpdate %d, elapsed time:%.2f ms", @@ -817,6 +1014,7 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const return TSDB_CODE_SUCCESS; _return: + ctgFreeJob(*job); CTG_RET(code); } @@ -835,8 +1033,10 @@ int32_t ctgDumpTbMetaRes(SCtgTask* pTask) { } SMetaRes res = {.code = pTask->code, .pRes = pTask->res}; - taosArrayPush(pJob->jobRes.pTableMeta, &res); - + if (NULL == taosArrayPush(pJob->jobRes.pTableMeta, &res)) { + CTG_ERR_RET(terrno); + } + return TSDB_CODE_SUCCESS; } @@ -866,7 +1066,9 @@ int32_t ctgDumpDbVgRes(SCtgTask* pTask) { } SMetaRes res = {.code = pTask->code, .pRes = pTask->res}; - taosArrayPush(pJob->jobRes.pDbVgroup, &res); + if (NULL == taosArrayPush(pJob->jobRes.pDbVgroup, &res)) { + CTG_ERR_RET(terrno); + } return TSDB_CODE_SUCCESS; } @@ -885,7 +1087,9 @@ int32_t ctgDumpTbHashRes(SCtgTask* pTask) { } SMetaRes res = {.code = pTask->code, .pRes = pTask->res}; - taosArrayPush(pJob->jobRes.pTableHash, &res); + if (NULL == taosArrayPush(pJob->jobRes.pTableHash, &res)) { + CTG_ERR_RET(terrno); + } return TSDB_CODE_SUCCESS; } @@ -920,7 +1124,9 @@ int32_t ctgDumpTbIndexRes(SCtgTask* pTask) { } SMetaRes res = {.code = pTask->code, .pRes = pTask->res}; - taosArrayPush(pJob->jobRes.pTableIndex, &res); + if (NULL == taosArrayPush(pJob->jobRes.pTableIndex, &res)) { + CTG_ERR_RET(terrno); + } return TSDB_CODE_SUCCESS; } @@ -943,7 +1149,9 @@ int32_t ctgDumpTbCfgRes(SCtgTask* pTask) { } SMetaRes res = {.code = pTask->code, .pRes = pTask->res}; - taosArrayPush(pJob->jobRes.pTableCfg, &res); + if (NULL == taosArrayPush(pJob->jobRes.pTableCfg, &res)) { + CTG_ERR_RET(terrno); + } return TSDB_CODE_SUCCESS; } @@ -966,7 +1174,9 @@ int32_t ctgDumpTbTagRes(SCtgTask* pTask) { } SMetaRes res = {.code = pTask->code, .pRes = pTask->res}; - taosArrayPush(pJob->jobRes.pTableTag, &res); + if (NULL == taosArrayPush(pJob->jobRes.pTableTag, &res)) { + CTG_ERR_RET(terrno); + } return TSDB_CODE_SUCCESS; } @@ -986,7 +1196,9 @@ int32_t ctgDumpIndexRes(SCtgTask* pTask) { } SMetaRes res = {.code = pTask->code, .pRes = pTask->res}; - taosArrayPush(pJob->jobRes.pIndex, &res); + if (NULL == taosArrayPush(pJob->jobRes.pIndex, &res)) { + CTG_ERR_RET(terrno); + } return TSDB_CODE_SUCCESS; } @@ -1005,7 +1217,9 @@ int32_t ctgDumpQnodeRes(SCtgTask* pTask) { } SMetaRes res = {.code = pTask->code, .pRes = pTask->res}; - taosArrayPush(pJob->jobRes.pQnodeList, &res); + if (NULL == taosArrayPush(pJob->jobRes.pQnodeList, &res)) { + CTG_ERR_RET(terrno); + } return TSDB_CODE_SUCCESS; } @@ -1024,7 +1238,9 @@ int32_t ctgDumpDnodeRes(SCtgTask* pTask) { } SMetaRes res = {.code = pTask->code, .pRes = pTask->res}; - taosArrayPush(pJob->jobRes.pDnodeList, &res); + if (NULL == taosArrayPush(pJob->jobRes.pDnodeList, &res)) { + CTG_ERR_RET(terrno); + } return TSDB_CODE_SUCCESS; } @@ -1043,7 +1259,9 @@ int32_t ctgDumpDbCfgRes(SCtgTask* pTask) { } SMetaRes res = {.code = pTask->code, .pRes = pTask->res}; - taosArrayPush(pJob->jobRes.pDbCfg, &res); + if (NULL == taosArrayPush(pJob->jobRes.pDbCfg, &res)) { + CTG_ERR_RET(terrno); + } return TSDB_CODE_SUCCESS; } @@ -1062,7 +1280,9 @@ int32_t ctgDumpDbInfoRes(SCtgTask* pTask) { } SMetaRes res = {.code = pTask->code, .pRes = pTask->res}; - taosArrayPush(pJob->jobRes.pDbInfo, &res); + if (NULL == taosArrayPush(pJob->jobRes.pDbInfo, &res)) { + CTG_ERR_RET(terrno); + } return TSDB_CODE_SUCCESS; } @@ -1081,7 +1301,9 @@ int32_t ctgDumpUdfRes(SCtgTask* pTask) { } SMetaRes res = {.code = pTask->code, .pRes = pTask->res}; - taosArrayPush(pJob->jobRes.pUdfList, &res); + if (NULL == taosArrayPush(pJob->jobRes.pUdfList, &res)) { + CTG_ERR_RET(terrno); + } return TSDB_CODE_SUCCESS; } @@ -1100,7 +1322,9 @@ int32_t ctgDumpUserRes(SCtgTask* pTask) { } SMetaRes res = {.code = pTask->code, .pRes = pTask->res}; - taosArrayPush(pJob->jobRes.pUser, &res); + if (NULL == taosArrayPush(pJob->jobRes.pUser, &res)) { + CTG_ERR_RET(terrno); + } return TSDB_CODE_SUCCESS; } @@ -1147,7 +1371,11 @@ int32_t ctgCallSubCb(SCtgTask* pTask) { for (int32_t i = 0; i < parentNum; ++i) { SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); SCtgTask* pParent = taosArrayGetP(pTask->pParents, i); - + if (NULL == pParent) { + qError("taosArrayGetP the %dth parent failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + pParent->subRes.code = pTask->code; if (TSDB_CODE_SUCCESS == pTask->code) { code = (*gCtgAsyncFps[pTask->type].cloneFp)(pTask, &pParent->subRes.res); @@ -1178,7 +1406,7 @@ int32_t ctgCallUserCb(void* param) { qDebug("QID:0x%" PRIx64 " ctg end to call user cb", pJob->queryId); - taosRemoveRef(gCtgMgmt.jobPool, pJob->refId); + (void)taosRemoveRef(gCtgMgmt.jobPool, pJob->refId); return TSDB_CODE_SUCCESS; } @@ -1204,7 +1432,7 @@ int32_t ctgHandleTaskEnd(SCtgTask* pTask, int32_t rspCode) { pTask->code = rspCode; pTask->status = CTG_TASK_DONE; - ctgCallSubCb(pTask); + CTG_ERR_JRET(ctgCallSubCb(pTask)); int32_t taskDone = atomic_add_fetch_32(&pJob->taskDone, 1); if (taskDone < taosArrayGetSize(pJob->pTasks)) { @@ -1220,12 +1448,11 @@ int32_t ctgHandleTaskEnd(SCtgTask* pTask, int32_t rspCode) { _return: ctgUpdateJobErrCode(pJob, rspCode); - // pJob->jobResCode = code; - // taosSsleep(2); - // qDebug("QID:0x%" PRIx64 " ctg after sleep", pJob->queryId); - - taosAsyncExec(ctgCallUserCb, pJob, NULL); + int32_t newCode = taosAsyncExec(ctgCallUserCb, pJob, NULL); + if (TSDB_CODE_SUCCESS == code && TSDB_CODE_SUCCESS != newCode) { + code = newCode; + } CTG_RET(code); } @@ -1242,6 +1469,11 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf int32_t flag = ctx->flag; int32_t* vgId = &ctx->vgId; + if (NULL == pMsgCtx) { + ctgError("fail to get task msgCtx, taskType:%d", pTask->type); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + CTG_ERR_JRET(ctgProcessRspMsg(pMsgCtx->out, reqType, pMsg->pData, pMsg->len, rspCode, pMsgCtx->target)); switch (reqType) { @@ -1264,7 +1496,7 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf if (CTG_IS_META_NULL(pOut->metaType)) { if (CTG_FLAG_IS_STB(flag)) { char dbFName[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(pName, dbFName); + (void)tNameGetFullDbName(pName, dbFName); CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache)); if (NULL != dbCache) { @@ -1291,7 +1523,7 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf } ctgError("no tbmeta got, tbName:%s", tNameGetTableName(pName)); - ctgRemoveTbMetaFromCache(pCtg, pName, false); + (void)ctgRemoveTbMetaFromCache(pCtg, pName, false); // update cache not fatal error CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST); } @@ -1309,7 +1541,7 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf if (CTG_IS_META_NULL(pOut->metaType)) { ctgError("no tbmeta got, tbName:%s", tNameGetTableName(pName)); - ctgRemoveTbMetaFromCache(pCtg, pName, false); + (void)ctgRemoveTbMetaFromCache(pCtg, pName, false); // update cache not fatal error CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST); } @@ -1327,7 +1559,7 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf int32_t exist = 0; if (!CTG_FLAG_IS_FORCE_UPDATE(flag)) { SName stbName = *pName; - strcpy(stbName.tname, pOut->tbName); + TAOS_STRCPY(stbName.tname, pOut->tbName); SCtgTbMetaCtx stbCtx = {0}; stbCtx.flag = flag; stbCtx.pName = &stbName; @@ -1353,16 +1585,16 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf STableMetaOutput* pOut = (STableMetaOutput*)pMsgCtx->out; - ctgUpdateTbMetaToCache(pCtg, pOut, flag & CTG_FLAG_SYNC_OP); + CTG_ERR_JRET(ctgUpdateTbMetaToCache(pCtg, pOut, flag & CTG_FLAG_SYNC_OP)); if (CTG_IS_META_BOTH(pOut->metaType)) { - memcpy(pOut->tbMeta, &pOut->ctbMeta, sizeof(pOut->ctbMeta)); + TAOS_MEMCPY(pOut->tbMeta, &pOut->ctbMeta, sizeof(pOut->ctbMeta)); } /* else if (CTG_IS_META_CTABLE(pOut->metaType)) { SName stbName = *pName; - strcpy(stbName.tname, pOut->tbName); + TAOS_STRCPY(stbName.tname, pOut->tbName); SCtgTbMetaCtx stbCtx = {0}; stbCtx.flag = flag; stbCtx.pName = &stbName; @@ -1375,7 +1607,7 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf return TSDB_CODE_SUCCESS; } - memcpy(pOut->tbMeta, &pOut->ctbMeta, sizeof(pOut->ctbMeta)); + TAOS_MEMCPY(pOut->tbMeta, &pOut->ctbMeta, sizeof(pOut->ctbMeta)); } */ @@ -1392,7 +1624,10 @@ _return: tstrerror(code)); } if (pTask->res || code) { - ctgHandleTaskEnd(pTask, code); + int32_t newCode = ctgHandleTaskEnd(pTask, code); + if (TSDB_CODE_SUCCESS == code && TSDB_CODE_SUCCESS != newCode) { + code = newCode; + } } CTG_RET(code); @@ -1406,12 +1641,24 @@ int32_t ctgHandleGetTbMetasRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBu SRequestConnInfo* pConn = &pTask->pJob->conn; SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx); SCtgTbMetasCtx* ctx = (SCtgTbMetasCtx*)pTask->taskCtx; + bool taskDone = false; + + if (NULL == pMsgCtx) { + ctgError("fail to get task msgCtx, taskType:%d", pTask->type); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + SCtgFetch* pFetch = taosArrayGet(ctx->pFetchs, tReq->msgIdx); - SName* pName = ctgGetFetchName(ctx->pNames, pFetch); + if (NULL == pFetch) { + ctgError("fail to get the %dth fetch, fetchNum:%d", tReq->msgIdx, (int32_t)taosArrayGetSize(ctx->pFetchs)); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + int32_t flag = pFetch->flag; int32_t* vgId = &pFetch->vgId; - bool taskDone = false; - + SName* pName = NULL; + CTG_ERR_JRET(ctgGetFetchName(ctx->pNames, pFetch, &pName)); + CTG_ERR_JRET(ctgProcessRspMsg(pMsgCtx->out, reqType, pMsg->pData, pMsg->len, rspCode, pMsgCtx->target)); switch (reqType) { @@ -1434,7 +1681,7 @@ int32_t ctgHandleGetTbMetasRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBu if (CTG_IS_META_NULL(pOut->metaType)) { if (CTG_FLAG_IS_STB(flag)) { char dbFName[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(pName, dbFName); + (void)tNameGetFullDbName(pName, dbFName); CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache)); if (NULL != dbCache) { @@ -1461,7 +1708,7 @@ int32_t ctgHandleGetTbMetasRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBu } ctgTaskError("no tbmeta got, tbName:%s", tNameGetTableName(pName)); - ctgRemoveTbMetaFromCache(pCtg, pName, false); + (void)ctgRemoveTbMetaFromCache(pCtg, pName, false); // cache update not fatal error CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST); } @@ -1479,7 +1726,7 @@ int32_t ctgHandleGetTbMetasRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBu if (CTG_IS_META_NULL(pOut->metaType)) { ctgTaskError("no tbmeta got, tbName:%s", tNameGetTableName(pName)); - ctgRemoveTbMetaFromCache(pCtg, pName, false); + (void)ctgRemoveTbMetaFromCache(pCtg, pName, false); // cache update not fatal error CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST); } @@ -1497,13 +1744,13 @@ int32_t ctgHandleGetTbMetasRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBu int32_t exist = 0; if (!CTG_FLAG_IS_FORCE_UPDATE(flag)) { SName stbName = *pName; - strcpy(stbName.tname, pOut->tbName); + TAOS_STRCPY(stbName.tname, pOut->tbName); SCtgTbMetaCtx stbCtx = {0}; stbCtx.flag = flag; stbCtx.pName = &stbName; STableMeta* stbMeta = NULL; - (void)ctgReadTbMetaFromCache(pCtg, &stbCtx, &stbMeta); + CTG_ERR_JRET(ctgReadTbMetaFromCache(pCtg, &stbCtx, &stbMeta)); if (stbMeta && stbMeta->sversion >= pOut->tbMeta->sversion) { ctgTaskDebug("use cached stb meta, tbName:%s", tNameGetTableName(pName)); exist = 1; @@ -1529,16 +1776,16 @@ int32_t ctgHandleGetTbMetasRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBu STableMetaOutput* pOut = (STableMetaOutput*)pMsgCtx->out; - ctgUpdateTbMetaToCache(pCtg, pOut, false); + (void)ctgUpdateTbMetaToCache(pCtg, pOut, false); // cache update not fatal error if (CTG_IS_META_BOTH(pOut->metaType)) { - memcpy(pOut->tbMeta, &pOut->ctbMeta, sizeof(pOut->ctbMeta)); + TAOS_MEMCPY(pOut->tbMeta, &pOut->ctbMeta, sizeof(pOut->ctbMeta)); } /* else if (CTG_IS_META_CTABLE(pOut->metaType)) { SName stbName = *pName; - strcpy(stbName.tname, pOut->tbName); + TAOS_STRCPY(stbName.tname, pOut->tbName); SCtgTbMetaCtx stbCtx = {0}; stbCtx.flag = flag; stbCtx.pName = &stbName; @@ -1551,11 +1798,16 @@ int32_t ctgHandleGetTbMetasRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBu return TSDB_CODE_SUCCESS; } - memcpy(pOut->tbMeta, &pOut->ctbMeta, sizeof(pOut->ctbMeta)); + TAOS_MEMCPY(pOut->tbMeta, &pOut->ctbMeta, sizeof(pOut->ctbMeta)); } */ SMetaRes* pRes = taosArrayGet(ctx->pResList, pFetch->resIdx); + if (NULL == pRes) { + ctgTaskError("fail to get the %dth res in pResList, resNum:%d", pFetch->resIdx, (int32_t)taosArrayGetSize(ctx->pResList)); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + pRes->code = 0; pRes->pRes = pOut->tbMeta; pOut->tbMeta = NULL; @@ -1572,18 +1824,25 @@ _return: if (code) { SMetaRes* pRes = taosArrayGet(ctx->pResList, pFetch->resIdx); - pRes->code = code; - pRes->pRes = NULL; - ctgTaskError("Get table %d.%s.%s meta failed with error %s", pName->acctId, pName->dbname, pName->tname, - tstrerror(code)); - if (0 == atomic_sub_fetch_32(&ctx->fetchNum, 1)) { - TSWAP(pTask->res, ctx->pResList); - taskDone = true; + if (NULL == pRes) { + ctgTaskError("fail to get the %dth res in pResList, resNum:%d", pFetch->resIdx, (int32_t)taosArrayGetSize(ctx->pResList)); + } else { + pRes->code = code; + pRes->pRes = NULL; + ctgTaskError("Get table %d.%s.%s meta failed with error %s", pName->acctId, pName->dbname, pName->tname, + tstrerror(code)); + if (0 == atomic_sub_fetch_32(&ctx->fetchNum, 1)) { + TSWAP(pTask->res, ctx->pResList); + taskDone = true; + } } } if (pTask->res && taskDone) { - ctgHandleTaskEnd(pTask, code); + int32_t newCode = ctgHandleTaskEnd(pTask, code); + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } } CTG_RET(code); @@ -1594,6 +1853,7 @@ int32_t ctgHandleGetDbVgRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* SCtgTask* pTask = tReq->pTask; SCtgDbVgCtx* ctx = (SCtgDbVgCtx*)pTask->taskCtx; SCatalog* pCtg = pTask->pJob->pCtg; + int32_t newCode = TSDB_CODE_SUCCESS; CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); @@ -1616,7 +1876,10 @@ int32_t ctgHandleGetDbVgRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* _return: - ctgHandleTaskEnd(pTask, code); + newCode = ctgHandleTaskEnd(pTask, code); + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } CTG_RET(code); } @@ -1626,6 +1889,7 @@ int32_t ctgHandleGetTbHashRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf SCtgTask* pTask = tReq->pTask; SCtgTbHashCtx* ctx = (SCtgTbHashCtx*)pTask->taskCtx; SCatalog* pCtg = pTask->pJob->pCtg; + int32_t newCode = TSDB_CODE_SUCCESS; CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); @@ -1652,7 +1916,10 @@ int32_t ctgHandleGetTbHashRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf _return: - ctgHandleTaskEnd(pTask, code); + newCode = ctgHandleTaskEnd(pTask, code); + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } CTG_RET(code); } @@ -1662,9 +1929,18 @@ int32_t ctgHandleGetTbHashsRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBu SCtgTask* pTask = tReq->pTask; SCtgTbHashsCtx* ctx = (SCtgTbHashsCtx*)pTask->taskCtx; SCatalog* pCtg = pTask->pJob->pCtg; - SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx); - SCtgFetch* pFetch = taosArrayGet(ctx->pFetchs, tReq->msgIdx); bool taskDone = false; + SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx); + if (NULL == pMsgCtx) { + ctgError("fail to get task msgCtx, taskType:%d", pTask->type); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + SCtgFetch* pFetch = taosArrayGet(ctx->pFetchs, tReq->msgIdx); + if (NULL == pFetch) { + ctgError("fail to get the %dth fetch, fetchNum:%d", tReq->msgIdx, (int32_t)taosArrayGetSize(ctx->pFetchs)); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } CTG_ERR_JRET(ctgProcessRspMsg(pMsgCtx->out, reqType, pMsg->pData, pMsg->len, rspCode, pMsgCtx->target)); @@ -1673,6 +1949,11 @@ int32_t ctgHandleGetTbHashsRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBu SUseDbOutput* pOut = (SUseDbOutput*)pMsgCtx->out; STablesReq* pReq = taosArrayGet(ctx->pNames, pFetch->dbIdx); + if (NULL == pReq) { + ctgError("fail to get the %dth tb in ctx->pNames, reqNum:%d", pFetch->dbIdx, (int32_t)taosArrayGetSize(ctx->pNames)); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + CTG_ERR_JRET(ctgGetVgInfosFromHashValue(pCtg, &pTask->pJob->conn.mgmtEps, tReq, pOut->dbVgroup, ctx, pMsgCtx->target, pReq->pTables, true)); CTG_ERR_JRET(ctgUpdateVgroupEnqueue(pCtg, pMsgCtx->target, pOut->dbId, pOut->dbVgroup, false)); @@ -1694,21 +1975,32 @@ _return: if (code) { STablesReq* pReq = taosArrayGet(ctx->pNames, pFetch->dbIdx); - int32_t num = taosArrayGetSize(pReq->pTables); - for (int32_t i = 0; i < num; ++i) { - SMetaRes* pRes = taosArrayGet(ctx->pResList, pFetch->resIdx + i); - pRes->code = code; - pRes->pRes = NULL; - } + if (NULL == pReq) { + ctgError("fail to get the %dth tb in ctx->pNames, reqNum:%d", pFetch->dbIdx, (int32_t)taosArrayGetSize(ctx->pNames)); + } else { + int32_t num = taosArrayGetSize(pReq->pTables); + for (int32_t i = 0; i < num; ++i) { + SMetaRes* pRes = taosArrayGet(ctx->pResList, pFetch->resIdx + i); + if (NULL == pRes) { + ctgError("fail to get the %dth res in ctx->pResList, resNum:%d", pFetch->resIdx + i, (int32_t)taosArrayGetSize(ctx->pResList)); + } else { + pRes->code = code; + pRes->pRes = NULL; + } + } - if (0 == atomic_sub_fetch_32(&ctx->fetchNum, 1)) { - TSWAP(pTask->res, ctx->pResList); - taskDone = true; + if (0 == atomic_sub_fetch_32(&ctx->fetchNum, 1)) { + TSWAP(pTask->res, ctx->pResList); + taskDone = true; + } } } if (pTask->res && taskDone) { - ctgHandleTaskEnd(pTask, code); + int32_t newCode = ctgHandleTaskEnd(pTask, code); + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } } CTG_RET(code); @@ -1717,6 +2009,8 @@ _return: int32_t ctgHandleGetTbIndexRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) { int32_t code = 0; SCtgTask* pTask = tReq->pTask; + int32_t newCode = TSDB_CODE_SUCCESS; + CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); STableIndex* pOut = (STableIndex*)pTask->msgCtx.out; @@ -1731,7 +2025,11 @@ _return: if (TSDB_CODE_MND_DB_INDEX_NOT_EXIST == code) { code = TSDB_CODE_SUCCESS; } - ctgHandleTaskEnd(pTask, code); + + newCode = ctgHandleTaskEnd(pTask, code); + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } CTG_RET(code); } @@ -1739,13 +2037,18 @@ _return: int32_t ctgHandleGetTbCfgRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) { int32_t code = 0; SCtgTask* pTask = tReq->pTask; + int32_t newCode = TSDB_CODE_SUCCESS; + CTG_ERR_JRET(ctgProcessRspMsg(&pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); TSWAP(pTask->res, pTask->msgCtx.out); _return: - ctgHandleTaskEnd(pTask, code); + newCode = ctgHandleTaskEnd(pTask, code); + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } CTG_RET(code); } @@ -1755,6 +2058,8 @@ int32_t ctgHandleGetTbTagRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* int32_t code = 0; SCtgTask* pTask = tReq->pTask; SCatalog* pCtg = pTask->pJob->pCtg; + int32_t newCode = TSDB_CODE_SUCCESS; + CTG_ERR_JRET(ctgProcessRspMsg(&pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); STableCfgRsp* pRsp = (STableCfgRsp*)pTask->msgCtx.out; @@ -1778,7 +2083,11 @@ int32_t ctgHandleGetTbTagRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* tagVal.type = TSDB_DATA_TYPE_JSON; tagVal.pData = pJson; tagVal.nData = strlen(pJson); - taosArrayPush(pTagVals, &tagVal); + if (NULL == taosArrayPush(pTagVals, &tagVal)) { + taosMemoryFree(pJson); + taosArrayDestroy(pTagVals); + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } } else { CTG_ERR_JRET(tTagToValArray((const STag*)pRsp->pTags, &pTagVals)); } @@ -1787,7 +2096,10 @@ int32_t ctgHandleGetTbTagRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* _return: - ctgHandleTaskEnd(pTask, code); + newCode = ctgHandleTaskEnd(pTask, code); + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } CTG_RET(code); } @@ -1797,10 +2109,12 @@ int32_t ctgHandleGetDbCfgRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* int32_t code = 0; SCtgTask* pTask = tReq->pTask; SCtgDbCfgCtx* ctx = pTask->taskCtx; + int32_t newCode = TSDB_CODE_SUCCESS; CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); - SDbCfgInfo* pCfg = ctgCloneDbCfgInfo(pTask->msgCtx.out); + SDbCfgInfo* pCfg = NULL; + CTG_ERR_JRET(ctgCloneDbCfgInfo(pTask->msgCtx.out, &pCfg)); CTG_ERR_RET(ctgUpdateDbCfgEnqueue(pTask->pJob->pCtg, ctx->dbFName, pCfg->dbId, pCfg, false)); @@ -1808,7 +2122,10 @@ int32_t ctgHandleGetDbCfgRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* _return: - ctgHandleTaskEnd(pTask, code); + newCode = ctgHandleTaskEnd(pTask, code); + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } CTG_RET(code); } @@ -1820,13 +2137,17 @@ int32_t ctgHandleGetDbInfoRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf int32_t ctgHandleGetQnodeRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) { int32_t code = 0; SCtgTask* pTask = tReq->pTask; + int32_t newCode = TSDB_CODE_SUCCESS; CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); TSWAP(pTask->res, pTask->msgCtx.out); _return: - ctgHandleTaskEnd(pTask, code); + newCode = ctgHandleTaskEnd(pTask, code); + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } CTG_RET(code); } @@ -1834,13 +2155,18 @@ _return: int32_t ctgHandleGetDnodeRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) { int32_t code = 0; SCtgTask* pTask = tReq->pTask; + int32_t newCode = TSDB_CODE_SUCCESS; + CTG_ERR_JRET(ctgProcessRspMsg(&pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); TSWAP(pTask->res, pTask->msgCtx.out); _return: - ctgHandleTaskEnd(pTask, code); + newCode = ctgHandleTaskEnd(pTask, code); + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } CTG_RET(code); } @@ -1848,13 +2174,17 @@ _return: int32_t ctgHandleGetIndexRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) { int32_t code = 0; SCtgTask* pTask = tReq->pTask; + int32_t newCode = TSDB_CODE_SUCCESS; CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); TSWAP(pTask->res, pTask->msgCtx.out); _return: - ctgHandleTaskEnd(pTask, code); + newCode = ctgHandleTaskEnd(pTask, code); + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } CTG_RET(code); } @@ -1862,13 +2192,17 @@ _return: int32_t ctgHandleGetUdfRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) { int32_t code = 0; SCtgTask* pTask = tReq->pTask; + int32_t newCode = TSDB_CODE_SUCCESS; CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); TSWAP(pTask->res, pTask->msgCtx.out); _return: - ctgHandleTaskEnd(pTask, code); + newCode = ctgHandleTaskEnd(pTask, code); + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } CTG_RET(code); } @@ -1878,10 +2212,11 @@ int32_t ctgHandleGetUserRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* SCtgTask* pTask = tReq->pTask; SCatalog* pCtg = pTask->pJob->pCtg; SGetUserAuthRsp* pOut = (SGetUserAuthRsp*)pTask->msgCtx.out; + int32_t newCode = TSDB_CODE_SUCCESS; CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); - ctgUpdateUserEnqueue(pCtg, pOut, true); + (void)ctgUpdateUserEnqueue(pCtg, pOut, true); // cache update not fatal error taosMemoryFreeClear(pTask->msgCtx.out); CTG_ERR_JRET((*gCtgAsyncFps[pTask->type].launchFp)(pTask)); @@ -1890,7 +2225,10 @@ int32_t ctgHandleGetUserRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* _return: - ctgHandleTaskEnd(pTask, code); + newCode = ctgHandleTaskEnd(pTask, code); + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } CTG_RET(code); } @@ -1898,6 +2236,7 @@ _return: int32_t ctgHandleGetSvrVerRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) { int32_t code = 0; SCtgTask* pTask = tReq->pTask; + int32_t newCode = TSDB_CODE_SUCCESS; CTG_ERR_JRET(ctgProcessRspMsg(&pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); @@ -1905,7 +2244,10 @@ int32_t ctgHandleGetSvrVerRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf _return: - ctgHandleTaskEnd(pTask, code); + newCode = ctgHandleTaskEnd(pTask, code); + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } CTG_RET(code); } @@ -1918,10 +2260,23 @@ int32_t ctgHandleGetViewsRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx); SCtgViewsCtx* ctx = (SCtgViewsCtx*)pTask->taskCtx; SCtgFetch* pFetch = taosArrayGet(ctx->pFetchs, tReq->msgIdx); - SName* pName = ctgGetFetchName(ctx->pNames, pFetch); + bool taskDone = false; + + if (NULL == pMsgCtx) { + ctgError("fail to get task msgCtx, taskType:%d", pTask->type); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + if (NULL == pFetch) { + ctgError("fail to get the %dth fetch, fetchNum:%d", tReq->msgIdx, (int32_t)taosArrayGetSize(ctx->pFetchs)); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + int32_t flag = pFetch->flag; int32_t* vgId = &pFetch->vgId; - bool taskDone = false; + SName* pName = NULL; + + CTG_ERR_JRET(ctgGetFetchName(ctx->pNames, pFetch, &pName)); CTG_ERR_JRET(ctgProcessRspMsg(pMsgCtx->out, reqType, pMsg->pData, pMsg->len, rspCode, pMsgCtx->target)); @@ -1931,18 +2286,24 @@ int32_t ctgHandleGetViewsRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } - if (TSDB_CODE_SUCCESS != (code = dupViewMetaFromRsp(pRsp, pViewMeta))) { + code = dupViewMetaFromRsp(pRsp, pViewMeta); + if (TSDB_CODE_SUCCESS != code) { ctgFreeSViewMeta(pViewMeta); taosMemoryFree(pViewMeta); CTG_ERR_JRET(code); } ctgDebug("start to update view meta to cache, view:%s, querySQL:%s", pRsp->name, pRsp->querySql); - ctgUpdateViewMetaToCache(pCtg, pRsp, false); + (void)ctgUpdateViewMetaToCache(pCtg, pRsp, false); // cache update not fatal error taosMemoryFreeClear(pMsgCtx->out); pRsp = NULL; SMetaRes* pRes = taosArrayGet(ctx->pResList, pFetch->resIdx); + if (NULL == pRes) { + ctgTaskError("fail to get the %dth res in ctx->pResList, totalResNum:%d", pFetch->resIdx, (int32_t)taosArrayGetSize(ctx->pResList)); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + pRes->code = 0; pRes->pRes = pViewMeta; if (0 == atomic_sub_fetch_32(&ctx->fetchNum, 1)) { @@ -1954,23 +2315,401 @@ _return: if (code) { SMetaRes* pRes = taosArrayGet(ctx->pResList, pFetch->resIdx); - pRes->code = code; - pRes->pRes = NULL; - if (TSDB_CODE_MND_VIEW_NOT_EXIST == code) { - ctgTaskDebug("Get view %d.%s.%s meta failed with %s", pName->acctId, pName->dbname, pName->tname, - tstrerror(code)); + if (NULL == pRes) { + ctgTaskError("fail to get the %dth res in ctx->pResList, totalResNum:%d", pFetch->resIdx, (int32_t)taosArrayGetSize(ctx->pResList)); } else { - ctgTaskError("Get view %d.%s.%s meta failed with error %s", pName->acctId, pName->dbname, pName->tname, - tstrerror(code)); - } - if (0 == atomic_sub_fetch_32(&ctx->fetchNum, 1)) { - TSWAP(pTask->res, ctx->pResList); - taskDone = true; + pRes->code = code; + pRes->pRes = NULL; + if (TSDB_CODE_MND_VIEW_NOT_EXIST == code) { + ctgTaskDebug("Get view %d.%s.%s meta failed with %s", pName->acctId, pName->dbname, pName->tname, + tstrerror(code)); + } else { + ctgTaskError("Get view %d.%s.%s meta failed with error %s", pName->acctId, pName->dbname, pName->tname, + tstrerror(code)); + } + if (0 == atomic_sub_fetch_32(&ctx->fetchNum, 1)) { + TSWAP(pTask->res, ctx->pResList); + taskDone = true; + } } } if (pTask->res && taskDone) { - ctgHandleTaskEnd(pTask, code); + int32_t newCode = ctgHandleTaskEnd(pTask, code); + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } + } + + CTG_RET(code); +} + + +static int32_t ctgTsmaFetchStreamProgress(SCtgTaskReq* tReq, SHashObj* pVgHash, const STableTSMAInfoRsp* pTsmas) { + int32_t code = 0; + SCtgTask* pTask = tReq->pTask; + SCatalog* pCtg = pTask->pJob->pCtg; + int32_t subFetchIdx = 0; + SCtgTbTSMACtx* pCtx = pTask->taskCtx; + SRequestConnInfo* pConn = &pTask->pJob->conn; + SVgroupInfo* pVgInfo = NULL; + SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx); + if (NULL == pFetch) { + ctgError("fail to get the %dth SCtgTSMAFetch, totalNum:%d", tReq->msgIdx, (int32_t)taosArrayGetSize(pCtx->pFetches)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx); + if (NULL == pTbReq) { + ctgError("fail to get the %dth STablesReq, totalNum:%d", pFetch->dbIdx, (int32_t)taosArrayGetSize(pCtx->pNames)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + const SName* pTbName = taosArrayGet(pTbReq->pTables, pFetch->tbIdx); + if (NULL == pTbName) { + ctgError("fail to get the %dth SName, totalNum:%d", pFetch->tbIdx, (int32_t)taosArrayGetSize(pTbReq->pTables)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + pFetch->vgNum = taosHashGetSize(pVgHash); + for (int32_t i = 0; i < taosArrayGetSize(pTsmas->pTsmas); ++i) { + STableTSMAInfo* pTsmaInfo = taosArrayGetP(pTsmas->pTsmas, i); + if (NULL == pTsmaInfo) { + ctgError("fail to get the %dth STableTSMAInfo, totalNum:%d", i, (int32_t)taosArrayGetSize(pTsmas->pTsmas)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + pVgInfo = taosHashIterate(pVgHash, NULL); + pTsmaInfo->reqTs = taosGetTimestampMs(); + while (pVgInfo) { + // make StreamProgressReq, send it + SStreamProgressReq req = {.fetchIdx = pFetch->fetchIdx, + .streamId = pTsmaInfo->streamUid, + .subFetchIdx = subFetchIdx++, + .vgId = pVgInfo->vgId}; + CTG_ERR_JRET(ctgGetStreamProgressFromVnode(pCtg, pConn, pTbName, pVgInfo, NULL, tReq, &req)); + pFetch->subFetchNum++; + + pVgInfo = taosHashIterate(pVgHash, pVgInfo); + } + } + +_return: + + CTG_RET(code); +} + + +int32_t ctgHandleGetTSMARsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) { + int32_t code = 0; + SCtgTask* pTask = tReq->pTask; + SCatalog* pCtg = pTask->pJob->pCtg; + SCtgTbTSMACtx* pCtx = pTask->taskCtx; + int32_t newCode = TSDB_CODE_SUCCESS; + SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx); + if (NULL == pMsgCtx) { + ctgError("fail to get the %dth SCtgMsgCtx", tReq->msgIdx); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + SMetaRes* pRes = taosArrayGet(pCtx->pResList, 0); + if (NULL == pRes) { + ctgError("fail to get the 0th SMetaRes, totalNum:%d", (int32_t)taosArrayGetSize(pCtx->pResList)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + STablesReq* pTbReq = taosArrayGet(pCtx->pNames, 0); + if (NULL == pTbReq) { + ctgError("fail to get the 0th STablesReq, totalNum:%d", (int32_t)taosArrayGetSize(pCtx->pNames)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + SName* pName = taosArrayGet(pTbReq->pTables, 0); + if (NULL == pName) { + ctgError("fail to get the 0th SName, totalNum:%d", (int32_t)taosArrayGetSize(pTbReq->pTables)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + SRequestConnInfo* pConn = &pTask->pJob->conn; + CTG_ERR_JRET(ctgProcessRspMsg(pMsgCtx->out, reqType, pMsg->pData, pMsg->len, rspCode, pMsgCtx->target)); + + switch (reqType) { + case TDMT_MND_TABLE_META: { + STableMetaOutput* pOut = (STableMetaOutput*)pMsgCtx->out; + if (!CTG_IS_META_NULL(pOut->metaType)) { + CTG_ERR_JRET(ctgUpdateTbMetaToCache(pCtg, pOut, CTG_FLAG_SYNC_OP)); + } + + break; + } + case TDMT_MND_GET_TSMA: { + STableTSMAInfoRsp* pOut = pMsgCtx->out; + pRes->code = 0; + if (pOut->pTsmas->size > 0) { + ASSERT(pOut->pTsmas->size == 1); + pRes->pRes = pOut; + pMsgCtx->out = NULL; + TSWAP(pTask->res, pCtx->pResList); + + STableTSMAInfo* pTsma = taosArrayGetP(pOut->pTsmas, 0); + if (NULL == pTsma) { + ctgError("fail to get the 0th STableTSMAInfo, totalNum:%d", (int32_t)taosArrayGetSize(pOut->pTsmas)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + int32_t exists = false; + CTG_ERR_JRET(ctgTbMetaExistInCache(pCtg, pTsma->targetDbFName, pTsma->targetTb, &exists)); + if (!exists) { + TSWAP(pMsgCtx->lastOut, pMsgCtx->out); + CTG_RET(ctgGetTbMetaFromMnodeImpl(pCtg, pConn, pTsma->targetDbFName, pTsma->targetTb, NULL, tReq)); + } + } + + break; + } + default: + ASSERT(0); + } + +_return: + + if (code) { + if (TSDB_CODE_MND_SMA_NOT_EXIST == code) { + code = TSDB_CODE_SUCCESS; + } else { + ctgTaskError("Get tsma for %d.%s.%s failed with err: %s", pName->acctId, pName->dbname, pName->tname, + tstrerror(code)); + } + } + + newCode = ctgHandleTaskEnd(pTask, code); + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } + + CTG_RET(code); +} + + +int32_t ctgHandleGetTbTSMARsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) { + bool taskDone = false; + int32_t code = 0; + SCtgTask* pTask = tReq->pTask; + SCatalog* pCtg = pTask->pJob->pCtg; + SCtgTbTSMACtx* pCtx = pTask->taskCtx; + SArray* pTsmas = NULL; + SHashObj* pVgHash = NULL; + SCtgDBCache* pDbCache = NULL; + STableTSMAInfo* pTsma = NULL; + SRequestConnInfo* pConn = &pTask->pJob->conn; + + SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx); + if (NULL == pMsgCtx) { + ctgError("fail to get the %dth SCtgMsgCtx", tReq->msgIdx); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx); + if (NULL == pFetch) { + ctgError("fail to get the %dth SCtgTSMAFetch, totalNum:%d", tReq->msgIdx, (int32_t)taosArrayGetSize(pCtx->pFetches)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + SMetaRes* pRes = taosArrayGet(pCtx->pResList, pFetch->resIdx); + if (NULL == pRes) { + ctgError("fail to get the %dth SMetaRes, totalNum:%d", pFetch->resIdx, (int32_t)taosArrayGetSize(pCtx->pResList)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx); + if (NULL == pTbReq) { + ctgError("fail to get the %dth STablesReq, totalNum:%d", pFetch->dbIdx, (int32_t)taosArrayGetSize(pCtx->pNames)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + SName* pTbName = taosArrayGet(pTbReq->pTables, pFetch->tbIdx); + if (NULL == pTbName) { + ctgError("fail to get the %dth SName, totalNum:%d", pFetch->tbIdx, (int32_t)taosArrayGetSize(pTbReq->pTables)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + if (reqType != TDMT_VND_GET_STREAM_PROGRESS) + CTG_ERR_JRET(ctgProcessRspMsg(pMsgCtx->out, reqType, pMsg->pData, pMsg->len, rspCode, pMsgCtx->target)); + + switch (reqType) { + case TDMT_MND_GET_TABLE_TSMA: { + STableTSMAInfoRsp* pOut = pMsgCtx->out; + pFetch->fetchType = FETCH_TSMA_STREAM_PROGRESS; + pRes->pRes = pOut; + pMsgCtx->out = NULL; + + if (pOut->pTsmas && taosArrayGetSize(pOut->pTsmas) > 0) { + // fetch progress + (void)ctgAcquireVgInfoFromCache(pCtg, pTbReq->dbFName, &pDbCache); // ignore cache error + + if (!pDbCache) { + // do not know which vnodes to fetch, fetch vnode list first + SBuildUseDBInput input = {0}; + tstrncpy(input.db, pTbReq->dbFName, tListLen(input.db)); + input.vgVersion = CTG_DEFAULT_INVALID_VERSION; + CTG_ERR_JRET(ctgGetDBVgInfoFromMnode(pCtg, pConn, &input, NULL, tReq)); + } else { + // fetch progress from every vnode + CTG_ERR_JRET(ctgTsmaFetchStreamProgress(tReq, pDbCache->vgCache.vgInfo->vgHash, pOut)); + ctgReleaseVgInfoToCache(pCtg, pDbCache); + pDbCache = NULL; + } + } else { + // no tsmas + if (atomic_sub_fetch_32(&pCtx->fetchNum, 1) == 0) { + TSWAP(pTask->res, pCtx->pResList); + taskDone = true; + } + } + + break; + } + case TDMT_VND_GET_STREAM_PROGRESS: { + SStreamProgressRsp rsp = {0}; + CTG_ERR_JRET(ctgProcessRspMsg(&rsp, reqType, pMsg->pData, pMsg->len, rspCode, pMsgCtx->target)); + + // update progress into res + STableTSMAInfoRsp* pTsmasRsp = pRes->pRes; + SArray* pTsmas = pTsmasRsp->pTsmas; + SStreamProgressRsp* pRsp = &rsp; + int32_t tsmaIdx = pRsp->subFetchIdx / pFetch->vgNum; + STableTSMAInfo* pTsmaInfo = taosArrayGetP(pTsmas, tsmaIdx); + if (NULL == pTsmaInfo) { + ctgError("fail to get the %dth STableTSMAInfo, totalNum:%d", tsmaIdx, (int32_t)taosArrayGetSize(pTsmas)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + if (pTsmaInfo->rspTs == 0) { + pTsmaInfo->fillHistoryFinished = true; + } + + pTsmaInfo->rspTs = taosGetTimestampMs(); + pTsmaInfo->delayDuration = TMAX(pRsp->progressDelay, pTsmaInfo->delayDuration); + pTsmaInfo->fillHistoryFinished = pTsmaInfo->fillHistoryFinished && pRsp->fillHisFinished; + + qDebug("received stream progress for tsma %s rsp history: %d vnode: %d, delay: %" PRId64, pTsmaInfo->name, + pRsp->fillHisFinished, pRsp->subFetchIdx, pRsp->progressDelay); + + if (atomic_add_fetch_32(&pFetch->finishedSubFetchNum, 1) == pFetch->subFetchNum) { + // subfetch all finished + for (int32_t i = 0; i < taosArrayGetSize(pTsmas); ++i) { + STableTSMAInfo* pInfo = taosArrayGetP(pTsmas, i); + CTG_ERR_JRET(tCloneTbTSMAInfo(pInfo, &pTsma)); + CTG_ERR_JRET(ctgUpdateTbTSMAEnqueue(pCtg, &pTsma, 0, false)); + } + + if (atomic_sub_fetch_32(&pCtx->fetchNum, 1) == 0) { + TSWAP(pTask->res, pCtx->pResList); + taskDone = true; + } + } + + break; + } + case TDMT_MND_USE_DB: { + SUseDbOutput* pOut = (SUseDbOutput*)pMsgCtx->out; + + switch (pFetch->fetchType) { + case FETCH_TSMA_SOURCE_TB_META: { + SVgroupInfo vgInfo = {0}; + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, &pConn->mgmtEps, pOut->dbVgroup, pTbName, &vgInfo)); + + pFetch->vgId = vgInfo.vgId; + CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, pTbName, &vgInfo, NULL, tReq)); + + break; + } + case FETCH_TSMA_STREAM_PROGRESS: { + STableTSMAInfoRsp* pTsmas = pRes->pRes; + TSWAP(pOut->dbVgroup->vgHash, pVgHash); + CTG_ERR_JRET(ctgTsmaFetchStreamProgress(tReq, pVgHash, pTsmas)); + + break; + } + default: + ASSERT(0); + } + + break; + } + case TDMT_VND_TABLE_META: { + // handle source tb meta + ASSERT(pFetch->fetchType == FETCH_TSMA_SOURCE_TB_META); + STableMetaOutput* pOut = (STableMetaOutput*)pMsgCtx->out; + pFetch->fetchType = FETCH_TB_TSMA; + pFetch->tsmaSourceTbName = *pTbName; + + if (CTG_IS_META_NULL(pOut->metaType)) { + ctgTaskError("no tbmeta found when fetching tsma source tb meta: %s.%s", pTbName->dbname, pTbName->tname); + (void)ctgRemoveTbMetaFromCache(pCtg, pTbName, false); // ignore cache error + CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST); + } + + if (META_TYPE_BOTH_TABLE == pOut->metaType) { + // rewrite tsma fetch table with it's super table name + (void)sprintf(pFetch->tsmaSourceTbName.tname, "%s", pOut->tbName); + } + + CTG_ERR_JRET(ctgGetTbTSMAFromMnode(pCtg, pConn, &pFetch->tsmaSourceTbName, NULL, tReq, TDMT_MND_GET_TABLE_TSMA)); + + break; + } + default: + ASSERT(0); + } + +_return: + + if (pDbCache) { + ctgReleaseVgInfoToCache(pCtg, pDbCache); + } + if (pTsma) { + tFreeTableTSMAInfo(pTsma); + pTsma = NULL; + } + if (pVgHash) { + taosHashCleanup(pVgHash); + } + if (code) { + int32_t newCode = TSDB_CODE_SUCCESS; + SMetaRes* pRes = taosArrayGet(pCtx->pResList, pFetch->resIdx); + if (NULL == pRes) { + ctgError("fail to get the %dth SMetaRes, totalNum:%d", pFetch->resIdx, (int32_t)taosArrayGetSize(pCtx->pResList)); + newCode = TSDB_CODE_CTG_INTERNAL_ERROR; + } else { + pRes->code = code; + if (TSDB_CODE_MND_SMA_NOT_EXIST == code) { + code = TSDB_CODE_SUCCESS; + } else { + ctgTaskError("Get tsma for %d.%s.%s faield with err: %s", pTbName->acctId, pTbName->dbname, pTbName->tname, + tstrerror(code)); + } + } + + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } + + bool allSubFetchFinished = false; + if (pMsgCtx->reqType == TDMT_VND_GET_STREAM_PROGRESS) { + allSubFetchFinished = atomic_add_fetch_32(&pFetch->finishedSubFetchNum, 1) >= pFetch->subFetchNum; + } + if ((allSubFetchFinished || pFetch->subFetchNum == 0) && 0 == atomic_sub_fetch_32(&pCtx->fetchNum, 1)) { + TSWAP(pTask->res, pCtx->pResList); + taskDone = true; + } + } + + if (pTask->res && taskDone) { + int32_t newCode = ctgHandleTaskEnd(pTask, code); + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } } CTG_RET(code); @@ -1998,7 +2737,7 @@ int32_t ctgAsyncRefreshTbMeta(SCtgTaskReq* tReq, int32_t flag, SName* pName, int SCtgDBCache* dbCache = NULL; char dbFName[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(pName, dbFName); + (void)tNameGetFullDbName(pName, dbFName); CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache)); if (dbCache) { @@ -2032,6 +2771,11 @@ int32_t ctgLaunchGetTbMetaTask(SCtgTask* pTask) { SRequestConnInfo* pConn = &pTask->pJob->conn; SCtgJob* pJob = pTask->pJob; SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx) { + ctgError("fail to get task msgCtx, taskType:%d", pTask->type); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (NULL == pMsgCtx->pBatchs) { pMsgCtx->pBatchs = pJob->pBatchs; } @@ -2056,12 +2800,18 @@ int32_t ctgLaunchGetTbMetasTask(SCtgTask* pTask) { SRequestConnInfo* pConn = &pTask->pJob->conn; SCtgTbMetasCtx* pCtx = (SCtgTbMetasCtx*)pTask->taskCtx; SCtgJob* pJob = pTask->pJob; + SName* pName = NULL; int32_t dbNum = taosArrayGetSize(pCtx->pNames); int32_t fetchIdx = 0; int32_t baseResIdx = 0; for (int32_t i = 0; i < dbNum; ++i) { STablesReq* pReq = taosArrayGet(pCtx->pNames, i); + if (NULL == pReq) { + ctgError("fail to get the %dth STablesReq, num:%d", i, dbNum); + CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); + } + ctgDebug("start to check tb metas in db %s, tbNum %ld", pReq->dbFName, taosArrayGetSize(pReq->pTables)); CTG_ERR_RET(ctgGetTbMetasFromCache(pCtg, pConn, pCtx, i, &fetchIdx, baseResIdx, pReq->pTables)); baseResIdx += taosArrayGetSize(pReq->pTables); @@ -2075,11 +2825,27 @@ int32_t ctgLaunchGetTbMetasTask(SCtgTask* pTask) { return TSDB_CODE_SUCCESS; } - pTask->msgCtxs = taosArrayInit_s(sizeof(SCtgMsgCtx), pCtx->fetchNum); + pTask->msgCtxs = taosArrayInit_s(pCtx->fetchNum, sizeof(SCtgMsgCtx)); + if (NULL == pTask->msgCtxs) { + ctgError("taosArrayInit_s %d SCtgMsgCtx %d failed", pCtx->fetchNum, (int32_t)sizeof(SCtgMsgCtx)); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + for (int32_t i = 0; i < pCtx->fetchNum; ++i) { - SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, i); - SName* pName = ctgGetFetchName(pCtx->pNames, pFetch); + SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, i); + if (NULL == pFetch) { + ctgError("fail to get the %dth fetch in pCtx->pFetchs, fetchNum:%d", i, pCtx->fetchNum); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + CTG_ERR_RET(ctgGetFetchName(pCtx->pNames, pFetch, &pName)); + SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, i); + if (NULL == pMsgCtx) { + ctgError("fail to get the %dth pMsgCtx", i); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (NULL == pMsgCtx->pBatchs) { pMsgCtx->pBatchs = pJob->pBatchs; } @@ -2101,6 +2867,11 @@ int32_t ctgLaunchGetDbVgTask(SCtgTask* pTask) { SCtgDbVgCtx* pCtx = (SCtgDbVgCtx*)pTask->taskCtx; SCtgJob* pJob = pTask->pJob; SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx) { + ctgError("fail to get the %dth pMsgCtx", -1); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (NULL == pMsgCtx->pBatchs) { pMsgCtx->pBatchs = pJob->pBatchs; } @@ -2142,6 +2913,11 @@ int32_t ctgLaunchGetTbHashTask(SCtgTask* pTask) { SCtgTbHashCtx* pCtx = (SCtgTbHashCtx*)pTask->taskCtx; SCtgJob* pJob = pTask->pJob; SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx) { + ctgError("fail to get the %dth pMsgCtx", -1); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (NULL == pMsgCtx->pBatchs) { pMsgCtx->pBatchs = pJob->pBatchs; } @@ -2192,6 +2968,10 @@ int32_t ctgLaunchGetTbHashsTask(SCtgTask* pTask) { for (int32_t i = 0; i < dbNum; ++i) { STablesReq* pReq = taosArrayGet(pCtx->pNames, i); + if (NULL == pReq) { + ctgError("fail to get the %dth STablesReq, dbNum:%d", i, dbNum); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, pReq->dbFName, &dbCache)); @@ -2207,12 +2987,14 @@ int32_t ctgLaunchGetTbHashsTask(SCtgTask* pTask) { baseResIdx += taosArrayGetSize(pReq->pTables); } else { - ctgAddFetch(&pCtx->pFetchs, i, -1, &fetchIdx, baseResIdx, 0); + CTG_ERR_JRET(ctgAddFetch(&pCtx->pFetchs, i, -1, &fetchIdx, baseResIdx, 0)); baseResIdx += taosArrayGetSize(pReq->pTables); int32_t inc = baseResIdx - taosArrayGetSize(pCtx->pResList); for(int32_t j = 0; j < inc; ++j) { - taosArrayPush(pCtx->pResList, &(SMetaRes){0}); + if (NULL == taosArrayPush(pCtx->pResList, &(SMetaRes){0})) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } } } } @@ -2221,22 +3003,41 @@ int32_t ctgLaunchGetTbHashsTask(SCtgTask* pTask) { if (pCtx->fetchNum <= 0) { TSWAP(pTask->res, pCtx->pResList); - CTG_ERR_RET(ctgHandleTaskEnd(pTask, 0)); + CTG_ERR_JRET(ctgHandleTaskEnd(pTask, 0)); return TSDB_CODE_SUCCESS; } pTask->msgCtxs = taosArrayInit_s(sizeof(SCtgMsgCtx), pCtx->fetchNum); + if (NULL == pTask->msgCtxs) { + ctgError("taosArrayInit_s %d SCtgMsgCtx %d failed", pCtx->fetchNum, (int32_t)sizeof(SCtgMsgCtx)); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } for (int32_t i = 0; i < pCtx->fetchNum; ++i) { SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, i); + if (NULL == pFetch) { + ctgError("fail to get the %dth SCtgFetch, fetchNum:%d", i, pCtx->fetchNum); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + STablesReq* pReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx); + if (NULL == pFetch) { + ctgError("fail to get the %dth SCtgFetch, fetchNum:%d", i, pCtx->fetchNum); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, i); + if (NULL == pFetch) { + ctgError("fail to get the %dth SCtgMsgCtx", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (NULL == pMsgCtx->pBatchs) { pMsgCtx->pBatchs = pJob->pBatchs; } SBuildUseDBInput input = {0}; - strcpy(input.db, pReq->dbFName); + TAOS_STRCPY(input.db, pReq->dbFName); input.vgVersion = CTG_DEFAULT_INVALID_VERSION; @@ -2263,6 +3064,11 @@ int32_t ctgLaunchGetTbIndexTask(SCtgTask* pTask) { SArray* pRes = NULL; SCtgJob* pJob = pTask->pJob; SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx) { + ctgError("fail to get the %dth pMsgCtx", -1); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (NULL == pMsgCtx->pBatchs) { pMsgCtx->pBatchs = pJob->pBatchs; } @@ -2285,10 +3091,16 @@ int32_t ctgLaunchGetTbCfgTask(SCtgTask* pTask) { SRequestConnInfo* pConn = &pTask->pJob->conn; SCtgTbCfgCtx* pCtx = (SCtgTbCfgCtx*)pTask->taskCtx; SArray* pRes = NULL; + SCtgJob* pJob = pTask->pJob; char dbFName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(pCtx->pName, dbFName); - SCtgJob* pJob = pTask->pJob; + (void)tNameGetFullDbName(pCtx->pName, dbFName); + SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx) { + ctgError("fail to get the %dth pMsgCtx", -1); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (NULL == pMsgCtx->pBatchs) { pMsgCtx->pBatchs = pJob->pBatchs; } @@ -2325,7 +3137,10 @@ int32_t ctgLaunchGetTbCfgTask(SCtgTask* pTask) { _return: if (CTG_TASK_LAUNCHED == pTask->status) { - ctgHandleTaskEnd(pTask, code); + int32_t newCode = ctgHandleTaskEnd(pTask, code); + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } } CTG_RET(code); @@ -2338,10 +3153,16 @@ int32_t ctgLaunchGetTbTagTask(SCtgTask* pTask) { SRequestConnInfo* pConn = &pTask->pJob->conn; SCtgTbTagCtx* pCtx = (SCtgTbTagCtx*)pTask->taskCtx; SArray* pRes = NULL; + SCtgJob* pJob = pTask->pJob; char dbFName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(pCtx->pName, dbFName); - SCtgJob* pJob = pTask->pJob; + (void)tNameGetFullDbName(pCtx->pName, dbFName); + SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx) { + ctgError("fail to get the %dth pMsgCtx", -1); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (NULL == pMsgCtx->pBatchs) { pMsgCtx->pBatchs = pJob->pBatchs; } @@ -2363,7 +3184,10 @@ int32_t ctgLaunchGetTbTagTask(SCtgTask* pTask) { _return: if (CTG_TASK_LAUNCHED == pTask->status) { - ctgHandleTaskEnd(pTask, code); + int32_t newCode = ctgHandleTaskEnd(pTask, code); + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } } CTG_RET(code); @@ -2375,12 +3199,18 @@ int32_t ctgLaunchGetQnodeTask(SCtgTask* pTask) { SRequestConnInfo* pConn = &pTask->pJob->conn; SCtgJob* pJob = pTask->pJob; SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx) { + ctgError("fail to get the %dth pMsgCtx", -1); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (NULL == pMsgCtx->pBatchs) { pMsgCtx->pBatchs = pJob->pBatchs; } CTG_CACHE_NHIT_INC(CTG_CI_QNODE, 1); CTG_ERR_RET(ctgGetQnodeListFromMnode(pCtg, pConn, NULL, pTask)); + return TSDB_CODE_SUCCESS; } @@ -2389,12 +3219,18 @@ int32_t ctgLaunchGetDnodeTask(SCtgTask* pTask) { SRequestConnInfo* pConn = &pTask->pJob->conn; SCtgJob* pJob = pTask->pJob; SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx) { + ctgError("fail to get the %dth pMsgCtx", -1); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (NULL == pMsgCtx->pBatchs) { pMsgCtx->pBatchs = pJob->pBatchs; } CTG_CACHE_NHIT_INC(CTG_CI_DNODE, 1); CTG_ERR_RET(ctgGetDnodeListFromMnode(pCtg, pConn, NULL, pTask)); + return TSDB_CODE_SUCCESS; } @@ -2404,11 +3240,16 @@ int32_t ctgLaunchGetDbCfgTask(SCtgTask* pTask) { SCtgDbCfgCtx* pCtx = (SCtgDbCfgCtx*)pTask->taskCtx; SCtgJob* pJob = pTask->pJob; SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); - SDbCfgInfo cfgInfo; + if (NULL == pMsgCtx) { + ctgError("fail to get the %dth pMsgCtx", -1); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (NULL == pMsgCtx->pBatchs) { pMsgCtx->pBatchs = pJob->pBatchs; } + SDbCfgInfo cfgInfo; CTG_ERR_RET(ctgReadDBCfgFromCache(pCtg, pCtx->dbFName, &cfgInfo)); if (cfgInfo.cfgVersion < 0) { @@ -2419,7 +3260,7 @@ int32_t ctgLaunchGetDbCfgTask(SCtgTask* pTask) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(pTask->res, &cfgInfo, sizeof(cfgInfo)); + TAOS_MEMCPY(pTask->res, &cfgInfo, sizeof(cfgInfo)); CTG_ERR_RET(ctgHandleTaskEnd(pTask, 0)); } @@ -2433,6 +3274,11 @@ int32_t ctgLaunchGetDbInfoTask(SCtgTask* pTask) { SCtgDbInfoCtx* pCtx = (SCtgDbInfoCtx*)pTask->taskCtx; SCtgJob* pJob = pTask->pJob; SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx) { + ctgError("fail to get the %dth pMsgCtx", -1); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (NULL == pMsgCtx->pBatchs) { pMsgCtx->pBatchs = pJob->pBatchs; } @@ -2473,6 +3319,11 @@ int32_t ctgLaunchGetIndexTask(SCtgTask* pTask) { SCtgIndexCtx* pCtx = (SCtgIndexCtx*)pTask->taskCtx; SCtgJob* pJob = pTask->pJob; SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx) { + ctgError("fail to get the %dth pMsgCtx", -1); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (NULL == pMsgCtx->pBatchs) { pMsgCtx->pBatchs = pJob->pBatchs; } @@ -2490,6 +3341,11 @@ int32_t ctgLaunchGetUdfTask(SCtgTask* pTask) { SCtgUdfCtx* pCtx = (SCtgUdfCtx*)pTask->taskCtx; SCtgJob* pJob = pTask->pJob; SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx) { + ctgError("fail to get the %dth pMsgCtx", -1); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (NULL == pMsgCtx->pBatchs) { pMsgCtx->pBatchs = pJob->pBatchs; } @@ -2511,6 +3367,11 @@ int32_t ctgLaunchGetUserTask(SCtgTask* pTask) { SCtgJob* pJob = pTask->pJob; bool tbNotExists = false; SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx) { + ctgError("fail to get the %dth pMsgCtx", -1); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (NULL == pMsgCtx->pBatchs) { pMsgCtx->pBatchs = pJob->pBatchs; } @@ -2559,6 +3420,11 @@ int32_t ctgLaunchGetSvrVerTask(SCtgTask* pTask) { SRequestConnInfo* pConn = &pTask->pJob->conn; SCtgJob* pJob = pTask->pJob; SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx) { + ctgError("fail to get the %dth pMsgCtx", -1); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (NULL == pMsgCtx->pBatchs) { pMsgCtx->pBatchs = pJob->pBatchs; } @@ -2576,6 +3442,7 @@ int32_t ctgLaunchGetViewsTask(SCtgTask* pTask) { SCtgViewsCtx* pCtx = (SCtgViewsCtx*)pTask->taskCtx; SCtgJob* pJob = pTask->pJob; bool tbMetaDone = false; + SName* pName = NULL; /* ctgIsTaskDone(pJob, CTG_TASK_GET_TB_META_BATCH, &tbMetaDone); @@ -2593,6 +3460,11 @@ int32_t ctgLaunchGetViewsTask(SCtgTask* pTask) { int32_t baseResIdx = 0; for (int32_t i = 0; i < dbNum; ++i) { STablesReq* pReq = taosArrayGet(pCtx->pNames, i); + if (NULL == pReq) { + ctgError("fail to get the %dth STablesReq, dbNum:%d", i, dbNum); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + ctgDebug("start to check views in db %s, viewNum %ld", pReq->dbFName, taosArrayGetSize(pReq->pTables)); CTG_ERR_RET(ctgGetViewsFromCache(pCtg, pConn, pCtx, i, &fetchIdx, baseResIdx, pReq->pTables)); baseResIdx += taosArrayGetSize(pReq->pTables); @@ -2607,10 +3479,26 @@ int32_t ctgLaunchGetViewsTask(SCtgTask* pTask) { } pTask->msgCtxs = taosArrayInit_s(sizeof(SCtgMsgCtx), pCtx->fetchNum); + if (NULL == pTask->msgCtxs) { + ctgError("taosArrayInit_s %d SCtgMsgCtx %d failed", pCtx->fetchNum, (int32_t)sizeof(SCtgMsgCtx)); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + for (int32_t i = 0; i < pCtx->fetchNum; ++i) { SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, i); - SName* pName = ctgGetFetchName(pCtx->pNames, pFetch); + if (NULL == pFetch) { + ctgError("fail to get the %dth SCtgFetch, fetchNum:%d", i, pCtx->fetchNum); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + CTG_ERR_RET(ctgGetFetchName(pCtx->pNames, pFetch, &pName)); + SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, i); + if (NULL == pMsgCtx) { + ctgError("fail to get the %dth SCtgMsgCtx", i); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (NULL == pMsgCtx->pBatchs) { pMsgCtx->pBatchs = pJob->pBatchs; } @@ -2633,10 +3521,13 @@ int32_t ctgAsyncRefreshTbTsma(SCtgTaskReq* pReq, const SCtgTSMAFetch* pFetch) { SCtgDBCache* pDbCache = NULL; STablesReq* pTbReq = taosArrayGet(pTaskCtx->pNames, pFetch->dbIdx); + if (NULL == pTbReq) { + ctgError("fail to get the %dth STablesReq, totalNum:%d", pFetch->dbIdx, (int32_t)taosArrayGetSize(pTaskCtx->pNames)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } - ctgAcquireVgInfoFromCache(pCtg, pTbReq->dbFName, &pDbCache); + (void)ctgAcquireVgInfoFromCache(pCtg, pTbReq->dbFName, &pDbCache); // ignore error if (pDbCache) { - ctgReleaseVgInfoToCache(pCtg, pDbCache); } else { SBuildUseDBInput input = {0}; @@ -2645,7 +3536,9 @@ int32_t ctgAsyncRefreshTbTsma(SCtgTaskReq* pReq, const SCtgTSMAFetch* pFetch) { CTG_ERR_JRET(ctgGetDBVgInfoFromMnode(pCtg, pConn, &input, NULL, pReq)); } + _return: + return code; } @@ -2661,9 +3554,15 @@ int32_t ctgLaunchGetTbTSMATask(SCtgTask* pTask) { for (int32_t idx = 0; idx < dbNum; ++idx) { STablesReq* pReq = taosArrayGet(pCtx->pNames, idx); + if (NULL == pReq) { + ctgError("fail to get the %dth STablesReq, dbNum:%d", idx, dbNum); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + CTG_ERR_RET(ctgGetTbTSMAFromCache(pCtg, pCtx, idx, &fetchIdx, baseResIdx, pReq->pTables)); baseResIdx += taosArrayGetSize(pReq->pTables); } + pCtx->fetchNum = taosArrayGetSize(pCtx->pFetches); if (pCtx->fetchNum <= 0) { TSWAP(pTask->res, pCtx->pResList); @@ -2672,12 +3571,39 @@ int32_t ctgLaunchGetTbTSMATask(SCtgTask* pTask) { } pTask->msgCtxs = taosArrayInit_s(sizeof(SCtgMsgCtx), pCtx->fetchNum); + if (NULL == pTask->msgCtxs) { + ctgError("taosArrayInit_s %d SCtgMsgCtx %d failed", pCtx->fetchNum, (int32_t)sizeof(SCtgMsgCtx)); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + for (int32_t i = 0; i < pCtx->fetchNum; ++i) { SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, i); + if (NULL == pFetch) { + ctgError("fail to get the %dth SCtgTSMAFetch, fetchNum:%d", i, pCtx->fetchNum); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + STablesReq* pReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx); + if (NULL == pReq) { + ctgError("fail to get the %dth STablesReq, totalNum:%d", pFetch->dbIdx, (int32_t)taosArrayGetSize(pCtx->pNames)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + SName* pName = taosArrayGet(pReq->pTables, pFetch->tbIdx); + if (NULL == pName) { + ctgError("fail to get the %dth SName, totalNum:%d", pFetch->tbIdx, (int32_t)taosArrayGetSize(pReq->pTables)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, i); - if (!pMsgCtx->pBatchs) pMsgCtx->pBatchs = pJob->pBatchs; + if (NULL == pMsgCtx) { + ctgError("fail to get the %dth pMsgCtx", i); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + if (!pMsgCtx->pBatchs) { + pMsgCtx->pBatchs = pJob->pBatchs; + } SCtgTaskReq tReq; tReq.pTask = pTask; @@ -2686,11 +3612,13 @@ int32_t ctgLaunchGetTbTSMATask(SCtgTask* pTask) { switch (pFetch->fetchType) { case FETCH_TSMA_SOURCE_TB_META: { CTG_ERR_RET(ctgAsyncRefreshTbMeta(&tReq, pFetch->flag, pName, &pFetch->vgId)); - } break; + break; + } case FETCH_TB_TSMA: { CTG_ERR_RET( ctgGetTbTSMAFromMnode(pCtg, pConn, &pFetch->tsmaSourceTbName, NULL, &tReq, TDMT_MND_GET_TABLE_TSMA)); - } break; + break; + } default: ASSERT(0); break; @@ -2706,24 +3634,58 @@ int32_t ctgLaunchGetTSMATask(SCtgTask* pTask) { SRequestConnInfo* pConn = &pTask->pJob->conn; SArray* pRes = NULL; SCtgJob* pJob = pTask->pJob; + // currently, only support fetching one tsma ASSERT(pCtx->pNames->size == 1); STablesReq* pReq = taosArrayGet(pCtx->pNames, 0); + if (NULL == pReq) { + ctgError("fail to get the 0th STablesReq, totalNum:%d", (int32_t)taosArrayGetSize(pCtx->pNames)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + ASSERT(pReq->pTables->size == 1); SName* pTsmaName = taosArrayGet(pReq->pTables, 0); + if (NULL == pReq) { + ctgError("fail to get the 0th SName, totalNum:%d", (int32_t)taosArrayGetSize(pReq->pTables)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + CTG_ERR_RET(ctgGetTSMAFromCache(pCtg, pCtx, pTsmaName)); if (pCtx->pResList->size == 0) { SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, 0); - if (!pMsgCtx->pBatchs) pMsgCtx->pBatchs = pJob->pBatchs; + if (NULL == pMsgCtx) { + ctgError("fail to get the 0th SCtgMsgCtx, taskType:%d", pTask->type); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + if (!pMsgCtx->pBatchs) { + pMsgCtx->pBatchs = pJob->pBatchs; + } + SCtgTaskReq tReq = {.pTask = pTask, .msgIdx = 0}; - taosArrayPush(pCtx->pResList, &(SMetaRes){0}); + if (NULL == taosArrayPush(pCtx->pResList, &(SMetaRes){0})) { + ctgError("taosArrayPush SMetaRes failed, code:%x", terrno); + CTG_ERR_RET(terrno); + } + CTG_ERR_RET(ctgGetTbTSMAFromMnode(pCtg, pConn, pTsmaName, NULL, &tReq, TDMT_MND_GET_TSMA)); } else { SMetaRes* pRes = taosArrayGet(pCtx->pResList, 0); + if (NULL == pRes) { + ctgError("fail to get the 0th SMetaRes, totalNum:%d", (int32_t)taosArrayGetSize(pCtx->pResList)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + STableTSMAInfoRsp* pRsp = (STableTSMAInfoRsp*)pRes->pRes; ASSERT(pRsp->pTsmas->size == 1); + const STSMACache* pTsma = taosArrayGetP(pRsp->pTsmas, 0); + if (NULL == pTsma) { + ctgError("fail to get the 0th STSMACache, totalNum:%d", (int32_t)taosArrayGetSize(pRsp->pTsmas)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + TSWAP(pTask->res, pCtx->pResList); // get tsma target stable meta if not existed in cache int32_t exists = false; @@ -2736,253 +3698,11 @@ int32_t ctgLaunchGetTSMATask(SCtgTask* pTask) { } else { CTG_ERR_RET(ctgHandleTaskEnd(pTask, 0)); } + return TSDB_CODE_SUCCESS; } - return 0; -} - -int32_t ctgHandleGetTSMARsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) { - int32_t code = 0; - SCtgTask* pTask = tReq->pTask; - SCatalog* pCtg = pTask->pJob->pCtg; - SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx); - SCtgTbTSMACtx* pCtx = pTask->taskCtx; - SMetaRes* pRes = taosArrayGet(pCtx->pResList, 0); - STablesReq* pTbReq = taosArrayGet(pCtx->pNames, 0); - SName* pName = taosArrayGet(pTbReq->pTables, 0); - SRequestConnInfo* pConn = &pTask->pJob->conn; - CTG_ERR_JRET(ctgProcessRspMsg(pMsgCtx->out, reqType, pMsg->pData, pMsg->len, rspCode, pMsgCtx->target)); - - switch (reqType) { - case TDMT_MND_TABLE_META: { - STableMetaOutput* pOut = (STableMetaOutput*)pMsgCtx->out; - if (!CTG_IS_META_NULL(pOut->metaType)) { - CTG_ERR_JRET(ctgUpdateTbMetaToCache(pCtg, pOut, CTG_FLAG_SYNC_OP)); - } - } break; - case TDMT_MND_GET_TSMA: { - STableTSMAInfoRsp* pOut = pMsgCtx->out; - pRes->code = 0; - if (pOut->pTsmas->size > 0) { - ASSERT(pOut->pTsmas->size == 1); - pRes->pRes = pOut; - pMsgCtx->out = NULL; - TSWAP(pTask->res, pCtx->pResList); - - STableTSMAInfo* pTsma = taosArrayGetP(pOut->pTsmas, 0); - int32_t exists = false; - CTG_ERR_JRET(ctgTbMetaExistInCache(pCtg, pTsma->targetDbFName, pTsma->targetTb, &exists)); - if (!exists) { - TSWAP(pMsgCtx->lastOut, pMsgCtx->out); - CTG_RET(ctgGetTbMetaFromMnodeImpl(pCtg, pConn, pTsma->targetDbFName, pTsma->targetTb, NULL, tReq)); - } - } - } break; - default: - ASSERT(0); - } - -_return: - if (code) { - if (TSDB_CODE_MND_SMA_NOT_EXIST == code) { - code = TSDB_CODE_SUCCESS; - } else { - ctgTaskError("Get tsma for %d.%s.%s failed with err: %s", pName->acctId, pName->dbname, pName->tname, - tstrerror(code)); - } - } - ctgHandleTaskEnd(pTask, code); - CTG_RET(code); -} - -static int32_t ctgTsmaFetchStreamProgress(SCtgTaskReq* tReq, SHashObj* pVgHash, const STableTSMAInfoRsp* pTsmas) { - int32_t code = 0; - SCtgTask* pTask = tReq->pTask; - SCatalog* pCtg = pTask->pJob->pCtg; - int32_t subFetchIdx = 0; - SCtgTbTSMACtx* pCtx = pTask->taskCtx; - SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx); - SRequestConnInfo* pConn = &pTask->pJob->conn; - STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx); - const SName* pTbName = taosArrayGet(pTbReq->pTables, pFetch->tbIdx); - SVgroupInfo* pVgInfo = NULL; - - pFetch->vgNum = taosHashGetSize(pVgHash); - for (int32_t i = 0; i < taosArrayGetSize(pTsmas->pTsmas); ++i) { - STableTSMAInfo* pTsmaInfo = taosArrayGetP(pTsmas->pTsmas, i); - pVgInfo = taosHashIterate(pVgHash, NULL); - pTsmaInfo->reqTs = taosGetTimestampMs(); - while (pVgInfo) { - // make StreamProgressReq, send it - SStreamProgressReq req = {.fetchIdx = pFetch->fetchIdx, - .streamId = pTsmaInfo->streamUid, - .subFetchIdx = subFetchIdx++, - .vgId = pVgInfo->vgId}; - CTG_ERR_JRET(ctgGetStreamProgressFromVnode(pCtg, pConn, pTbName, pVgInfo, NULL, tReq, &req)); - pFetch->subFetchNum++; - pVgInfo = taosHashIterate(pVgHash, pVgInfo); - } - } -_return: - CTG_RET(code); -} - -int32_t ctgHandleGetTbTSMARsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) { - bool taskDone = false; - int32_t code = 0; - SCtgTask* pTask = tReq->pTask; - SCatalog* pCtg = pTask->pJob->pCtg; - SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx); - SCtgTbTSMACtx* pCtx = pTask->taskCtx; - SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx); - SArray* pTsmas = NULL; - SMetaRes* pRes = taosArrayGet(pCtx->pResList, pFetch->resIdx); - SHashObj* pVgHash = NULL; - SCtgDBCache* pDbCache = NULL; - STableTSMAInfo* pTsma = NULL; - SRequestConnInfo* pConn = &pTask->pJob->conn; - STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx); - SName* pTbName = taosArrayGet(pTbReq->pTables, pFetch->tbIdx); - - if (reqType != TDMT_VND_GET_STREAM_PROGRESS) - CTG_ERR_JRET(ctgProcessRspMsg(pMsgCtx->out, reqType, pMsg->pData, pMsg->len, rspCode, pMsgCtx->target)); - - switch (reqType) { - case TDMT_MND_GET_TABLE_TSMA: { - STableTSMAInfoRsp* pOut = pMsgCtx->out; - pFetch->fetchType = FETCH_TSMA_STREAM_PROGRESS; - pRes->pRes = pOut; - pMsgCtx->out = NULL; - - if (pOut->pTsmas && taosArrayGetSize(pOut->pTsmas) > 0) { - // fetch progress - ctgAcquireVgInfoFromCache(pCtg, pTbReq->dbFName, &pDbCache); - if (!pDbCache) { - // do not know which vnodes to fetch, fetch vnode list first - SBuildUseDBInput input = {0}; - tstrncpy(input.db, pTbReq->dbFName, tListLen(input.db)); - input.vgVersion = CTG_DEFAULT_INVALID_VERSION; - CTG_ERR_JRET(ctgGetDBVgInfoFromMnode(pCtg, pConn, &input, NULL, tReq)); - } else { - // fetch progress from every vnode - CTG_ERR_JRET(ctgTsmaFetchStreamProgress(tReq, pDbCache->vgCache.vgInfo->vgHash, pOut)); - ctgReleaseVgInfoToCache(pCtg, pDbCache); - pDbCache = NULL; - } - } else { - // no tsmas - if (atomic_sub_fetch_32(&pCtx->fetchNum, 1) == 0) { - TSWAP(pTask->res, pCtx->pResList); - taskDone = true; - } - } - } break; - case TDMT_VND_GET_STREAM_PROGRESS: { - SStreamProgressRsp rsp = {0}; - CTG_ERR_JRET(ctgProcessRspMsg(&rsp, reqType, pMsg->pData, pMsg->len, rspCode, pMsgCtx->target)); - // update progress into res - STableTSMAInfoRsp* pTsmasRsp = pRes->pRes; - SArray* pTsmas = pTsmasRsp->pTsmas; - SStreamProgressRsp* pRsp = &rsp; - int32_t tsmaIdx = pRsp->subFetchIdx / pFetch->vgNum; - STableTSMAInfo* pTsmaInfo = taosArrayGetP(pTsmas, tsmaIdx); - if (pTsmaInfo->rspTs == 0) pTsmaInfo->fillHistoryFinished = true; - pTsmaInfo->rspTs = taosGetTimestampMs(); - pTsmaInfo->delayDuration = TMAX(pRsp->progressDelay, pTsmaInfo->delayDuration); - pTsmaInfo->fillHistoryFinished = pTsmaInfo->fillHistoryFinished && pRsp->fillHisFinished; - qDebug("received stream progress for tsma %s rsp history: %d vnode: %d, delay: %" PRId64, pTsmaInfo->name, - pRsp->fillHisFinished, pRsp->subFetchIdx, pRsp->progressDelay); - - if (atomic_add_fetch_32(&pFetch->finishedSubFetchNum, 1) == pFetch->subFetchNum) { - // subfetch all finished - for (int32_t i = 0; i < taosArrayGetSize(pTsmas); ++i) { - STableTSMAInfo* pInfo = taosArrayGetP(pTsmas, i); - CTG_ERR_JRET(tCloneTbTSMAInfo(pInfo, &pTsma)); - CTG_ERR_JRET(ctgUpdateTbTSMAEnqueue(pCtg, &pTsma, 0, false)); - } - - if (atomic_sub_fetch_32(&pCtx->fetchNum, 1) == 0) { - TSWAP(pTask->res, pCtx->pResList); - taskDone = true; - } - } - } break; - case TDMT_MND_USE_DB: { - SUseDbOutput* pOut = (SUseDbOutput*)pMsgCtx->out; - - switch (pFetch->fetchType) { - case FETCH_TSMA_SOURCE_TB_META: { - SVgroupInfo vgInfo = {0}; - CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, &pConn->mgmtEps, pOut->dbVgroup, pTbName, &vgInfo)); - - pFetch->vgId = vgInfo.vgId; - CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, pTbName, &vgInfo, NULL, tReq)); - } break; - case FETCH_TSMA_STREAM_PROGRESS: { - STableTSMAInfoRsp* pTsmas = pRes->pRes; - TSWAP(pOut->dbVgroup->vgHash, pVgHash); - CTG_ERR_JRET(ctgTsmaFetchStreamProgress(tReq, pVgHash, pTsmas)); - } break; - default: - ASSERT(0); - } - } break; - case TDMT_VND_TABLE_META: { - // handle source tb meta - ASSERT(pFetch->fetchType == FETCH_TSMA_SOURCE_TB_META); - STableMetaOutput* pOut = (STableMetaOutput*)pMsgCtx->out; - pFetch->fetchType = FETCH_TB_TSMA; - pFetch->tsmaSourceTbName = *pTbName; - if (CTG_IS_META_NULL(pOut->metaType)) { - ctgTaskError("no tbmeta found when fetching tsma source tb meta: %s.%s", pTbName->dbname, pTbName->tname); - ctgRemoveTbMetaFromCache(pCtg, pTbName, false); - CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST); - } - if (META_TYPE_BOTH_TABLE == pOut->metaType) { - // rewrite tsma fetch table with it's super table name - sprintf(pFetch->tsmaSourceTbName.tname, "%s", pOut->tbName); - } - CTG_ERR_JRET(ctgGetTbTSMAFromMnode(pCtg, pConn, &pFetch->tsmaSourceTbName, NULL, tReq, TDMT_MND_GET_TABLE_TSMA)); - } break; - default: - ASSERT(0); - } - -_return: - if (pDbCache) { - ctgReleaseVgInfoToCache(pCtg, pDbCache); - } - if (pTsma) { - tFreeTableTSMAInfo(pTsma); - pTsma = NULL; - } - if (pVgHash) { - taosHashCleanup(pVgHash); - } - if (code) { - SMetaRes* pRes = taosArrayGet(pCtx->pResList, pFetch->resIdx); - pRes->code = code; - if (TSDB_CODE_MND_SMA_NOT_EXIST == code) { - code = TSDB_CODE_SUCCESS; - } else { - ctgTaskError("Get tsma for %d.%s.%s faield with err: %s", pTbName->acctId, pTbName->dbname, pTbName->tname, - tstrerror(code)); - } - bool allSubFetchFinished = false; - if (pMsgCtx->reqType == TDMT_VND_GET_STREAM_PROGRESS) { - allSubFetchFinished = atomic_add_fetch_32(&pFetch->finishedSubFetchNum, 1) >= pFetch->subFetchNum; - } - if ((allSubFetchFinished || pFetch->subFetchNum == 0) && 0 == atomic_sub_fetch_32(&pCtx->fetchNum, 1)) { - TSWAP(pTask->res, pCtx->pResList); - taskDone = true; - } - } - if (pTask->res && taskDone) { - ctgHandleTaskEnd(pTask, code); - } - - CTG_RET(code); + return TSDB_CODE_SUCCESS; } int32_t ctgDumpTbTSMARes(SCtgTask* pTask) { @@ -3025,6 +3745,10 @@ int32_t ctgGetTbCfgCb(SCtgTask* pTask) { SDBVgInfo* pDb = (SDBVgInfo*)pTask->subRes.res; pCtx->pVgInfo = taosMemoryCalloc(1, sizeof(SVgroupInfo)); + if (NULL == pCtx->pVgInfo) { + CTG_ERR_JRET(terrno); + } + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pTask->pJob->pCtg, &pTask->pJob->conn.mgmtEps, pDb, pCtx->pName, pCtx->pVgInfo)); } @@ -3045,6 +3769,10 @@ int32_t ctgGetTbTagCb(SCtgTask* pTask) { if (NULL == pCtx->pVgInfo) { pCtx->pVgInfo = taosMemoryCalloc(1, sizeof(SVgroupInfo)); + if (NULL == pCtx->pVgInfo) { + CTG_ERR_JRET(terrno); + } + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pTask->pJob->pCtg, &pTask->pJob->conn.mgmtEps, pDb, pCtx->pName, pCtx->pVgInfo)); } @@ -3148,6 +3876,11 @@ int32_t ctgSearchExistingTask(SCtgJob* pJob, CTG_TASK_TYPE type, void* param, in int32_t taskNum = taosArrayGetSize(pJob->pTasks); for (int32_t i = 0; i < taskNum; ++i) { pTask = taosArrayGet(pJob->pTasks, i); + if (NULL == pTask) { + qError("fail to get the %dth task, taskNum:%d", i, taskNum); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (type != pTask->type) { continue; } @@ -3176,16 +3909,31 @@ int32_t ctgSetSubTaskCb(SCtgTask* pSub, SCtgTask* pTask) { pTask->subRes.code = pSub->code; CTG_ERR_JRET((*gCtgAsyncFps[pTask->type].cloneFp)(pSub, &pTask->subRes.res)); SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx) { + qError("fail to get the -1th SCtgMsgCtx"); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + SCtgMsgCtx* pSubMsgCtx = CTG_GET_TASK_MSGCTX(pSub, -1); + if (NULL == pSubMsgCtx) { + qError("fail to get the -1th sub SCtgMsgCtx"); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + pMsgCtx->pBatchs = pSubMsgCtx->pBatchs; CTG_ERR_JRET(pTask->subRes.fp(pTask)); } else { if (NULL == pSub->pParents) { pSub->pParents = taosArrayInit(4, POINTER_BYTES); + if (NULL == pSub->pParents) { + CTG_ERR_JRET(terrno); + } } - taosArrayPush(pSub->pParents, &pTask); + if (NULL == taosArrayPush(pSub->pParents, &pTask)) { + CTG_ERR_JRET(terrno); + } } _return: @@ -3227,6 +3975,11 @@ int32_t ctgLaunchSubTask(SCtgTask** ppTask, CTG_TASK_TYPE type, ctgSubTaskCbFp f } SCtgTask* pSub = taosArrayGet(pJob->pTasks, subTaskId); + if (NULL == pSub) { + qError("fail to get the %dth sub SCtgTask, taskNum:%d", subTaskId, (int32_t)taosArrayGetSize(pJob->pTasks)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (newTask) { pSub->subTask = true; } @@ -3235,7 +3988,17 @@ int32_t ctgLaunchSubTask(SCtgTask** ppTask, CTG_TASK_TYPE type, ctgSubTaskCbFp f if (newTask) { SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(*ppTask, -1); + if (NULL == pMsgCtx) { + qError("fail to get the -1th SCtgMsgCtx"); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + SCtgMsgCtx* pSubMsgCtx = CTG_GET_TASK_MSGCTX(pSub, -1); + if (NULL == pSubMsgCtx) { + qError("fail to get the -1th sub SCtgMsgCtx"); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + pSubMsgCtx->pBatchs = pMsgCtx->pBatchs; CTG_ERR_RET((*gCtgAsyncFps[pSub->type].launchFp)(pSub)); @@ -3250,21 +4013,30 @@ int32_t ctgLaunchJob(SCtgJob* pJob) { for (int32_t i = 0; i < taskNum; ++i) { SCtgTask* pTask = taosArrayGet(pJob->pTasks, i); - + if (NULL == pTask) { + qError("fail to get the %dth task, taskNum:%d", i, taskNum); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + qDebug("QID:0x%" PRIx64 " ctg launch [%dth] task", pJob->queryId, pTask->taskId); CTG_ERR_RET((*gCtgAsyncFps[pTask->type].launchFp)(pTask)); pTask = taosArrayGet(pJob->pTasks, i); - atomic_val_compare_exchange_32((int32_t*)&pTask->status, 0, CTG_TASK_LAUNCHED); + if (NULL == pTask) { + qError("fail to get the %dth task, taskNum:%d", i, taskNum); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + (void)atomic_val_compare_exchange_32((int32_t*)&pTask->status, 0, CTG_TASK_LAUNCHED); } if (taskNum <= 0) { qDebug("QID:0x%" PRIx64 " ctg call user callback with rsp %s", pJob->queryId, tstrerror(pJob->jobResCode)); - taosAsyncExec(ctgCallUserCb, pJob, NULL); + CTG_ERR_RET(taosAsyncExec(ctgCallUserCb, pJob, NULL)); #if CTG_BATCH_FETCH } else { - ctgLaunchBatchs(pJob->pCtg, pJob, pJob->pBatchs); + CTG_ERR_RET(ctgLaunchBatchs(pJob->pCtg, pJob, pJob->pBatchs)); #endif } diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 9f9f783458..b023695937 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -764,12 +764,10 @@ int32_t ctgReadDBCfgFromCache(SCatalog *pCtg, const char* dbFName, SDbCfgInfo* p CTG_LOCK(CTG_READ, &dbCache->cfgCache.cfgLock); if (dbCache->cfgCache.cfgInfo) { - SDbCfgInfo *pInfo = ctgCloneDbCfgInfo(dbCache->cfgCache.cfgInfo); - if (NULL == pInfo) { - CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); - } + SDbCfgInfo *pInfo = NULL; + CTG_ERR_JRET(ctgCloneDbCfgInfo(dbCache->cfgCache.cfgInfo, &pInfo)); - memcpy(pDbCfg, pInfo, sizeof(*pInfo)); + TAOS_MEMCPY(pDbCfg, pInfo, sizeof(*pInfo)); taosMemoryFree(pInfo); CTG_CACHE_HIT_INC(CTG_CI_DB_CFG, 1); } else { diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c index bbd9b39f6c..f5de7fdc7a 100644 --- a/source/libs/catalog/src/ctgRemote.c +++ b/source/libs/catalog/src/ctgRemote.c @@ -556,7 +556,7 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT if (CTG_TASK_GET_TB_META_BATCH == pTask->type) { SCtgTbMetasCtx* ctx = (SCtgTbMetasCtx*)pTask->taskCtx; SCtgFetch* fetch = taosArrayGet(ctx->pFetchs, tReq->msgIdx); - pName = ctgGetFetchName(ctx->pNames, fetch); + CTG_ERR_JRET(ctgGetFetchName(ctx->pNames, fetch, &pName)); } else if (CTG_TASK_GET_TB_TSMA == pTask->type){ SCtgTbTSMACtx* pCtx = pTask->taskCtx; SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx); @@ -616,7 +616,7 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT if (CTG_TASK_GET_TB_META_BATCH == pTask->type) { SCtgTbMetasCtx* ctx = (SCtgTbMetasCtx*)pTask->taskCtx; SCtgFetch* fetch = taosArrayGet(ctx->pFetchs, tReq->msgIdx); - pName = ctgGetFetchName(ctx->pNames, fetch); + CTG_ERR_JRET(ctgGetFetchName(ctx->pNames, fetch, &pName)); } else if (CTG_TASK_GET_TB_TSMA == pTask->type){ SCtgTbTSMACtx* pCtx = pTask->taskCtx; SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx); diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index eac716339b..0098c79384 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -1627,23 +1627,41 @@ int32_t ctgAddFetch(SArray** pFetchs, int32_t dbIdx, int32_t tbIdx, int32_t* fet return TSDB_CODE_SUCCESS; } -SName* ctgGetFetchName(SArray* pNames, SCtgFetch* pFetch) { +int32_t ctgGetFetchName(SArray* pNames, SCtgFetch* pFetch, SName** ppName) { STablesReq* pReq = (STablesReq*)taosArrayGet(pNames, pFetch->dbIdx); - return (SName*)taosArrayGet(pReq->pTables, pFetch->tbIdx); + if (NULL == pReq) { + qError("fail to get the %dth tb in pTables, tbNum:%d", pFetch->tbIdx, (int32_t)taosArrayGetSize(pReq->pTables)); + return TSDB_CODE_CTG_INTERNAL_ERROR; + } + + *ppName = (SName*)taosArrayGet(pReq->pTables, pFetch->tbIdx); + if (NULL == *ppName) { + qError("fail to get the %dth tb in pTables, tbNum:%d", pFetch->tbIdx, (int32_t)taosArrayGetSize(pReq->pTables)); + return TSDB_CODE_CTG_INTERNAL_ERROR; + } + + return TSDB_CODE_SUCCESS; } static void* ctgCloneDbVgroup(void* pSrc) { return taosArrayDup((const SArray*)pSrc, NULL); } static void ctgFreeDbVgroup(void* p) { taosArrayDestroy((SArray*)((SMetaRes*)p)->pRes); } -void* ctgCloneDbCfgInfo(void* pSrc) { +int32_t ctgCloneDbCfgInfo(void* pSrc, SDbCfgInfo** ppDst) { SDbCfgInfo* pDst = taosMemoryMalloc(sizeof(SDbCfgInfo)); if (NULL == pDst) { - return NULL; + return terrno; } - memcpy(pDst, pSrc, sizeof(SDbCfgInfo)); + TAOS_MEMCPY(pDst, pSrc, sizeof(SDbCfgInfo)); pDst->pRetensions = taosArrayDup(((SDbCfgInfo *)pSrc)->pRetensions, NULL); - return pDst; + if (NULL == pDst->pRetensions) { + taosMemoryFree(pDst); + return terrno; + } + + *ppDst = pDst; + + return TSDB_CODE_SUCCESS; } static void ctgFreeDbCfgInfo(void* p) { diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index f906e3ec3e..37860ac202 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -77,10 +77,10 @@ void schtInitLogFile() { const int32_t maxLogFileNum = 10; tsAsyncLog = 0; qDebugFlag = 159; - strcpy(tsLogDir, TD_LOG_DIR_PATH); + TAOS_STRCPY(tsLogDir, TD_LOG_DIR_PATH); if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { - printf("failed to open log file in directory:%s\n", tsLogDir); + (void)printf("failed to open log file in directory:%s\n", tsLogDir); } } @@ -628,7 +628,7 @@ void schtFreeQueryJob(int32_t freeThread) { schedulerFreeJob(&job, 0); if (freeThread) { if (++freeNum % schtTestPrintNum == 0) { - printf("FreeNum:%d\n", freeNum); + (void)printf("FreeNum:%d\n", freeNum); } } } @@ -667,7 +667,7 @@ void *schtRunJobThread(void *aa) { SQueryNodeLoad load = {0}; load.addr.epSet.numOfEps = 1; - strcpy(load.addr.epSet.eps[0].fqdn, "qnode0.ep"); + TAOS_STRCPY(load.addr.epSet.eps[0].fqdn, "qnode0.ep"); load.addr.epSet.eps[0].port = 6031; if (NULL == taosArrayPush(qnodeList, &load)) { assert(0); @@ -800,7 +800,7 @@ void *schtRunJobThread(void *aa) { schtFreeQueryDag(dag); if (++jobFinished % schtTestPrintNum == 0) { - printf("jobFinished:%d\n", jobFinished); + (void)printf("jobFinished:%d\n", jobFinished); } ++schtQueryId; @@ -834,9 +834,9 @@ TEST(queryTest, normalCase) { SQueryNodeLoad load = {0}; load.addr.epSet.numOfEps = 1; - strcpy(load.addr.epSet.eps[0].fqdn, "qnode0.ep"); + TAOS_STRCPY(load.addr.epSet.eps[0].fqdn, "qnode0.ep"); load.addr.epSet.eps[0].port = 6031; - taosArrayPush(qnodeList, &load); + assert(taosArrayPush(qnodeList, &load) != NULL); int32_t code = schedulerInit(); ASSERT_EQ(code, 0); @@ -872,7 +872,7 @@ TEST(queryTest, normalCase) { SDataBuf msg = {0}; void *rmsg = NULL; - schtBuildQueryRspMsg(&msg.len, &rmsg); + assert(0 == schtBuildQueryRspMsg(&msg.len, &rmsg)); msg.msgType = TDMT_SCH_QUERY_RSP; msg.pData = rmsg; @@ -888,7 +888,7 @@ TEST(queryTest, normalCase) { if (JOB_TASK_STATUS_EXEC == task->status) { SDataBuf msg = {0}; void *rmsg = NULL; - schtBuildQueryRspMsg(&msg.len, &rmsg); + assert(0 == schtBuildQueryRspMsg(&msg.len, &rmsg)); msg.msgType = TDMT_SCH_QUERY_RSP; msg.pData = rmsg; @@ -909,10 +909,10 @@ TEST(queryTest, normalCase) { } TdThreadAttr thattr; - taosThreadAttrInit(&thattr); + assert(0 == taosThreadAttrInit(&thattr)); TdThread thread1; - taosThreadCreate(&(thread1), &thattr, schtCreateFetchRspThread, &job); + assert(0 == taosThreadCreate(&(thread1), &thattr, schtCreateFetchRspThread, &job)); void *data = NULL; req.syncReq = true; @@ -926,13 +926,13 @@ TEST(queryTest, normalCase) { ASSERT_EQ(pRsp->numOfRows, 10); taosMemoryFreeClear(data); - schReleaseJob(job); + (void)schReleaseJob(job); schedulerDestroy(); schedulerFreeJob(&job, 0); - taosThreadJoin(thread1, NULL); + (void)taosThreadJoin(thread1, NULL); } TEST(queryTest, readyFirstCase) { @@ -948,9 +948,9 @@ TEST(queryTest, readyFirstCase) { SQueryNodeLoad load = {0}; load.addr.epSet.numOfEps = 1; - strcpy(load.addr.epSet.eps[0].fqdn, "qnode0.ep"); + TAOS_STRCPY(load.addr.epSet.eps[0].fqdn, "qnode0.ep"); load.addr.epSet.eps[0].port = 6031; - taosArrayPush(qnodeList, &load); + assert(NULL != taosArrayPush(qnodeList, &load)); int32_t code = schedulerInit(); ASSERT_EQ(code, 0); @@ -985,7 +985,7 @@ TEST(queryTest, readyFirstCase) { SDataBuf msg = {0}; void *rmsg = NULL; - schtBuildQueryRspMsg(&msg.len, &rmsg); + assert(0 == schtBuildQueryRspMsg(&msg.len, &rmsg)); msg.msgType = TDMT_SCH_QUERY_RSP; msg.pData = rmsg; @@ -1002,7 +1002,7 @@ TEST(queryTest, readyFirstCase) { if (JOB_TASK_STATUS_EXEC == task->status) { SDataBuf msg = {0}; void *rmsg = NULL; - schtBuildQueryRspMsg(&msg.len, &rmsg); + assert(0 == schtBuildQueryRspMsg(&msg.len, &rmsg)); msg.msgType = TDMT_SCH_QUERY_RSP; msg.pData = rmsg; @@ -1023,10 +1023,10 @@ TEST(queryTest, readyFirstCase) { } TdThreadAttr thattr; - taosThreadAttrInit(&thattr); + assert(0 == taosThreadAttrInit(&thattr)); TdThread thread1; - taosThreadCreate(&(thread1), &thattr, schtCreateFetchRspThread, &job); + assert(0 == taosThreadCreate(&(thread1), &thattr, schtCreateFetchRspThread, &job)); void *data = NULL; req.syncReq = true; @@ -1039,13 +1039,13 @@ TEST(queryTest, readyFirstCase) { ASSERT_EQ(pRsp->numOfRows, 10); taosMemoryFreeClear(data); - schReleaseJob(job); + (void)schReleaseJob(job); schedulerDestroy(); schedulerFreeJob(&job, 0); - taosThreadJoin(thread1, NULL); + (void)taosThreadJoin(thread1, NULL); } TEST(queryTest, flowCtrlCase) { @@ -1065,9 +1065,9 @@ TEST(queryTest, flowCtrlCase) { SQueryNodeLoad load = {0}; load.addr.epSet.numOfEps = 1; - strcpy(load.addr.epSet.eps[0].fqdn, "qnode0.ep"); + TAOS_STRCPY(load.addr.epSet.eps[0].fqdn, "qnode0.ep"); load.addr.epSet.eps[0].port = 6031; - taosArrayPush(qnodeList, &load); + assert(NULL != taosArrayPush(qnodeList, &load)); int32_t code = schedulerInit(); ASSERT_EQ(code, 0); @@ -1078,7 +1078,7 @@ TEST(queryTest, flowCtrlCase) { schtSetExecNode(); schtSetAsyncSendMsgToServer(); - initTaskQueue(); + assert(0 == initTaskQueue()); int32_t queryDone = 0; SRequestConnInfo conn = {0}; @@ -1106,7 +1106,7 @@ TEST(queryTest, flowCtrlCase) { if (JOB_TASK_STATUS_EXEC == task->status && 0 != task->lastMsgType) { SDataBuf msg = {0}; void *rmsg = NULL; - schtBuildQueryRspMsg(&msg.len, &rmsg); + assert(0 == schtBuildQueryRspMsg(&msg.len, &rmsg)); msg.msgType = TDMT_SCH_QUERY_RSP; msg.pData = rmsg; @@ -1120,10 +1120,10 @@ TEST(queryTest, flowCtrlCase) { } TdThreadAttr thattr; - taosThreadAttrInit(&thattr); + assert(0 == taosThreadAttrInit(&thattr)); TdThread thread1; - taosThreadCreate(&(thread1), &thattr, schtCreateFetchRspThread, &job); + assert(0 == taosThreadCreate(&(thread1), &thattr, schtCreateFetchRspThread, &job)); void *data = NULL; req.syncReq = true; @@ -1136,13 +1136,13 @@ TEST(queryTest, flowCtrlCase) { ASSERT_EQ(pRsp->numOfRows, 10); taosMemoryFreeClear(data); - schReleaseJob(job); + (void)schReleaseJob(job); schedulerDestroy(); schedulerFreeJob(&job, 0); - taosThreadJoin(thread1, NULL); + (void)taosThreadJoin(thread1, NULL); } TEST(insertTest, normalCase) { @@ -1158,9 +1158,9 @@ TEST(insertTest, normalCase) { SQueryNodeLoad load = {0}; load.addr.epSet.numOfEps = 1; - strcpy(load.addr.epSet.eps[0].fqdn, "qnode0.ep"); + TAOS_STRCPY(load.addr.epSet.eps[0].fqdn, "qnode0.ep"); load.addr.epSet.eps[0].port = 6031; - taosArrayPush(qnodeList, &load); + assert(NULL != taosArrayPush(qnodeList, &load)); int32_t code = schedulerInit(); ASSERT_EQ(code, 0); @@ -1171,12 +1171,12 @@ TEST(insertTest, normalCase) { schtSetAsyncSendMsgToServer(); TdThreadAttr thattr; - taosThreadAttrInit(&thattr); + assert(0 == taosThreadAttrInit(&thattr)); schtJobDone = false; TdThread thread1; - taosThreadCreate(&(thread1), &thattr, schtSendRsp, &insertJobRefId); + assert(0 == taosThreadCreate(&(thread1), &thattr, schtSendRsp, &insertJobRefId)); int32_t queryDone = 0; SRequestConnInfo conn = {0}; @@ -1204,17 +1204,17 @@ TEST(insertTest, normalCase) { schedulerDestroy(); - taosThreadJoin(thread1, NULL); + (void)taosThreadJoin(thread1, NULL); } TEST(multiThread, forceFree) { TdThreadAttr thattr; - taosThreadAttrInit(&thattr); + assert(0 == taosThreadAttrInit(&thattr)); TdThread thread1, thread2, thread3; - taosThreadCreate(&(thread1), &thattr, schtRunJobThread, NULL); + assert(0 == taosThreadCreate(&(thread1), &thattr, schtRunJobThread, NULL)); // taosThreadCreate(&(thread2), &thattr, schtFreeJobThread, NULL); - taosThreadCreate(&(thread3), &thattr, schtFetchRspThread, NULL); + assert(0 == taosThreadCreate(&(thread3), &thattr, schtFetchRspThread, NULL)); while (true) { if (schtTestDeadLoop) { @@ -1231,7 +1231,7 @@ TEST(multiThread, forceFree) { TEST(otherTest, otherCase) { // excpet test - schReleaseJob(0); + (void)schReleaseJob(0); schFreeRpcCtx(NULL); char* ep = NULL;