diff --git a/include/os/osDef.h b/include/os/osDef.h index 335b151cac..1052722492 100644 --- a/include/os/osDef.h +++ b/include/os/osDef.h @@ -220,7 +220,7 @@ void syslog(int unused, const char *format, ...); // Linux, length of name must <= 16 (the last '\0' included) #define setThreadName(name) \ do { \ - prctl(PR_SET_NAME, (name)); \ + (void)prctl(PR_SET_NAME, (name)); \ } while (0) #define getThreadName(name) \ do { \ diff --git a/include/os/osMemory.h b/include/os/osMemory.h index c6a5ce27c4..7f2963f15a 100644 --- a/include/os/osMemory.h +++ b/include/os/osMemory.h @@ -55,6 +55,7 @@ void *taosMemoryMallocAlign(uint32_t alignment, int64_t size); #define TAOS_MEMSET(_s, _c, _n) ((void)memset(_s, _c, _n)) #define TAOS_MEMCPY(_d, _s, _n) ((void)memcpy(_d, _s, _n)) +#define TAOS_MEMMOVE(_d, _s, _n) ((void)memmove(_d, _s, _n)) #define taosMemoryFreeClear(ptr) \ do { \ diff --git a/include/os/osString.h b/include/os/osString.h index c834eb8e65..e53aceb83a 100644 --- a/include/os/osString.h +++ b/include/os/osString.h @@ -60,6 +60,7 @@ typedef enum { M2C = 0, C2M } ConvType; } while (0) #define TAOS_STRCPY(_dst, _src) ((void)strcpy(_dst, _src)) +#define TAOS_STRNCPY(_dst, _src, _size) ((void)strncpy(_dst, _src, _size)) char *tstrdup(const char *src); int32_t taosUcs4len(TdUcs4 *ucs4); diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 6aaa79bb31..afadc8d8c2 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -25,6 +25,7 @@ extern "C" { #include "tcommon.h" #include "ttimer.h" #include "tglobal.h" +#include "os.h" #define CTG_DEFAULT_CACHE_CLUSTER_NUMBER 6 #define CTG_DEFAULT_CACHE_VGROUP_NUMBER 100 @@ -669,12 +670,12 @@ typedef struct SCtgCacheItemInfo { #define CTG_AUTH_READ(_t) ((_t) == AUTH_TYPE_READ || (_t) == AUTH_TYPE_READ_OR_WRITE) #define CTG_AUTH_WRITE(_t) ((_t) == AUTH_TYPE_WRITE || (_t) == AUTH_TYPE_READ_OR_WRITE) -#define CTG_QUEUE_INC() atomic_add_fetch_64(&gCtgMgmt.queue.qRemainNum, 1) -#define CTG_QUEUE_DEC() atomic_sub_fetch_64(&gCtgMgmt.queue.qRemainNum, 1) +#define CTG_QUEUE_INC() (void)atomic_add_fetch_64(&gCtgMgmt.queue.qRemainNum, 1) +#define CTG_QUEUE_DEC() (void)atomic_sub_fetch_64(&gCtgMgmt.queue.qRemainNum, 1) -#define CTG_STAT_INC(_item, _n) atomic_add_fetch_64(&(_item), _n) -#define CTG_STAT_DEC(_item, _n) atomic_sub_fetch_64(&(_item), _n) -#define CTG_STAT_GET(_item) atomic_load_64(&(_item)) +#define CTG_STAT_INC(_item, _n) (void)atomic_add_fetch_64(&(_item), _n) +#define CTG_STAT_DEC(_item, _n) (void)atomic_sub_fetch_64(&(_item), _n) +#define CTG_STAT_GET(_item) (void)atomic_load_64(&(_item)) #define CTG_STAT_API_INC(item, n) (CTG_STAT_INC(gCtgMgmt.statInfo.api.item, n)) #define CTG_STAT_RT_INC(item, n) (CTG_STAT_INC(gCtgMgmt.statInfo.runtime.item, n)) @@ -971,7 +972,7 @@ int32_t ctgRemoveTbMetaFromCache(SCatalog* pCtg, SName* pTableName, bool syncReq int32_t ctgGetTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta); int32_t ctgGetTbMetasFromCache(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetasCtx* ctx, int32_t dbIdx, int32_t* fetchIdx, int32_t baseResIdx, SArray* pList); -void* ctgCloneDbCfgInfo(void* pSrc); +int32_t ctgCloneDbCfgInfo(void* pSrc, SDbCfgInfo** ppDst); int32_t ctgOpUpdateVgroup(SCtgCacheOperation* action); int32_t ctgOpUpdateDbCfg(SCtgCacheOperation *operation); @@ -1112,7 +1113,7 @@ int32_t ctgRemoveTbMeta(SCatalog* pCtg, SName* pTableName); int32_t ctgRemoveCacheUser(SCatalog* pCtg, SCtgUserAuth* pUser, const char* user); int32_t ctgGetTbHashVgroup(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SVgroupInfo* pVgroup, bool* exists); -SName* ctgGetFetchName(SArray* pNames, SCtgFetch* pFetch); +int32_t ctgGetFetchName(SArray* pNames, SCtgFetch* pFetch, SName** ppName); int32_t ctgdGetOneHandle(SCatalog** pHandle); int ctgVgInfoComp(const void* lp, const void* rp); int32_t ctgMakeVgArray(SDBVgInfo* dbInfo); @@ -1165,6 +1166,7 @@ int32_t ctgGetStreamProgressFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, void* bInput); int32_t ctgAddTSMAFetch(SArray** pFetchs, int32_t dbIdx, int32_t tbIdx, int32_t* fetchIdx, int32_t resIdx, int32_t flag, CTG_TSMA_FETCH_TYPE fetchType, const SName* sourceTbName); +void ctgFreeTask(SCtgTask* pTask, bool freeRes); extern SCatalogMgmt gCtgMgmt; extern SCtgDebug gCTGDebug; diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 4048c8841b..07982afd5d 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -86,7 +86,7 @@ int32_t ctgRefreshDBVgInfo(SCatalog* pCtg, SRequestConnInfo* pConn, const char* if (code) { if (CTG_DB_NOT_EXIST(code) && (NULL != dbCache)) { ctgDebug("db no longer exist, dbFName:%s, dbId:0x%" PRIx64, input.db, input.dbId); - ctgDropDbCacheEnqueue(pCtg, input.db, input.dbId); + CTG_ERR_RET(ctgDropDbCacheEnqueue(pCtg, input.db, input.dbId)); } CTG_ERR_RET(code); @@ -116,8 +116,7 @@ int32_t ctgRefreshTbMeta(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetaCtx* if (CTG_FLAG_IS_SYS_DB(ctx->flag)) { ctgDebug("will refresh tbmeta, supposed in information_schema, tbName:%s", tNameGetTableName(ctx->pName)); - CTG_ERR_JRET( - ctgGetTbMetaFromMnodeImpl(pCtg, pConn, (char*)ctx->pName->dbname, (char*)ctx->pName->tname, output, NULL)); + CTG_ERR_JRET(ctgGetTbMetaFromMnodeImpl(pCtg, pConn, (char*)ctx->pName->dbname, (char*)ctx->pName->tname, output, NULL)); } else if (CTG_FLAG_IS_STB(ctx->flag)) { ctgDebug("will refresh tbmeta, supposed to be stb, tbName:%s", tNameGetTableName(ctx->pName)); @@ -128,8 +127,7 @@ int32_t ctgRefreshTbMeta(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetaCtx* CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, ctx->pName, &vgroupInfo, output, NULL)); } } else { - ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(ctx->pName), - ctx->flag); + ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(ctx->pName), ctx->flag); // if get from vnode failed or no table meta, will not try mnode CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, ctx->pName, &vgroupInfo, output, NULL)); @@ -166,7 +164,7 @@ int32_t ctgRefreshTbMeta(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetaCtx* if (CTG_IS_META_NULL(output->metaType)) { ctgError("no tbmeta got, tbName:%s", tNameGetTableName(ctx->pName)); - ctgRemoveTbMetaFromCache(pCtg, ctx->pName, false); + CTG_ERR_JRET(ctgRemoveTbMetaFromCache(pCtg, ctx->pName, false)); CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST); } @@ -216,7 +214,7 @@ int32_t ctgGetTbMeta(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetaCtx* ctx } if (CTG_IS_META_BOTH(output->metaType)) { - memcpy(output->tbMeta, &output->ctbMeta, sizeof(output->ctbMeta)); + TAOS_MEMCPY(output->tbMeta, &output->ctbMeta, sizeof(output->ctbMeta)); *pTableMeta = output->tbMeta; goto _return; @@ -233,7 +231,7 @@ int32_t ctgGetTbMeta(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetaCtx* ctx taosMemoryFreeClear(output->tbMeta); SName stbName = *ctx->pName; - strcpy(stbName.tname, output->tbName); + TAOS_STRCPY(stbName.tname, output->tbName); SCtgTbMetaCtx stbCtx = {0}; stbCtx.flag = ctx->flag; stbCtx.pName = &stbName; @@ -244,7 +242,7 @@ int32_t ctgGetTbMeta(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetaCtx* ctx continue; } - memcpy(*pTableMeta, &output->ctbMeta, sizeof(output->ctbMeta)); + TAOS_MEMCPY(*pTableMeta, &output->ctbMeta, sizeof(output->ctbMeta)); break; } @@ -254,15 +252,15 @@ _return: if (CTG_TABLE_NOT_EXIST(code) && ctx->tbInfo.inCache) { char dbFName[TSDB_DB_FNAME_LEN] = {0}; if (CTG_FLAG_IS_SYS_DB(ctx->flag)) { - strcpy(dbFName, ctx->pName->dbname); + TAOS_STRCPY(dbFName, ctx->pName->dbname); } else { - tNameGetFullDbName(ctx->pName, dbFName); + (void)tNameGetFullDbName(ctx->pName, dbFName); } if (TSDB_SUPER_TABLE == ctx->tbInfo.tbType) { - ctgDropStbMetaEnqueue(pCtg, dbFName, ctx->tbInfo.dbId, ctx->pName->tname, ctx->tbInfo.suid, false); + (void)ctgDropStbMetaEnqueue(pCtg, dbFName, ctx->tbInfo.dbId, ctx->pName->tname, ctx->tbInfo.suid, false); // already in error } else { - ctgDropTbMetaEnqueue(pCtg, dbFName, ctx->tbInfo.dbId, ctx->pName->tname, false); + (void)ctgDropTbMetaEnqueue(pCtg, dbFName, ctx->tbInfo.dbId, ctx->pName->tname, false); // already in error } } @@ -285,18 +283,18 @@ int32_t ctgUpdateTbMeta(SCatalog* pCtg, STableMetaRsp* rspMsg, bool syncOp) { int32_t code = 0; - strcpy(output->dbFName, rspMsg->dbFName); + TAOS_STRCPY(output->dbFName, rspMsg->dbFName); output->dbId = rspMsg->dbId; if (TSDB_CHILD_TABLE == rspMsg->tableType && NULL == rspMsg->pSchemas) { - strcpy(output->ctbName, rspMsg->tbName); + TAOS_STRCPY(output->ctbName, rspMsg->tbName); SET_META_TYPE_CTABLE(output->metaType); CTG_ERR_JRET(queryCreateCTableMetaFromMsg(rspMsg, &output->ctbMeta)); } else { - strcpy(output->tbName, rspMsg->tbName); + TAOS_STRCPY(output->tbName, rspMsg->tbName); SET_META_TYPE_TABLE(output->metaType); @@ -348,14 +346,14 @@ int32_t ctgChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, SUserAuthInfo *pReq, _return: - ctgUpdateUserEnqueue(pCtg, &req.authInfo, false); + (void)ctgUpdateUserEnqueue(pCtg, &req.authInfo, false); // cache update not fatal error CTG_RET(code); } int32_t ctgGetTbType(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTableName, int32_t* tbType) { char dbFName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(pTableName, dbFName); + (void)tNameGetFullDbName(pTableName, dbFName); CTG_ERR_RET(ctgReadTbTypeFromCache(pCtg, dbFName, pTableName->tname, tbType)); if (*tbType > 0) { return TSDB_CODE_SUCCESS; @@ -454,7 +452,11 @@ int32_t ctgGetTbTag(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTableName, tagVal.type = TSDB_DATA_TYPE_JSON; tagVal.pData = pJson; tagVal.nData = strlen(pJson); - taosArrayPush(pTagVals, &tagVal); + if (NULL == taosArrayPush(pTagVals, &tagVal)) { + taosMemoryFree(pJson); + taosArrayDestroy(pTagVals); + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } } else { CTG_ERR_JRET(tTagToValArray((const STag*)pCfg->pTags, &pTagVals)); } @@ -484,7 +486,7 @@ int32_t ctgGetTbDistVgInfo(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTabl CTG_ERR_JRET(ctgGetTbMeta(pCtg, pConn, &ctx, &tbMeta)); char db[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(pTableName, db); + (void)tNameGetFullDbName(pTableName, db); SHashObj* vgHash = NULL; CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pConn, db, &dbCache, &vgInfo, NULL)); @@ -555,7 +557,7 @@ int32_t ctgGetTbHashVgroup(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* SCtgDBCache* dbCache = NULL; int32_t code = 0; char db[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(pTableName, db); + (void)tNameGetFullDbName(pTableName, db); SDBVgInfo* vgInfo = NULL; CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pConn, db, &dbCache, &vgInfo, exists)); @@ -590,7 +592,7 @@ int32_t ctgGetTbsHashVgId(SCatalog* pCtg, SRequestConnInfo* pConn, int32_t acctI SCtgDBCache* dbCache = NULL; int32_t code = 0; char dbFName[TSDB_DB_FNAME_LEN] = {0}; - snprintf(dbFName, TSDB_DB_FNAME_LEN, "%d.%s", acctId, pDb); + (void)snprintf(dbFName, TSDB_DB_FNAME_LEN, "%d.%s", acctId, pDb); SDBVgInfo* vgInfo = NULL; CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pConn, dbFName, &dbCache, &vgInfo, NULL)); @@ -615,7 +617,7 @@ _return: int32_t ctgGetCachedTbVgMeta(SCatalog* pCtg, const SName* pTableName, SVgroupInfo* pVgroup, STableMeta** pTableMeta) { int32_t code = 0; char db[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(pTableName, db); + (void)tNameGetFullDbName(pTableName, db); SCtgDBCache *dbCache = NULL; SCtgTbCache *tbCache = NULL; @@ -688,8 +690,8 @@ _return: void ctgProcessTimerEvent(void *param, void *tmrId) { CTG_API_NENTER(); - ctgdShowCacheInfo(); - ctgdShowStatInfo(); + (void)ctgdShowCacheInfo(); + (void)ctgdShowStatInfo(); int32_t cacheMaxSize = atomic_load_32(&tsMetaCacheMaxSize); if (cacheMaxSize >= 0) { @@ -703,7 +705,7 @@ void ctgProcessTimerEvent(void *param, void *tmrId) { int32_t code = ctgClearCacheEnqueue(NULL, true, false, false, false); if (code) { qError("clear cache enqueue failed, error:%s", tstrerror(code)); - taosTmrReset(ctgProcessTimerEvent, CTG_DEFAULT_CACHE_MON_MSEC, NULL, gCtgMgmt.timer, &gCtgMgmt.cacheTimer); + (void)taosTmrReset(ctgProcessTimerEvent, CTG_DEFAULT_CACHE_MON_MSEC, NULL, gCtgMgmt.timer, &gCtgMgmt.cacheTimer); } goto _return; @@ -711,7 +713,7 @@ void ctgProcessTimerEvent(void *param, void *tmrId) { } qTrace("reset catalog timer"); - taosTmrReset(ctgProcessTimerEvent, CTG_DEFAULT_CACHE_MON_MSEC, NULL, gCtgMgmt.timer, &gCtgMgmt.cacheTimer); + (void)taosTmrReset(ctgProcessTimerEvent, CTG_DEFAULT_CACHE_MON_MSEC, NULL, gCtgMgmt.timer, &gCtgMgmt.cacheTimer); _return: @@ -723,10 +725,8 @@ int32_t ctgGetDBCfg(SCatalog* pCtg, SRequestConnInfo* pConn, const char* dbFName if (pDbCfg->cfgVersion < 0) { CTG_ERR_RET(ctgGetDBCfgFromMnode(pCtg, pConn, dbFName, pDbCfg, NULL)); - SDbCfgInfo *pCfg = ctgCloneDbCfgInfo(pDbCfg); - if (NULL == pCfg) { - return TSDB_CODE_OUT_OF_MEMORY; - } + SDbCfgInfo *pCfg = NULL; + CTG_ERR_RET(ctgCloneDbCfgInfo(pDbCfg, &pCfg)); CTG_ERR_RET(ctgUpdateDbCfgEnqueue(pCtg, dbFName, pDbCfg->dbId, pCfg, false)); } @@ -735,6 +735,59 @@ int32_t ctgGetDBCfg(SCatalog* pCtg, SRequestConnInfo* pConn, const char* dbFName } +int32_t ctgGetTbTsmas(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTableName, SArray** ppRes) { + STableTSMAInfoRsp tsmasRsp = {0}; + int32_t code = ctgGetTbTSMAFromMnode(pCtg, pConn, pTableName, &tsmasRsp, NULL, TDMT_MND_GET_TABLE_TSMA); + if (code == TSDB_CODE_MND_SMA_NOT_EXIST) { + code = 0; + goto _return; + } + + CTG_ERR_JRET(code); + + *ppRes = tsmasRsp.pTsmas; + tsmasRsp.pTsmas = NULL; + + for (int32_t i = 0; i < (*ppRes)->size; ++i) { + CTG_ERR_JRET(ctgUpdateTbTSMAEnqueue(pCtg, taosArrayGet((*ppRes), i), 0, false)); + } + + return TSDB_CODE_SUCCESS; + +_return: + + if (tsmasRsp.pTsmas) { + tFreeTableTSMAInfoRsp(&tsmasRsp); + } + CTG_RET(code); +} + + +int32_t ctgGetTsma(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTsmaName, STableTSMAInfo** pTsma) { + STableTSMAInfoRsp tsmaRsp = {0}; + int32_t code = ctgGetTbTSMAFromMnode(pCtg, pConn, pTsmaName, &tsmaRsp, NULL, TDMT_MND_GET_TSMA); + if (code == TSDB_CODE_MND_SMA_NOT_EXIST) { + code = 0; + goto _return; + } + + CTG_ERR_JRET(code); + + ASSERT(tsmaRsp.pTsmas && tsmaRsp.pTsmas->size == 1); + + *pTsma = taosArrayGetP(tsmaRsp.pTsmas, 0); + taosArrayDestroy(tsmaRsp.pTsmas); + tsmaRsp.pTsmas = NULL; + +_return: + + if (tsmaRsp.pTsmas) { + tFreeTableTSMAInfoRsp(&tsmaRsp); + } + CTG_RET(code); +} + + int32_t catalogInit(SCatalogCfg* cfg) { qDebug("catalogInit start"); if (gCtgMgmt.pCluster) { @@ -742,10 +795,10 @@ int32_t catalogInit(SCatalogCfg* cfg) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } - memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); + TAOS_MEMSET(&gCtgMgmt, 0, sizeof(gCtgMgmt)); if (cfg) { - memcpy(&gCtgMgmt.cfg, cfg, sizeof(*cfg)); + TAOS_MEMCPY(&gCtgMgmt.cfg, cfg, sizeof(*cfg)); if (gCtgMgmt.cfg.maxDBCacheNum == 0) { gCtgMgmt.cfg.maxDBCacheNum = CTG_DEFAULT_CACHE_DB_NUMBER; @@ -1129,7 +1182,7 @@ int32_t catalogUpdateTableIndex(SCatalog* pCtg, STableIndexRsp* pRsp) { CTG_API_LEAVE(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(pIndex, pRsp, sizeof(STableIndex)); + TAOS_MEMCPY(pIndex, pRsp, sizeof(STableIndex)); CTG_ERR_JRET(ctgUpdateTbIndexEnqueue(pCtg, &pIndex, false)); @@ -1247,17 +1300,26 @@ int32_t catalogChkTbMetaVersion(SCatalog* pCtg, SRequestConnInfo* pConn, SArray* CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } + int32_t code = TSDB_CODE_SUCCESS; SName name = {0}; int32_t sver = 0; int32_t tver = 0; int32_t tbNum = taosArrayGetSize(pTables); for (int32_t i = 0; i < tbNum; ++i) { STbSVersion* pTb = (STbSVersion*)taosArrayGet(pTables, i); + if (NULL == pTb) { + ctgError("fail to get the %dth table, tbNum:%d", i, tbNum); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } + if (NULL == pTb->tbFName || 0 == pTb->tbFName[0]) { continue; } - tNameFromString(&name, pTb->tbFName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + if (tNameFromString(&name, pTb->tbFName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE)) { + ctgError("invalid tbFName format, tbFName:%s, idx:%d", pTb->tbFName, i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } if (IS_SYS_DBNAME(name.dbname)) { continue; @@ -1266,18 +1328,18 @@ int32_t catalogChkTbMetaVersion(SCatalog* pCtg, SRequestConnInfo* pConn, SArray* int32_t tbType = 0; uint64_t suid = 0; char stbName[TSDB_TABLE_FNAME_LEN]; - ctgReadTbVerFromCache(pCtg, &name, &sver, &tver, &tbType, &suid, stbName); + CTG_ERR_JRET(ctgReadTbVerFromCache(pCtg, &name, &sver, &tver, &tbType, &suid, stbName)); if ((sver >= 0 && sver < pTb->sver) || (tver >= 0 && tver < pTb->tver)) { switch (tbType) { case TSDB_CHILD_TABLE: { SName stb = name; tstrncpy(stb.tname, stbName, sizeof(stb.tname)); - ctgRemoveTbMeta(pCtg, &stb); + CTG_ERR_JRET(ctgRemoveTbMeta(pCtg, &stb)); break; } case TSDB_SUPER_TABLE: case TSDB_NORMAL_TABLE: - ctgRemoveTbMeta(pCtg, &name); + CTG_ERR_JRET(ctgRemoveTbMeta(pCtg, &name)); break; default: ctgError("ignore table type %d", tbType); @@ -1286,7 +1348,9 @@ int32_t catalogChkTbMetaVersion(SCatalog* pCtg, SRequestConnInfo* pConn, SArray* } } - CTG_API_LEAVE(TSDB_CODE_SUCCESS); +_return: + + CTG_API_LEAVE(code); } int32_t catalogRefreshDBVgInfo(SCatalog* pCtg, SRequestConnInfo* pConn, const char* dbFName) { @@ -1454,10 +1518,10 @@ int32_t catalogAsyncGetAllMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SC _return: if (pJob) { - taosReleaseRef(gCtgMgmt.jobPool, pJob->refId); + (void)taosReleaseRef(gCtgMgmt.jobPool, pJob->refId); if (code) { - taosRemoveRef(gCtgMgmt.jobPool, pJob->refId); + (void)taosRemoveRef(gCtgMgmt.jobPool, pJob->refId); } } @@ -1558,7 +1622,7 @@ int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion** users, uint32_ while (pAuth != NULL) { size_t len = 0; void* key = taosHashGetKey(pAuth, &len); - strncpy((*users)[i].user, key, len); + TAOS_STRNCPY((*users)[i].user, key, len); (*users)[i].user[len] = 0; (*users)[i].version = pAuth->userAuth.version; ++i; @@ -1795,10 +1859,12 @@ int32_t catalogAsyncUpdateTSMA(SCatalog* pCtg, STableTSMAInfo** ppTsma, int32_t if (!pCtg || !ppTsma) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } + int32_t code = 0; CTG_ERR_JRET(ctgUpdateTbTSMAEnqueue(pCtg, ppTsma, tsmaVersion, false)); _return: + CTG_API_LEAVE(code); } @@ -1807,10 +1873,12 @@ int32_t catalogUpdateTSMA(SCatalog* pCtg, STableTSMAInfo** pTsma) { if (!pCtg || !pTsma) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } + int32_t code = 0; CTG_ERR_JRET(ctgUpdateTbTSMAEnqueue(pCtg, pTsma, 0, true)); _return: + CTG_API_LEAVE(code); } @@ -1823,36 +1891,14 @@ int32_t catalogRemoveTSMA(SCatalog* pCtg, const STableTSMAInfo* pTsma) { } if (!pCtg->dbCache) { - return TSDB_CODE_SUCCESS; - } - CTG_ERR_JRET(ctgDropTbTSMAEnqueue(pCtg, pTsma, true)); -_return: - CTG_API_LEAVE(code); -} - -int32_t ctgGetTbTsmas(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTableName, SArray** ppRes) { - STableTSMAInfoRsp tsmasRsp = {0}; - int32_t code = ctgGetTbTSMAFromMnode(pCtg, pConn, pTableName, &tsmasRsp, NULL, TDMT_MND_GET_TABLE_TSMA); - if (code == TSDB_CODE_MND_SMA_NOT_EXIST) { - code = 0; goto _return; } - CTG_ERR_JRET(code); - assert(tsmasRsp.pTsmas); - assert(tsmasRsp.pTsmas->size > 0); - *ppRes = tsmasRsp.pTsmas; - tsmasRsp.pTsmas = NULL; - - for (int32_t i = 0; i < (*ppRes)->size; ++i) { - CTG_ERR_JRET(ctgUpdateTbTSMAEnqueue(pCtg, taosArrayGet((*ppRes), i), 0, false)); - } - return TSDB_CODE_SUCCESS; - + + CTG_ERR_JRET(ctgDropTbTSMAEnqueue(pCtg, pTsma, true)); + _return: - if (tsmasRsp.pTsmas) { - tFreeTableTSMAInfoRsp(&tsmasRsp); - } - CTG_RET(code); + + CTG_API_LEAVE(code); } int32_t catalogGetTableTsmas(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SArray** pRes) { @@ -1870,27 +1916,6 @@ _return: CTG_API_LEAVE(code); } -int32_t ctgGetTsma(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTsmaName, STableTSMAInfo** pTsma) { - STableTSMAInfoRsp tsmaRsp = {0}; - int32_t code = ctgGetTbTSMAFromMnode(pCtg, pConn, pTsmaName, &tsmaRsp, NULL, TDMT_MND_GET_TSMA); - if (code == TSDB_CODE_MND_SMA_NOT_EXIST) { - code = 0; - goto _return; - } - - CTG_ERR_JRET(code); - - ASSERT(tsmaRsp.pTsmas && tsmaRsp.pTsmas->size == 1); - *pTsma = taosArrayGetP(tsmaRsp.pTsmas, 0); - taosArrayDestroy(tsmaRsp.pTsmas); - tsmaRsp.pTsmas = NULL; - -_return: - if (tsmaRsp.pTsmas) { - tFreeTableTSMAInfoRsp(&tsmaRsp); - } - CTG_RET(code); -} int32_t catalogGetTsma(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTsmaName, STableTSMAInfo** pTsma) { CTG_API_ENTER(); @@ -1903,6 +1928,7 @@ int32_t catalogGetTsma(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTs CTG_ERR_JRET(ctgGetTsma(pCtg, pConn, pTsmaName, pTsma)); _return: + CTG_API_LEAVE(code); } @@ -1930,7 +1956,7 @@ void catalogDestroy(void) { } if (gCtgMgmt.cacheTimer) { - taosTmrStop(gCtgMgmt.cacheTimer); + (void)taosTmrStop(gCtgMgmt.cacheTimer); gCtgMgmt.cacheTimer = NULL; taosTmrCleanUp(gCtgMgmt.timer); gCtgMgmt.timer = NULL; @@ -1939,8 +1965,8 @@ void catalogDestroy(void) { atomic_store_8((int8_t*)&gCtgMgmt.exit, true); if (!taosCheckCurrentInDll()) { - ctgClearCacheEnqueue(NULL, false, true, true, true); - taosThreadJoin(gCtgMgmt.updateThread, NULL); + (void)ctgClearCacheEnqueue(NULL, false, true, true, true); + (void)taosThreadJoin(gCtgMgmt.updateThread, NULL); } taosHashCleanup(gCtgMgmt.pCluster); diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 601e01f7e9..13bf2d08e6 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -65,10 +65,13 @@ int32_t ctgInitGetTbMetaTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(ctx->pName, name, sizeof(*name)); + TAOS_MEMCPY(ctx->pName, name, sizeof(*name)); ctx->flag = pParam->flag | CTG_FLAG_UNKNOWN_STB; - taosArrayPush(pJob->pTasks, &task); + if (NULL == taosArrayPush(pJob->pTasks, &task)) { + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tbName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name->tname); @@ -91,8 +94,16 @@ int32_t ctgInitGetTbMetasTask(SCtgJob* pJob, int32_t taskIdx, void* param) { SCtgTbMetasCtx* ctx = task.taskCtx; ctx->pNames = param; ctx->pResList = taosArrayInit(pJob->tbMetaNum, sizeof(SMetaRes)); - - taosArrayPush(pJob->pTasks, &task); + if (NULL == ctx->pResList) { + qError("QID:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->tbMetaNum, (int32_t)sizeof(SMetaRes)); + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + if (NULL == taosArrayPush(pJob->pTasks, &task)) { + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbNum:%lu, tbNum:%d", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), taosArrayGetSize(ctx->pNames), pJob->tbMetaNum); @@ -115,9 +126,12 @@ int32_t ctgInitGetDbVgTask(SCtgJob* pJob, int32_t taskIdx, void* param) { SCtgDbVgCtx* ctx = task.taskCtx; - memcpy(ctx->dbFName, dbFName, sizeof(ctx->dbFName)); + TAOS_MEMCPY(ctx->dbFName, dbFName, sizeof(ctx->dbFName)); - taosArrayPush(pJob->pTasks, &task); + if (NULL == taosArrayPush(pJob->pTasks, &task)) { + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbFName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), dbFName); @@ -140,9 +154,12 @@ int32_t ctgInitGetDbCfgTask(SCtgJob* pJob, int32_t taskIdx, void* param) { SCtgDbCfgCtx* ctx = task.taskCtx; - memcpy(ctx->dbFName, dbFName, sizeof(ctx->dbFName)); + TAOS_MEMCPY(ctx->dbFName, dbFName, sizeof(ctx->dbFName)); - taosArrayPush(pJob->pTasks, &task); + if (NULL == taosArrayPush(pJob->pTasks, &task)) { + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbFName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), dbFName); @@ -165,9 +182,12 @@ int32_t ctgInitGetDbInfoTask(SCtgJob* pJob, int32_t taskIdx, void* param) { SCtgDbInfoCtx* ctx = task.taskCtx; - memcpy(ctx->dbFName, dbFName, sizeof(ctx->dbFName)); + TAOS_MEMCPY(ctx->dbFName, dbFName, sizeof(ctx->dbFName)); - taosArrayPush(pJob->pTasks, &task); + if (NULL == taosArrayPush(pJob->pTasks, &task)) { + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbFName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), dbFName); @@ -195,10 +215,13 @@ int32_t ctgInitGetTbHashTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(ctx->pName, name, sizeof(*name)); - tNameGetFullDbName(ctx->pName, ctx->dbFName); + TAOS_MEMCPY(ctx->pName, name, sizeof(*name)); + (void)tNameGetFullDbName(ctx->pName, ctx->dbFName); - taosArrayPush(pJob->pTasks, &task); + if (NULL == taosArrayPush(pJob->pTasks, &task)) { + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tableName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name->tname); @@ -221,8 +244,16 @@ int32_t ctgInitGetTbHashsTask(SCtgJob* pJob, int32_t taskIdx, void* param) { SCtgTbHashsCtx* ctx = task.taskCtx; ctx->pNames = param; ctx->pResList = taosArrayInit(pJob->tbHashNum, sizeof(SMetaRes)); - - taosArrayPush(pJob->pTasks, &task); + if (NULL == ctx->pResList) { + qError("QID:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->tbHashNum, (int32_t)sizeof(SMetaRes)); + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + if (NULL == taosArrayPush(pJob->pTasks, &task)) { + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbNum:%lu, tbNum:%d", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), taosArrayGetSize(ctx->pNames), pJob->tbHashNum); @@ -238,7 +269,10 @@ int32_t ctgInitGetQnodeTask(SCtgJob* pJob, int32_t taskIdx, void* param) { task.pJob = pJob; task.taskCtx = NULL; - taosArrayPush(pJob->pTasks, &task); + if (NULL == taosArrayPush(pJob->pTasks, &task)) { + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type)); @@ -253,7 +287,10 @@ int32_t ctgInitGetDnodeTask(SCtgJob* pJob, int32_t taskIdx, void* param) { task.pJob = pJob; task.taskCtx = NULL; - taosArrayPush(pJob->pTasks, &task); + if (NULL == taosArrayPush(pJob->pTasks, &task)) { + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type)); @@ -277,7 +314,10 @@ int32_t ctgInitGetIndexTask(SCtgJob* pJob, int32_t taskIdx, void* param) { tstrncpy(ctx->indexFName, name, sizeof(ctx->indexFName)); - taosArrayPush(pJob->pTasks, &task); + if (NULL == taosArrayPush(pJob->pTasks, &task)) { + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, indexFName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name); @@ -302,7 +342,10 @@ int32_t ctgInitGetUdfTask(SCtgJob* pJob, int32_t taskIdx, void* param) { tstrncpy(ctx->udfName, name, sizeof(ctx->udfName)); - taosArrayPush(pJob->pTasks, &task); + if (NULL == taosArrayPush(pJob->pTasks, &task)) { + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, udfName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name); @@ -325,9 +368,12 @@ int32_t ctgInitGetUserTask(SCtgJob* pJob, int32_t taskIdx, void* param) { SCtgUserCtx* ctx = task.taskCtx; - memcpy(&ctx->user, user, sizeof(*user)); + TAOS_MEMCPY(&ctx->user, user, sizeof(*user)); - taosArrayPush(pJob->pTasks, &task); + if (NULL == taosArrayPush(pJob->pTasks, &task)) { + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, user:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), user->user); @@ -342,7 +388,10 @@ int32_t ctgInitGetSvrVerTask(SCtgJob* pJob, int32_t taskIdx, void* param) { task.taskId = taskIdx; task.pJob = pJob; - taosArrayPush(pJob->pTasks, &task); + if (NULL == taosArrayPush(pJob->pTasks, &task)) { + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type)); @@ -369,9 +418,12 @@ int32_t ctgInitGetTbIndexTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(ctx->pName, name, sizeof(*name)); + TAOS_MEMCPY(ctx->pName, name, sizeof(*name)); - taosArrayPush(pJob->pTasks, &task); + if (NULL == taosArrayPush(pJob->pTasks, &task)) { + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tbName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name->tname); @@ -399,9 +451,12 @@ int32_t ctgInitGetTbCfgTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(ctx->pName, name, sizeof(*name)); + TAOS_MEMCPY(ctx->pName, name, sizeof(*name)); - taosArrayPush(pJob->pTasks, &task); + if (NULL == taosArrayPush(pJob->pTasks, &task)) { + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tbName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name->tname); @@ -429,9 +484,12 @@ int32_t ctgInitGetTbTagTask(SCtgJob* pJob, int32_t taskIdx, void* param) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(ctx->pName, name, sizeof(*name)); + TAOS_MEMCPY(ctx->pName, name, sizeof(*name)); - taosArrayPush(pJob->pTasks, &task); + if (NULL == taosArrayPush(pJob->pTasks, &task)) { + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tbName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name->tname); @@ -454,8 +512,16 @@ int32_t ctgInitGetViewsTask(SCtgJob* pJob, int32_t taskIdx, void* param) { SCtgViewsCtx* ctx = task.taskCtx; ctx->pNames = param; ctx->pResList = taosArrayInit(pJob->viewNum, sizeof(SMetaRes)); + if (NULL == ctx->pResList) { + qError("QID:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->viewNum, (int32_t)sizeof(SMetaRes)); + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } - taosArrayPush(pJob->pTasks, &task); + if (NULL == taosArrayPush(pJob->pTasks, &task)) { + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbNum:%lu, viewNum:%d", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), taosArrayGetSize(ctx->pNames), pJob->viewNum); @@ -463,19 +529,6 @@ int32_t ctgInitGetViewsTask(SCtgJob* pJob, int32_t taskIdx, void* param) { return TSDB_CODE_SUCCESS; } -int32_t ctgHandleForceUpdateView(SCatalog* pCtg, const SCatalogReq* pReq) { - int32_t viewNum = taosArrayGetSize(pReq->pView); - for (int32_t i = 0; i < viewNum; ++i) { - STablesReq* p = taosArrayGet(pReq->pView, i); - int32_t viewNum = taosArrayGetSize(p->pTables); - for (int32_t m = 0; m < viewNum; ++m) { - SName* name = taosArrayGet(p->pTables, m); - ctgDropViewMetaEnqueue(pCtg, p->dbFName, 0, name->tname, 0, true); - } - } - return TSDB_CODE_SUCCESS; -} - int32_t ctgInitGetTbTSMATask(SCtgJob* pJob, int32_t taskId, void* param) { SCtgTask task = {0}; task.type = CTG_TASK_GET_TB_TSMA; @@ -483,12 +536,24 @@ int32_t ctgInitGetTbTSMATask(SCtgJob* pJob, int32_t taskId, void* param) { task.pJob = pJob; SCtgTbTSMACtx* pTaskCtx = taosMemoryCalloc(1, sizeof(SCtgTbTSMACtx)); - if (!pTaskCtx) CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + if (NULL == pTaskCtx) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + task.taskCtx = pTaskCtx; pTaskCtx->pNames = param; pTaskCtx->pResList = taosArrayInit(pJob->tbTsmaNum, sizeof(SMetaRes)); - - taosArrayPush(pJob->pTasks, &task); + if (NULL == pTaskCtx->pResList) { + qError("QID:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->tbTsmaNum, (int32_t)sizeof(SMetaRes)); + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + if (NULL == taosArrayPush(pJob->pTasks, &task)) { + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + return TSDB_CODE_SUCCESS; } @@ -499,18 +564,51 @@ int32_t ctgInitGetTSMATask(SCtgJob* pJob, int32_t taskId, void* param) { task.pJob = pJob; SCtgTbTSMACtx* pTaskCtx = taosMemoryCalloc(1, sizeof(SCtgTbTSMACtx)); - if (!pTaskCtx) CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + if (NULL == pTaskCtx) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } task.taskCtx = pTaskCtx; pTaskCtx->pNames = param; pTaskCtx->pResList = taosArrayInit(pJob->tsmaNum, sizeof(SMetaRes)); + if (NULL == pTaskCtx->pResList) { + qError("QID:0x%" PRIx64 " taosArrayInit %d SMetaRes %d failed", pJob->queryId, pJob->tsmaNum, (int32_t)sizeof(SMetaRes)); + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } - taosArrayPush(pJob->pTasks, &task); - return 0; + if (NULL == taosArrayPush(pJob->pTasks, &task)) { + ctgFreeTask(&task, true); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + return TSDB_CODE_SUCCESS; } + +int32_t ctgHandleForceUpdateView(SCatalog* pCtg, const SCatalogReq* pReq) { + int32_t viewNum = taosArrayGetSize(pReq->pView); + for (int32_t i = 0; i < viewNum; ++i) { + STablesReq* p = taosArrayGet(pReq->pView, i); + if (NULL == p) { + qError("taosArrayGet the %dth view in req failed", i); + CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); + } + + int32_t viewNum = taosArrayGetSize(p->pTables); + for (int32_t m = 0; m < viewNum; ++m) { + SName* name = taosArrayGet(p->pTables, m); + CTG_ERR_RET(ctgDropViewMetaEnqueue(pCtg, p->dbFName, 0, name->tname, 0, true)); + } + } + + return TSDB_CODE_SUCCESS; +} + + int32_t ctgHandleForceUpdate(SCatalog* pCtg, int32_t taskNum, SCtgJob* pJob, const SCatalogReq* pReq) { - SHashObj* pDb = taosHashInit(taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - SHashObj* pTb = taosHashInit(taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + int32_t code = TSDB_CODE_SUCCESS; + SHashObj* pDb = taosHashInit(taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + SHashObj* pTb = taosHashInit(taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); if (NULL == pDb || NULL == pTb) { taosHashCleanup(pDb); taosHashCleanup(pTb); @@ -519,101 +617,167 @@ int32_t ctgHandleForceUpdate(SCatalog* pCtg, int32_t taskNum, SCtgJob* pJob, con for (int32_t i = 0; i < pJob->dbVgNum; ++i) { char* dbFName = taosArrayGet(pReq->pDbVgroup, i); - taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN); + if (NULL == dbFName) { + qError("taosArrayGet the %dth db in req failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } + CTG_ERR_JRET(taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN)); } for (int32_t i = 0; i < pJob->dbCfgNum; ++i) { char* dbFName = taosArrayGet(pReq->pDbCfg, i); - taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN); + if (NULL == dbFName) { + qError("taosArrayGet the %dth db in req failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } + CTG_ERR_JRET(taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN)); } for (int32_t i = 0; i < pJob->dbInfoNum; ++i) { char* dbFName = taosArrayGet(pReq->pDbInfo, i); - taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN); + if (NULL == dbFName) { + qError("taosArrayGet the %dth db in req failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } + CTG_ERR_JRET(taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN)); } int32_t dbNum = taosArrayGetSize(pReq->pTableMeta); for (int32_t i = 0; i < dbNum; ++i) { STablesReq* p = taosArrayGet(pReq->pTableMeta, i); - taosHashPut(pDb, p->dbFName, strlen(p->dbFName), p->dbFName, TSDB_DB_FNAME_LEN); + if (NULL == p) { + qError("taosArrayGet the %dth tb in req failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } + CTG_ERR_JRET(taosHashPut(pDb, p->dbFName, strlen(p->dbFName), p->dbFName, TSDB_DB_FNAME_LEN)); + int32_t tbNum = taosArrayGetSize(p->pTables); for (int32_t m = 0; m < tbNum; ++m) { SName* name = taosArrayGet(p->pTables, m); - taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName)); + if (NULL == name) { + qError("taosArrayGet the %dth tb in req failed", m); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } + CTG_ERR_JRET(taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName))); } } dbNum = taosArrayGetSize(pReq->pTableHash); for (int32_t i = 0; i < dbNum; ++i) { STablesReq* p = taosArrayGet(pReq->pTableHash, i); - taosHashPut(pDb, p->dbFName, strlen(p->dbFName), p->dbFName, TSDB_DB_FNAME_LEN); + if (NULL == p) { + qError("taosArrayGet the %dth tb in req failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } + + CTG_ERR_JRET(taosHashPut(pDb, p->dbFName, strlen(p->dbFName), p->dbFName, TSDB_DB_FNAME_LEN)); + int32_t tbNum = taosArrayGetSize(p->pTables); for (int32_t m = 0; m < tbNum; ++m) { SName* name = taosArrayGet(p->pTables, m); - taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName)); + if (NULL == name) { + qError("taosArrayGet the %dth tb in req failed", m); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } + CTG_ERR_JRET(taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName))); } } dbNum = taosArrayGetSize(pReq->pView); for (int32_t i = 0; i < dbNum; ++i) { STablesReq* p = taosArrayGet(pReq->pView, i); - taosHashPut(pDb, p->dbFName, strlen(p->dbFName), p->dbFName, TSDB_DB_FNAME_LEN); + if (NULL == p) { + qError("taosArrayGet the %dth db in req failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } + CTG_ERR_JRET(taosHashPut(pDb, p->dbFName, strlen(p->dbFName), p->dbFName, TSDB_DB_FNAME_LEN)); } for (int32_t i = 0; i < pJob->tbCfgNum; ++i) { SName* name = taosArrayGet(pReq->pTableCfg, i); + if (NULL == name) { + qError("taosArrayGet the %dth tb in req failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } + char dbFName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(name, dbFName); - taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN); - taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName)); + (void)tNameGetFullDbName(name, dbFName); + CTG_ERR_JRET(taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN)); + CTG_ERR_JRET(taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName))); } for (int32_t i = 0; i < pJob->tbTagNum; ++i) { SName* name = taosArrayGet(pReq->pTableTag, i); + if (NULL == name) { + qError("taosArrayGet the %dth tb in req failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } + char dbFName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(name, dbFName); - taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN); - taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName)); + (void)tNameGetFullDbName(name, dbFName); + CTG_ERR_JRET(taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN)); + CTG_ERR_JRET(taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName))); } char* dbFName = taosHashIterate(pDb, NULL); while (dbFName) { - ctgDropDbVgroupEnqueue(pCtg, dbFName, true); + CTG_ERR_JRET(ctgDropDbVgroupEnqueue(pCtg, dbFName, true)); dbFName = taosHashIterate(pDb, dbFName); } taosHashCleanup(pDb); + pDb = NULL; // REFRESH TABLE META for (int32_t i = 0; i < pJob->tbCfgNum; ++i) { SName* name = taosArrayGet(pReq->pTableCfg, i); - taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName)); + if (NULL == name) { + qError("taosArrayGet the %dth tb in req failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } + CTG_ERR_JRET(taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName))); } SName* name = taosHashIterate(pTb, NULL); while (name) { - ctgRemoveTbMeta(pCtg, name); + CTG_ERR_JRET(ctgRemoveTbMeta(pCtg, name)); name = taosHashIterate(pTb, name); } taosHashCleanup(pTb); + pTb = NULL; for (int32_t i = 0; i < pJob->tbIndexNum; ++i) { SName* name = taosArrayGet(pReq->pTableIndex, i); - ctgDropTbIndexEnqueue(pCtg, name, true); + if (NULL == name) { + qError("taosArrayGet the %dth tb in req failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } + CTG_ERR_JRET(ctgDropTbIndexEnqueue(pCtg, name, true)); } for (int32_t i = 0; i < pJob->tbTsmaNum; ++i) { STablesReq* pTbReq = taosArrayGet(pReq->pTableTSMAs, i); + if (NULL == pTbReq) { + qError("taosArrayGet the %dth tb in req failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } for (int32_t j = 0; j < pTbReq->pTables->size; ++j) { SName* name = taosArrayGet(pTbReq->pTables, j); - ctgDropTSMAForTbEnqueue(pCtg, name, true); + CTG_ERR_JRET(ctgDropTSMAForTbEnqueue(pCtg, name, true)); } } // REFRESH VIEW META - return ctgHandleForceUpdateView(pCtg, pReq); + CTG_ERR_JRET(ctgHandleForceUpdateView(pCtg, pReq)); + +_return: + + taosHashCleanup(pDb); + taosHashCleanup(pTb); + + return code; } int32_t ctgInitTask(SCtgJob* pJob, CTG_TASK_TYPE type, void* param, int32_t* taskId) { @@ -628,6 +792,7 @@ int32_t ctgInitTask(SCtgJob* pJob, CTG_TASK_TYPE type, void* param, int32_t* tas } _return: + CTG_UNLOCK(CTG_WRITE, &pJob->taskLock); return code; @@ -714,16 +879,28 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const for (int32_t i = 0; i < dbVgNum; ++i) { char* dbFName = taosArrayGet(pReq->pDbVgroup, i); + if (NULL == dbFName) { + qError("taosArrayGet the %dth db in pDbVgroup failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_DB_VGROUP, dbFName, NULL)); } for (int32_t i = 0; i < dbCfgNum; ++i) { char* dbFName = taosArrayGet(pReq->pDbCfg, i); + if (NULL == dbFName) { + qError("taosArrayGet the %dth db in pDbCfg failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_DB_CFG, dbFName, NULL)); } for (int32_t i = 0; i < dbInfoNum; ++i) { char* dbFName = taosArrayGet(pReq->pDbInfo, i); + if (NULL == dbFName) { + qError("taosArrayGet the %dth db in pDbInfo failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_DB_INFO, dbFName, NULL)); } @@ -751,54 +928,74 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const for (int32_t i = 0; i < tbIndexNum; ++i) { SName* name = taosArrayGet(pReq->pTableIndex, i); + if (NULL == name) { + qError("taosArrayGet the %dth tb in pTableIndex failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_TB_SMA_INDEX, name, NULL)); } for (int32_t i = 0; i < tbCfgNum; ++i) { SName* name = taosArrayGet(pReq->pTableCfg, i); + if (NULL == name) { + qError("taosArrayGet the %dth tb in pTableCfg failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_TB_CFG, name, NULL)); } for (int32_t i = 0; i < tbTagNum; ++i) { SName* name = taosArrayGet(pReq->pTableTag, i); + if (NULL == name) { + qError("taosArrayGet the %dth tb in pTableTag failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_TB_TAG, name, NULL)); } for (int32_t i = 0; i < indexNum; ++i) { char* indexName = taosArrayGet(pReq->pIndex, i); + if (NULL == indexName) { + qError("taosArrayGet the %dth index in pIndex failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_INDEX_INFO, indexName, NULL)); } for (int32_t i = 0; i < udfNum; ++i) { char* udfName = taosArrayGet(pReq->pUdf, i); + if (NULL == udfName) { + qError("taosArrayGet the %dth udf in pUdf failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_UDF, udfName, NULL)); } for (int32_t i = 0; i < userNum; ++i) { SUserAuthInfo* user = taosArrayGet(pReq->pUser, i); + if (NULL == user) { + qError("taosArrayGet the %dth user in pUser failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_USER, user, NULL)); } if (viewNum > 0) { CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_VIEW, pReq->pView, NULL)); } - if (tbTsmaNum > 0) { CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_TB_TSMA, pReq->pTableTSMAs, NULL)); } if (tsmaNum > 0) { CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_TSMA, pReq->pTSMAs, NULL)); } - if (qnodeNum) { CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_QNODE, NULL, NULL)); } - if (dnodeNum) { CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_DNODE, NULL, NULL)); } - if (svrVerNum) { CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_SVR_VER, NULL, NULL)); } @@ -809,7 +1006,7 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const CTG_ERR_JRET(terrno); } - taosAcquireRef(gCtgMgmt.jobPool, pJob->refId); + (void)taosAcquireRef(gCtgMgmt.jobPool, pJob->refId); double el = (taosGetTimestampUs() - st) / 1000.0; qDebug("QID:0x%" PRIx64 ", jobId: 0x%" PRIx64 " initialized, task num %d, forceUpdate %d, elapsed time:%.2f ms", @@ -817,7 +1014,10 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const return TSDB_CODE_SUCCESS; _return: + ctgFreeJob(*job); + *job = NULL; + CTG_RET(code); } @@ -835,8 +1035,10 @@ int32_t ctgDumpTbMetaRes(SCtgTask* pTask) { } SMetaRes res = {.code = pTask->code, .pRes = pTask->res}; - taosArrayPush(pJob->jobRes.pTableMeta, &res); - + if (NULL == taosArrayPush(pJob->jobRes.pTableMeta, &res)) { + CTG_ERR_RET(terrno); + } + return TSDB_CODE_SUCCESS; } @@ -866,7 +1068,9 @@ int32_t ctgDumpDbVgRes(SCtgTask* pTask) { } SMetaRes res = {.code = pTask->code, .pRes = pTask->res}; - taosArrayPush(pJob->jobRes.pDbVgroup, &res); + if (NULL == taosArrayPush(pJob->jobRes.pDbVgroup, &res)) { + CTG_ERR_RET(terrno); + } return TSDB_CODE_SUCCESS; } @@ -885,7 +1089,9 @@ int32_t ctgDumpTbHashRes(SCtgTask* pTask) { } SMetaRes res = {.code = pTask->code, .pRes = pTask->res}; - taosArrayPush(pJob->jobRes.pTableHash, &res); + if (NULL == taosArrayPush(pJob->jobRes.pTableHash, &res)) { + CTG_ERR_RET(terrno); + } return TSDB_CODE_SUCCESS; } @@ -920,7 +1126,9 @@ int32_t ctgDumpTbIndexRes(SCtgTask* pTask) { } SMetaRes res = {.code = pTask->code, .pRes = pTask->res}; - taosArrayPush(pJob->jobRes.pTableIndex, &res); + if (NULL == taosArrayPush(pJob->jobRes.pTableIndex, &res)) { + CTG_ERR_RET(terrno); + } return TSDB_CODE_SUCCESS; } @@ -943,7 +1151,9 @@ int32_t ctgDumpTbCfgRes(SCtgTask* pTask) { } SMetaRes res = {.code = pTask->code, .pRes = pTask->res}; - taosArrayPush(pJob->jobRes.pTableCfg, &res); + if (NULL == taosArrayPush(pJob->jobRes.pTableCfg, &res)) { + CTG_ERR_RET(terrno); + } return TSDB_CODE_SUCCESS; } @@ -966,7 +1176,9 @@ int32_t ctgDumpTbTagRes(SCtgTask* pTask) { } SMetaRes res = {.code = pTask->code, .pRes = pTask->res}; - taosArrayPush(pJob->jobRes.pTableTag, &res); + if (NULL == taosArrayPush(pJob->jobRes.pTableTag, &res)) { + CTG_ERR_RET(terrno); + } return TSDB_CODE_SUCCESS; } @@ -986,7 +1198,9 @@ int32_t ctgDumpIndexRes(SCtgTask* pTask) { } SMetaRes res = {.code = pTask->code, .pRes = pTask->res}; - taosArrayPush(pJob->jobRes.pIndex, &res); + if (NULL == taosArrayPush(pJob->jobRes.pIndex, &res)) { + CTG_ERR_RET(terrno); + } return TSDB_CODE_SUCCESS; } @@ -1005,7 +1219,9 @@ int32_t ctgDumpQnodeRes(SCtgTask* pTask) { } SMetaRes res = {.code = pTask->code, .pRes = pTask->res}; - taosArrayPush(pJob->jobRes.pQnodeList, &res); + if (NULL == taosArrayPush(pJob->jobRes.pQnodeList, &res)) { + CTG_ERR_RET(terrno); + } return TSDB_CODE_SUCCESS; } @@ -1024,7 +1240,9 @@ int32_t ctgDumpDnodeRes(SCtgTask* pTask) { } SMetaRes res = {.code = pTask->code, .pRes = pTask->res}; - taosArrayPush(pJob->jobRes.pDnodeList, &res); + if (NULL == taosArrayPush(pJob->jobRes.pDnodeList, &res)) { + CTG_ERR_RET(terrno); + } return TSDB_CODE_SUCCESS; } @@ -1043,7 +1261,9 @@ int32_t ctgDumpDbCfgRes(SCtgTask* pTask) { } SMetaRes res = {.code = pTask->code, .pRes = pTask->res}; - taosArrayPush(pJob->jobRes.pDbCfg, &res); + if (NULL == taosArrayPush(pJob->jobRes.pDbCfg, &res)) { + CTG_ERR_RET(terrno); + } return TSDB_CODE_SUCCESS; } @@ -1062,7 +1282,9 @@ int32_t ctgDumpDbInfoRes(SCtgTask* pTask) { } SMetaRes res = {.code = pTask->code, .pRes = pTask->res}; - taosArrayPush(pJob->jobRes.pDbInfo, &res); + if (NULL == taosArrayPush(pJob->jobRes.pDbInfo, &res)) { + CTG_ERR_RET(terrno); + } return TSDB_CODE_SUCCESS; } @@ -1081,7 +1303,9 @@ int32_t ctgDumpUdfRes(SCtgTask* pTask) { } SMetaRes res = {.code = pTask->code, .pRes = pTask->res}; - taosArrayPush(pJob->jobRes.pUdfList, &res); + if (NULL == taosArrayPush(pJob->jobRes.pUdfList, &res)) { + CTG_ERR_RET(terrno); + } return TSDB_CODE_SUCCESS; } @@ -1100,7 +1324,9 @@ int32_t ctgDumpUserRes(SCtgTask* pTask) { } SMetaRes res = {.code = pTask->code, .pRes = pTask->res}; - taosArrayPush(pJob->jobRes.pUser, &res); + if (NULL == taosArrayPush(pJob->jobRes.pUser, &res)) { + CTG_ERR_RET(terrno); + } return TSDB_CODE_SUCCESS; } @@ -1147,7 +1373,11 @@ int32_t ctgCallSubCb(SCtgTask* pTask) { for (int32_t i = 0; i < parentNum; ++i) { SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); SCtgTask* pParent = taosArrayGetP(pTask->pParents, i); - + if (NULL == pParent) { + qError("taosArrayGetP the %dth parent failed", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + pParent->subRes.code = pTask->code; if (TSDB_CODE_SUCCESS == pTask->code) { code = (*gCtgAsyncFps[pTask->type].cloneFp)(pTask, &pParent->subRes.res); @@ -1178,7 +1408,7 @@ int32_t ctgCallUserCb(void* param) { qDebug("QID:0x%" PRIx64 " ctg end to call user cb", pJob->queryId); - taosRemoveRef(gCtgMgmt.jobPool, pJob->refId); + (void)taosRemoveRef(gCtgMgmt.jobPool, pJob->refId); return TSDB_CODE_SUCCESS; } @@ -1204,7 +1434,7 @@ int32_t ctgHandleTaskEnd(SCtgTask* pTask, int32_t rspCode) { pTask->code = rspCode; pTask->status = CTG_TASK_DONE; - ctgCallSubCb(pTask); + CTG_ERR_JRET(ctgCallSubCb(pTask)); int32_t taskDone = atomic_add_fetch_32(&pJob->taskDone, 1); if (taskDone < taosArrayGetSize(pJob->pTasks)) { @@ -1220,12 +1450,11 @@ int32_t ctgHandleTaskEnd(SCtgTask* pTask, int32_t rspCode) { _return: ctgUpdateJobErrCode(pJob, rspCode); - // pJob->jobResCode = code; - // taosSsleep(2); - // qDebug("QID:0x%" PRIx64 " ctg after sleep", pJob->queryId); - - taosAsyncExec(ctgCallUserCb, pJob, NULL); + int32_t newCode = taosAsyncExec(ctgCallUserCb, pJob, NULL); + if (TSDB_CODE_SUCCESS == code && TSDB_CODE_SUCCESS != newCode) { + code = newCode; + } CTG_RET(code); } @@ -1242,6 +1471,11 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf int32_t flag = ctx->flag; int32_t* vgId = &ctx->vgId; + if (NULL == pMsgCtx) { + ctgError("fail to get task msgCtx, taskType:%d", pTask->type); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + CTG_ERR_JRET(ctgProcessRspMsg(pMsgCtx->out, reqType, pMsg->pData, pMsg->len, rspCode, pMsgCtx->target)); switch (reqType) { @@ -1264,7 +1498,7 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf if (CTG_IS_META_NULL(pOut->metaType)) { if (CTG_FLAG_IS_STB(flag)) { char dbFName[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(pName, dbFName); + (void)tNameGetFullDbName(pName, dbFName); CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache)); if (NULL != dbCache) { @@ -1291,7 +1525,7 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf } ctgError("no tbmeta got, tbName:%s", tNameGetTableName(pName)); - ctgRemoveTbMetaFromCache(pCtg, pName, false); + (void)ctgRemoveTbMetaFromCache(pCtg, pName, false); // update cache not fatal error CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST); } @@ -1309,7 +1543,7 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf if (CTG_IS_META_NULL(pOut->metaType)) { ctgError("no tbmeta got, tbName:%s", tNameGetTableName(pName)); - ctgRemoveTbMetaFromCache(pCtg, pName, false); + (void)ctgRemoveTbMetaFromCache(pCtg, pName, false); // update cache not fatal error CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST); } @@ -1327,7 +1561,7 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf int32_t exist = 0; if (!CTG_FLAG_IS_FORCE_UPDATE(flag)) { SName stbName = *pName; - strcpy(stbName.tname, pOut->tbName); + TAOS_STRCPY(stbName.tname, pOut->tbName); SCtgTbMetaCtx stbCtx = {0}; stbCtx.flag = flag; stbCtx.pName = &stbName; @@ -1353,16 +1587,16 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf STableMetaOutput* pOut = (STableMetaOutput*)pMsgCtx->out; - ctgUpdateTbMetaToCache(pCtg, pOut, flag & CTG_FLAG_SYNC_OP); + CTG_ERR_JRET(ctgUpdateTbMetaToCache(pCtg, pOut, flag & CTG_FLAG_SYNC_OP)); if (CTG_IS_META_BOTH(pOut->metaType)) { - memcpy(pOut->tbMeta, &pOut->ctbMeta, sizeof(pOut->ctbMeta)); + TAOS_MEMCPY(pOut->tbMeta, &pOut->ctbMeta, sizeof(pOut->ctbMeta)); } /* else if (CTG_IS_META_CTABLE(pOut->metaType)) { SName stbName = *pName; - strcpy(stbName.tname, pOut->tbName); + TAOS_STRCPY(stbName.tname, pOut->tbName); SCtgTbMetaCtx stbCtx = {0}; stbCtx.flag = flag; stbCtx.pName = &stbName; @@ -1375,7 +1609,7 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf return TSDB_CODE_SUCCESS; } - memcpy(pOut->tbMeta, &pOut->ctbMeta, sizeof(pOut->ctbMeta)); + TAOS_MEMCPY(pOut->tbMeta, &pOut->ctbMeta, sizeof(pOut->ctbMeta)); } */ @@ -1392,7 +1626,10 @@ _return: tstrerror(code)); } if (pTask->res || code) { - ctgHandleTaskEnd(pTask, code); + int32_t newCode = ctgHandleTaskEnd(pTask, code); + if (TSDB_CODE_SUCCESS == code && TSDB_CODE_SUCCESS != newCode) { + code = newCode; + } } CTG_RET(code); @@ -1406,12 +1643,24 @@ int32_t ctgHandleGetTbMetasRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBu SRequestConnInfo* pConn = &pTask->pJob->conn; SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx); SCtgTbMetasCtx* ctx = (SCtgTbMetasCtx*)pTask->taskCtx; + bool taskDone = false; + + if (NULL == pMsgCtx) { + ctgError("fail to get task msgCtx, taskType:%d", pTask->type); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + SCtgFetch* pFetch = taosArrayGet(ctx->pFetchs, tReq->msgIdx); - SName* pName = ctgGetFetchName(ctx->pNames, pFetch); + if (NULL == pFetch) { + ctgError("fail to get the %dth fetch, fetchNum:%d", tReq->msgIdx, (int32_t)taosArrayGetSize(ctx->pFetchs)); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + int32_t flag = pFetch->flag; int32_t* vgId = &pFetch->vgId; - bool taskDone = false; - + SName* pName = NULL; + CTG_ERR_JRET(ctgGetFetchName(ctx->pNames, pFetch, &pName)); + CTG_ERR_JRET(ctgProcessRspMsg(pMsgCtx->out, reqType, pMsg->pData, pMsg->len, rspCode, pMsgCtx->target)); switch (reqType) { @@ -1434,7 +1683,7 @@ int32_t ctgHandleGetTbMetasRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBu if (CTG_IS_META_NULL(pOut->metaType)) { if (CTG_FLAG_IS_STB(flag)) { char dbFName[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(pName, dbFName); + (void)tNameGetFullDbName(pName, dbFName); CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache)); if (NULL != dbCache) { @@ -1461,7 +1710,7 @@ int32_t ctgHandleGetTbMetasRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBu } ctgTaskError("no tbmeta got, tbName:%s", tNameGetTableName(pName)); - ctgRemoveTbMetaFromCache(pCtg, pName, false); + (void)ctgRemoveTbMetaFromCache(pCtg, pName, false); // cache update not fatal error CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST); } @@ -1479,7 +1728,7 @@ int32_t ctgHandleGetTbMetasRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBu if (CTG_IS_META_NULL(pOut->metaType)) { ctgTaskError("no tbmeta got, tbName:%s", tNameGetTableName(pName)); - ctgRemoveTbMetaFromCache(pCtg, pName, false); + (void)ctgRemoveTbMetaFromCache(pCtg, pName, false); // cache update not fatal error CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST); } @@ -1497,13 +1746,13 @@ int32_t ctgHandleGetTbMetasRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBu int32_t exist = 0; if (!CTG_FLAG_IS_FORCE_UPDATE(flag)) { SName stbName = *pName; - strcpy(stbName.tname, pOut->tbName); + TAOS_STRCPY(stbName.tname, pOut->tbName); SCtgTbMetaCtx stbCtx = {0}; stbCtx.flag = flag; stbCtx.pName = &stbName; STableMeta* stbMeta = NULL; - (void)ctgReadTbMetaFromCache(pCtg, &stbCtx, &stbMeta); + CTG_ERR_JRET(ctgReadTbMetaFromCache(pCtg, &stbCtx, &stbMeta)); if (stbMeta && stbMeta->sversion >= pOut->tbMeta->sversion) { ctgTaskDebug("use cached stb meta, tbName:%s", tNameGetTableName(pName)); exist = 1; @@ -1529,16 +1778,16 @@ int32_t ctgHandleGetTbMetasRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBu STableMetaOutput* pOut = (STableMetaOutput*)pMsgCtx->out; - ctgUpdateTbMetaToCache(pCtg, pOut, false); + (void)ctgUpdateTbMetaToCache(pCtg, pOut, false); // cache update not fatal error if (CTG_IS_META_BOTH(pOut->metaType)) { - memcpy(pOut->tbMeta, &pOut->ctbMeta, sizeof(pOut->ctbMeta)); + TAOS_MEMCPY(pOut->tbMeta, &pOut->ctbMeta, sizeof(pOut->ctbMeta)); } /* else if (CTG_IS_META_CTABLE(pOut->metaType)) { SName stbName = *pName; - strcpy(stbName.tname, pOut->tbName); + TAOS_STRCPY(stbName.tname, pOut->tbName); SCtgTbMetaCtx stbCtx = {0}; stbCtx.flag = flag; stbCtx.pName = &stbName; @@ -1551,11 +1800,16 @@ int32_t ctgHandleGetTbMetasRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBu return TSDB_CODE_SUCCESS; } - memcpy(pOut->tbMeta, &pOut->ctbMeta, sizeof(pOut->ctbMeta)); + TAOS_MEMCPY(pOut->tbMeta, &pOut->ctbMeta, sizeof(pOut->ctbMeta)); } */ SMetaRes* pRes = taosArrayGet(ctx->pResList, pFetch->resIdx); + if (NULL == pRes) { + ctgTaskError("fail to get the %dth res in pResList, resNum:%d", pFetch->resIdx, (int32_t)taosArrayGetSize(ctx->pResList)); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + pRes->code = 0; pRes->pRes = pOut->tbMeta; pOut->tbMeta = NULL; @@ -1572,18 +1826,25 @@ _return: if (code) { SMetaRes* pRes = taosArrayGet(ctx->pResList, pFetch->resIdx); - pRes->code = code; - pRes->pRes = NULL; - ctgTaskError("Get table %d.%s.%s meta failed with error %s", pName->acctId, pName->dbname, pName->tname, - tstrerror(code)); - if (0 == atomic_sub_fetch_32(&ctx->fetchNum, 1)) { - TSWAP(pTask->res, ctx->pResList); - taskDone = true; + if (NULL == pRes) { + ctgTaskError("fail to get the %dth res in pResList, resNum:%d", pFetch->resIdx, (int32_t)taosArrayGetSize(ctx->pResList)); + } else { + pRes->code = code; + pRes->pRes = NULL; + ctgTaskError("Get table %d.%s.%s meta failed with error %s", pName->acctId, pName->dbname, pName->tname, + tstrerror(code)); + if (0 == atomic_sub_fetch_32(&ctx->fetchNum, 1)) { + TSWAP(pTask->res, ctx->pResList); + taskDone = true; + } } } if (pTask->res && taskDone) { - ctgHandleTaskEnd(pTask, code); + int32_t newCode = ctgHandleTaskEnd(pTask, code); + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } } CTG_RET(code); @@ -1594,6 +1855,7 @@ int32_t ctgHandleGetDbVgRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* SCtgTask* pTask = tReq->pTask; SCtgDbVgCtx* ctx = (SCtgDbVgCtx*)pTask->taskCtx; SCatalog* pCtg = pTask->pJob->pCtg; + int32_t newCode = TSDB_CODE_SUCCESS; CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); @@ -1616,7 +1878,10 @@ int32_t ctgHandleGetDbVgRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* _return: - ctgHandleTaskEnd(pTask, code); + newCode = ctgHandleTaskEnd(pTask, code); + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } CTG_RET(code); } @@ -1626,6 +1891,7 @@ int32_t ctgHandleGetTbHashRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf SCtgTask* pTask = tReq->pTask; SCtgTbHashCtx* ctx = (SCtgTbHashCtx*)pTask->taskCtx; SCatalog* pCtg = pTask->pJob->pCtg; + int32_t newCode = TSDB_CODE_SUCCESS; CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); @@ -1652,7 +1918,10 @@ int32_t ctgHandleGetTbHashRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf _return: - ctgHandleTaskEnd(pTask, code); + newCode = ctgHandleTaskEnd(pTask, code); + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } CTG_RET(code); } @@ -1662,9 +1931,18 @@ int32_t ctgHandleGetTbHashsRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBu SCtgTask* pTask = tReq->pTask; SCtgTbHashsCtx* ctx = (SCtgTbHashsCtx*)pTask->taskCtx; SCatalog* pCtg = pTask->pJob->pCtg; - SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx); - SCtgFetch* pFetch = taosArrayGet(ctx->pFetchs, tReq->msgIdx); bool taskDone = false; + SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx); + if (NULL == pMsgCtx) { + ctgError("fail to get task msgCtx, taskType:%d", pTask->type); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + SCtgFetch* pFetch = taosArrayGet(ctx->pFetchs, tReq->msgIdx); + if (NULL == pFetch) { + ctgError("fail to get the %dth fetch, fetchNum:%d", tReq->msgIdx, (int32_t)taosArrayGetSize(ctx->pFetchs)); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } CTG_ERR_JRET(ctgProcessRspMsg(pMsgCtx->out, reqType, pMsg->pData, pMsg->len, rspCode, pMsgCtx->target)); @@ -1673,6 +1951,11 @@ int32_t ctgHandleGetTbHashsRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBu SUseDbOutput* pOut = (SUseDbOutput*)pMsgCtx->out; STablesReq* pReq = taosArrayGet(ctx->pNames, pFetch->dbIdx); + if (NULL == pReq) { + ctgError("fail to get the %dth tb in ctx->pNames, reqNum:%d", pFetch->dbIdx, (int32_t)taosArrayGetSize(ctx->pNames)); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + CTG_ERR_JRET(ctgGetVgInfosFromHashValue(pCtg, &pTask->pJob->conn.mgmtEps, tReq, pOut->dbVgroup, ctx, pMsgCtx->target, pReq->pTables, true)); CTG_ERR_JRET(ctgUpdateVgroupEnqueue(pCtg, pMsgCtx->target, pOut->dbId, pOut->dbVgroup, false)); @@ -1694,21 +1977,32 @@ _return: if (code) { STablesReq* pReq = taosArrayGet(ctx->pNames, pFetch->dbIdx); - int32_t num = taosArrayGetSize(pReq->pTables); - for (int32_t i = 0; i < num; ++i) { - SMetaRes* pRes = taosArrayGet(ctx->pResList, pFetch->resIdx + i); - pRes->code = code; - pRes->pRes = NULL; - } + if (NULL == pReq) { + ctgError("fail to get the %dth tb in ctx->pNames, reqNum:%d", pFetch->dbIdx, (int32_t)taosArrayGetSize(ctx->pNames)); + } else { + int32_t num = taosArrayGetSize(pReq->pTables); + for (int32_t i = 0; i < num; ++i) { + SMetaRes* pRes = taosArrayGet(ctx->pResList, pFetch->resIdx + i); + if (NULL == pRes) { + ctgError("fail to get the %dth res in ctx->pResList, resNum:%d", pFetch->resIdx + i, (int32_t)taosArrayGetSize(ctx->pResList)); + } else { + pRes->code = code; + pRes->pRes = NULL; + } + } - if (0 == atomic_sub_fetch_32(&ctx->fetchNum, 1)) { - TSWAP(pTask->res, ctx->pResList); - taskDone = true; + if (0 == atomic_sub_fetch_32(&ctx->fetchNum, 1)) { + TSWAP(pTask->res, ctx->pResList); + taskDone = true; + } } } if (pTask->res && taskDone) { - ctgHandleTaskEnd(pTask, code); + int32_t newCode = ctgHandleTaskEnd(pTask, code); + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } } CTG_RET(code); @@ -1717,6 +2011,8 @@ _return: int32_t ctgHandleGetTbIndexRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) { int32_t code = 0; SCtgTask* pTask = tReq->pTask; + int32_t newCode = TSDB_CODE_SUCCESS; + CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); STableIndex* pOut = (STableIndex*)pTask->msgCtx.out; @@ -1731,7 +2027,11 @@ _return: if (TSDB_CODE_MND_DB_INDEX_NOT_EXIST == code) { code = TSDB_CODE_SUCCESS; } - ctgHandleTaskEnd(pTask, code); + + newCode = ctgHandleTaskEnd(pTask, code); + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } CTG_RET(code); } @@ -1739,13 +2039,18 @@ _return: int32_t ctgHandleGetTbCfgRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) { int32_t code = 0; SCtgTask* pTask = tReq->pTask; + int32_t newCode = TSDB_CODE_SUCCESS; + CTG_ERR_JRET(ctgProcessRspMsg(&pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); TSWAP(pTask->res, pTask->msgCtx.out); _return: - ctgHandleTaskEnd(pTask, code); + newCode = ctgHandleTaskEnd(pTask, code); + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } CTG_RET(code); } @@ -1755,6 +2060,8 @@ int32_t ctgHandleGetTbTagRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* int32_t code = 0; SCtgTask* pTask = tReq->pTask; SCatalog* pCtg = pTask->pJob->pCtg; + int32_t newCode = TSDB_CODE_SUCCESS; + CTG_ERR_JRET(ctgProcessRspMsg(&pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); STableCfgRsp* pRsp = (STableCfgRsp*)pTask->msgCtx.out; @@ -1778,7 +2085,11 @@ int32_t ctgHandleGetTbTagRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* tagVal.type = TSDB_DATA_TYPE_JSON; tagVal.pData = pJson; tagVal.nData = strlen(pJson); - taosArrayPush(pTagVals, &tagVal); + if (NULL == taosArrayPush(pTagVals, &tagVal)) { + taosMemoryFree(pJson); + taosArrayDestroy(pTagVals); + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } } else { CTG_ERR_JRET(tTagToValArray((const STag*)pRsp->pTags, &pTagVals)); } @@ -1787,7 +2098,10 @@ int32_t ctgHandleGetTbTagRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* _return: - ctgHandleTaskEnd(pTask, code); + newCode = ctgHandleTaskEnd(pTask, code); + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } CTG_RET(code); } @@ -1797,10 +2111,12 @@ int32_t ctgHandleGetDbCfgRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* int32_t code = 0; SCtgTask* pTask = tReq->pTask; SCtgDbCfgCtx* ctx = pTask->taskCtx; + int32_t newCode = TSDB_CODE_SUCCESS; CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); - SDbCfgInfo* pCfg = ctgCloneDbCfgInfo(pTask->msgCtx.out); + SDbCfgInfo* pCfg = NULL; + CTG_ERR_JRET(ctgCloneDbCfgInfo(pTask->msgCtx.out, &pCfg)); CTG_ERR_RET(ctgUpdateDbCfgEnqueue(pTask->pJob->pCtg, ctx->dbFName, pCfg->dbId, pCfg, false)); @@ -1808,7 +2124,10 @@ int32_t ctgHandleGetDbCfgRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* _return: - ctgHandleTaskEnd(pTask, code); + newCode = ctgHandleTaskEnd(pTask, code); + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } CTG_RET(code); } @@ -1820,13 +2139,17 @@ int32_t ctgHandleGetDbInfoRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf int32_t ctgHandleGetQnodeRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) { int32_t code = 0; SCtgTask* pTask = tReq->pTask; + int32_t newCode = TSDB_CODE_SUCCESS; CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); TSWAP(pTask->res, pTask->msgCtx.out); _return: - ctgHandleTaskEnd(pTask, code); + newCode = ctgHandleTaskEnd(pTask, code); + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } CTG_RET(code); } @@ -1834,13 +2157,18 @@ _return: int32_t ctgHandleGetDnodeRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) { int32_t code = 0; SCtgTask* pTask = tReq->pTask; + int32_t newCode = TSDB_CODE_SUCCESS; + CTG_ERR_JRET(ctgProcessRspMsg(&pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); TSWAP(pTask->res, pTask->msgCtx.out); _return: - ctgHandleTaskEnd(pTask, code); + newCode = ctgHandleTaskEnd(pTask, code); + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } CTG_RET(code); } @@ -1848,13 +2176,17 @@ _return: int32_t ctgHandleGetIndexRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) { int32_t code = 0; SCtgTask* pTask = tReq->pTask; + int32_t newCode = TSDB_CODE_SUCCESS; CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); TSWAP(pTask->res, pTask->msgCtx.out); _return: - ctgHandleTaskEnd(pTask, code); + newCode = ctgHandleTaskEnd(pTask, code); + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } CTG_RET(code); } @@ -1862,13 +2194,17 @@ _return: int32_t ctgHandleGetUdfRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) { int32_t code = 0; SCtgTask* pTask = tReq->pTask; + int32_t newCode = TSDB_CODE_SUCCESS; CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); TSWAP(pTask->res, pTask->msgCtx.out); _return: - ctgHandleTaskEnd(pTask, code); + newCode = ctgHandleTaskEnd(pTask, code); + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } CTG_RET(code); } @@ -1878,10 +2214,11 @@ int32_t ctgHandleGetUserRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* SCtgTask* pTask = tReq->pTask; SCatalog* pCtg = pTask->pJob->pCtg; SGetUserAuthRsp* pOut = (SGetUserAuthRsp*)pTask->msgCtx.out; + int32_t newCode = TSDB_CODE_SUCCESS; CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); - ctgUpdateUserEnqueue(pCtg, pOut, true); + (void)ctgUpdateUserEnqueue(pCtg, pOut, true); // cache update not fatal error taosMemoryFreeClear(pTask->msgCtx.out); CTG_ERR_JRET((*gCtgAsyncFps[pTask->type].launchFp)(pTask)); @@ -1890,7 +2227,10 @@ int32_t ctgHandleGetUserRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* _return: - ctgHandleTaskEnd(pTask, code); + newCode = ctgHandleTaskEnd(pTask, code); + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } CTG_RET(code); } @@ -1898,6 +2238,7 @@ _return: int32_t ctgHandleGetSvrVerRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) { int32_t code = 0; SCtgTask* pTask = tReq->pTask; + int32_t newCode = TSDB_CODE_SUCCESS; CTG_ERR_JRET(ctgProcessRspMsg(&pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); @@ -1905,7 +2246,10 @@ int32_t ctgHandleGetSvrVerRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf _return: - ctgHandleTaskEnd(pTask, code); + newCode = ctgHandleTaskEnd(pTask, code); + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } CTG_RET(code); } @@ -1918,10 +2262,23 @@ int32_t ctgHandleGetViewsRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx); SCtgViewsCtx* ctx = (SCtgViewsCtx*)pTask->taskCtx; SCtgFetch* pFetch = taosArrayGet(ctx->pFetchs, tReq->msgIdx); - SName* pName = ctgGetFetchName(ctx->pNames, pFetch); + bool taskDone = false; + + if (NULL == pMsgCtx) { + ctgError("fail to get task msgCtx, taskType:%d", pTask->type); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + if (NULL == pFetch) { + ctgError("fail to get the %dth fetch, fetchNum:%d", tReq->msgIdx, (int32_t)taosArrayGetSize(ctx->pFetchs)); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + int32_t flag = pFetch->flag; int32_t* vgId = &pFetch->vgId; - bool taskDone = false; + SName* pName = NULL; + + CTG_ERR_JRET(ctgGetFetchName(ctx->pNames, pFetch, &pName)); CTG_ERR_JRET(ctgProcessRspMsg(pMsgCtx->out, reqType, pMsg->pData, pMsg->len, rspCode, pMsgCtx->target)); @@ -1931,18 +2288,24 @@ int32_t ctgHandleGetViewsRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } - if (TSDB_CODE_SUCCESS != (code = dupViewMetaFromRsp(pRsp, pViewMeta))) { + code = dupViewMetaFromRsp(pRsp, pViewMeta); + if (TSDB_CODE_SUCCESS != code) { ctgFreeSViewMeta(pViewMeta); taosMemoryFree(pViewMeta); CTG_ERR_JRET(code); } ctgDebug("start to update view meta to cache, view:%s, querySQL:%s", pRsp->name, pRsp->querySql); - ctgUpdateViewMetaToCache(pCtg, pRsp, false); + (void)ctgUpdateViewMetaToCache(pCtg, pRsp, false); // cache update not fatal error taosMemoryFreeClear(pMsgCtx->out); pRsp = NULL; SMetaRes* pRes = taosArrayGet(ctx->pResList, pFetch->resIdx); + if (NULL == pRes) { + ctgTaskError("fail to get the %dth res in ctx->pResList, totalResNum:%d", pFetch->resIdx, (int32_t)taosArrayGetSize(ctx->pResList)); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + pRes->code = 0; pRes->pRes = pViewMeta; if (0 == atomic_sub_fetch_32(&ctx->fetchNum, 1)) { @@ -1954,23 +2317,401 @@ _return: if (code) { SMetaRes* pRes = taosArrayGet(ctx->pResList, pFetch->resIdx); - pRes->code = code; - pRes->pRes = NULL; - if (TSDB_CODE_MND_VIEW_NOT_EXIST == code) { - ctgTaskDebug("Get view %d.%s.%s meta failed with %s", pName->acctId, pName->dbname, pName->tname, - tstrerror(code)); + if (NULL == pRes) { + ctgTaskError("fail to get the %dth res in ctx->pResList, totalResNum:%d", pFetch->resIdx, (int32_t)taosArrayGetSize(ctx->pResList)); } else { - ctgTaskError("Get view %d.%s.%s meta failed with error %s", pName->acctId, pName->dbname, pName->tname, - tstrerror(code)); - } - if (0 == atomic_sub_fetch_32(&ctx->fetchNum, 1)) { - TSWAP(pTask->res, ctx->pResList); - taskDone = true; + pRes->code = code; + pRes->pRes = NULL; + if (TSDB_CODE_MND_VIEW_NOT_EXIST == code) { + ctgTaskDebug("Get view %d.%s.%s meta failed with %s", pName->acctId, pName->dbname, pName->tname, + tstrerror(code)); + } else { + ctgTaskError("Get view %d.%s.%s meta failed with error %s", pName->acctId, pName->dbname, pName->tname, + tstrerror(code)); + } + if (0 == atomic_sub_fetch_32(&ctx->fetchNum, 1)) { + TSWAP(pTask->res, ctx->pResList); + taskDone = true; + } } } if (pTask->res && taskDone) { - ctgHandleTaskEnd(pTask, code); + int32_t newCode = ctgHandleTaskEnd(pTask, code); + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } + } + + CTG_RET(code); +} + + +static int32_t ctgTsmaFetchStreamProgress(SCtgTaskReq* tReq, SHashObj* pVgHash, const STableTSMAInfoRsp* pTsmas) { + int32_t code = 0; + SCtgTask* pTask = tReq->pTask; + SCatalog* pCtg = pTask->pJob->pCtg; + int32_t subFetchIdx = 0; + SCtgTbTSMACtx* pCtx = pTask->taskCtx; + SRequestConnInfo* pConn = &pTask->pJob->conn; + SVgroupInfo* pVgInfo = NULL; + SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx); + if (NULL == pFetch) { + ctgError("fail to get the %dth SCtgTSMAFetch, totalNum:%d", tReq->msgIdx, (int32_t)taosArrayGetSize(pCtx->pFetches)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx); + if (NULL == pTbReq) { + ctgError("fail to get the %dth STablesReq, totalNum:%d", pFetch->dbIdx, (int32_t)taosArrayGetSize(pCtx->pNames)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + const SName* pTbName = taosArrayGet(pTbReq->pTables, pFetch->tbIdx); + if (NULL == pTbName) { + ctgError("fail to get the %dth SName, totalNum:%d", pFetch->tbIdx, (int32_t)taosArrayGetSize(pTbReq->pTables)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + pFetch->vgNum = taosHashGetSize(pVgHash); + for (int32_t i = 0; i < taosArrayGetSize(pTsmas->pTsmas); ++i) { + STableTSMAInfo* pTsmaInfo = taosArrayGetP(pTsmas->pTsmas, i); + if (NULL == pTsmaInfo) { + ctgError("fail to get the %dth STableTSMAInfo, totalNum:%d", i, (int32_t)taosArrayGetSize(pTsmas->pTsmas)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + pVgInfo = taosHashIterate(pVgHash, NULL); + pTsmaInfo->reqTs = taosGetTimestampMs(); + while (pVgInfo) { + // make StreamProgressReq, send it + SStreamProgressReq req = {.fetchIdx = pFetch->fetchIdx, + .streamId = pTsmaInfo->streamUid, + .subFetchIdx = subFetchIdx++, + .vgId = pVgInfo->vgId}; + CTG_ERR_JRET(ctgGetStreamProgressFromVnode(pCtg, pConn, pTbName, pVgInfo, NULL, tReq, &req)); + pFetch->subFetchNum++; + + pVgInfo = taosHashIterate(pVgHash, pVgInfo); + } + } + +_return: + + CTG_RET(code); +} + + +int32_t ctgHandleGetTSMARsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) { + int32_t code = 0; + SCtgTask* pTask = tReq->pTask; + SCatalog* pCtg = pTask->pJob->pCtg; + SCtgTbTSMACtx* pCtx = pTask->taskCtx; + int32_t newCode = TSDB_CODE_SUCCESS; + SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx); + if (NULL == pMsgCtx) { + ctgError("fail to get the %dth SCtgMsgCtx", tReq->msgIdx); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + STablesReq* pTbReq = taosArrayGet(pCtx->pNames, 0); + if (NULL == pTbReq) { + ctgError("fail to get the 0th STablesReq, totalNum:%d", (int32_t)taosArrayGetSize(pCtx->pNames)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + SName* pName = taosArrayGet(pTbReq->pTables, 0); + if (NULL == pName) { + ctgError("fail to get the 0th SName, totalNum:%d", (int32_t)taosArrayGetSize(pTbReq->pTables)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + SRequestConnInfo* pConn = &pTask->pJob->conn; + CTG_ERR_JRET(ctgProcessRspMsg(pMsgCtx->out, reqType, pMsg->pData, pMsg->len, rspCode, pMsgCtx->target)); + + switch (reqType) { + case TDMT_MND_TABLE_META: { + STableMetaOutput* pOut = (STableMetaOutput*)pMsgCtx->out; + if (!CTG_IS_META_NULL(pOut->metaType)) { + CTG_ERR_JRET(ctgUpdateTbMetaToCache(pCtg, pOut, CTG_FLAG_SYNC_OP)); + } + + break; + } + case TDMT_MND_GET_TSMA: { + SMetaRes* pRes = taosArrayGet(pCtx->pResList, 0); + if (NULL == pRes) { + ctgError("fail to get the 0th SMetaRes, totalNum:%d", (int32_t)taosArrayGetSize(pCtx->pResList)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + STableTSMAInfoRsp* pOut = pMsgCtx->out; + pRes->code = 0; + if (pOut->pTsmas->size > 0) { + ASSERT(pOut->pTsmas->size == 1); + pRes->pRes = pOut; + pMsgCtx->out = NULL; + TSWAP(pTask->res, pCtx->pResList); + + STableTSMAInfo* pTsma = taosArrayGetP(pOut->pTsmas, 0); + if (NULL == pTsma) { + ctgError("fail to get the 0th STableTSMAInfo, totalNum:%d", (int32_t)taosArrayGetSize(pOut->pTsmas)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + int32_t exists = false; + CTG_ERR_JRET(ctgTbMetaExistInCache(pCtg, pTsma->targetDbFName, pTsma->targetTb, &exists)); + if (!exists) { + TSWAP(pMsgCtx->lastOut, pMsgCtx->out); + CTG_RET(ctgGetTbMetaFromMnodeImpl(pCtg, pConn, pTsma->targetDbFName, pTsma->targetTb, NULL, tReq)); + } + } + + break; + } + default: + ASSERT(0); + } + +_return: + + if (code) { + if (TSDB_CODE_MND_SMA_NOT_EXIST == code) { + code = TSDB_CODE_SUCCESS; + } else { + ctgTaskError("Get tsma for %d.%s.%s failed with err: %s", pName->acctId, pName->dbname, pName->tname, + tstrerror(code)); + } + } + + newCode = ctgHandleTaskEnd(pTask, code); + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } + + CTG_RET(code); +} + + +int32_t ctgHandleGetTbTSMARsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) { + bool taskDone = false; + int32_t code = 0; + SCtgTask* pTask = tReq->pTask; + SCatalog* pCtg = pTask->pJob->pCtg; + SCtgTbTSMACtx* pCtx = pTask->taskCtx; + SArray* pTsmas = NULL; + SHashObj* pVgHash = NULL; + SCtgDBCache* pDbCache = NULL; + STableTSMAInfo* pTsma = NULL; + SRequestConnInfo* pConn = &pTask->pJob->conn; + + SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx); + if (NULL == pMsgCtx) { + ctgError("fail to get the %dth SCtgMsgCtx", tReq->msgIdx); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx); + if (NULL == pFetch) { + ctgError("fail to get the %dth SCtgTSMAFetch, totalNum:%d", tReq->msgIdx, (int32_t)taosArrayGetSize(pCtx->pFetches)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + SMetaRes* pRes = taosArrayGet(pCtx->pResList, pFetch->resIdx); + if (NULL == pRes) { + ctgError("fail to get the %dth SMetaRes, totalNum:%d", pFetch->resIdx, (int32_t)taosArrayGetSize(pCtx->pResList)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx); + if (NULL == pTbReq) { + ctgError("fail to get the %dth STablesReq, totalNum:%d", pFetch->dbIdx, (int32_t)taosArrayGetSize(pCtx->pNames)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + SName* pTbName = taosArrayGet(pTbReq->pTables, pFetch->tbIdx); + if (NULL == pTbName) { + ctgError("fail to get the %dth SName, totalNum:%d", pFetch->tbIdx, (int32_t)taosArrayGetSize(pTbReq->pTables)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + if (reqType != TDMT_VND_GET_STREAM_PROGRESS) + CTG_ERR_JRET(ctgProcessRspMsg(pMsgCtx->out, reqType, pMsg->pData, pMsg->len, rspCode, pMsgCtx->target)); + + switch (reqType) { + case TDMT_MND_GET_TABLE_TSMA: { + STableTSMAInfoRsp* pOut = pMsgCtx->out; + pFetch->fetchType = FETCH_TSMA_STREAM_PROGRESS; + pRes->pRes = pOut; + pMsgCtx->out = NULL; + + if (pOut->pTsmas && taosArrayGetSize(pOut->pTsmas) > 0) { + // fetch progress + (void)ctgAcquireVgInfoFromCache(pCtg, pTbReq->dbFName, &pDbCache); // ignore cache error + + if (!pDbCache) { + // do not know which vnodes to fetch, fetch vnode list first + SBuildUseDBInput input = {0}; + tstrncpy(input.db, pTbReq->dbFName, tListLen(input.db)); + input.vgVersion = CTG_DEFAULT_INVALID_VERSION; + CTG_ERR_JRET(ctgGetDBVgInfoFromMnode(pCtg, pConn, &input, NULL, tReq)); + } else { + // fetch progress from every vnode + CTG_ERR_JRET(ctgTsmaFetchStreamProgress(tReq, pDbCache->vgCache.vgInfo->vgHash, pOut)); + ctgReleaseVgInfoToCache(pCtg, pDbCache); + pDbCache = NULL; + } + } else { + // no tsmas + if (atomic_sub_fetch_32(&pCtx->fetchNum, 1) == 0) { + TSWAP(pTask->res, pCtx->pResList); + taskDone = true; + } + } + + break; + } + case TDMT_VND_GET_STREAM_PROGRESS: { + SStreamProgressRsp rsp = {0}; + CTG_ERR_JRET(ctgProcessRspMsg(&rsp, reqType, pMsg->pData, pMsg->len, rspCode, pMsgCtx->target)); + + // update progress into res + STableTSMAInfoRsp* pTsmasRsp = pRes->pRes; + SArray* pTsmas = pTsmasRsp->pTsmas; + SStreamProgressRsp* pRsp = &rsp; + int32_t tsmaIdx = pRsp->subFetchIdx / pFetch->vgNum; + STableTSMAInfo* pTsmaInfo = taosArrayGetP(pTsmas, tsmaIdx); + if (NULL == pTsmaInfo) { + ctgError("fail to get the %dth STableTSMAInfo, totalNum:%d", tsmaIdx, (int32_t)taosArrayGetSize(pTsmas)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + if (pTsmaInfo->rspTs == 0) { + pTsmaInfo->fillHistoryFinished = true; + } + + pTsmaInfo->rspTs = taosGetTimestampMs(); + pTsmaInfo->delayDuration = TMAX(pRsp->progressDelay, pTsmaInfo->delayDuration); + pTsmaInfo->fillHistoryFinished = pTsmaInfo->fillHistoryFinished && pRsp->fillHisFinished; + + qDebug("received stream progress for tsma %s rsp history: %d vnode: %d, delay: %" PRId64, pTsmaInfo->name, + pRsp->fillHisFinished, pRsp->subFetchIdx, pRsp->progressDelay); + + if (atomic_add_fetch_32(&pFetch->finishedSubFetchNum, 1) == pFetch->subFetchNum) { + // subfetch all finished + for (int32_t i = 0; i < taosArrayGetSize(pTsmas); ++i) { + STableTSMAInfo* pInfo = taosArrayGetP(pTsmas, i); + CTG_ERR_JRET(tCloneTbTSMAInfo(pInfo, &pTsma)); + CTG_ERR_JRET(ctgUpdateTbTSMAEnqueue(pCtg, &pTsma, 0, false)); + } + + if (atomic_sub_fetch_32(&pCtx->fetchNum, 1) == 0) { + TSWAP(pTask->res, pCtx->pResList); + taskDone = true; + } + } + + break; + } + case TDMT_MND_USE_DB: { + SUseDbOutput* pOut = (SUseDbOutput*)pMsgCtx->out; + + switch (pFetch->fetchType) { + case FETCH_TSMA_SOURCE_TB_META: { + SVgroupInfo vgInfo = {0}; + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, &pConn->mgmtEps, pOut->dbVgroup, pTbName, &vgInfo)); + + pFetch->vgId = vgInfo.vgId; + CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, pTbName, &vgInfo, NULL, tReq)); + + break; + } + case FETCH_TSMA_STREAM_PROGRESS: { + STableTSMAInfoRsp* pTsmas = pRes->pRes; + TSWAP(pOut->dbVgroup->vgHash, pVgHash); + CTG_ERR_JRET(ctgTsmaFetchStreamProgress(tReq, pVgHash, pTsmas)); + + break; + } + default: + ASSERT(0); + } + + break; + } + case TDMT_VND_TABLE_META: { + // handle source tb meta + ASSERT(pFetch->fetchType == FETCH_TSMA_SOURCE_TB_META); + STableMetaOutput* pOut = (STableMetaOutput*)pMsgCtx->out; + pFetch->fetchType = FETCH_TB_TSMA; + pFetch->tsmaSourceTbName = *pTbName; + + if (CTG_IS_META_NULL(pOut->metaType)) { + ctgTaskError("no tbmeta found when fetching tsma source tb meta: %s.%s", pTbName->dbname, pTbName->tname); + (void)ctgRemoveTbMetaFromCache(pCtg, pTbName, false); // ignore cache error + CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST); + } + + if (META_TYPE_BOTH_TABLE == pOut->metaType) { + // rewrite tsma fetch table with it's super table name + (void)sprintf(pFetch->tsmaSourceTbName.tname, "%s", pOut->tbName); + } + + CTG_ERR_JRET(ctgGetTbTSMAFromMnode(pCtg, pConn, &pFetch->tsmaSourceTbName, NULL, tReq, TDMT_MND_GET_TABLE_TSMA)); + + break; + } + default: + ASSERT(0); + } + +_return: + + if (pDbCache) { + ctgReleaseVgInfoToCache(pCtg, pDbCache); + } + if (pTsma) { + tFreeTableTSMAInfo(pTsma); + pTsma = NULL; + } + if (pVgHash) { + taosHashCleanup(pVgHash); + } + if (code) { + int32_t newCode = TSDB_CODE_SUCCESS; + SMetaRes* pRes = taosArrayGet(pCtx->pResList, pFetch->resIdx); + if (NULL == pRes) { + ctgError("fail to get the %dth SMetaRes, totalNum:%d", pFetch->resIdx, (int32_t)taosArrayGetSize(pCtx->pResList)); + newCode = TSDB_CODE_CTG_INTERNAL_ERROR; + } else { + pRes->code = code; + if (TSDB_CODE_MND_SMA_NOT_EXIST == code) { + code = TSDB_CODE_SUCCESS; + } else { + ctgTaskError("Get tsma for %d.%s.%s faield with err: %s", pTbName->acctId, pTbName->dbname, pTbName->tname, + tstrerror(code)); + } + } + + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } + + bool allSubFetchFinished = false; + if (pMsgCtx->reqType == TDMT_VND_GET_STREAM_PROGRESS) { + allSubFetchFinished = atomic_add_fetch_32(&pFetch->finishedSubFetchNum, 1) >= pFetch->subFetchNum; + } + if ((allSubFetchFinished || pFetch->subFetchNum == 0) && 0 == atomic_sub_fetch_32(&pCtx->fetchNum, 1)) { + TSWAP(pTask->res, pCtx->pResList); + taskDone = true; + } + } + + if (pTask->res && taskDone) { + int32_t newCode = ctgHandleTaskEnd(pTask, code); + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } } CTG_RET(code); @@ -1998,7 +2739,7 @@ int32_t ctgAsyncRefreshTbMeta(SCtgTaskReq* tReq, int32_t flag, SName* pName, int SCtgDBCache* dbCache = NULL; char dbFName[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(pName, dbFName); + (void)tNameGetFullDbName(pName, dbFName); CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache)); if (dbCache) { @@ -2032,6 +2773,11 @@ int32_t ctgLaunchGetTbMetaTask(SCtgTask* pTask) { SRequestConnInfo* pConn = &pTask->pJob->conn; SCtgJob* pJob = pTask->pJob; SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx) { + ctgError("fail to get task msgCtx, taskType:%d", pTask->type); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (NULL == pMsgCtx->pBatchs) { pMsgCtx->pBatchs = pJob->pBatchs; } @@ -2056,12 +2802,18 @@ int32_t ctgLaunchGetTbMetasTask(SCtgTask* pTask) { SRequestConnInfo* pConn = &pTask->pJob->conn; SCtgTbMetasCtx* pCtx = (SCtgTbMetasCtx*)pTask->taskCtx; SCtgJob* pJob = pTask->pJob; + SName* pName = NULL; int32_t dbNum = taosArrayGetSize(pCtx->pNames); int32_t fetchIdx = 0; int32_t baseResIdx = 0; for (int32_t i = 0; i < dbNum; ++i) { STablesReq* pReq = taosArrayGet(pCtx->pNames, i); + if (NULL == pReq) { + ctgError("fail to get the %dth STablesReq, num:%d", i, dbNum); + CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); + } + ctgDebug("start to check tb metas in db %s, tbNum %ld", pReq->dbFName, taosArrayGetSize(pReq->pTables)); CTG_ERR_RET(ctgGetTbMetasFromCache(pCtg, pConn, pCtx, i, &fetchIdx, baseResIdx, pReq->pTables)); baseResIdx += taosArrayGetSize(pReq->pTables); @@ -2076,10 +2828,26 @@ int32_t ctgLaunchGetTbMetasTask(SCtgTask* pTask) { } pTask->msgCtxs = taosArrayInit_s(sizeof(SCtgMsgCtx), pCtx->fetchNum); + if (NULL == pTask->msgCtxs) { + ctgError("taosArrayInit_s %d SCtgMsgCtx %d failed", pCtx->fetchNum, (int32_t)sizeof(SCtgMsgCtx)); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + for (int32_t i = 0; i < pCtx->fetchNum; ++i) { - SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, i); - SName* pName = ctgGetFetchName(pCtx->pNames, pFetch); + SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, i); + if (NULL == pFetch) { + ctgError("fail to get the %dth fetch in pCtx->pFetchs, fetchNum:%d", i, pCtx->fetchNum); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + CTG_ERR_RET(ctgGetFetchName(pCtx->pNames, pFetch, &pName)); + SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, i); + if (NULL == pMsgCtx) { + ctgError("fail to get the %dth pMsgCtx", i); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (NULL == pMsgCtx->pBatchs) { pMsgCtx->pBatchs = pJob->pBatchs; } @@ -2101,6 +2869,11 @@ int32_t ctgLaunchGetDbVgTask(SCtgTask* pTask) { SCtgDbVgCtx* pCtx = (SCtgDbVgCtx*)pTask->taskCtx; SCtgJob* pJob = pTask->pJob; SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx) { + ctgError("fail to get the %dth pMsgCtx", -1); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (NULL == pMsgCtx->pBatchs) { pMsgCtx->pBatchs = pJob->pBatchs; } @@ -2142,6 +2915,11 @@ int32_t ctgLaunchGetTbHashTask(SCtgTask* pTask) { SCtgTbHashCtx* pCtx = (SCtgTbHashCtx*)pTask->taskCtx; SCtgJob* pJob = pTask->pJob; SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx) { + ctgError("fail to get the %dth pMsgCtx", -1); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (NULL == pMsgCtx->pBatchs) { pMsgCtx->pBatchs = pJob->pBatchs; } @@ -2192,6 +2970,10 @@ int32_t ctgLaunchGetTbHashsTask(SCtgTask* pTask) { for (int32_t i = 0; i < dbNum; ++i) { STablesReq* pReq = taosArrayGet(pCtx->pNames, i); + if (NULL == pReq) { + ctgError("fail to get the %dth STablesReq, dbNum:%d", i, dbNum); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, pReq->dbFName, &dbCache)); @@ -2207,12 +2989,14 @@ int32_t ctgLaunchGetTbHashsTask(SCtgTask* pTask) { baseResIdx += taosArrayGetSize(pReq->pTables); } else { - ctgAddFetch(&pCtx->pFetchs, i, -1, &fetchIdx, baseResIdx, 0); + CTG_ERR_JRET(ctgAddFetch(&pCtx->pFetchs, i, -1, &fetchIdx, baseResIdx, 0)); baseResIdx += taosArrayGetSize(pReq->pTables); int32_t inc = baseResIdx - taosArrayGetSize(pCtx->pResList); for(int32_t j = 0; j < inc; ++j) { - taosArrayPush(pCtx->pResList, &(SMetaRes){0}); + if (NULL == taosArrayPush(pCtx->pResList, &(SMetaRes){0})) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } } } } @@ -2221,22 +3005,41 @@ int32_t ctgLaunchGetTbHashsTask(SCtgTask* pTask) { if (pCtx->fetchNum <= 0) { TSWAP(pTask->res, pCtx->pResList); - CTG_ERR_RET(ctgHandleTaskEnd(pTask, 0)); + CTG_ERR_JRET(ctgHandleTaskEnd(pTask, 0)); return TSDB_CODE_SUCCESS; } pTask->msgCtxs = taosArrayInit_s(sizeof(SCtgMsgCtx), pCtx->fetchNum); + if (NULL == pTask->msgCtxs) { + ctgError("taosArrayInit_s %d SCtgMsgCtx %d failed", pCtx->fetchNum, (int32_t)sizeof(SCtgMsgCtx)); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } for (int32_t i = 0; i < pCtx->fetchNum; ++i) { SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, i); + if (NULL == pFetch) { + ctgError("fail to get the %dth SCtgFetch, fetchNum:%d", i, pCtx->fetchNum); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + STablesReq* pReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx); + if (NULL == pFetch) { + ctgError("fail to get the %dth SCtgFetch, fetchNum:%d", i, pCtx->fetchNum); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, i); + if (NULL == pFetch) { + ctgError("fail to get the %dth SCtgMsgCtx", i); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (NULL == pMsgCtx->pBatchs) { pMsgCtx->pBatchs = pJob->pBatchs; } SBuildUseDBInput input = {0}; - strcpy(input.db, pReq->dbFName); + TAOS_STRCPY(input.db, pReq->dbFName); input.vgVersion = CTG_DEFAULT_INVALID_VERSION; @@ -2263,6 +3066,11 @@ int32_t ctgLaunchGetTbIndexTask(SCtgTask* pTask) { SArray* pRes = NULL; SCtgJob* pJob = pTask->pJob; SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx) { + ctgError("fail to get the %dth pMsgCtx", -1); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (NULL == pMsgCtx->pBatchs) { pMsgCtx->pBatchs = pJob->pBatchs; } @@ -2285,10 +3093,16 @@ int32_t ctgLaunchGetTbCfgTask(SCtgTask* pTask) { SRequestConnInfo* pConn = &pTask->pJob->conn; SCtgTbCfgCtx* pCtx = (SCtgTbCfgCtx*)pTask->taskCtx; SArray* pRes = NULL; + SCtgJob* pJob = pTask->pJob; char dbFName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(pCtx->pName, dbFName); - SCtgJob* pJob = pTask->pJob; + (void)tNameGetFullDbName(pCtx->pName, dbFName); + SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx) { + ctgError("fail to get the %dth pMsgCtx", -1); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (NULL == pMsgCtx->pBatchs) { pMsgCtx->pBatchs = pJob->pBatchs; } @@ -2325,7 +3139,10 @@ int32_t ctgLaunchGetTbCfgTask(SCtgTask* pTask) { _return: if (CTG_TASK_LAUNCHED == pTask->status) { - ctgHandleTaskEnd(pTask, code); + int32_t newCode = ctgHandleTaskEnd(pTask, code); + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } } CTG_RET(code); @@ -2338,10 +3155,16 @@ int32_t ctgLaunchGetTbTagTask(SCtgTask* pTask) { SRequestConnInfo* pConn = &pTask->pJob->conn; SCtgTbTagCtx* pCtx = (SCtgTbTagCtx*)pTask->taskCtx; SArray* pRes = NULL; + SCtgJob* pJob = pTask->pJob; char dbFName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(pCtx->pName, dbFName); - SCtgJob* pJob = pTask->pJob; + (void)tNameGetFullDbName(pCtx->pName, dbFName); + SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx) { + ctgError("fail to get the %dth pMsgCtx", -1); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (NULL == pMsgCtx->pBatchs) { pMsgCtx->pBatchs = pJob->pBatchs; } @@ -2363,7 +3186,10 @@ int32_t ctgLaunchGetTbTagTask(SCtgTask* pTask) { _return: if (CTG_TASK_LAUNCHED == pTask->status) { - ctgHandleTaskEnd(pTask, code); + int32_t newCode = ctgHandleTaskEnd(pTask, code); + if (newCode && TSDB_CODE_SUCCESS == code) { + code = newCode; + } } CTG_RET(code); @@ -2375,12 +3201,18 @@ int32_t ctgLaunchGetQnodeTask(SCtgTask* pTask) { SRequestConnInfo* pConn = &pTask->pJob->conn; SCtgJob* pJob = pTask->pJob; SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx) { + ctgError("fail to get the %dth pMsgCtx", -1); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (NULL == pMsgCtx->pBatchs) { pMsgCtx->pBatchs = pJob->pBatchs; } CTG_CACHE_NHIT_INC(CTG_CI_QNODE, 1); CTG_ERR_RET(ctgGetQnodeListFromMnode(pCtg, pConn, NULL, pTask)); + return TSDB_CODE_SUCCESS; } @@ -2389,12 +3221,18 @@ int32_t ctgLaunchGetDnodeTask(SCtgTask* pTask) { SRequestConnInfo* pConn = &pTask->pJob->conn; SCtgJob* pJob = pTask->pJob; SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx) { + ctgError("fail to get the %dth pMsgCtx", -1); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (NULL == pMsgCtx->pBatchs) { pMsgCtx->pBatchs = pJob->pBatchs; } CTG_CACHE_NHIT_INC(CTG_CI_DNODE, 1); CTG_ERR_RET(ctgGetDnodeListFromMnode(pCtg, pConn, NULL, pTask)); + return TSDB_CODE_SUCCESS; } @@ -2404,11 +3242,16 @@ int32_t ctgLaunchGetDbCfgTask(SCtgTask* pTask) { SCtgDbCfgCtx* pCtx = (SCtgDbCfgCtx*)pTask->taskCtx; SCtgJob* pJob = pTask->pJob; SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); - SDbCfgInfo cfgInfo; + if (NULL == pMsgCtx) { + ctgError("fail to get the %dth pMsgCtx", -1); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (NULL == pMsgCtx->pBatchs) { pMsgCtx->pBatchs = pJob->pBatchs; } + SDbCfgInfo cfgInfo; CTG_ERR_RET(ctgReadDBCfgFromCache(pCtg, pCtx->dbFName, &cfgInfo)); if (cfgInfo.cfgVersion < 0) { @@ -2419,7 +3262,7 @@ int32_t ctgLaunchGetDbCfgTask(SCtgTask* pTask) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(pTask->res, &cfgInfo, sizeof(cfgInfo)); + TAOS_MEMCPY(pTask->res, &cfgInfo, sizeof(cfgInfo)); CTG_ERR_RET(ctgHandleTaskEnd(pTask, 0)); } @@ -2433,6 +3276,11 @@ int32_t ctgLaunchGetDbInfoTask(SCtgTask* pTask) { SCtgDbInfoCtx* pCtx = (SCtgDbInfoCtx*)pTask->taskCtx; SCtgJob* pJob = pTask->pJob; SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx) { + ctgError("fail to get the %dth pMsgCtx", -1); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (NULL == pMsgCtx->pBatchs) { pMsgCtx->pBatchs = pJob->pBatchs; } @@ -2473,6 +3321,11 @@ int32_t ctgLaunchGetIndexTask(SCtgTask* pTask) { SCtgIndexCtx* pCtx = (SCtgIndexCtx*)pTask->taskCtx; SCtgJob* pJob = pTask->pJob; SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx) { + ctgError("fail to get the %dth pMsgCtx", -1); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (NULL == pMsgCtx->pBatchs) { pMsgCtx->pBatchs = pJob->pBatchs; } @@ -2490,6 +3343,11 @@ int32_t ctgLaunchGetUdfTask(SCtgTask* pTask) { SCtgUdfCtx* pCtx = (SCtgUdfCtx*)pTask->taskCtx; SCtgJob* pJob = pTask->pJob; SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx) { + ctgError("fail to get the %dth pMsgCtx", -1); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (NULL == pMsgCtx->pBatchs) { pMsgCtx->pBatchs = pJob->pBatchs; } @@ -2511,6 +3369,11 @@ int32_t ctgLaunchGetUserTask(SCtgTask* pTask) { SCtgJob* pJob = pTask->pJob; bool tbNotExists = false; SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx) { + ctgError("fail to get the %dth pMsgCtx", -1); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (NULL == pMsgCtx->pBatchs) { pMsgCtx->pBatchs = pJob->pBatchs; } @@ -2559,6 +3422,11 @@ int32_t ctgLaunchGetSvrVerTask(SCtgTask* pTask) { SRequestConnInfo* pConn = &pTask->pJob->conn; SCtgJob* pJob = pTask->pJob; SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx) { + ctgError("fail to get the %dth pMsgCtx", -1); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (NULL == pMsgCtx->pBatchs) { pMsgCtx->pBatchs = pJob->pBatchs; } @@ -2576,6 +3444,7 @@ int32_t ctgLaunchGetViewsTask(SCtgTask* pTask) { SCtgViewsCtx* pCtx = (SCtgViewsCtx*)pTask->taskCtx; SCtgJob* pJob = pTask->pJob; bool tbMetaDone = false; + SName* pName = NULL; /* ctgIsTaskDone(pJob, CTG_TASK_GET_TB_META_BATCH, &tbMetaDone); @@ -2593,6 +3462,11 @@ int32_t ctgLaunchGetViewsTask(SCtgTask* pTask) { int32_t baseResIdx = 0; for (int32_t i = 0; i < dbNum; ++i) { STablesReq* pReq = taosArrayGet(pCtx->pNames, i); + if (NULL == pReq) { + ctgError("fail to get the %dth STablesReq, dbNum:%d", i, dbNum); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + ctgDebug("start to check views in db %s, viewNum %ld", pReq->dbFName, taosArrayGetSize(pReq->pTables)); CTG_ERR_RET(ctgGetViewsFromCache(pCtg, pConn, pCtx, i, &fetchIdx, baseResIdx, pReq->pTables)); baseResIdx += taosArrayGetSize(pReq->pTables); @@ -2607,10 +3481,26 @@ int32_t ctgLaunchGetViewsTask(SCtgTask* pTask) { } pTask->msgCtxs = taosArrayInit_s(sizeof(SCtgMsgCtx), pCtx->fetchNum); + if (NULL == pTask->msgCtxs) { + ctgError("taosArrayInit_s %d SCtgMsgCtx %d failed", pCtx->fetchNum, (int32_t)sizeof(SCtgMsgCtx)); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + for (int32_t i = 0; i < pCtx->fetchNum; ++i) { SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, i); - SName* pName = ctgGetFetchName(pCtx->pNames, pFetch); + if (NULL == pFetch) { + ctgError("fail to get the %dth SCtgFetch, fetchNum:%d", i, pCtx->fetchNum); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + CTG_ERR_RET(ctgGetFetchName(pCtx->pNames, pFetch, &pName)); + SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, i); + if (NULL == pMsgCtx) { + ctgError("fail to get the %dth SCtgMsgCtx", i); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (NULL == pMsgCtx->pBatchs) { pMsgCtx->pBatchs = pJob->pBatchs; } @@ -2633,10 +3523,13 @@ int32_t ctgAsyncRefreshTbTsma(SCtgTaskReq* pReq, const SCtgTSMAFetch* pFetch) { SCtgDBCache* pDbCache = NULL; STablesReq* pTbReq = taosArrayGet(pTaskCtx->pNames, pFetch->dbIdx); + if (NULL == pTbReq) { + ctgError("fail to get the %dth STablesReq, totalNum:%d", pFetch->dbIdx, (int32_t)taosArrayGetSize(pTaskCtx->pNames)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } - ctgAcquireVgInfoFromCache(pCtg, pTbReq->dbFName, &pDbCache); + (void)ctgAcquireVgInfoFromCache(pCtg, pTbReq->dbFName, &pDbCache); // ignore error if (pDbCache) { - ctgReleaseVgInfoToCache(pCtg, pDbCache); } else { SBuildUseDBInput input = {0}; @@ -2645,7 +3538,9 @@ int32_t ctgAsyncRefreshTbTsma(SCtgTaskReq* pReq, const SCtgTSMAFetch* pFetch) { CTG_ERR_JRET(ctgGetDBVgInfoFromMnode(pCtg, pConn, &input, NULL, pReq)); } + _return: + return code; } @@ -2661,9 +3556,15 @@ int32_t ctgLaunchGetTbTSMATask(SCtgTask* pTask) { for (int32_t idx = 0; idx < dbNum; ++idx) { STablesReq* pReq = taosArrayGet(pCtx->pNames, idx); + if (NULL == pReq) { + ctgError("fail to get the %dth STablesReq, dbNum:%d", idx, dbNum); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + CTG_ERR_RET(ctgGetTbTSMAFromCache(pCtg, pCtx, idx, &fetchIdx, baseResIdx, pReq->pTables)); baseResIdx += taosArrayGetSize(pReq->pTables); } + pCtx->fetchNum = taosArrayGetSize(pCtx->pFetches); if (pCtx->fetchNum <= 0) { TSWAP(pTask->res, pCtx->pResList); @@ -2672,12 +3573,39 @@ int32_t ctgLaunchGetTbTSMATask(SCtgTask* pTask) { } pTask->msgCtxs = taosArrayInit_s(sizeof(SCtgMsgCtx), pCtx->fetchNum); + if (NULL == pTask->msgCtxs) { + ctgError("taosArrayInit_s %d SCtgMsgCtx %d failed", pCtx->fetchNum, (int32_t)sizeof(SCtgMsgCtx)); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + for (int32_t i = 0; i < pCtx->fetchNum; ++i) { SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, i); + if (NULL == pFetch) { + ctgError("fail to get the %dth SCtgTSMAFetch, fetchNum:%d", i, pCtx->fetchNum); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + STablesReq* pReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx); + if (NULL == pReq) { + ctgError("fail to get the %dth STablesReq, totalNum:%d", pFetch->dbIdx, (int32_t)taosArrayGetSize(pCtx->pNames)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + SName* pName = taosArrayGet(pReq->pTables, pFetch->tbIdx); + if (NULL == pName) { + ctgError("fail to get the %dth SName, totalNum:%d", pFetch->tbIdx, (int32_t)taosArrayGetSize(pReq->pTables)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, i); - if (!pMsgCtx->pBatchs) pMsgCtx->pBatchs = pJob->pBatchs; + if (NULL == pMsgCtx) { + ctgError("fail to get the %dth pMsgCtx", i); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + if (!pMsgCtx->pBatchs) { + pMsgCtx->pBatchs = pJob->pBatchs; + } SCtgTaskReq tReq; tReq.pTask = pTask; @@ -2686,11 +3614,13 @@ int32_t ctgLaunchGetTbTSMATask(SCtgTask* pTask) { switch (pFetch->fetchType) { case FETCH_TSMA_SOURCE_TB_META: { CTG_ERR_RET(ctgAsyncRefreshTbMeta(&tReq, pFetch->flag, pName, &pFetch->vgId)); - } break; + break; + } case FETCH_TB_TSMA: { CTG_ERR_RET( ctgGetTbTSMAFromMnode(pCtg, pConn, &pFetch->tsmaSourceTbName, NULL, &tReq, TDMT_MND_GET_TABLE_TSMA)); - } break; + break; + } default: ASSERT(0); break; @@ -2706,24 +3636,58 @@ int32_t ctgLaunchGetTSMATask(SCtgTask* pTask) { SRequestConnInfo* pConn = &pTask->pJob->conn; SArray* pRes = NULL; SCtgJob* pJob = pTask->pJob; + // currently, only support fetching one tsma ASSERT(pCtx->pNames->size == 1); STablesReq* pReq = taosArrayGet(pCtx->pNames, 0); + if (NULL == pReq) { + ctgError("fail to get the 0th STablesReq, totalNum:%d", (int32_t)taosArrayGetSize(pCtx->pNames)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + ASSERT(pReq->pTables->size == 1); SName* pTsmaName = taosArrayGet(pReq->pTables, 0); + if (NULL == pReq) { + ctgError("fail to get the 0th SName, totalNum:%d", (int32_t)taosArrayGetSize(pReq->pTables)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + CTG_ERR_RET(ctgGetTSMAFromCache(pCtg, pCtx, pTsmaName)); if (pCtx->pResList->size == 0) { SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, 0); - if (!pMsgCtx->pBatchs) pMsgCtx->pBatchs = pJob->pBatchs; + if (NULL == pMsgCtx) { + ctgError("fail to get the 0th SCtgMsgCtx, taskType:%d", pTask->type); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + if (!pMsgCtx->pBatchs) { + pMsgCtx->pBatchs = pJob->pBatchs; + } + SCtgTaskReq tReq = {.pTask = pTask, .msgIdx = 0}; - taosArrayPush(pCtx->pResList, &(SMetaRes){0}); + if (NULL == taosArrayPush(pCtx->pResList, &(SMetaRes){0})) { + ctgError("taosArrayPush SMetaRes failed, code:%x", terrno); + CTG_ERR_RET(terrno); + } + CTG_ERR_RET(ctgGetTbTSMAFromMnode(pCtg, pConn, pTsmaName, NULL, &tReq, TDMT_MND_GET_TSMA)); } else { SMetaRes* pRes = taosArrayGet(pCtx->pResList, 0); + if (NULL == pRes) { + ctgError("fail to get the 0th SMetaRes, totalNum:%d", (int32_t)taosArrayGetSize(pCtx->pResList)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + STableTSMAInfoRsp* pRsp = (STableTSMAInfoRsp*)pRes->pRes; ASSERT(pRsp->pTsmas->size == 1); + const STSMACache* pTsma = taosArrayGetP(pRsp->pTsmas, 0); + if (NULL == pTsma) { + ctgError("fail to get the 0th STSMACache, totalNum:%d", (int32_t)taosArrayGetSize(pRsp->pTsmas)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + TSWAP(pTask->res, pCtx->pResList); // get tsma target stable meta if not existed in cache int32_t exists = false; @@ -2736,253 +3700,11 @@ int32_t ctgLaunchGetTSMATask(SCtgTask* pTask) { } else { CTG_ERR_RET(ctgHandleTaskEnd(pTask, 0)); } + return TSDB_CODE_SUCCESS; } - return 0; -} - -int32_t ctgHandleGetTSMARsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) { - int32_t code = 0; - SCtgTask* pTask = tReq->pTask; - SCatalog* pCtg = pTask->pJob->pCtg; - SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx); - SCtgTbTSMACtx* pCtx = pTask->taskCtx; - SMetaRes* pRes = taosArrayGet(pCtx->pResList, 0); - STablesReq* pTbReq = taosArrayGet(pCtx->pNames, 0); - SName* pName = taosArrayGet(pTbReq->pTables, 0); - SRequestConnInfo* pConn = &pTask->pJob->conn; - CTG_ERR_JRET(ctgProcessRspMsg(pMsgCtx->out, reqType, pMsg->pData, pMsg->len, rspCode, pMsgCtx->target)); - - switch (reqType) { - case TDMT_MND_TABLE_META: { - STableMetaOutput* pOut = (STableMetaOutput*)pMsgCtx->out; - if (!CTG_IS_META_NULL(pOut->metaType)) { - CTG_ERR_JRET(ctgUpdateTbMetaToCache(pCtg, pOut, CTG_FLAG_SYNC_OP)); - } - } break; - case TDMT_MND_GET_TSMA: { - STableTSMAInfoRsp* pOut = pMsgCtx->out; - pRes->code = 0; - if (pOut->pTsmas->size > 0) { - ASSERT(pOut->pTsmas->size == 1); - pRes->pRes = pOut; - pMsgCtx->out = NULL; - TSWAP(pTask->res, pCtx->pResList); - - STableTSMAInfo* pTsma = taosArrayGetP(pOut->pTsmas, 0); - int32_t exists = false; - CTG_ERR_JRET(ctgTbMetaExistInCache(pCtg, pTsma->targetDbFName, pTsma->targetTb, &exists)); - if (!exists) { - TSWAP(pMsgCtx->lastOut, pMsgCtx->out); - CTG_RET(ctgGetTbMetaFromMnodeImpl(pCtg, pConn, pTsma->targetDbFName, pTsma->targetTb, NULL, tReq)); - } - } - } break; - default: - ASSERT(0); - } - -_return: - if (code) { - if (TSDB_CODE_MND_SMA_NOT_EXIST == code) { - code = TSDB_CODE_SUCCESS; - } else { - ctgTaskError("Get tsma for %d.%s.%s failed with err: %s", pName->acctId, pName->dbname, pName->tname, - tstrerror(code)); - } - } - ctgHandleTaskEnd(pTask, code); - CTG_RET(code); -} - -static int32_t ctgTsmaFetchStreamProgress(SCtgTaskReq* tReq, SHashObj* pVgHash, const STableTSMAInfoRsp* pTsmas) { - int32_t code = 0; - SCtgTask* pTask = tReq->pTask; - SCatalog* pCtg = pTask->pJob->pCtg; - int32_t subFetchIdx = 0; - SCtgTbTSMACtx* pCtx = pTask->taskCtx; - SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx); - SRequestConnInfo* pConn = &pTask->pJob->conn; - STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx); - const SName* pTbName = taosArrayGet(pTbReq->pTables, pFetch->tbIdx); - SVgroupInfo* pVgInfo = NULL; - - pFetch->vgNum = taosHashGetSize(pVgHash); - for (int32_t i = 0; i < taosArrayGetSize(pTsmas->pTsmas); ++i) { - STableTSMAInfo* pTsmaInfo = taosArrayGetP(pTsmas->pTsmas, i); - pVgInfo = taosHashIterate(pVgHash, NULL); - pTsmaInfo->reqTs = taosGetTimestampMs(); - while (pVgInfo) { - // make StreamProgressReq, send it - SStreamProgressReq req = {.fetchIdx = pFetch->fetchIdx, - .streamId = pTsmaInfo->streamUid, - .subFetchIdx = subFetchIdx++, - .vgId = pVgInfo->vgId}; - CTG_ERR_JRET(ctgGetStreamProgressFromVnode(pCtg, pConn, pTbName, pVgInfo, NULL, tReq, &req)); - pFetch->subFetchNum++; - pVgInfo = taosHashIterate(pVgHash, pVgInfo); - } - } -_return: - CTG_RET(code); -} - -int32_t ctgHandleGetTbTSMARsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) { - bool taskDone = false; - int32_t code = 0; - SCtgTask* pTask = tReq->pTask; - SCatalog* pCtg = pTask->pJob->pCtg; - SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx); - SCtgTbTSMACtx* pCtx = pTask->taskCtx; - SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx); - SArray* pTsmas = NULL; - SMetaRes* pRes = taosArrayGet(pCtx->pResList, pFetch->resIdx); - SHashObj* pVgHash = NULL; - SCtgDBCache* pDbCache = NULL; - STableTSMAInfo* pTsma = NULL; - SRequestConnInfo* pConn = &pTask->pJob->conn; - STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx); - SName* pTbName = taosArrayGet(pTbReq->pTables, pFetch->tbIdx); - - if (reqType != TDMT_VND_GET_STREAM_PROGRESS) - CTG_ERR_JRET(ctgProcessRspMsg(pMsgCtx->out, reqType, pMsg->pData, pMsg->len, rspCode, pMsgCtx->target)); - - switch (reqType) { - case TDMT_MND_GET_TABLE_TSMA: { - STableTSMAInfoRsp* pOut = pMsgCtx->out; - pFetch->fetchType = FETCH_TSMA_STREAM_PROGRESS; - pRes->pRes = pOut; - pMsgCtx->out = NULL; - - if (pOut->pTsmas && taosArrayGetSize(pOut->pTsmas) > 0) { - // fetch progress - ctgAcquireVgInfoFromCache(pCtg, pTbReq->dbFName, &pDbCache); - if (!pDbCache) { - // do not know which vnodes to fetch, fetch vnode list first - SBuildUseDBInput input = {0}; - tstrncpy(input.db, pTbReq->dbFName, tListLen(input.db)); - input.vgVersion = CTG_DEFAULT_INVALID_VERSION; - CTG_ERR_JRET(ctgGetDBVgInfoFromMnode(pCtg, pConn, &input, NULL, tReq)); - } else { - // fetch progress from every vnode - CTG_ERR_JRET(ctgTsmaFetchStreamProgress(tReq, pDbCache->vgCache.vgInfo->vgHash, pOut)); - ctgReleaseVgInfoToCache(pCtg, pDbCache); - pDbCache = NULL; - } - } else { - // no tsmas - if (atomic_sub_fetch_32(&pCtx->fetchNum, 1) == 0) { - TSWAP(pTask->res, pCtx->pResList); - taskDone = true; - } - } - } break; - case TDMT_VND_GET_STREAM_PROGRESS: { - SStreamProgressRsp rsp = {0}; - CTG_ERR_JRET(ctgProcessRspMsg(&rsp, reqType, pMsg->pData, pMsg->len, rspCode, pMsgCtx->target)); - // update progress into res - STableTSMAInfoRsp* pTsmasRsp = pRes->pRes; - SArray* pTsmas = pTsmasRsp->pTsmas; - SStreamProgressRsp* pRsp = &rsp; - int32_t tsmaIdx = pRsp->subFetchIdx / pFetch->vgNum; - STableTSMAInfo* pTsmaInfo = taosArrayGetP(pTsmas, tsmaIdx); - if (pTsmaInfo->rspTs == 0) pTsmaInfo->fillHistoryFinished = true; - pTsmaInfo->rspTs = taosGetTimestampMs(); - pTsmaInfo->delayDuration = TMAX(pRsp->progressDelay, pTsmaInfo->delayDuration); - pTsmaInfo->fillHistoryFinished = pTsmaInfo->fillHistoryFinished && pRsp->fillHisFinished; - qDebug("received stream progress for tsma %s rsp history: %d vnode: %d, delay: %" PRId64, pTsmaInfo->name, - pRsp->fillHisFinished, pRsp->subFetchIdx, pRsp->progressDelay); - - if (atomic_add_fetch_32(&pFetch->finishedSubFetchNum, 1) == pFetch->subFetchNum) { - // subfetch all finished - for (int32_t i = 0; i < taosArrayGetSize(pTsmas); ++i) { - STableTSMAInfo* pInfo = taosArrayGetP(pTsmas, i); - CTG_ERR_JRET(tCloneTbTSMAInfo(pInfo, &pTsma)); - CTG_ERR_JRET(ctgUpdateTbTSMAEnqueue(pCtg, &pTsma, 0, false)); - } - - if (atomic_sub_fetch_32(&pCtx->fetchNum, 1) == 0) { - TSWAP(pTask->res, pCtx->pResList); - taskDone = true; - } - } - } break; - case TDMT_MND_USE_DB: { - SUseDbOutput* pOut = (SUseDbOutput*)pMsgCtx->out; - - switch (pFetch->fetchType) { - case FETCH_TSMA_SOURCE_TB_META: { - SVgroupInfo vgInfo = {0}; - CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, &pConn->mgmtEps, pOut->dbVgroup, pTbName, &vgInfo)); - - pFetch->vgId = vgInfo.vgId; - CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, pTbName, &vgInfo, NULL, tReq)); - } break; - case FETCH_TSMA_STREAM_PROGRESS: { - STableTSMAInfoRsp* pTsmas = pRes->pRes; - TSWAP(pOut->dbVgroup->vgHash, pVgHash); - CTG_ERR_JRET(ctgTsmaFetchStreamProgress(tReq, pVgHash, pTsmas)); - } break; - default: - ASSERT(0); - } - } break; - case TDMT_VND_TABLE_META: { - // handle source tb meta - ASSERT(pFetch->fetchType == FETCH_TSMA_SOURCE_TB_META); - STableMetaOutput* pOut = (STableMetaOutput*)pMsgCtx->out; - pFetch->fetchType = FETCH_TB_TSMA; - pFetch->tsmaSourceTbName = *pTbName; - if (CTG_IS_META_NULL(pOut->metaType)) { - ctgTaskError("no tbmeta found when fetching tsma source tb meta: %s.%s", pTbName->dbname, pTbName->tname); - ctgRemoveTbMetaFromCache(pCtg, pTbName, false); - CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST); - } - if (META_TYPE_BOTH_TABLE == pOut->metaType) { - // rewrite tsma fetch table with it's super table name - sprintf(pFetch->tsmaSourceTbName.tname, "%s", pOut->tbName); - } - CTG_ERR_JRET(ctgGetTbTSMAFromMnode(pCtg, pConn, &pFetch->tsmaSourceTbName, NULL, tReq, TDMT_MND_GET_TABLE_TSMA)); - } break; - default: - ASSERT(0); - } - -_return: - if (pDbCache) { - ctgReleaseVgInfoToCache(pCtg, pDbCache); - } - if (pTsma) { - tFreeTableTSMAInfo(pTsma); - pTsma = NULL; - } - if (pVgHash) { - taosHashCleanup(pVgHash); - } - if (code) { - SMetaRes* pRes = taosArrayGet(pCtx->pResList, pFetch->resIdx); - pRes->code = code; - if (TSDB_CODE_MND_SMA_NOT_EXIST == code) { - code = TSDB_CODE_SUCCESS; - } else { - ctgTaskError("Get tsma for %d.%s.%s faield with err: %s", pTbName->acctId, pTbName->dbname, pTbName->tname, - tstrerror(code)); - } - bool allSubFetchFinished = false; - if (pMsgCtx->reqType == TDMT_VND_GET_STREAM_PROGRESS) { - allSubFetchFinished = atomic_add_fetch_32(&pFetch->finishedSubFetchNum, 1) >= pFetch->subFetchNum; - } - if ((allSubFetchFinished || pFetch->subFetchNum == 0) && 0 == atomic_sub_fetch_32(&pCtx->fetchNum, 1)) { - TSWAP(pTask->res, pCtx->pResList); - taskDone = true; - } - } - if (pTask->res && taskDone) { - ctgHandleTaskEnd(pTask, code); - } - - CTG_RET(code); + return TSDB_CODE_SUCCESS; } int32_t ctgDumpTbTSMARes(SCtgTask* pTask) { @@ -3025,6 +3747,10 @@ int32_t ctgGetTbCfgCb(SCtgTask* pTask) { SDBVgInfo* pDb = (SDBVgInfo*)pTask->subRes.res; pCtx->pVgInfo = taosMemoryCalloc(1, sizeof(SVgroupInfo)); + if (NULL == pCtx->pVgInfo) { + CTG_ERR_JRET(terrno); + } + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pTask->pJob->pCtg, &pTask->pJob->conn.mgmtEps, pDb, pCtx->pName, pCtx->pVgInfo)); } @@ -3045,6 +3771,10 @@ int32_t ctgGetTbTagCb(SCtgTask* pTask) { if (NULL == pCtx->pVgInfo) { pCtx->pVgInfo = taosMemoryCalloc(1, sizeof(SVgroupInfo)); + if (NULL == pCtx->pVgInfo) { + CTG_ERR_JRET(terrno); + } + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pTask->pJob->pCtg, &pTask->pJob->conn.mgmtEps, pDb, pCtx->pName, pCtx->pVgInfo)); } @@ -3148,6 +3878,11 @@ int32_t ctgSearchExistingTask(SCtgJob* pJob, CTG_TASK_TYPE type, void* param, in int32_t taskNum = taosArrayGetSize(pJob->pTasks); for (int32_t i = 0; i < taskNum; ++i) { pTask = taosArrayGet(pJob->pTasks, i); + if (NULL == pTask) { + qError("fail to get the %dth task, taskNum:%d", i, taskNum); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (type != pTask->type) { continue; } @@ -3176,16 +3911,31 @@ int32_t ctgSetSubTaskCb(SCtgTask* pSub, SCtgTask* pTask) { pTask->subRes.code = pSub->code; CTG_ERR_JRET((*gCtgAsyncFps[pTask->type].cloneFp)(pSub, &pTask->subRes.res)); SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx) { + qError("fail to get the -1th SCtgMsgCtx"); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + SCtgMsgCtx* pSubMsgCtx = CTG_GET_TASK_MSGCTX(pSub, -1); + if (NULL == pSubMsgCtx) { + qError("fail to get the -1th sub SCtgMsgCtx"); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + pMsgCtx->pBatchs = pSubMsgCtx->pBatchs; CTG_ERR_JRET(pTask->subRes.fp(pTask)); } else { if (NULL == pSub->pParents) { pSub->pParents = taosArrayInit(4, POINTER_BYTES); + if (NULL == pSub->pParents) { + CTG_ERR_JRET(terrno); + } } - taosArrayPush(pSub->pParents, &pTask); + if (NULL == taosArrayPush(pSub->pParents, &pTask)) { + CTG_ERR_JRET(terrno); + } } _return: @@ -3227,6 +3977,11 @@ int32_t ctgLaunchSubTask(SCtgTask** ppTask, CTG_TASK_TYPE type, ctgSubTaskCbFp f } SCtgTask* pSub = taosArrayGet(pJob->pTasks, subTaskId); + if (NULL == pSub) { + qError("fail to get the %dth sub SCtgTask, taskNum:%d", subTaskId, (int32_t)taosArrayGetSize(pJob->pTasks)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (newTask) { pSub->subTask = true; } @@ -3235,7 +3990,17 @@ int32_t ctgLaunchSubTask(SCtgTask** ppTask, CTG_TASK_TYPE type, ctgSubTaskCbFp f if (newTask) { SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(*ppTask, -1); + if (NULL == pMsgCtx) { + qError("fail to get the -1th SCtgMsgCtx"); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + SCtgMsgCtx* pSubMsgCtx = CTG_GET_TASK_MSGCTX(pSub, -1); + if (NULL == pSubMsgCtx) { + qError("fail to get the -1th sub SCtgMsgCtx"); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + pSubMsgCtx->pBatchs = pMsgCtx->pBatchs; CTG_ERR_RET((*gCtgAsyncFps[pSub->type].launchFp)(pSub)); @@ -3250,21 +4015,30 @@ int32_t ctgLaunchJob(SCtgJob* pJob) { for (int32_t i = 0; i < taskNum; ++i) { SCtgTask* pTask = taosArrayGet(pJob->pTasks, i); - + if (NULL == pTask) { + qError("fail to get the %dth task, taskNum:%d", i, taskNum); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + qDebug("QID:0x%" PRIx64 " ctg launch [%dth] task", pJob->queryId, pTask->taskId); CTG_ERR_RET((*gCtgAsyncFps[pTask->type].launchFp)(pTask)); pTask = taosArrayGet(pJob->pTasks, i); - atomic_val_compare_exchange_32((int32_t*)&pTask->status, 0, CTG_TASK_LAUNCHED); + if (NULL == pTask) { + qError("fail to get the %dth task, taskNum:%d", i, taskNum); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + (void)atomic_val_compare_exchange_32((int32_t*)&pTask->status, 0, CTG_TASK_LAUNCHED); } if (taskNum <= 0) { qDebug("QID:0x%" PRIx64 " ctg call user callback with rsp %s", pJob->queryId, tstrerror(pJob->jobResCode)); - taosAsyncExec(ctgCallUserCb, pJob, NULL); + CTG_ERR_RET(taosAsyncExec(ctgCallUserCb, pJob, NULL)); #if CTG_BATCH_FETCH } else { - ctgLaunchBatchs(pJob->pCtg, pJob, pJob->pBatchs); + CTG_ERR_RET(ctgLaunchBatchs(pJob->pCtg, pJob, pJob->pBatchs)); #endif } diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 9f9f783458..565dcd1739 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -224,15 +224,16 @@ void ctgReleaseTSMAToCache(SCatalog* pCtg, SCtgDBCache* dbCache, SCtgTSMACache* } int32_t ctgAcquireVgInfoFromCache(SCatalog *pCtg, const char *dbFName, SCtgDBCache **pCache) { + int32_t code = TSDB_CODE_SUCCESS; SCtgDBCache *dbCache = NULL; - ctgAcquireDBCache(pCtg, dbFName, &dbCache); + CTG_ERR_JRET(ctgAcquireDBCache(pCtg, dbFName, &dbCache)); if (NULL == dbCache) { ctgDebug("db %s not in cache", dbFName); goto _return; } bool inCache = false; - ctgRLockVgInfo(pCtg, dbCache, &inCache); + CTG_ERR_JRET(ctgRLockVgInfo(pCtg, dbCache, &inCache)); if (!inCache) { ctgDebug("vgInfo of db %s not in cache", dbFName); goto _return; @@ -256,13 +257,15 @@ _return: CTG_CACHE_NHIT_INC(CTG_CI_DB_VGROUP, 1); - return TSDB_CODE_SUCCESS; + return code; } int32_t ctgAcquireTbMetaFromCache(SCatalog *pCtg, const char *dbFName, const char *tbName, SCtgDBCache **pDb, SCtgTbCache **pTb) { SCtgDBCache *dbCache = NULL; SCtgTbCache *pCache = NULL; - ctgAcquireDBCache(pCtg, dbFName, &dbCache); + int32_t code = TSDB_CODE_SUCCESS; + + CTG_ERR_JRET(ctgAcquireDBCache(pCtg, dbFName, &dbCache)); if (NULL == dbCache) { ctgDebug("db %s not in cache", dbFName); goto _return; @@ -303,15 +306,16 @@ int32_t ctgAcquireVgMetaFromCache(SCatalog *pCtg, const char *dbFName, const cha SCtgDBCache *dbCache = NULL; SCtgTbCache *tbCache = NULL; bool vgInCache = false; + int32_t code = TSDB_CODE_SUCCESS; - ctgAcquireDBCache(pCtg, dbFName, &dbCache); + CTG_ERR_JRET(ctgAcquireDBCache(pCtg, dbFName, &dbCache)); if (NULL == dbCache) { ctgDebug("db %s not in cache", dbFName); CTG_CACHE_NHIT_INC(CTG_CI_DB_VGROUP, 1); goto _return; } - ctgRLockVgInfo(pCtg, dbCache, &vgInCache); + CTG_ERR_JRET(ctgRLockVgInfo(pCtg, dbCache, &vgInCache)); if (!vgInCache) { ctgDebug("vgInfo of db %s not in cache", dbFName); CTG_CACHE_NHIT_INC(CTG_CI_DB_VGROUP, 1); @@ -466,7 +470,9 @@ _return: int32_t ctgAcquireTbIndexFromCache(SCatalog *pCtg, char *dbFName, char *tbName, SCtgDBCache **pDb, SCtgTbCache **pTb) { SCtgDBCache *dbCache = NULL; SCtgTbCache *pCache = NULL; - ctgAcquireDBCache(pCtg, dbFName, &dbCache); + int32_t code = TSDB_CODE_SUCCESS; + + CTG_ERR_JRET(ctgAcquireDBCache(pCtg, dbFName, &dbCache)); if (NULL == dbCache) { ctgDebug("db %s not in cache", dbFName); goto _return; @@ -506,7 +512,8 @@ _return: int32_t ctgTbMetaExistInCache(SCatalog *pCtg, const char *dbFName, const char *tbName, int32_t *exist) { SCtgDBCache *dbCache = NULL; SCtgTbCache *tbCache = NULL; - ctgAcquireTbMetaFromCache(pCtg, dbFName, tbName, &dbCache, &tbCache); + + CTG_ERR_RET(ctgAcquireTbMetaFromCache(pCtg, dbFName, tbName, &dbCache, &tbCache)); if (NULL == tbCache) { ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); @@ -541,10 +548,10 @@ int32_t ctgCopyTbMeta(SCatalog *pCtg, SCtgTbMetaCtx *ctx, SCtgDBCache **pDb, SCt CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(*pTableMeta, tbMeta, metaSize); + TAOS_MEMCPY(*pTableMeta, tbMeta, metaSize); if (tbMeta->schemaExt != NULL) { (*pTableMeta)->schemaExt = (SSchemaExt *)((char *)*pTableMeta + metaSize); - memcpy((*pTableMeta)->schemaExt, tbMeta->schemaExt, schemaExtSize); + TAOS_MEMCPY((*pTableMeta)->schemaExt, tbMeta->schemaExt, schemaExtSize); } else { (*pTableMeta)->schemaExt = NULL; } @@ -561,7 +568,7 @@ int32_t ctgCopyTbMeta(SCatalog *pCtg, SCtgTbMetaCtx *ctx, SCtgDBCache **pDb, SCt CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(*pTableMeta, tbMeta, metaSize); + TAOS_MEMCPY(*pTableMeta, tbMeta, metaSize); // ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); @@ -572,7 +579,7 @@ int32_t ctgCopyTbMeta(SCatalog *pCtg, SCtgTbMetaCtx *ctx, SCtgDBCache **pDb, SCt ctgDebug("Got ctb %s meta from cache, will continue to get its stb meta, type:%d, dbFName:%s", ctx->pName->tname, ctx->tbInfo.tbType, dbFName); - ctgAcquireStbMetaFromCache(dbCache, pCtg, dbFName, ctx->tbInfo.suid, &tbCache); + CTG_ERR_RET(ctgAcquireStbMetaFromCache(dbCache, pCtg, dbFName, ctx->tbInfo.suid, &tbCache)); if (NULL == tbCache) { taosMemoryFreeClear(*pTableMeta); *pDb = NULL; @@ -595,7 +602,7 @@ int32_t ctgCopyTbMeta(SCatalog *pCtg, SCtgTbMetaCtx *ctx, SCtgDBCache **pDb, SCt CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(&(*pTableMeta)->sversion, &stbMeta->sversion, metaSize - sizeof(SCTableMeta)); + TAOS_MEMCPY(&(*pTableMeta)->sversion, &stbMeta->sversion, metaSize - sizeof(SCTableMeta)); (*pTableMeta)->schemaExt = NULL; return TSDB_CODE_SUCCESS; @@ -609,12 +616,12 @@ int32_t ctgReadTbMetaFromCache(SCatalog *pCtg, SCtgTbMetaCtx *ctx, STableMeta ** char dbFName[TSDB_DB_FNAME_LEN] = {0}; if (CTG_FLAG_IS_SYS_DB(ctx->flag)) { - strcpy(dbFName, ctx->pName->dbname); + TAOS_STRCPY(dbFName, ctx->pName->dbname); } else { - tNameGetFullDbName(ctx->pName, dbFName); + (void)tNameGetFullDbName(ctx->pName, dbFName); } - ctgAcquireTbMetaFromCache(pCtg, dbFName, ctx->pName->tname, &dbCache, &tbCache); + CTG_ERR_JRET(ctgAcquireTbMetaFromCache(pCtg, dbFName, ctx->pName->tname, &dbCache, &tbCache)); if (NULL == tbCache) { ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); return TSDB_CODE_SUCCESS; @@ -645,9 +652,9 @@ int32_t ctgReadTbVerFromCache(SCatalog *pCtg, SName *pTableName, int32_t *sver, SCtgDBCache *dbCache = NULL; SCtgTbCache *tbCache = NULL; char dbFName[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(pTableName, dbFName); + (void)tNameGetFullDbName(pTableName, dbFName); - ctgAcquireTbMetaFromCache(pCtg, dbFName, pTableName->tname, &dbCache, &tbCache); + CTG_ERR_RET(ctgAcquireTbMetaFromCache(pCtg, dbFName, pTableName->tname, &dbCache, &tbCache)); if (NULL == tbCache) { ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); return TSDB_CODE_SUCCESS; @@ -678,7 +685,7 @@ int32_t ctgReadTbVerFromCache(SCatalog *pCtg, SName *pTableName, int32_t *sver, ctgDebug("Got ctb %s ver from cache, will continue to get its stb ver, dbFName:%s", pTableName->tname, dbFName); - ctgAcquireStbMetaFromCache(dbCache, pCtg, dbFName, *suid, &tbCache); + CTG_ERR_RET(ctgAcquireStbMetaFromCache(dbCache, pCtg, dbFName, *suid, &tbCache)); if (NULL == tbCache) { // ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); ctgDebug("stb 0x%" PRIx64 " meta not in cache", *suid); @@ -695,7 +702,7 @@ int32_t ctgReadTbVerFromCache(SCatalog *pCtg, SName *pTableName, int32_t *sver, size_t nameLen = 0; char *name = taosHashGetKey(tbCache, &nameLen); - strncpy(stbName, name, nameLen); + TAOS_STRNCPY(stbName, name, nameLen); stbName[nameLen] = 0; *sver = stbMeta->sversion; @@ -731,11 +738,11 @@ int32_t ctgReadTbIndexFromCache(SCatalog *pCtg, SName *pTableName, SArray **pRes SCtgDBCache *dbCache = NULL; SCtgTbCache *tbCache = NULL; char dbFName[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(pTableName, dbFName); + (void)tNameGetFullDbName(pTableName, dbFName); *pRes = NULL; - ctgAcquireTbIndexFromCache(pCtg, dbFName, pTableName->tname, &dbCache, &tbCache); + CTG_ERR_RET(ctgAcquireTbIndexFromCache(pCtg, dbFName, pTableName->tname, &dbCache, &tbCache)); if (NULL == tbCache) { ctgReleaseTbIndexToCache(pCtg, dbCache, tbCache); return TSDB_CODE_SUCCESS; @@ -753,7 +760,8 @@ _return: int32_t ctgReadDBCfgFromCache(SCatalog *pCtg, const char* dbFName, SDbCfgInfo* pDbCfg) { int32_t code = 0; SCtgDBCache *dbCache = NULL; - ctgAcquireDBCache(pCtg, dbFName, &dbCache); + + CTG_ERR_RET(ctgAcquireDBCache(pCtg, dbFName, &dbCache)); if (NULL == dbCache) { ctgDebug("db %s not in cache", dbFName); pDbCfg->cfgVersion = -1; @@ -764,12 +772,10 @@ int32_t ctgReadDBCfgFromCache(SCatalog *pCtg, const char* dbFName, SDbCfgInfo* p CTG_LOCK(CTG_READ, &dbCache->cfgCache.cfgLock); if (dbCache->cfgCache.cfgInfo) { - SDbCfgInfo *pInfo = ctgCloneDbCfgInfo(dbCache->cfgCache.cfgInfo); - if (NULL == pInfo) { - CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); - } + SDbCfgInfo *pInfo = NULL; + CTG_ERR_JRET(ctgCloneDbCfgInfo(dbCache->cfgCache.cfgInfo, &pInfo)); - memcpy(pDbCfg, pInfo, sizeof(*pInfo)); + TAOS_MEMCPY(pDbCfg, pInfo, sizeof(*pInfo)); taosMemoryFree(pInfo); CTG_CACHE_HIT_INC(CTG_CI_DB_CFG, 1); } else { @@ -788,10 +794,11 @@ _return: } int32_t ctgGetCachedStbNameFromSuid(SCatalog* pCtg, char* dbFName, uint64_t suid, char **stbName) { + int32_t code = TSDB_CODE_SUCCESS; *stbName = NULL; SCtgDBCache *dbCache = NULL; - ctgAcquireDBCache(pCtg, dbFName, &dbCache); + CTG_ERR_RET(ctgAcquireDBCache(pCtg, dbFName, &dbCache)); if (NULL == dbCache) { ctgDebug("db %s not in cache", dbFName); return TSDB_CODE_SUCCESS; @@ -805,11 +812,14 @@ int32_t ctgGetCachedStbNameFromSuid(SCatalog* pCtg, char* dbFName, uint64_t suid } *stbName = taosStrdup(stb); + if (NULL == *stbName) { + code = TSDB_CODE_OUT_OF_MEMORY; + } taosHashRelease(dbCache->stbCache, stb); ctgReleaseDBCache(pCtg, dbCache); - return TSDB_CODE_SUCCESS; + return code; } @@ -833,7 +843,7 @@ int32_t ctgChkAuthFromCache(SCatalog *pCtg, SUserAuthInfo *pReq, bool tbNotExist req.tbNotExists = tbNotExists; CTG_LOCK(CTG_READ, &pUser->lock); - memcpy(&req.authInfo, &pUser->userAuth, sizeof(pUser->userAuth)); + TAOS_MEMCPY(&req.authInfo, &pUser->userAuth, sizeof(pUser->userAuth)); code = ctgChkSetAuthRes(pCtg, &req, pRes); CTG_UNLOCK(CTG_READ, &pUser->lock); CTG_ERR_JRET(code); @@ -867,6 +877,7 @@ void ctgDequeue(SCtgCacheOperation **op) { } int32_t ctgEnqueue(SCatalog *pCtg, SCtgCacheOperation *operation) { + int32_t code = TSDB_CODE_SUCCESS; SCtgQNode *node = taosMemoryCalloc(1, sizeof(SCtgQNode)); if (NULL == node) { qError("calloc %d failed", (int32_t)sizeof(SCtgQNode)); @@ -875,13 +886,19 @@ int32_t ctgEnqueue(SCatalog *pCtg, SCtgCacheOperation *operation) { CTG_RET(TSDB_CODE_OUT_OF_MEMORY); } + node->op = operation; + bool syncOp = operation->syncOp; char *opName = gCtgCacheOperation[operation->opId].name; if (operation->syncOp) { - tsem_init(&operation->rspSem, 0, 0); + code = tsem_init(&operation->rspSem, 0, 0); + if (TSDB_CODE_SUCCESS != code) { + qError("tsem_init failed, code:%x", code); + ctgFreeQNode(node); + CTG_RET(code); + } } - node->op = operation; CTG_LOCK(CTG_WRITE, &gCtgMgmt.queue.qlock); @@ -903,25 +920,34 @@ int32_t ctgEnqueue(SCatalog *pCtg, SCtgCacheOperation *operation) { CTG_QUEUE_INC(); CTG_STAT_RT_INC(numOfOpEnqueue, 1); - tsem_post(&gCtgMgmt.queue.reqSem); + code = tsem_post(&gCtgMgmt.queue.reqSem); + if (TSDB_CODE_SUCCESS != code) { + qError("tsem_post failed, code:%x", code); + CTG_RET(code); + } if (syncOp) { if (!operation->unLocked) { CTG_UNLOCK(CTG_READ, &gCtgMgmt.lock); } - tsem_wait(&operation->rspSem); + code = tsem_wait(&operation->rspSem); if (!operation->unLocked) { CTG_LOCK(CTG_READ, &gCtgMgmt.lock); } taosMemoryFree(operation); } - return TSDB_CODE_SUCCESS; + return code; } int32_t ctgDropDbCacheEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId) { int32_t code = 0; SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); + if (NULL == op) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgCacheOperation)); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + op->opId = CTG_OP_DROP_DB_CACHE; op->syncOp = true; @@ -955,6 +981,11 @@ _return: int32_t ctgDropDbVgroupEnqueue(SCatalog *pCtg, const char *dbFName, bool syncOp) { int32_t code = 0; SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); + if (NULL == op) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgCacheOperation)); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + op->opId = CTG_OP_DROP_DB_VGROUP; op->syncOp = syncOp; @@ -988,6 +1019,11 @@ int32_t ctgDropStbMetaEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId, bool syncOp) { int32_t code = 0; SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); + if (NULL == op) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgCacheOperation)); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + op->opId = CTG_OP_DROP_STB_META; op->syncOp = syncOp; @@ -1018,6 +1054,11 @@ _return: int32_t ctgDropTbMetaEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId, const char *tbName, bool syncOp) { int32_t code = 0; SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); + if (NULL == op) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgCacheOperation)); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + op->opId = CTG_OP_DROP_TB_META; op->syncOp = syncOp; @@ -1047,6 +1088,11 @@ _return: int32_t ctgUpdateVgroupEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId, SDBVgInfo *dbInfo, bool syncOp) { int32_t code = 0; SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); + if (NULL == op) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgCacheOperation)); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + op->opId = CTG_OP_UPDATE_VGROUP; op->syncOp = syncOp; @@ -1092,6 +1138,11 @@ _return: int32_t ctgUpdateDbCfgEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId, SDbCfgInfo *cfgInfo, bool syncOp) { int32_t code = 0; SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); + if (NULL == op) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgCacheOperation)); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + op->opId = CTG_OP_UPDATE_DB_CFG; op->syncOp = syncOp; @@ -1129,6 +1180,11 @@ _return: int32_t ctgUpdateTbMetaEnqueue(SCatalog *pCtg, STableMetaOutput *output, bool syncOp) { int32_t code = 0; SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); + if (NULL == op) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgCacheOperation)); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + op->opId = CTG_OP_UPDATE_TB_META; op->syncOp = syncOp; @@ -1142,7 +1198,7 @@ int32_t ctgUpdateTbMetaEnqueue(SCatalog *pCtg, STableMetaOutput *output, bool sy char *p = strchr(output->dbFName, '.'); if (p && IS_SYS_DBNAME(p + 1)) { int32_t len = strlen(p + 1); - memmove(output->dbFName, p + 1, len >= TSDB_DB_FNAME_LEN ? TSDB_DB_FNAME_LEN - 1 : len); + TAOS_MEMMOVE(output->dbFName, p + 1, len >= TSDB_DB_FNAME_LEN ? TSDB_DB_FNAME_LEN - 1 : len); } msg->pCtg = pCtg; @@ -1167,6 +1223,11 @@ _return: int32_t ctgUpdateVgEpsetEnqueue(SCatalog *pCtg, char *dbFName, int32_t vgId, SEpSet *pEpSet) { int32_t code = 0; SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); + if (NULL == op) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgCacheOperation)); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + op->opId = CTG_OP_UPDATE_VG_EPSET; SCtgUpdateEpsetMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateEpsetMsg)); @@ -1195,6 +1256,11 @@ _return: int32_t ctgUpdateUserEnqueue(SCatalog *pCtg, SGetUserAuthRsp *pAuth, bool syncOp) { int32_t code = 0; SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); + if (NULL == op) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgCacheOperation)); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + op->opId = CTG_OP_UPDATE_USER; op->syncOp = syncOp; @@ -1224,6 +1290,11 @@ _return: int32_t ctgUpdateTbIndexEnqueue(SCatalog *pCtg, STableIndex **pIndex, bool syncOp) { int32_t code = 0; SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); + if (NULL == op) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgCacheOperation)); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + op->opId = CTG_OP_UPDATE_TB_INDEX; op->syncOp = syncOp; @@ -1255,6 +1326,11 @@ _return: int32_t ctgDropTbIndexEnqueue(SCatalog *pCtg, SName *pName, bool syncOp) { int32_t code = 0; SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); + if (NULL == op) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgCacheOperation)); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + op->opId = CTG_OP_DROP_TB_INDEX; op->syncOp = syncOp; @@ -1266,8 +1342,8 @@ int32_t ctgDropTbIndexEnqueue(SCatalog *pCtg, SName *pName, bool syncOp) { } msg->pCtg = pCtg; - tNameGetFullDbName(pName, msg->dbFName); - strcpy(msg->tbName, pName->tname); + (void)tNameGetFullDbName(pName, msg->dbFName); + TAOS_STRCPY(msg->tbName, pName->tname); op->data = msg; @@ -1283,6 +1359,11 @@ _return: int32_t ctgClearCacheEnqueue(SCatalog *pCtg, bool clearMeta, bool freeCtg, bool stopQueue, bool syncOp) { int32_t code = 0; SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); + if (NULL == op) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgCacheOperation)); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + op->opId = CTG_OP_CLEAR_CACHE; op->syncOp = syncOp; op->stopQueue = stopQueue; @@ -1312,6 +1393,11 @@ _return: int32_t ctgUpdateViewMetaEnqueue(SCatalog *pCtg, SViewMetaRsp *pRsp, bool syncOp) { int32_t code = 0; SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); + if (NULL == op) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgCacheOperation)); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + op->opId = CTG_OP_UPDATE_VIEW_META; op->syncOp = syncOp; @@ -1325,7 +1411,7 @@ int32_t ctgUpdateViewMetaEnqueue(SCatalog *pCtg, SViewMetaRsp *pRsp, bool syncOp char *p = strchr(pRsp->dbFName, '.'); if (p && IS_SYS_DBNAME(p + 1)) { int32_t len = strlen(p + 1); - memmove(pRsp->dbFName, p + 1, len >= TSDB_DB_FNAME_LEN ? TSDB_DB_FNAME_LEN - 1 : len); + TAOS_MEMMOVE(pRsp->dbFName, p + 1, len >= TSDB_DB_FNAME_LEN ? TSDB_DB_FNAME_LEN - 1 : len); } msg->pCtg = pCtg; @@ -1351,6 +1437,11 @@ _return: int32_t ctgDropViewMetaEnqueue(SCatalog *pCtg, const char *dbFName, uint64_t dbId, const char *viewName, uint64_t viewId, bool syncOp) { int32_t code = 0; SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); + if (NULL == op) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgCacheOperation)); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + op->opId = CTG_OP_DROP_VIEW_META; op->syncOp = syncOp; @@ -1378,6 +1469,165 @@ _return: CTG_RET(code); } +int32_t ctgUpdateTbTSMAEnqueue(SCatalog *pCtg, STSMACache **pTsma, int32_t tsmaVersion, bool syncOp) { + int32_t code = 0; + SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); + if (NULL == op) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgCacheOperation)); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + op->opId = CTG_OP_UPDATE_TB_TSMA; + op->syncOp = syncOp; + + SCtgUpdateTbTSMAMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateTbTSMAMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateTbTSMAMsg)); + taosMemoryFree(op); + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + + msg->pCtg = pCtg; + msg->pTsma = *pTsma; + msg->dbTsmaVersion = tsmaVersion; + msg->dbId = (*pTsma)->dbId; + + op->data = msg; + + CTG_ERR_JRET(ctgEnqueue(pCtg, op)); + + *pTsma = NULL; + return TSDB_CODE_SUCCESS; + +_return: + + CTG_RET(code); +} + +int32_t ctgDropTbTSMAEnqueue(SCatalog* pCtg, const STSMACache* pTsma, bool syncOp) { + int32_t code = 0; + SCtgCacheOperation* op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); + if (NULL == op) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgCacheOperation)); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + op->opId = CTG_OP_DROP_TB_TSMA; + op->syncOp = syncOp; + + SCtgDropTbTSMAMsg* msg = taosMemoryCalloc(1, sizeof(SCtgDropTbTSMAMsg)); + if (!msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgDropTbTSMAMsg)); + taosMemoryFree(op); + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + + msg->pCtg = pCtg; + msg->dbId = pTsma->dbId; + msg->tbId = pTsma->suid; + msg->tsmaId = pTsma->tsmaId; + tstrncpy(msg->dbFName, pTsma->dbFName, TSDB_DB_FNAME_LEN); + tstrncpy(msg->tbName, pTsma->tb, TSDB_TABLE_NAME_LEN); + tstrncpy(msg->tsmaName, pTsma->name, TSDB_TABLE_NAME_LEN); + + op->data = msg; + CTG_ERR_JRET(ctgEnqueue(pCtg, op)); + + return TSDB_CODE_SUCCESS; + +_return: + + CTG_RET(code); +} + + +static int32_t createDropAllTbTsmaCtgCacheOp(SCatalog* pCtg, const STSMACache* pCache, bool syncOp, SCtgCacheOperation** ppOp) { + SCtgCacheOperation* pOp = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); + if (NULL == pOp) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgCacheOperation)); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + SCtgDropTbTSMAMsg* pMsg = taosMemoryCalloc(1, sizeof(SCtgDropTbTSMAMsg)); + if (NULL == pMsg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgDropTbTSMAMsg)); + taosMemoryFree(pOp); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + pOp->opId = CTG_OP_DROP_TB_TSMA; + pOp->syncOp = syncOp; + pMsg->pCtg = pCtg; + pMsg->dbId = pCache->dbId; + pMsg->tbId = pCache->suid; + pMsg->tsmaId = pCache->tsmaId; + pMsg->dropAllForTb = true; + tstrncpy(pMsg->tsmaName, pCache->name, TSDB_TABLE_NAME_LEN); + tstrncpy(pMsg->dbFName, pCache->dbFName, TSDB_DB_FNAME_LEN); + tstrncpy(pMsg->tbName, pCache->tb, TSDB_TABLE_NAME_LEN); + pOp->data = pMsg; + + *ppOp = pOp; + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgDropTSMAForTbEnqueue(SCatalog *pCtg, SName *pName, bool syncOp) { + ctgDebug("drop tsma meta for tb: %s.%s", pName->dbname, pName->tname); + + int32_t code = 0; + SCtgDBCache *pDbCache = NULL; + SCtgCacheOperation *pOp = NULL; + char dbFName[TSDB_DB_FNAME_LEN]; + SCtgTSMACache *pCtgCache = NULL; + (void)tNameGetFullDbName(pName, dbFName); + + CTG_ERR_JRET(ctgGetDBCache(pCtg, dbFName, &pDbCache)); + if (NULL == pDbCache || !pDbCache->tsmaCache) { + goto _return; + } + + pCtgCache = taosHashAcquire(pDbCache->tsmaCache, pName->tname, strlen(pName->tname)); + if (!pCtgCache) { + goto _return; + } + + CTG_LOCK(CTG_READ, &pCtgCache->tsmaLock); + if (!pCtgCache->pTsmas || pCtgCache->pTsmas->size == 0) { + CTG_UNLOCK(CTG_READ, &pCtgCache->tsmaLock); + goto _return; + } + + STSMACache *pCache = taosArrayGetP(pCtgCache->pTsmas, 0); + if (NULL == pCache) { + ctgError("fail to get the 0th STSMACache, total:%d", (int32_t)pCtgCache->pTsmas->size); + code = TSDB_CODE_CTG_INTERNAL_ERROR; + } + if (TSDB_CODE_SUCCESS == code) { + code = createDropAllTbTsmaCtgCacheOp(pCtg, pCache, syncOp, &pOp); + } + CTG_UNLOCK(CTG_READ, &pCtgCache->tsmaLock); + + CTG_ERR_JRET(code); + + CTG_ERR_JRET(ctgEnqueue(pCtg, pOp)); + taosHashRelease(pDbCache->tsmaCache, pCtgCache); + + return TSDB_CODE_SUCCESS; + +_return: + + if (pCtgCache) { + taosHashRelease(pDbCache->tsmaCache, pCtgCache); + } + if (pOp) { + taosMemoryFree(pOp->data); + taosMemoryFree(pOp); + } + + CTG_RET(code); +} + int32_t ctgAddNewDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId) { int32_t code = 0; @@ -1478,7 +1728,7 @@ int32_t ctgRemoveDBFromCache(SCatalog *pCtg, SCtgDBCache *dbCache, const char *d int32_t ctgGetAddDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId, SCtgDBCache **pCache) { int32_t code = 0; SCtgDBCache *dbCache = NULL; - ctgGetDBCache(pCtg, dbFName, &dbCache); + CTG_ERR_RET(ctgGetDBCache(pCtg, dbFName, &dbCache)); if (dbCache) { // TODO OPEN IT @@ -1509,7 +1759,7 @@ int32_t ctgGetAddDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId, SCt CTG_ERR_RET(ctgAddNewDBCache(pCtg, dbFName, dbId)); - ctgGetDBCache(pCtg, dbFName, &dbCache); + CTG_ERR_RET(ctgGetDBCache(pCtg, dbFName, &dbCache)); *pCache = dbCache; @@ -1547,7 +1797,7 @@ int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNam ctgError("stb not exist in stbCache, dbFName:%s, stb:%s, suid:0x%" PRIx64, dbFName, tbName, orig->suid); } else { ctgDebug("stb removed from stbCache, dbFName:%s, stb:%s, suid:0x%" PRIx64, dbFName, tbName, orig->suid); - atomic_sub_fetch_64(&dbCache->dbCacheSize, metaSize); + (void)atomic_sub_fetch_64(&dbCache->dbCacheSize, metaSize); } } } @@ -1562,16 +1812,19 @@ int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNam CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - atomic_add_fetch_64(&dbCache->dbCacheSize, strlen(tbName) + sizeof(SCtgTbCache) + ctgGetTbMetaCacheSize(meta)); + (void)atomic_add_fetch_64(&dbCache->dbCacheSize, strlen(tbName) + sizeof(SCtgTbCache) + ctgGetTbMetaCacheSize(meta)); pCache = taosHashGet(dbCache->tbCache, tbName, strlen(tbName)); + if (NULL == pCache) { + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } } else { CTG_LOCK(CTG_WRITE, &pCache->metaLock); if (orig) { CTG_META_NUM_DEC(origType); } - atomic_add_fetch_64(&dbCache->dbCacheSize, ctgGetTbMetaCacheSize(meta) - ctgGetTbMetaCacheSize(pCache->pMeta)); + (void)atomic_add_fetch_64(&dbCache->dbCacheSize, ctgGetTbMetaCacheSize(meta) - ctgGetTbMetaCacheSize(pCache->pMeta)); taosMemoryFree(pCache->pMeta); pCache->pMeta = meta; @@ -1593,7 +1846,7 @@ int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNam CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - atomic_add_fetch_64(&dbCache->dbCacheSize, sizeof(meta->suid) + strlen(tbName) + 1); + (void)atomic_add_fetch_64(&dbCache->dbCacheSize, sizeof(meta->suid) + strlen(tbName) + 1); ctgDebug("stb 0x%" PRIx64 " updated to cache, dbFName:%s, tbName:%s, tbType:%d", meta->suid, dbFName, tbName, meta->tableType); @@ -1625,7 +1878,7 @@ int32_t ctgWriteTbIndexToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNa CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - atomic_add_fetch_64(&dbCache->dbCacheSize, strlen(tbName) + sizeof(SCtgTbCache) + ctgGetTbIndexCacheSize(pIndex)); + (void)atomic_add_fetch_64(&dbCache->dbCacheSize, strlen(tbName) + sizeof(SCtgTbCache) + ctgGetTbIndexCacheSize(pIndex)); CTG_DB_NUM_INC(CTG_CI_TBL_SMA); @@ -1643,7 +1896,7 @@ int32_t ctgWriteTbIndexToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNa CTG_LOCK(CTG_WRITE, &pCache->indexLock); if (pCache->pIndex) { - atomic_sub_fetch_64(&dbCache->dbCacheSize, ctgGetTbIndexCacheSize(pCache->pIndex)); + (void)atomic_sub_fetch_64(&dbCache->dbCacheSize, ctgGetTbIndexCacheSize(pCache->pIndex)); if (0 == suid) { suid = pCache->pIndex->suid; } @@ -1654,7 +1907,7 @@ int32_t ctgWriteTbIndexToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNa pCache->pIndex = pIndex; CTG_UNLOCK(CTG_WRITE, &pCache->indexLock); - atomic_add_fetch_64(&dbCache->dbCacheSize, ctgGetTbIndexCacheSize(pIndex)); + (void)atomic_add_fetch_64(&dbCache->dbCacheSize, ctgGetTbIndexCacheSize(pIndex)); *index = NULL; @@ -1685,7 +1938,7 @@ int32_t ctgWriteViewMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFN CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } - atomic_add_fetch_64(&dbCache->dbCacheSize, strlen(viewName) + sizeof(SCtgViewCache) + ctgGetViewMetaCacheSize(pMeta)); + (void)atomic_add_fetch_64(&dbCache->dbCacheSize, strlen(viewName) + sizeof(SCtgViewCache) + ctgGetViewMetaCacheSize(pMeta)); CTG_DB_NUM_INC(CTG_CI_VIEW); @@ -1700,7 +1953,7 @@ int32_t ctgWriteViewMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFN CTG_LOCK(CTG_WRITE, &pCache->viewLock); if (pCache->pMeta) { - atomic_sub_fetch_64(&dbCache->dbCacheSize, ctgGetViewMetaCacheSize(pCache->pMeta)); + (void)atomic_sub_fetch_64(&dbCache->dbCacheSize, ctgGetViewMetaCacheSize(pCache->pMeta)); ctgFreeSViewMeta(pCache->pMeta); taosMemoryFree(pCache->pMeta); } @@ -1708,7 +1961,7 @@ int32_t ctgWriteViewMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFN pCache->pMeta = pMeta; CTG_UNLOCK(CTG_WRITE, &pCache->viewLock); - atomic_add_fetch_64(&dbCache->dbCacheSize, ctgGetViewMetaCacheSize(pMeta)); + (void)atomic_add_fetch_64(&dbCache->dbCacheSize, ctgGetViewMetaCacheSize(pMeta)); ctgDebug("view meta updated to cache, view:%s, id:%" PRIu64 ", ver:%d, effectiveUser:%s, querySQL:%s", viewName, pMeta->viewId, pMeta->version, pMeta->user, pMeta->querySql); @@ -1794,6 +2047,103 @@ int32_t ctgVgInfoIdComp(void const *lp, void const *rp) { return 0; } + + + +int32_t ctgWriteTbTSMAToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, char *tbName, + STSMACache **ppTsmaCache) { + if (NULL == dbCache->tsmaCache) { + ctgError("db is dropping, dbId:0x%" PRIx64, dbCache->dbId); + CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); + } + + STSMACache *pTsmaCache = *ppTsmaCache; + int32_t code = TSDB_CODE_SUCCESS; + + SCtgTSMACache* pCache = taosHashGet(dbCache->tsmaCache, tbName, strlen(tbName)); + if (!pCache) { + SCtgTSMACache cache = {0}; + cache.pTsmas = taosArrayInit(4, sizeof(POINTER_BYTES)); + if (NULL == cache.pTsmas) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + + if (NULL == taosArrayPush(cache.pTsmas, &pTsmaCache)) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + + if (taosHashPut(dbCache->tsmaCache, tbName, strlen(tbName), &cache, sizeof(cache))) { + ctgError("taosHashPut new tsmacache for tb: %s.%s failed", dbFName, tbName); + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + + (void)atomic_add_fetch_64(&dbCache->dbCacheSize, strlen(tbName) + sizeof(STSMACache) + ctgGetTbTSMACacheSize(pTsmaCache)); + + CTG_DB_NUM_INC(CTG_CI_TBL_TSMA); + ctgDebug("tb %s tsma updated to cache, name: %s", tbName, pTsmaCache->name); + + CTG_ERR_JRET(ctgUpdateRentTSMAVersion(pCtg, dbFName, pTsmaCache)); + *ppTsmaCache = NULL; + + goto _return; + } + + CTG_LOCK(CTG_WRITE, &pCache->tsmaLock); + + if (pCache->pTsmas) { + uint64_t cacheSize = 0; + for (int32_t i = 0; i < pCache->pTsmas->size; ++i) { + STableTSMAInfo* pInfo = taosArrayGetP(pCache->pTsmas, i); + if (NULL == pInfo) { + ctgError("fail to get the %dth STableTSMAInfo, total:%d", i, (int32_t)pCache->pTsmas->size); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + if (pInfo->tsmaId == pTsmaCache->tsmaId) { + ctgDebug("tsma: %s removed from cache, history from %d to %d, reqTs from %" PRId64 " to %" PRId64 + "rspTs from %" PRId64 " to %" PRId64 " delay from %" PRId64 " to %" PRId64, + pInfo->name, pInfo->fillHistoryFinished, pTsmaCache->fillHistoryFinished, pInfo->reqTs, + pTsmaCache->reqTs, pInfo->rspTs, pTsmaCache->rspTs, pInfo->delayDuration, pTsmaCache->delayDuration); + + cacheSize = ctgGetTbTSMACacheSize(pInfo); + taosArrayRemove(pCache->pTsmas, i); + (void)atomic_sub_fetch_64(&dbCache->dbCacheSize, cacheSize); + + tFreeTableTSMAInfo(pInfo); + taosMemoryFreeClear(pInfo); + + break; + } + } + } else { + pCache->pTsmas = taosArrayInit(4, sizeof(POINTER_BYTES)); + if (!pCache->pTsmas) { + CTG_UNLOCK(CTG_WRITE, &pCache->tsmaLock); + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + } + + // push the new cache + if (NULL == taosArrayPush(pCache->pTsmas, &pTsmaCache)) { + code = TSDB_CODE_OUT_OF_MEMORY; + } else { + *ppTsmaCache = NULL; + + (void)atomic_add_fetch_64(&dbCache->dbCacheSize, ctgGetTbTSMACacheSize(pTsmaCache)); + + CTG_ERR_RET(ctgUpdateRentTSMAVersion(pCtg, dbFName, pTsmaCache)); + + ctgDebug("table %s tsma updated to cache, tsma: %s", tbName, pTsmaCache->name); + } + + CTG_UNLOCK(CTG_WRITE, &pCache->tsmaLock); + +_return: + + CTG_RET(code); +} + + int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) { int32_t code = 0; SCtgUpdateVgMsg *msg = operation->data; @@ -1849,7 +2199,7 @@ int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) { uint64_t groupCacheSize = ctgGetDbVgroupCacheSize(vgCache->vgInfo); ctgDebug("sub dbGroupCacheSize %" PRIu64 " from db, dbFName:%s", groupCacheSize, dbFName); - atomic_sub_fetch_64(&dbCache->dbCacheSize, groupCacheSize); + (void)atomic_sub_fetch_64(&dbCache->dbCacheSize, groupCacheSize); freeVgInfo(vgInfo); CTG_DB_NUM_RESET(CTG_CI_DB_VGROUP); @@ -1870,7 +2220,7 @@ int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) { ctgWUnlockVgInfo(dbCache); uint64_t groupCacheSize = ctgGetDbVgroupCacheSize(vgCache->vgInfo); - atomic_add_fetch_64(&dbCache->dbCacheSize, groupCacheSize); + (void)atomic_add_fetch_64(&dbCache->dbCacheSize, groupCacheSize); ctgDebug("add dbGroupCacheSize %" PRIu64 " from db, dbFName:%s", groupCacheSize, dbFName); dbCache = NULL; @@ -1961,7 +2311,7 @@ int32_t ctgOpDropDbCache(SCtgCacheOperation *operation) { } SCtgDBCache *dbCache = NULL; - ctgGetDBCache(msg->pCtg, msg->dbFName, &dbCache); + CTG_ERR_RET(ctgGetDBCache(msg->pCtg, msg->dbFName, &dbCache)); if (NULL == dbCache) { goto _return; } @@ -1991,14 +2341,14 @@ int32_t ctgOpDropDbVgroup(SCtgCacheOperation *operation) { } SCtgDBCache *dbCache = NULL; - ctgGetDBCache(msg->pCtg, msg->dbFName, &dbCache); + CTG_ERR_RET(ctgGetDBCache(msg->pCtg, msg->dbFName, &dbCache)); if (NULL == dbCache) { goto _return; } CTG_ERR_JRET(ctgWLockVgInfo(pCtg, dbCache)); - atomic_sub_fetch_64(&dbCache->dbCacheSize, ctgGetDbVgroupCacheSize(dbCache->vgCache.vgInfo)); + (void)atomic_sub_fetch_64(&dbCache->dbCacheSize, ctgGetDbVgroupCacheSize(dbCache->vgCache.vgInfo)); freeVgInfo(dbCache->vgCache.vgInfo); dbCache->vgCache.vgInfo = NULL; @@ -2053,7 +2403,7 @@ int32_t ctgOpUpdateTbMeta(SCtgCacheOperation *operation) { if (NULL == ctbMeta) { CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(ctbMeta, &pMeta->ctbMeta, sizeof(SCTableMeta)); + TAOS_MEMCPY(ctbMeta, &pMeta->ctbMeta, sizeof(SCTableMeta)); CTG_ERR_JRET(ctgWriteTbMetaToCache(pCtg, dbCache, pMeta->dbFName, pMeta->dbId, pMeta->ctbName, (STableMeta *)ctbMeta)); } @@ -2079,7 +2429,7 @@ int32_t ctgOpDropStbMeta(SCtgCacheOperation *operation) { } SCtgDBCache *dbCache = NULL; - ctgGetDBCache(pCtg, msg->dbFName, &dbCache); + CTG_ERR_JRET(ctgGetDBCache(pCtg, msg->dbFName, &dbCache)); if (NULL == dbCache) { goto _return; } @@ -2097,7 +2447,7 @@ int32_t ctgOpDropStbMeta(SCtgCacheOperation *operation) { ctgDebug("stb not exist in stbCache, may be removed, dbFName:%s, stb:%s, suid:0x%" PRIx64, msg->dbFName, msg->stbName, msg->suid); } else { - atomic_sub_fetch_64(&dbCache->dbCacheSize, metaSize); + (void)atomic_sub_fetch_64(&dbCache->dbCacheSize, metaSize); } } @@ -2108,7 +2458,7 @@ int32_t ctgOpDropStbMeta(SCtgCacheOperation *operation) { } tblType = pTbCache->pMeta->tableType; - atomic_sub_fetch_64(&dbCache->dbCacheSize, + (void)atomic_sub_fetch_64(&dbCache->dbCacheSize, ctgGetTbMetaCacheSize(pTbCache->pMeta) + ctgGetTbIndexCacheSize(pTbCache->pIndex)); ctgFreeTbCacheImpl(pTbCache, true); @@ -2116,7 +2466,7 @@ int32_t ctgOpDropStbMeta(SCtgCacheOperation *operation) { ctgError("stb not exist in cache, dbFName:%s, stb:%s, suid:0x%" PRIx64, msg->dbFName, msg->stbName, msg->suid); } else { CTG_META_NUM_DEC(tblType); - atomic_sub_fetch_64(&dbCache->dbCacheSize, sizeof(*pTbCache) + strlen(msg->stbName)); + (void)atomic_sub_fetch_64(&dbCache->dbCacheSize, sizeof(*pTbCache) + strlen(msg->stbName)); } ctgInfo("stb removed from cache, dbFName:%s, stbName:%s, suid:0x%" PRIx64, msg->dbFName, msg->stbName, msg->suid); @@ -2143,7 +2493,7 @@ int32_t ctgOpDropTbMeta(SCtgCacheOperation *operation) { } SCtgDBCache *dbCache = NULL; - ctgGetDBCache(pCtg, msg->dbFName, &dbCache); + CTG_ERR_JRET(ctgGetDBCache(pCtg, msg->dbFName, &dbCache)); if (NULL == dbCache) { goto _return; } @@ -2161,7 +2511,7 @@ int32_t ctgOpDropTbMeta(SCtgCacheOperation *operation) { } tblType = pTbCache->pMeta->tableType; - atomic_sub_fetch_64(&dbCache->dbCacheSize, ctgGetTbMetaCacheSize(pTbCache->pMeta) + + (void)atomic_sub_fetch_64(&dbCache->dbCacheSize, ctgGetTbMetaCacheSize(pTbCache->pMeta) + ctgGetTbIndexCacheSize(pTbCache->pIndex)); ctgFreeTbCacheImpl(pTbCache, true); @@ -2169,7 +2519,7 @@ int32_t ctgOpDropTbMeta(SCtgCacheOperation *operation) { ctgError("tb %s not exist in cache, dbFName:%s", msg->tbName, msg->dbFName); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } else { - atomic_sub_fetch_64(&dbCache->dbCacheSize, sizeof(*pTbCache) + strlen(msg->tbName)); + (void)atomic_sub_fetch_64(&dbCache->dbCacheSize, sizeof(*pTbCache) + strlen(msg->tbName)); CTG_META_NUM_DEC(tblType); } @@ -2199,7 +2549,7 @@ int32_t ctgOpUpdateUser(SCtgCacheOperation *operation) { SCtgUserAuth userAuth = {0}; - memcpy(&userAuth.userAuth, &msg->userAuth, sizeof(msg->userAuth)); + TAOS_MEMCPY(&userAuth.userAuth, &msg->userAuth, sizeof(msg->userAuth)); userAuth.userCacheSize = ctgGetUserCacheSize(&userAuth.userAuth); if (taosHashPut(pCtg->userCache, msg->userAuth.user, strlen(msg->userAuth.user), &userAuth, sizeof(userAuth))) { @@ -2232,7 +2582,7 @@ int32_t ctgOpUpdateUser(SCtgCacheOperation *operation) { taosHashCleanup(pUser->userAuth.alterViews); taosHashCleanup(pUser->userAuth.useDbs); - memcpy(&pUser->userAuth, &msg->userAuth, sizeof(msg->userAuth)); + TAOS_MEMCPY(&pUser->userAuth, &msg->userAuth, sizeof(msg->userAuth)); msg->userAuth.createdDbs = NULL; msg->userAuth.readDbs = NULL; @@ -2247,7 +2597,7 @@ int32_t ctgOpUpdateUser(SCtgCacheOperation *operation) { CTG_UNLOCK(CTG_WRITE, &pUser->lock); - atomic_store_64(&pUser->userCacheSize, ctgGetUserCacheSize(&pUser->userAuth)); + (void)atomic_store_64(&pUser->userCacheSize, ctgGetUserCacheSize(&pUser->userAuth)); _return: @@ -2369,8 +2719,8 @@ int32_t ctgOpDropTbIndex(SCtgCacheOperation *operation) { if (NULL == pIndex) { CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } - strcpy(pIndex->tbName, msg->tbName); - strcpy(pIndex->dbFName, msg->dbFName); + TAOS_STRCPY(pIndex->tbName, msg->tbName); + TAOS_STRCPY(pIndex->dbFName, msg->dbFName); pIndex->version = -1; CTG_ERR_JRET(ctgWriteTbIndexToCache(pCtg, dbCache, pIndex->dbFName, pIndex->tbName, &pIndex)); @@ -2439,7 +2789,7 @@ int32_t ctgOpDropViewMeta(SCtgCacheOperation *operation) { } SCtgDBCache *dbCache = NULL; - ctgGetDBCache(pCtg, msg->dbFName, &dbCache); + CTG_ERR_JRET(ctgGetDBCache(pCtg, msg->dbFName, &dbCache)); if (NULL == dbCache) { goto _return; } @@ -2462,14 +2812,14 @@ int32_t ctgOpDropViewMeta(SCtgCacheOperation *operation) { goto _return; } - atomic_sub_fetch_64(&dbCache->dbCacheSize, ctgGetViewMetaCacheSize(pViewCache->pMeta)); + (void)atomic_sub_fetch_64(&dbCache->dbCacheSize, ctgGetViewMetaCacheSize(pViewCache->pMeta)); ctgFreeViewCacheImpl(pViewCache, true); if (taosHashRemove(dbCache->viewCache, msg->viewName, strlen(msg->viewName))) { ctgError("view %s not exist in cache, dbFName:%s", msg->viewName, msg->dbFName); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } else { - atomic_sub_fetch_64(&dbCache->dbCacheSize, sizeof(SCtgViewCache) + strlen(msg->viewName)); + (void)atomic_sub_fetch_64(&dbCache->dbCacheSize, sizeof(SCtgViewCache) + strlen(msg->viewName)); CTG_DB_NUM_DEC(CTG_CI_VIEW); } @@ -2529,21 +2879,21 @@ void ctgClearMetaCache(SCtgCacheOperation *operation) { if (CTG_CACHE_LOW(remainSize, cacheMaxSize)) { qDebug("catalog finish meta clear, remainSize:%" PRId64 ", cacheMaxSize:%dMB", remainSize, cacheMaxSize); - taosTmrReset(ctgProcessTimerEvent, CTG_DEFAULT_CACHE_MON_MSEC, NULL, gCtgMgmt.timer, &gCtgMgmt.cacheTimer); + (void)taosTmrReset(ctgProcessTimerEvent, CTG_DEFAULT_CACHE_MON_MSEC, NULL, gCtgMgmt.timer, &gCtgMgmt.cacheTimer); return; } if (!roundDone) { qDebug("catalog all meta cleared, remainSize:%" PRId64 ", cacheMaxSize:%dMB, to clear handle", remainSize, cacheMaxSize); ctgClearFreeCache(operation); - taosTmrReset(ctgProcessTimerEvent, CTG_DEFAULT_CACHE_MON_MSEC, NULL, gCtgMgmt.timer, &gCtgMgmt.cacheTimer); + (void)taosTmrReset(ctgProcessTimerEvent, CTG_DEFAULT_CACHE_MON_MSEC, NULL, gCtgMgmt.timer, &gCtgMgmt.cacheTimer); return; } int32_t code = ctgClearCacheEnqueue(NULL, true, false, false, false); if (code) { qError("clear cache enqueue failed, error:%s", tstrerror(code)); - taosTmrReset(ctgProcessTimerEvent, CTG_DEFAULT_CACHE_MON_MSEC, NULL, gCtgMgmt.timer, &gCtgMgmt.cacheTimer); + (void)taosTmrReset(ctgProcessTimerEvent, CTG_DEFAULT_CACHE_MON_MSEC, NULL, gCtgMgmt.timer, &gCtgMgmt.cacheTimer); } } @@ -2564,6 +2914,150 @@ _return: CTG_RET(code); } + +int32_t ctgOpDropTbTSMA(SCtgCacheOperation *operation) { + int32_t code = 0; + SCtgDropTbTSMAMsg * msg = operation->data; + SCatalog *pCtg = msg->pCtg; + SCtgDBCache *dbCache = NULL; + + if (pCtg->stopUpdate) { + goto _return; + } + + CTG_ERR_JRET(ctgGetDBCache(pCtg, msg->dbFName, &dbCache)); + if (NULL == dbCache || !dbCache->tsmaCache || (msg->dbId != dbCache->dbId && msg->dbId != 0)) { + goto _return; + } + + SCtgTSMACache* pCtgCache = taosHashGet(dbCache->tsmaCache, msg->tbName, strlen(msg->tbName)); + if (!pCtgCache || !pCtgCache->pTsmas || pCtgCache->pTsmas->size == 0) { + goto _return; + } + + uint64_t cacheSize = 0; + STSMACache *pCache = NULL; + if (msg->dropAllForTb) { + CTG_LOCK(CTG_WRITE, &pCtgCache->tsmaLock); + + for (int32_t i = 0; i < pCtgCache->pTsmas->size; ++i) { + pCache = taosArrayGetP(pCtgCache->pTsmas, i); + if (NULL == pCache) { + ctgError("fail to the %dth tsma in pTsmas, total:%d", i, (int32_t)pCtgCache->pTsmas->size); + continue; + } + + cacheSize += ctgGetTbTSMACacheSize(pCache); + (void)ctgMetaRentRemove(&msg->pCtg->tsmaRent, pCache->tsmaId, ctgTSMAVersionSearchCompare, ctgTSMAVersionSearchCompare); + + CTG_DB_NUM_DEC(CTG_CI_TBL_TSMA); + } + + taosArrayDestroyP(pCtgCache->pTsmas, tFreeAndClearTableTSMAInfo); + pCtgCache->pTsmas = NULL; + + ctgDebug("all tsmas for table dropped: %s.%s", msg->dbFName, msg->tbName); + (void)taosHashRemove(dbCache->tsmaCache, msg->tbName, TSDB_TABLE_NAME_LEN); + + CTG_UNLOCK(CTG_WRITE, &pCtgCache->tsmaLock); + } else { + CTG_LOCK(CTG_WRITE, &pCtgCache->tsmaLock); + + pCache = taosArrayGetP(pCtgCache->pTsmas, 0); + if (NULL == pCache) { + ctgError("fail to the 0th tsma in pTsmas, total:%d", (int32_t)pCtgCache->pTsmas->size); + code = TSDB_CODE_CTG_INTERNAL_ERROR; + } else { + if (msg->tbId != 0 && pCache->suid != msg->tbId) { + // table id mismatch, skip drops + CTG_UNLOCK(CTG_WRITE, &pCtgCache->tsmaLock); + goto _return; + } + + for (int32_t i = 0; i < pCtgCache->pTsmas->size; ++i) { + pCache = taosArrayGetP(pCtgCache->pTsmas, i); + if (NULL == pCache) { + ctgError("fail to the %dth tsma in pTsmas, total:%d", i, (int32_t)pCtgCache->pTsmas->size); + code = TSDB_CODE_CTG_INTERNAL_ERROR; + continue; + } + + if (pCache->tsmaId != msg->tsmaId) { + continue; + } + + cacheSize = ctgGetTbTSMACacheSize(pCache); + (void)ctgMetaRentRemove(&msg->pCtg->tsmaRent, pCache->tsmaId, ctgTSMAVersionSearchCompare, ctgTSMAVersionSearchCompare); + + taosArrayRemove(pCtgCache->pTsmas, i); + tFreeAndClearTableTSMAInfo(pCache); + + CTG_DB_NUM_DEC(CTG_CI_TBL_TSMA); + + break; + } + } + + CTG_UNLOCK(CTG_WRITE, &pCtgCache->tsmaLock); + } + + (void)atomic_sub_fetch_64(&dbCache->dbCacheSize, cacheSize); + +_return: + + taosMemoryFreeClear(msg); + + CTG_RET(code); +} + +int32_t ctgOpUpdateTbTSMA(SCtgCacheOperation *operation) { + int32_t code = 0; + SCtgUpdateTbTSMAMsg *msg = operation->data; + SCatalog * pCtg = msg->pCtg; + STableTSMAInfo * pTsmaInfo = msg->pTsma; + SCtgDBCache * dbCache = NULL; + + if (pCtg->stopUpdate) { + goto _return; + } + + CTG_ERR_JRET(ctgGetAddDBCache(pCtg, pTsmaInfo->dbFName, pTsmaInfo->dbId, &dbCache)); + CTG_ERR_JRET(ctgWriteTbTSMAToCache(pCtg, dbCache, pTsmaInfo->dbFName, pTsmaInfo->tb, &pTsmaInfo)); + + if (dbCache && msg->dbTsmaVersion > 0) { + dbCache->tsmaVersion = msg->dbTsmaVersion; + SDbCacheInfo cacheInfo = {0}; + cacheInfo.dbId = dbCache->dbId; + + if (dbCache->cfgCache.cfgInfo) { + cacheInfo.cfgVersion = dbCache->cfgCache.cfgInfo->cfgVersion; + tstrncpy(cacheInfo.dbFName, dbCache->cfgCache.cfgInfo->db, TSDB_DB_FNAME_LEN); + } + + if (dbCache->vgCache.vgInfo) { + cacheInfo.vgVersion = dbCache->vgCache.vgInfo->vgVersion; + cacheInfo.numOfTable = dbCache->vgCache.vgInfo->numOfTable; + cacheInfo.stateTs = dbCache->vgCache.vgInfo->stateTs; + } + + cacheInfo.tsmaVersion = dbCache->tsmaVersion; + CTG_ERR_JRET(ctgMetaRentUpdate(&msg->pCtg->dbRent, &cacheInfo, cacheInfo.dbId, sizeof(SDbCacheInfo), + ctgDbCacheInfoSortCompare, ctgDbCacheInfoSearchCompare)); + } + +_return: + + if (pTsmaInfo) { + tFreeTableTSMAInfo(pTsmaInfo); + taosMemoryFreeClear(pTsmaInfo); + } + + taosMemoryFreeClear(msg); + + CTG_RET(code); +} + + void ctgFreeCacheOperationData(SCtgCacheOperation *op) { if (NULL == op || NULL == op->data) { return; @@ -2648,7 +3142,7 @@ void ctgCleanupCacheQueue(void) { if (op->stopQueue) { SCatalog *pCtg = ((SCtgUpdateMsgHeader *)op->data)->pCtg; ctgDebug("process [%s] operation", gCtgCacheOperation[op->opId].name); - (*gCtgCacheOperation[op->opId].func)(op); + (void)(*gCtgCacheOperation[op->opId].func)(op); // ignore any error stopQueue = true; CTG_STAT_RT_INC(numOfOpDequeue, 1); } else { @@ -2657,7 +3151,7 @@ void ctgCleanupCacheQueue(void) { } if (op->syncOp) { - tsem_post(&op->rspSem); + (void)tsem_post(&op->rspSem); } else { taosMemoryFree(op); } @@ -2701,18 +3195,18 @@ void *ctgUpdateThreadFunc(void *param) { ctgDebug("process [%s] operation", gCtgCacheOperation[operation->opId].name); - (*gCtgCacheOperation[operation->opId].func)(operation); + (void)(*gCtgCacheOperation[operation->opId].func)(operation); // ignore any error if (operation->syncOp) { - tsem_post(&operation->rspSem); + (void)tsem_post(&operation->rspSem); } else { taosMemoryFreeClear(operation); } CTG_STAT_RT_INC(numOfOpDequeue, 1); - ctgdShowCacheInfo(); - ctgdShowStatInfo(); + (void)ctgdShowCacheInfo(); + (void)ctgdShowStatInfo(); } qInfo("catalog update thread stopped"); @@ -2721,16 +3215,25 @@ void *ctgUpdateThreadFunc(void *param) { } int32_t ctgStartUpdateThread() { + int32_t code = TSDB_CODE_SUCCESS; TdThreadAttr thAttr; - taosThreadAttrInit(&thAttr); - taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); + CTG_ERR_JRET(taosThreadAttrInit(&thAttr)); + CTG_ERR_JRET(taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE)); if (taosThreadCreate(&gCtgMgmt.updateThread, &thAttr, ctgUpdateThreadFunc, NULL) != 0) { terrno = TAOS_SYSTEM_ERROR(errno); CTG_ERR_RET(terrno); } - taosThreadAttrDestroy(&thAttr); + (void)taosThreadAttrDestroy(&thAttr); + +_return: + + if (code) { + terrno = TAOS_SYSTEM_ERROR(errno); + CTG_ERR_RET(terrno); + } + return TSDB_CODE_SUCCESS; } @@ -2818,28 +3321,35 @@ int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe int32_t ctgGetTbMetasFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgTbMetasCtx *ctx, int32_t dbIdx, int32_t *fetchIdx, int32_t baseResIdx, SArray *pList) { int32_t tbNum = taosArrayGetSize(pList); - SName *pName = taosArrayGet(pList, 0); char dbFName[TSDB_DB_FNAME_LEN] = {0}; int32_t flag = CTG_FLAG_UNKNOWN_STB; + int32_t code = TSDB_CODE_SUCCESS; uint64_t lastSuid = 0; STableMeta *lastTableMeta = NULL; - + SName *pName = taosArrayGet(pList, 0); + if (NULL == pName) { + ctgError("fail to get the 0th SName from tableList, tableNum:%d", (int32_t)taosArrayGetSize(pList)); + return TSDB_CODE_CTG_INVALID_INPUT; + } + if (IS_SYS_DBNAME(pName->dbname)) { CTG_FLAG_SET_SYS_DB(flag); - strcpy(dbFName, pName->dbname); + TAOS_STRCPY(dbFName, pName->dbname); } else { - tNameGetFullDbName(pName, dbFName); + (void)tNameGetFullDbName(pName, dbFName); } SCtgDBCache *dbCache = NULL; SCtgTbCache *pCache = NULL; - ctgAcquireDBCache(pCtg, dbFName, &dbCache); + CTG_ERR_RET(ctgAcquireDBCache(pCtg, dbFName, &dbCache)); if (NULL == dbCache) { ctgDebug("db %s not in cache", dbFName); for (int32_t i = 0; i < tbNum; ++i) { - ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag); - taosArrayPush(ctx->pResList, &(SMetaData){0}); + CTG_ERR_JRET(ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag)); + if (NULL == taosArrayPush(ctx->pResList, &(SMetaData){0})) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } } return TSDB_CODE_SUCCESS; @@ -2847,12 +3357,19 @@ int32_t ctgGetTbMetasFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgTbMe for (int32_t i = 0; i < tbNum; ++i) { pName = taosArrayGet(pList, i); + if (NULL == pName) { + ctgError("fail to get the %dth SName from tableList, tableNum:%d", i, (int32_t)taosArrayGetSize(pList)); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } pCache = taosHashAcquire(dbCache->tbCache, pName->tname, strlen(pName->tname)); if (NULL == pCache) { ctgDebug("tb %s not in cache, dbFName:%s", pName->tname, dbFName); - ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag); - taosArrayPush(ctx->pResList, &(SMetaRes){0}); + CTG_ERR_JRET(ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag)); + if (NULL == taosArrayPush(ctx->pResList, &(SMetaData){0})) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + CTG_META_NHIT_INC(); continue; @@ -2862,9 +3379,14 @@ int32_t ctgGetTbMetasFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgTbMe if (NULL == pCache->pMeta) { CTG_UNLOCK(CTG_READ, &pCache->metaLock); taosHashRelease(dbCache->tbCache, pCache); + ctgDebug("tb %s meta not in cache, dbFName:%s", pName->tname, dbFName); - ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag); - taosArrayPush(ctx->pResList, &(SMetaRes){0}); + + CTG_ERR_JRET(ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag)); + if (NULL == taosArrayPush(ctx->pResList, &(SMetaData){0})) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + CTG_META_NHIT_INC(); continue; @@ -2889,16 +3411,17 @@ int32_t ctgGetTbMetasFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgTbMe if (tbMeta->schemaExt != NULL) { schemaExtSize = tbMeta->tableInfo.numOfColumns * sizeof(SSchemaExt); } + pTableMeta = taosMemoryCalloc(1, metaSize + schemaExtSize); if (NULL == pTableMeta) { ctgReleaseTbMetaToCache(pCtg, dbCache, pCache); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(pTableMeta, tbMeta, metaSize); + TAOS_MEMCPY(pTableMeta, tbMeta, metaSize); if (tbMeta->schemaExt != NULL) { pTableMeta->schemaExt = (SSchemaExt *)((char *)pTableMeta + metaSize); - memcpy(pTableMeta->schemaExt, tbMeta->schemaExt, schemaExtSize); + TAOS_MEMCPY(pTableMeta->schemaExt, tbMeta->schemaExt, schemaExtSize); } else { pTableMeta->schemaExt = NULL; } @@ -2909,7 +3432,9 @@ int32_t ctgGetTbMetasFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgTbMe ctgDebug("Got tb %s meta from cache, type:%d, dbFName:%s", pName->tname, pTableMeta->tableType, dbFName); res.pRes = pTableMeta; - taosArrayPush(ctx->pResList, &res); + if (NULL == taosArrayPush(ctx->pResList, &res)) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } continue; } @@ -2917,8 +3442,14 @@ int32_t ctgGetTbMetasFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgTbMe // PROCESS FOR CHILD TABLE if (lastSuid && tbMeta->suid == lastSuid && lastTableMeta) { - cloneTableMeta(lastTableMeta, &pTableMeta); - memcpy(pTableMeta, tbMeta, sizeof(SCTableMeta)); + code = cloneTableMeta(lastTableMeta, &pTableMeta); + if (code) { + CTG_UNLOCK(CTG_READ, &pCache->metaLock); + taosHashRelease(dbCache->tbCache, pCache); + CTG_ERR_JRET(code); + } + + TAOS_MEMCPY(pTableMeta, tbMeta, sizeof(SCTableMeta)); CTG_UNLOCK(CTG_READ, &pCache->metaLock); taosHashRelease(dbCache->tbCache, pCache); @@ -2926,7 +3457,9 @@ int32_t ctgGetTbMetasFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgTbMe ctgDebug("Got tb %s meta from cache, type:%d, dbFName:%s", pName->tname, pTableMeta->tableType, dbFName); res.pRes = pTableMeta; - taosArrayPush(ctx->pResList, &res); + if (NULL == taosArrayPush(ctx->pResList, &res)) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } continue; } @@ -2938,7 +3471,7 @@ int32_t ctgGetTbMetasFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgTbMe CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(pTableMeta, tbMeta, metaSize); + TAOS_MEMCPY(pTableMeta, tbMeta, metaSize); CTG_UNLOCK(CTG_READ, &pCache->metaLock); taosHashRelease(dbCache->tbCache, pCache); @@ -2949,8 +3482,10 @@ int32_t ctgGetTbMetasFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgTbMe char *stName = taosHashAcquire(dbCache->stbCache, &pTableMeta->suid, sizeof(pTableMeta->suid)); if (NULL == stName) { ctgDebug("stb 0x%" PRIx64 " not in cache, dbFName:%s", pTableMeta->suid, dbFName); - ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag); - taosArrayPush(ctx->pResList, &(SMetaRes){0}); + CTG_ERR_JRET(ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag)); + if (NULL == taosArrayPush(ctx->pResList, &(SMetaRes){0})) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } taosMemoryFreeClear(pTableMeta); CTG_META_NHIT_INC(); @@ -2962,11 +3497,14 @@ int32_t ctgGetTbMetasFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgTbMe ctgDebug("stb 0x%" PRIx64 " name %s not in cache, dbFName:%s", pTableMeta->suid, stName, dbFName); taosHashRelease(dbCache->stbCache, stName); - ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag); - taosArrayPush(ctx->pResList, &(SMetaRes){0}); + CTG_ERR_JRET(ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag)); + if (NULL == taosArrayPush(ctx->pResList, &(SMetaRes){0})) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } taosMemoryFreeClear(pTableMeta); CTG_META_NHIT_INC(); + continue; } @@ -2978,8 +3516,11 @@ int32_t ctgGetTbMetasFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgTbMe CTG_UNLOCK(CTG_READ, &pCache->metaLock); taosHashRelease(dbCache->tbCache, pCache); - ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag); - taosArrayPush(ctx->pResList, &(SMetaRes){0}); + CTG_ERR_JRET(ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag)); + if (NULL == taosArrayPush(ctx->pResList, &(SMetaRes){0})) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + taosMemoryFreeClear(pTableMeta); CTG_META_NHIT_INC(); @@ -2994,8 +3535,11 @@ int32_t ctgGetTbMetasFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgTbMe ctgError("stb suid 0x%" PRIx64 " in stbCache mis-match, expected suid 0x%" PRIx64, stbMeta->suid, nctx.tbInfo.suid); - ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag); - taosArrayPush(ctx->pResList, &(SMetaRes){0}); + CTG_ERR_JRET(ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag)); + if (NULL == taosArrayPush(ctx->pResList, &(SMetaRes){0})) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + taosMemoryFreeClear(pTableMeta); CTG_META_NHIT_INC(); @@ -3009,7 +3553,7 @@ int32_t ctgGetTbMetasFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgTbMe CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(&pTableMeta->sversion, &stbMeta->sversion, metaSize - sizeof(SCTableMeta)); + TAOS_MEMCPY(&pTableMeta->sversion, &stbMeta->sversion, metaSize - sizeof(SCTableMeta)); pTableMeta->schemaExt = NULL; CTG_UNLOCK(CTG_READ, &pCache->metaLock); @@ -3018,15 +3562,19 @@ int32_t ctgGetTbMetasFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgTbMe CTG_META_HIT_INC(pTableMeta->tableType); res.pRes = pTableMeta; - taosArrayPush(ctx->pResList, &res); + if (NULL == taosArrayPush(ctx->pResList, &res)) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } lastSuid = pTableMeta->suid; lastTableMeta = pTableMeta; } +_return: + ctgReleaseDBCache(pCtg, dbCache); - return TSDB_CODE_SUCCESS; + return code; } int32_t ctgRemoveTbMetaFromCache(SCatalog *pCtg, SName *pTableName, bool syncReq) { @@ -3040,7 +3588,7 @@ int32_t ctgRemoveTbMetaFromCache(SCatalog *pCtg, SName *pTableName, bool syncReq if (NULL != tblMeta) { char dbFName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(pTableName, dbFName); + (void)tNameGetFullDbName(pTableName, dbFName); if (TSDB_SUPER_TABLE == tblMeta->tableType) { CTG_ERR_JRET(ctgDropStbMetaEnqueue(pCtg, dbFName, tbCtx.tbInfo.dbId, pTableName->tname, tblMeta->suid, syncReq)); @@ -3069,7 +3617,7 @@ int32_t ctgGetTbHashVgroupFromCache(SCatalog *pCtg, const SName *pTableName, SVg SCtgDBCache *dbCache = NULL; int32_t code = 0; char dbFName[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(pTableName, dbFName); + (void)tNameGetFullDbName(pTableName, dbFName); CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache)); @@ -3098,28 +3646,35 @@ _return: int32_t ctgGetViewsFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgViewsCtx *ctx, int32_t dbIdx, int32_t *fetchIdx, int32_t baseResIdx, SArray *pList) { int32_t tbNum = taosArrayGetSize(pList); - SName *pName = taosArrayGet(pList, 0); char dbFName[TSDB_DB_FNAME_LEN] = {0}; int32_t flag = CTG_FLAG_UNKNOWN_STB; uint64_t lastSuid = 0; STableMeta *lastTableMeta = NULL; + int32_t code = TSDB_CODE_SUCCESS; + SName *pName = taosArrayGet(pList, 0); + if (NULL == pName) { + ctgError("fail to get the 0th SName from viewList, viewNum:%d", (int32_t)taosArrayGetSize(pList)); + return TSDB_CODE_CTG_INVALID_INPUT; + } if (IS_SYS_DBNAME(pName->dbname)) { CTG_FLAG_SET_SYS_DB(flag); - strcpy(dbFName, pName->dbname); + TAOS_STRCPY(dbFName, pName->dbname); } else { - tNameGetFullDbName(pName, dbFName); + (void)tNameGetFullDbName(pName, dbFName); } SCtgDBCache *dbCache = NULL; SCtgViewCache *pCache = NULL; - ctgAcquireDBCache(pCtg, dbFName, &dbCache); + CTG_ERR_RET(ctgAcquireDBCache(pCtg, dbFName, &dbCache)); if (NULL == dbCache) { ctgDebug("db %s not in cache", dbFName); for (int32_t i = 0; i < tbNum; ++i) { - ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag); - taosArrayPush(ctx->pResList, &(SMetaData){0}); + CTG_ERR_RET(ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag)); + if (NULL == taosArrayPush(ctx->pResList, &(SMetaData){0})) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } } return TSDB_CODE_SUCCESS; @@ -3127,12 +3682,18 @@ int32_t ctgGetViewsFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgViewsC for (int32_t i = 0; i < tbNum; ++i) { pName = taosArrayGet(pList, i); + if (NULL == pName) { + ctgError("fail to get the %dth SName from viewList, viewNum:%d", i, (int32_t)taosArrayGetSize(pList)); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } pCache = taosHashAcquire(dbCache->viewCache, pName->tname, strlen(pName->tname)); if (NULL == pCache) { ctgDebug("view %s not in cache, dbFName:%s", pName->tname, dbFName); - ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag); - taosArrayPush(ctx->pResList, &(SMetaRes){0}); + CTG_ERR_JRET(ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag)); + if (NULL == taosArrayPush(ctx->pResList, &(SMetaRes){0})) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } CTG_CACHE_NHIT_INC(CTG_CI_VIEW, 1); continue; @@ -3143,8 +3704,10 @@ int32_t ctgGetViewsFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgViewsC CTG_UNLOCK(CTG_READ, &pCache->viewLock); taosHashRelease(dbCache->viewCache, pCache); ctgDebug("view %s meta not in cache, dbFName:%s", pName->tname, dbFName); - ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag); - taosArrayPush(ctx->pResList, &(SMetaRes){0}); + CTG_ERR_JRET(ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag)); + if (NULL == taosArrayPush(ctx->pResList, &(SMetaRes){0})) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } CTG_CACHE_NHIT_INC(CTG_CI_VIEW, 1); continue; @@ -3159,7 +3722,7 @@ int32_t ctgGetViewsFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgViewsC CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(pViewMeta, pCache->pMeta, sizeof(*pViewMeta)); + TAOS_MEMCPY(pViewMeta, pCache->pMeta, sizeof(*pViewMeta)); pViewMeta->querySql = tstrdup(pCache->pMeta->querySql); pViewMeta->user = tstrdup(pCache->pMeta->user); if (NULL == pViewMeta->querySql || NULL == pViewMeta->user) { @@ -3177,7 +3740,7 @@ int32_t ctgGetViewsFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgViewsC taosMemoryFree(pViewMeta); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(pViewMeta->pSchema, pCache->pMeta->pSchema, pViewMeta->numOfCols * sizeof(SSchema)); + TAOS_MEMCPY(pViewMeta->pSchema, pCache->pMeta->pSchema, pViewMeta->numOfCols * sizeof(SSchema)); CTG_UNLOCK(CTG_READ, &pCache->viewLock); taosHashRelease(dbCache->viewCache, pCache); @@ -3185,12 +3748,16 @@ int32_t ctgGetViewsFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgViewsC ctgDebug("Got view %s meta from cache, dbFName:%s", pName->tname, dbFName); res.pRes = pViewMeta; - taosArrayPush(ctx->pResList, &res); + if (NULL == taosArrayPush(ctx->pResList, &res)) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } } +_return: + ctgReleaseDBCache(pCtg, dbCache); - return TSDB_CODE_SUCCESS; + return code; } int32_t ctgGetTbTSMAFromCache(SCatalog* pCtg, SCtgTbTSMACtx* pCtx, int32_t dbIdx, int32_t* fetchIdx, int32_t baseResIdx, @@ -3202,48 +3769,70 @@ int32_t ctgGetTbTSMAFromCache(SCatalog* pCtg, SCtgTbTSMACtx* pCtx, int32_t dbIdx int32_t flag = CTG_FLAG_UNKNOWN_STB; uint64_t lastSuid = 0; STableMeta * pTableMeta = NULL; - SName * pName = taosArrayGet(pList, 0); int32_t tbNum = taosArrayGetSize(pList); SCtgTbCache * pTbCache = NULL; + SName * pName = taosArrayGet(pList, 0); + if (NULL == pName) { + ctgError("fail to get the 0th SName from tbTSMAList, num:%d", (int32_t)taosArrayGetSize(pList)); + return TSDB_CODE_CTG_INVALID_INPUT; + } if (IS_SYS_DBNAME(pName->dbname)) { return TSDB_CODE_SUCCESS; } - tNameGetFullDbName(pName, dbFName); + (void)tNameGetFullDbName(pName, dbFName); // get db cache CTG_ERR_RET(ctgAcquireDBCache(pCtg, dbFName, &dbCache)); if (!dbCache) { ctgDebug("DB %s not in cache", dbFName); for (int32_t i = 0; i < tbNum; ++i) { - ctgAddTSMAFetch(&pCtx->pFetches, dbIdx, i, fetchIdx, baseResIdx + i, flag, FETCH_TSMA_SOURCE_TB_META, NULL); - taosArrayPush(pCtx->pResList, &(SMetaData){0}); + CTG_ERR_RET(ctgAddTSMAFetch(&pCtx->pFetches, dbIdx, i, fetchIdx, baseResIdx + i, flag, FETCH_TSMA_SOURCE_TB_META, NULL)); + if (NULL == taosArrayPush(pCtx->pResList, &(SMetaData){0})) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } } + return TSDB_CODE_SUCCESS; } for (int32_t i = 0; i < tbNum; ++i) { // get tb cache pName = taosArrayGet(pList, i); + if (NULL == pName) { + ctgError("fail to get the %dth SName from tbTSMAList, num:%d", i, (int32_t)taosArrayGetSize(pList)); + CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); + } + pTbCache = taosHashAcquire(dbCache->tbCache, pName->tname, strlen(pName->tname)); if (!pTbCache) { ctgDebug("tb: %s.%s not in cache", dbFName, pName->tname); - ctgAddTSMAFetch(&pCtx->pFetches, dbIdx, i, fetchIdx, baseResIdx + i, flag, FETCH_TSMA_SOURCE_TB_META, NULL); - taosArrayPush(pCtx->pResList, &(SMetaRes){0}); + CTG_ERR_JRET(ctgAddTSMAFetch(&pCtx->pFetches, dbIdx, i, fetchIdx, baseResIdx + i, flag, FETCH_TSMA_SOURCE_TB_META, NULL)); + if (NULL == taosArrayPush(pCtx->pResList, &(SMetaRes){0})) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + continue; } + CTG_LOCK(CTG_READ, &pTbCache->metaLock); if (!pTbCache->pMeta) { CTG_UNLOCK(CTG_READ, &pTbCache->metaLock); ctgDebug("tb: %s.%s not in cache", dbFName, pName->tname); - ctgAddTSMAFetch(&pCtx->pFetches, dbIdx, i, fetchIdx, baseResIdx + i, flag, FETCH_TSMA_SOURCE_TB_META, NULL); - taosArrayPush(pCtx->pResList, &(SMetaRes){0}); + + CTG_ERR_JRET(ctgAddTSMAFetch(&pCtx->pFetches, dbIdx, i, fetchIdx, baseResIdx + i, flag, FETCH_TSMA_SOURCE_TB_META, NULL)); + if (NULL == taosArrayPush(pCtx->pResList, &(SMetaRes){0})) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + taosHashRelease(dbCache->tbCache, pTbCache); + continue; } uint64_t suid = pTbCache->pMeta->suid; int8_t tbType = pTbCache->pMeta->tableType; CTG_UNLOCK(CTG_READ, &pTbCache->metaLock); + taosHashRelease(dbCache->tbCache, pTbCache); SName tsmaSourceTbName = *pName; @@ -3251,12 +3840,16 @@ int32_t ctgGetTbTSMAFromCache(SCatalog* pCtg, SCtgTbTSMACtx* pCtx, int32_t dbIdx if (tbType == TSDB_CHILD_TABLE) { char* stbName = taosHashAcquire(dbCache->stbCache, &suid, sizeof(uint64_t)); if (stbName) { - snprintf(tsmaSourceTbName.tname, TMIN(TSDB_TABLE_NAME_LEN, strlen(stbName) + 1), "%s", stbName); + (void)snprintf(tsmaSourceTbName.tname, TMIN(TSDB_TABLE_NAME_LEN, strlen(stbName) + 1), "%s", stbName); taosHashRelease(dbCache->stbCache, stbName); } else { ctgDebug("stb in db: %s, uid: %" PRId64 " not in cache", dbFName, suid); - ctgAddTSMAFetch(&pCtx->pFetches, dbIdx, i, fetchIdx, baseResIdx + i, flag, FETCH_TSMA_SOURCE_TB_META, NULL); - taosArrayPush(pCtx->pResList, &(SMetaRes){0}); + + CTG_ERR_JRET(ctgAddTSMAFetch(&pCtx->pFetches, dbIdx, i, fetchIdx, baseResIdx + i, flag, FETCH_TSMA_SOURCE_TB_META, NULL)); + if (NULL == taosArrayPush(pCtx->pResList, &(SMetaRes){0})) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + continue; } } @@ -3264,7 +3857,11 @@ int32_t ctgGetTbTSMAFromCache(SCatalog* pCtg, SCtgTbTSMACtx* pCtx, int32_t dbIdx // get tsma cache pCache = taosHashAcquire(dbCache->tsmaCache, tsmaSourceTbName.tname, strlen(tsmaSourceTbName.tname)); if (!pCache || !pCache->pTsmas || pCache->pTsmas->size == 0) { - taosArrayPush(pCtx->pResList, &(SMetaRes){0}); + if (NULL == taosArrayPush(pCtx->pResList, &(SMetaRes){0})) { + ctgReleaseTSMAToCache(pCtg, dbCache, pCache); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + continue; } @@ -3272,9 +3869,14 @@ int32_t ctgGetTbTSMAFromCache(SCatalog* pCtg, SCtgTbTSMACtx* pCtx, int32_t dbIdx if (hasOutOfDateTSMACache(pCache->pTsmas)) { CTG_UNLOCK(CTG_READ, &pCache->tsmaLock); taosHashRelease(dbCache->tsmaCache, pCache); + ctgDebug("tsma for tb: %s.%s not in cache", tsmaSourceTbName.tname, dbFName); - ctgAddTSMAFetch(&pCtx->pFetches, dbIdx, i, fetchIdx, baseResIdx + i, flag, FETCH_TB_TSMA, &tsmaSourceTbName); - taosArrayPush(pCtx->pResList, &(SMetaRes){0}); + + CTG_ERR_JRET(ctgAddTSMAFetch(&pCtx->pFetches, dbIdx, i, fetchIdx, baseResIdx + i, flag, FETCH_TB_TSMA, &tsmaSourceTbName)); + if (NULL == taosArrayPush(pCtx->pResList, &(SMetaRes){0})) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + CTG_CACHE_NHIT_INC(CTG_CI_TBL_TSMA, 1); continue; } @@ -3286,31 +3888,45 @@ int32_t ctgGetTbTSMAFromCache(SCatalog* pCtg, SCtgTbTSMACtx* pCtx, int32_t dbIdx ctgReleaseTSMAToCache(pCtg, dbCache, pCache); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } + pRsp->pTsmas = taosArrayInit(pCache->pTsmas->size, POINTER_BYTES); if (!pRsp->pTsmas) { ctgReleaseTSMAToCache(pCtg, dbCache, pCache); taosMemoryFreeClear(pRsp); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } + SMetaRes res = {0}; for (int32_t i = 0; i < pCache->pTsmas->size; ++i) { STSMACache *pTsmaOut = NULL; STSMACache *pTsmaCache = taosArrayGetP(pCache->pTsmas, i); code = tCloneTbTSMAInfo(pTsmaCache, &pTsmaOut); - if (code) { + if (TSDB_CODE_SUCCESS != code) { ctgReleaseTSMAToCache(pCtg, dbCache, pCache); tFreeTableTSMAInfoRsp(pRsp); taosMemoryFreeClear(pRsp); CTG_ERR_RET(code); } - taosArrayPush(pRsp->pTsmas, &pTsmaOut); + + if (NULL == taosArrayPush(pRsp->pTsmas, &pTsmaOut)) { + ctgReleaseTSMAToCache(pCtg, dbCache, pCache); + tFreeTableTSMAInfoRsp(pRsp); + taosMemoryFreeClear(pRsp); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } } res.pRes = pRsp; - taosArrayPush(pCtx->pResList, &res); CTG_UNLOCK(CTG_READ, &pCache->tsmaLock); taosHashRelease(dbCache->tsmaCache, pCache); + if (NULL == taosArrayPush(pCtx->pResList, &res)) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } } + +_return: + ctgReleaseDBCache(pCtg, dbCache); + CTG_RET(code); } @@ -3322,7 +3938,7 @@ int32_t ctgGetTSMAFromCache(SCatalog* pCtg, SCtgTbTSMACtx* pCtx, SName* pTsmaNam bool found = false; STSMACache * pTsmaOut = NULL; - tNameGetFullDbName(pTsmaName, dbFName); + (void)tNameGetFullDbName(pTsmaName, dbFName); CTG_ERR_RET(ctgAcquireDBCache(pCtg, dbFName, &pDbCache)); if (!pDbCache) { @@ -3334,10 +3950,16 @@ int32_t ctgGetTSMAFromCache(SCatalog* pCtg, SCtgTbTSMACtx* pCtx, SName* pTsmaNam while (pIter && !found) { SCtgTSMACache* pCtgCache = pIter; + CTG_LOCK(CTG_READ, &pCtgCache->tsmaLock); int32_t size = pCtgCache ? (pCtgCache->pTsmas ? pCtgCache->pTsmas->size : 0) : 0; for (int32_t i = 0; i < size; ++i) { STSMACache* pCache = taosArrayGetP(pCtgCache->pTsmas, i); + if (NULL == pCache) { + ctgError("fail to the %dth tsma in pTsmas, total:%d", i, size); + code = TSDB_CODE_CTG_INTERNAL_ERROR; + break; + } if (memcmp(pCache->name, pTsmaName->tname, TSDB_TABLE_NAME_LEN) == 0) { found = true; CTG_CACHE_NHIT_INC(CTG_CI_TBL_TSMA, 1); @@ -3346,323 +3968,44 @@ int32_t ctgGetTSMAFromCache(SCatalog* pCtg, SCtgTbTSMACtx* pCtx, SName* pTsmaNam } } CTG_UNLOCK(CTG_READ, &pCtgCache->tsmaLock); + + if (TSDB_CODE_SUCCESS != code) { + break; + } + pIter = taosHashIterate(pDbCache->tsmaCache, pIter); } + taosHashCancelIterate(pDbCache->tsmaCache, pIter); + if (found && code == TSDB_CODE_SUCCESS) { res.pRes = taosMemoryCalloc(1, sizeof(STableTSMAInfoRsp)); if (!res.pRes) { tFreeAndClearTableTSMAInfo(pTsmaOut); - CTG_RET(TSDB_CODE_OUT_OF_MEMORY); + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } + STableTSMAInfoRsp* pRsp = res.pRes; pRsp->pTsmas = taosArrayInit(1, POINTER_BYTES); if (!pRsp->pTsmas) { tFreeAndClearTableTSMAInfo(pTsmaOut); - CTG_RET(TSDB_CODE_OUT_OF_MEMORY); + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } - taosArrayPush(pRsp->pTsmas, &pTsmaOut); - taosArrayPush(pCtx->pResList, &res); + if (NULL == taosArrayPush(pRsp->pTsmas, &pTsmaOut)) { + tFreeAndClearTableTSMAInfo(pTsmaOut); + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + + if (NULL == taosArrayPush(pCtx->pResList, &res)) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } } +_return: + ctgReleaseDBCache(pCtg, pDbCache); + CTG_RET(code); } -int32_t ctgUpdateTbTSMAEnqueue(SCatalog *pCtg, STSMACache **pTsma, int32_t tsmaVersion, bool syncOp) { - int32_t code = 0; - SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); - op->opId = CTG_OP_UPDATE_TB_TSMA; - op->syncOp = syncOp; - - SCtgUpdateTbTSMAMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateTbTSMAMsg)); - if (NULL == msg) { - ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateTbTSMAMsg)); - taosMemoryFree(op); - CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); - } - - msg->pCtg = pCtg; - msg->pTsma = *pTsma; - msg->dbTsmaVersion = tsmaVersion; - msg->dbId = (*pTsma)->dbId; - - op->data = msg; - - CTG_ERR_JRET(ctgEnqueue(pCtg, op)); - - *pTsma = NULL; - return TSDB_CODE_SUCCESS; - -_return: - CTG_RET(code); -} - -int32_t ctgDropTbTSMAEnqueue(SCatalog* pCtg, const STSMACache* pTsma, bool syncOp) { - int32_t code = 0; - SCtgCacheOperation* op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); - if (!op) CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); - - op->opId = CTG_OP_DROP_TB_TSMA; - op->syncOp = syncOp; - - SCtgDropTbTSMAMsg* msg = taosMemoryCalloc(1, sizeof(SCtgDropTbTSMAMsg)); - if (!msg) { - ctgError("malloc %d failed", (int32_t)sizeof(SCtgDropTbTSMAMsg)); - taosMemoryFree(op); - CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); - } - - msg->pCtg = pCtg; - msg->dbId = pTsma->dbId; - msg->tbId = pTsma->suid; - msg->tsmaId = pTsma->tsmaId; - tstrncpy(msg->dbFName, pTsma->dbFName, TSDB_DB_FNAME_LEN); - tstrncpy(msg->tbName, pTsma->tb, TSDB_TABLE_NAME_LEN); - tstrncpy(msg->tsmaName, pTsma->name, TSDB_TABLE_NAME_LEN); - - op->data = msg; - CTG_ERR_JRET(ctgEnqueue(pCtg, op)); - return TSDB_CODE_SUCCESS; -_return: - CTG_RET(code); -} - -static SCtgCacheOperation* createDropAllTbTsmaCtgCacheOp(SCatalog* pCtg, const STSMACache* pCache, bool syncOp) { - SCtgCacheOperation* pOp = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); - if (!pOp) return NULL; - - SCtgDropTbTSMAMsg* pMsg = taosMemoryCalloc(1, sizeof(SCtgDropTbTSMAMsg)); - if (!pMsg) { - taosMemoryFree(pOp); - return NULL; - } - pOp->opId = CTG_OP_DROP_TB_TSMA; - pOp->syncOp = syncOp; - pMsg->pCtg = pCtg; - pMsg->dbId = pCache->dbId; - pMsg->tbId = pCache->suid; - pMsg->tsmaId = pCache->tsmaId; - pMsg->dropAllForTb = true; - tstrncpy(pMsg->tsmaName, pCache->name, TSDB_TABLE_NAME_LEN); - tstrncpy(pMsg->dbFName, pCache->dbFName, TSDB_DB_FNAME_LEN); - tstrncpy(pMsg->tbName, pCache->tb, TSDB_TABLE_NAME_LEN); - pOp->data = pMsg; - return pOp; -} - -int32_t ctgDropTSMAForTbEnqueue(SCatalog *pCtg, SName *pName, bool syncOp) { - ctgDebug("drop tsma meta for tb: %s.%s", pName->dbname, pName->tname); - int32_t code = 0; - SCtgDBCache *pDbCache = NULL; - SCtgCacheOperation *pOp = NULL; - char dbFName[TSDB_DB_FNAME_LEN]; - SCtgTSMACache *pCtgCache = NULL; - tNameGetFullDbName(pName, dbFName); - CTG_ERR_JRET(ctgGetDBCache(pCtg, dbFName, &pDbCache)); - if (NULL == pDbCache || !pDbCache->tsmaCache) { - goto _return; - } - - pCtgCache = taosHashAcquire(pDbCache->tsmaCache, pName->tname, strlen(pName->tname)); - if (!pCtgCache) goto _return; - - CTG_LOCK(CTG_READ, &pCtgCache->tsmaLock); - if (!pCtgCache->pTsmas || pCtgCache->pTsmas->size == 0) { - CTG_UNLOCK(CTG_READ, &pCtgCache->tsmaLock); - goto _return; - } - STSMACache *pCache = taosArrayGetP(pCtgCache->pTsmas, 0); - pOp = createDropAllTbTsmaCtgCacheOp(pCtg, pCache, syncOp); - if (!pOp) { - code = TSDB_CODE_OUT_OF_MEMORY; - CTG_UNLOCK(CTG_READ, &pCtgCache->tsmaLock); - goto _return; - } - CTG_UNLOCK(CTG_READ, &pCtgCache->tsmaLock); - CTG_ERR_JRET(ctgEnqueue(pCtg, pOp)); - taosHashRelease(pDbCache->tsmaCache, pCtgCache); - return TSDB_CODE_SUCCESS; - -_return: - if (pCtgCache) taosHashRelease(pDbCache->tsmaCache, pCtgCache); - if (pOp) { - taosMemoryFree(pOp->data); - taosMemoryFree(pOp); - } - CTG_RET(code); -} - -int32_t ctgWriteTbTSMAToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, char *tbName, - STSMACache **ppTsmaCache) { - if (NULL == dbCache->tsmaCache) { - ctgError("db is dropping, dbId:0x%" PRIx64, dbCache->dbId); - CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); - } - - STSMACache *pTsmaCache = *ppTsmaCache; - int32_t code = TSDB_CODE_SUCCESS; - - SCtgTSMACache* pCache = taosHashGet(dbCache->tsmaCache, tbName, strlen(tbName)); - if (!pCache) { - SCtgTSMACache cache = {0}; - cache.pTsmas = taosArrayInit(4, sizeof(POINTER_BYTES)); - if (!cache.pTsmas) CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); - taosArrayPush(cache.pTsmas, &pTsmaCache); - if (taosHashPut(dbCache->tsmaCache, tbName, strlen(tbName), &cache, sizeof(cache))) { - ctgError("taosHashPut new tsmacache for tb: %s.%s failed", dbFName, tbName); - CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); - } - atomic_add_fetch_64(&dbCache->dbCacheSize, strlen(tbName) + sizeof(STSMACache) + ctgGetTbTSMACacheSize(pTsmaCache)); - CTG_DB_NUM_INC(CTG_CI_TBL_TSMA); - ctgDebug("tb %s tsma updated to cache, name: %s", tbName, pTsmaCache->name); - CTG_ERR_JRET(ctgUpdateRentTSMAVersion(pCtg, dbFName, pTsmaCache)); - *ppTsmaCache = NULL; - goto _return; - } - - CTG_LOCK(CTG_WRITE, &pCache->tsmaLock); - - if (pCache->pTsmas) { - uint64_t cacheSize = 0; - for (int32_t i = 0; i < pCache->pTsmas->size; ++i) { - STableTSMAInfo* pInfo = taosArrayGetP(pCache->pTsmas, i); - if (pInfo->tsmaId == pTsmaCache->tsmaId) { - ctgDebug("tsma: %s removed from cache, history from %d to %d, reqTs from %" PRId64 " to %" PRId64 - "rspTs from %" PRId64 " to %" PRId64 " delay from %" PRId64 " to %" PRId64, - pInfo->name, pInfo->fillHistoryFinished, pTsmaCache->fillHistoryFinished, pInfo->reqTs, - pTsmaCache->reqTs, pInfo->rspTs, pTsmaCache->rspTs, pInfo->delayDuration, pTsmaCache->delayDuration); - cacheSize = ctgGetTbTSMACacheSize(pInfo); - taosArrayRemove(pCache->pTsmas, i); - atomic_sub_fetch_64(&dbCache->dbCacheSize, cacheSize); - tFreeTableTSMAInfo(pInfo); - taosMemoryFreeClear(pInfo); - break; - } - } - } else { - pCache->pTsmas = taosArrayInit(4, sizeof(POINTER_BYTES)); - if (!pCache->pTsmas) { - CTG_UNLOCK(CTG_WRITE, &pCache->tsmaLock); - CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); - } - } - // push the new cache - taosArrayPush(pCache->pTsmas, &pTsmaCache); - *ppTsmaCache = NULL; - - atomic_add_fetch_64(&dbCache->dbCacheSize, ctgGetTbTSMACacheSize(pTsmaCache)); - CTG_ERR_RET(ctgUpdateRentTSMAVersion(pCtg, dbFName, pTsmaCache)); - CTG_UNLOCK(CTG_WRITE, &pCache->tsmaLock); - ctgDebug("table %s tsma updated to cache, tsma: %s", tbName, pTsmaCache->name); -_return: - CTG_RET(code); -} - -int32_t ctgOpDropTbTSMA(SCtgCacheOperation *operation) { - int32_t code = 0; - SCtgDropTbTSMAMsg * msg = operation->data; - SCatalog *pCtg = msg->pCtg; - SCtgDBCache *dbCache = NULL; - - if (pCtg->stopUpdate) { - goto _return; - } - - CTG_ERR_JRET(ctgGetDBCache(pCtg, msg->dbFName, &dbCache)); - if (NULL == dbCache || !dbCache->tsmaCache || (msg->dbId != dbCache->dbId && msg->dbId != 0)) { - goto _return; - } - - SCtgTSMACache* pCtgCache = taosHashGet(dbCache->tsmaCache, msg->tbName, strlen(msg->tbName)); - if (!pCtgCache || !pCtgCache->pTsmas || pCtgCache->pTsmas->size == 0) { - goto _return; - } - - uint64_t cacheSize = 0; - STSMACache *pCache = NULL; - if (msg->dropAllForTb) { - CTG_LOCK(CTG_WRITE, &pCtgCache->tsmaLock); - for (int32_t i = 0; i < pCtgCache->pTsmas->size; ++i) { - pCache = taosArrayGetP(pCtgCache->pTsmas, i); - cacheSize += ctgGetTbTSMACacheSize(pCache); - ctgMetaRentRemove(&msg->pCtg->tsmaRent, pCache->tsmaId, ctgTSMAVersionSearchCompare, ctgTSMAVersionSearchCompare); - CTG_DB_NUM_DEC(CTG_CI_TBL_TSMA); - } - taosArrayDestroyP(pCtgCache->pTsmas, tFreeAndClearTableTSMAInfo); - pCtgCache->pTsmas = NULL; - ctgDebug("all tsmas for table dropped: %s.%s", msg->dbFName, msg->tbName); - taosHashRemove(dbCache->tsmaCache, msg->tbName, TSDB_TABLE_NAME_LEN); - CTG_UNLOCK(CTG_WRITE, &pCtgCache->tsmaLock); - } else { - CTG_LOCK(CTG_WRITE, &pCtgCache->tsmaLock); - pCache = taosArrayGetP(pCtgCache->pTsmas, 0); - if (msg->tbId != 0 && pCache->suid != msg->tbId) { - // table id mismatch, skip drops - CTG_UNLOCK(CTG_WRITE, &pCtgCache->tsmaLock); - goto _return; - } - for (int32_t i = 0; i < pCtgCache->pTsmas->size; ++i) { - pCache = taosArrayGetP(pCtgCache->pTsmas, i); - if (pCache->tsmaId != msg->tsmaId) { - continue; - } - cacheSize = ctgGetTbTSMACacheSize(pCache); - ctgMetaRentRemove(&msg->pCtg->tsmaRent, pCache->tsmaId, ctgTSMAVersionSearchCompare, ctgTSMAVersionSearchCompare); - taosArrayRemove(pCtgCache->pTsmas, i); - tFreeAndClearTableTSMAInfo(pCache); - CTG_DB_NUM_DEC(CTG_CI_TBL_TSMA); - break; - } - CTG_UNLOCK(CTG_WRITE, &pCtgCache->tsmaLock); - } - atomic_sub_fetch_64(&dbCache->dbCacheSize, cacheSize); - -_return: - - taosMemoryFreeClear(msg); - CTG_RET(code); -} - -int32_t ctgOpUpdateTbTSMA(SCtgCacheOperation *operation) { - int32_t code = 0; - SCtgUpdateTbTSMAMsg *msg = operation->data; - SCatalog * pCtg = msg->pCtg; - STableTSMAInfo * pTsmaInfo = msg->pTsma; - SCtgDBCache * dbCache = NULL; - - if (pCtg->stopUpdate) { - goto _return; - } - - CTG_ERR_JRET(ctgGetAddDBCache(pCtg, pTsmaInfo->dbFName, pTsmaInfo->dbId, &dbCache)); - CTG_ERR_JRET(ctgWriteTbTSMAToCache(pCtg, dbCache, pTsmaInfo->dbFName, pTsmaInfo->tb, &pTsmaInfo)); - if (dbCache && msg->dbTsmaVersion > 0) { - dbCache->tsmaVersion = msg->dbTsmaVersion; - SDbCacheInfo cacheInfo = {0}; - cacheInfo.dbId = dbCache->dbId; - if (dbCache->cfgCache.cfgInfo) { - cacheInfo.cfgVersion = dbCache->cfgCache.cfgInfo->cfgVersion; - tstrncpy(cacheInfo.dbFName, dbCache->cfgCache.cfgInfo->db, TSDB_DB_FNAME_LEN); - } - if (dbCache->vgCache.vgInfo) { - cacheInfo.vgVersion = dbCache->vgCache.vgInfo->vgVersion; - cacheInfo.numOfTable = dbCache->vgCache.vgInfo->numOfTable; - cacheInfo.stateTs = dbCache->vgCache.vgInfo->stateTs; - } - cacheInfo.tsmaVersion = dbCache->tsmaVersion; - CTG_ERR_JRET(ctgMetaRentUpdate(&msg->pCtg->dbRent, &cacheInfo, cacheInfo.dbId, sizeof(SDbCacheInfo), - ctgDbCacheInfoSortCompare, ctgDbCacheInfoSearchCompare)); - } - -_return: - - if (pTsmaInfo) { - tFreeTableTSMAInfo(pTsmaInfo); - taosMemoryFreeClear(pTsmaInfo); - } - - taosMemoryFreeClear(msg); - CTG_RET(code); -} diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c index bbd9b39f6c..a312dce164 100644 --- a/source/libs/catalog/src/ctgRemote.c +++ b/source/libs/catalog/src/ctgRemote.c @@ -57,8 +57,23 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu for (int32_t i = 0; i < taskNum; ++i) { int32_t* taskId = taosArrayGet(cbParam->taskId, i); + if (NULL == taskId) { + ctgError("taosArrayGet %d taskId failed, total:%d", i, (int32_t)taosArrayGetSize(cbParam->taskId)); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + int32_t* msgIdx = taosArrayGet(cbParam->msgIdx, i); + if (NULL == msgIdx) { + ctgError("taosArrayGet %d msgIdx failed, total:%d", i, (int32_t)taosArrayGetSize(cbParam->msgIdx)); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + SCtgTask* pTask = taosArrayGet(pJob->pTasks, *taskId); + if (NULL == pTask) { + ctgError("taosArrayGet %d SCtgTask failed, total:%d", *taskId, (int32_t)taosArrayGetSize(pJob->pTasks)); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + if (msgNum > 0) { pRsp = taosArrayGet(batchRsp.pRsps, i); @@ -89,12 +104,17 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu tReq.pTask = pTask; tReq.msgIdx = pRsp->msgIdx; SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, tReq.msgIdx); + if (NULL == pMsgCtx) { + ctgError("get task %d SCtgMsgCtx failed, taskType:%d", tReq.msgIdx, pTask->type); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + pMsgCtx->pBatchs = pBatchs; ctgDebug("QID:0x%" PRIx64 " ctg task %d idx %d start to handle rsp %s, pBatchs: %p", pJob->queryId, pTask->taskId, pRsp->msgIdx, TMSG_INFO(taskMsg.msgType + 1), pBatchs); - (*gCtgAsyncFps[pTask->type].handleRspFp)(&tReq, pRsp->reqType, &taskMsg, (pRsp->rspCode ? pRsp->rspCode : rspCode)); + (void)(*gCtgAsyncFps[pTask->type].handleRspFp)(&tReq, pRsp->reqType, &taskMsg, (pRsp->rspCode ? pRsp->rspCode : rspCode)); // error handled internal } CTG_ERR_JRET(ctgLaunchBatchs(pJob->pCtg, pJob, pBatchs)); @@ -398,7 +418,16 @@ int32_t ctgHandleMsgCallback(void* param, SDataBuf* pMsg, int32_t rspCode) { CTG_ERR_JRET(ctgHandleBatchRsp(pJob, cbParam, pMsg, rspCode)); } else { int32_t* taskId = taosArrayGet(cbParam->taskId, 0); + if (NULL == taskId) { + ctgError("taosArrayGet %d taskId failed, total:%d", 0, (int32_t)taosArrayGetSize(cbParam->taskId)); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + SCtgTask* pTask = taosArrayGet(pJob->pTasks, *taskId); + if (NULL == pTask) { + ctgError("taosArrayGet %d SCtgTask failed, total:%d", *taskId, (int32_t)taosArrayGetSize(pJob->pTasks)); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } qDebug("QID:0x%" PRIx64 " ctg task %d start to handle rsp %s", pJob->queryId, pTask->taskId, TMSG_INFO(cbParam->reqType + 1)); @@ -412,6 +441,11 @@ int32_t ctgHandleMsgCallback(void* param, SDataBuf* pMsg, int32_t rspCode) { } SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx) { + ctgError("get task %d SCtgMsgCtx failed, taskType:%d", -1, pTask->type); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + pMsgCtx->pBatchs = pBatchs; #endif @@ -432,7 +466,7 @@ _return: taosMemoryFree(pMsg->pEpSet); if (pJob) { - taosReleaseRef(gCtgMgmt.jobPool, cbParam->refId); + (void)taosReleaseRef(gCtgMgmt.jobPool, cbParam->refId); } CTG_API_LEAVE(code); @@ -450,6 +484,7 @@ int32_t ctgMakeMsgSendInfo(SCtgJob* pJob, SArray* pTaskId, int32_t batchId, SArr SCtgTaskCallbackParam* param = taosMemoryCalloc(1, sizeof(SCtgTaskCallbackParam)); if (NULL == param) { qError("calloc %d failed", (int32_t)sizeof(SCtgTaskCallbackParam)); + taosMemoryFree(msgSendInfo); CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } @@ -482,7 +517,7 @@ int32_t ctgAsyncSendMsg(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob* pJob, SMsgSendInfo* pMsgSendInfo = NULL; CTG_ERR_JRET(ctgMakeMsgSendInfo(pJob, pTaskId, batchId, pMsgIdx, msgType, &pMsgSendInfo)); - ctgUpdateSendTargetInfo(pMsgSendInfo, msgType, dbFName, vgId); + CTG_ERR_JRET(ctgUpdateSendTargetInfo(pMsgSendInfo, msgType, dbFName, vgId)); pMsgSendInfo->requestId = pConn->requestId; pMsgSendInfo->requestObjRefId = pConn->requestObjRefId; @@ -515,18 +550,25 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT void* msg, uint32_t msgSize) { int32_t code = 0; SCtgTask* pTask = tReq->pTask; - SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx); - SHashObj* pBatchs = pMsgCtx->pBatchs; SCtgJob* pJob = pTask->pJob; - SCtgBatch* pBatch = taosHashGet(pBatchs, &vgId, sizeof(vgId)); SCtgBatch newBatch = {0}; SBatchMsg req = {0}; - + SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx); + if (NULL == pMsgCtx) { + ctgError("get task %d SCtgMsgCtx failed, taskType:%d", tReq->msgIdx, pTask->type); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + SHashObj* pBatchs = pMsgCtx->pBatchs; + SCtgBatch* pBatch = taosHashGet(pBatchs, &vgId, sizeof(vgId)); if (NULL == pBatch) { newBatch.pMsgs = taosArrayInit(pJob->subTaskNum, sizeof(SBatchMsg)); newBatch.pTaskIds = taosArrayInit(pJob->subTaskNum, sizeof(int32_t)); newBatch.pMsgIdxs = taosArrayInit(pJob->subTaskNum, sizeof(int32_t)); if (NULL == newBatch.pMsgs || NULL == newBatch.pTaskIds || NULL == newBatch.pMsgIdxs) { + taosArrayDestroy(newBatch.pMsgs); + taosArrayDestroy(newBatch.pTaskIds); + taosArrayDestroy(newBatch.pMsgIdxs); CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } @@ -556,12 +598,16 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT if (CTG_TASK_GET_TB_META_BATCH == pTask->type) { SCtgTbMetasCtx* ctx = (SCtgTbMetasCtx*)pTask->taskCtx; SCtgFetch* fetch = taosArrayGet(ctx->pFetchs, tReq->msgIdx); - pName = ctgGetFetchName(ctx->pNames, fetch); + CTG_ERR_JRET(ctgGetFetchName(ctx->pNames, fetch, &pName)); } else if (CTG_TASK_GET_TB_TSMA == pTask->type){ SCtgTbTSMACtx* pCtx = pTask->taskCtx; SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx); STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx); pName = taosArrayGet(pTbReq->pTables, pFetch->tbIdx); + if (NULL == pName) { + ctgError("fail to get %d SName, totalTables:%d", pFetch->tbIdx, (int32_t)taosArrayGetSize(pTbReq->pTables)); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } } else { SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx; pName = ctx->pName; @@ -569,14 +615,26 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT } else if (TDMT_VND_GET_STREAM_PROGRESS == msgType) { SCtgTbTSMACtx* pCtx = pTask->taskCtx; SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx); + if (NULL == pFetch) { + ctgError("fail to get %d SCtgTSMAFetch, totalFetchs:%d", tReq->msgIdx, (int32_t)taosArrayGetSize(pCtx->pFetches)); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx); + if (NULL == pTbReq) { + ctgError("fail to get %d STablesReq, totalTables:%d", pFetch->dbIdx, (int32_t)taosArrayGetSize(pCtx->pNames)); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } pName = taosArrayGet(pTbReq->pTables, pFetch->tbIdx); + if (NULL == pName) { + ctgError("fail to get %d SName, totalTables:%d", pFetch->tbIdx, (int32_t)taosArrayGetSize(pTbReq->pTables)); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } } else { ctgError("invalid vnode msgType %d", msgType); CTG_ERR_JRET(TSDB_CODE_APP_ERROR); } - tNameGetFullDbName(pName, newBatch.dbFName); + (void)tNameGetFullDbName(pName, newBatch.dbFName); } newBatch.msgType = (vgId > 0) ? TDMT_VND_BATCH_META : TDMT_MND_BATCH_META; @@ -616,12 +674,16 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT if (CTG_TASK_GET_TB_META_BATCH == pTask->type) { SCtgTbMetasCtx* ctx = (SCtgTbMetasCtx*)pTask->taskCtx; SCtgFetch* fetch = taosArrayGet(ctx->pFetchs, tReq->msgIdx); - pName = ctgGetFetchName(ctx->pNames, fetch); + CTG_ERR_JRET(ctgGetFetchName(ctx->pNames, fetch, &pName)); } else if (CTG_TASK_GET_TB_TSMA == pTask->type){ SCtgTbTSMACtx* pCtx = pTask->taskCtx; SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx); STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx); pName = taosArrayGet(pTbReq->pTables, pFetch->tbIdx); + if (NULL == pName) { + ctgError("fail to get %d SName, totalTables:%d", pFetch->tbIdx, (int32_t)taosArrayGetSize(pTbReq->pTables)); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } } else { SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx; pName = ctx->pName; @@ -629,14 +691,26 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT } else if (TDMT_VND_GET_STREAM_PROGRESS == msgType) { SCtgTbTSMACtx* pCtx = pTask->taskCtx; SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx); + if (NULL == pFetch) { + ctgError("fail to get %d SCtgTSMAFetch, totalFetchs:%d", tReq->msgIdx, (int32_t)taosArrayGetSize(pCtx->pFetches)); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx); + if (NULL == pTbReq) { + ctgError("fail to get %d STablesReq, totalTables:%d", pFetch->dbIdx, (int32_t)taosArrayGetSize(pCtx->pNames)); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } pName = taosArrayGet(pTbReq->pTables, pFetch->tbIdx); + if (NULL == pName) { + ctgError("fail to get %d SName, totalTables:%d", pFetch->tbIdx, (int32_t)taosArrayGetSize(pTbReq->pTables)); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } } else { ctgError("invalid vnode msgType %d", msgType); CTG_ERR_JRET(TSDB_CODE_APP_ERROR); } - tNameGetFullDbName(pName, pBatch->dbFName); + (void)tNameGetFullDbName(pName, pBatch->dbFName); } ctgDebug("task %d %s req added to batch %d, target vgId %d", pTask->taskId, TMSG_INFO(msgType), pBatch->batchId, @@ -752,7 +826,10 @@ int32_t ctgGetQnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SArray if (NULL == pTaskId) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - taosArrayPush(pTaskId, &pTask->taskId); + if (NULL == taosArrayPush(pTaskId, &pTask->taskId)) { + taosArrayDestroy(pTaskId); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif @@ -765,7 +842,7 @@ int32_t ctgGetQnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SArray }; SRpcMsg rpcRsp = {0}; - rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp); + CTG_ERR_RET(rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp)); CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, NULL)); @@ -801,7 +878,10 @@ int32_t ctgGetDnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SArray if (NULL == pTaskId) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - taosArrayPush(pTaskId, &pTask->taskId); + if (NULL == taosArrayPush(pTaskId, &pTask->taskId)) { + taosArrayDestroy(pTaskId); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif @@ -814,7 +894,7 @@ int32_t ctgGetDnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SArray }; SRpcMsg rpcRsp = {0}; - rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp); + CTG_ERR_RET(rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp)); CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, NULL)); @@ -854,7 +934,10 @@ int32_t ctgGetDBVgInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SBuildU if (NULL == pTaskId) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - taosArrayPush(pTaskId, &pTask->taskId); + if (NULL == taosArrayPush(pTaskId, &pTask->taskId)) { + taosArrayDestroy(pTaskId); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif @@ -867,7 +950,7 @@ int32_t ctgGetDBVgInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SBuildU }; SRpcMsg rpcRsp = {0}; - rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp); + CTG_ERR_RET(rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp)); CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, input->db)); @@ -909,7 +992,10 @@ int32_t ctgGetDBCfgFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const char if (NULL == pTaskId) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - taosArrayPush(pTaskId, &pTask->taskId); + if (NULL == taosArrayPush(pTaskId, &pTask->taskId)) { + taosArrayDestroy(pTaskId); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif @@ -922,7 +1008,7 @@ int32_t ctgGetDBCfgFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const char }; SRpcMsg rpcRsp = {0}; - rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp); + CTG_ERR_RET(rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp)); CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)dbFName)); @@ -964,7 +1050,10 @@ int32_t ctgGetIndexInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const if (NULL == pTaskId) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - taosArrayPush(pTaskId, &pTask->taskId); + if (NULL == taosArrayPush(pTaskId, &pTask->taskId)) { + taosArrayDestroy(pTaskId); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif @@ -977,7 +1066,7 @@ int32_t ctgGetIndexInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const }; SRpcMsg rpcRsp = {0}; - rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp); + CTG_ERR_RET(rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp)); CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)indexName)); @@ -993,7 +1082,7 @@ int32_t ctgGetTbIndexFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SName* n int32_t reqType = TDMT_MND_GET_TABLE_INDEX; void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; char tbFName[TSDB_TABLE_FNAME_LEN]; - tNameExtractFullName(name, tbFName); + (void)tNameExtractFullName(name, tbFName); ctgDebug("try to get tb index from mnode, tbFName:%s", tbFName); @@ -1021,7 +1110,10 @@ int32_t ctgGetTbIndexFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SName* n if (NULL == pTaskId) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - taosArrayPush(pTaskId, &pTask->taskId); + if (NULL == taosArrayPush(pTaskId, &pTask->taskId)) { + taosArrayDestroy(pTaskId); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif @@ -1034,7 +1126,7 @@ int32_t ctgGetTbIndexFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SName* n }; SRpcMsg rpcRsp = {0}; - rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp); + CTG_ERR_RET(rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp)); CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)tbFName)); @@ -1076,7 +1168,10 @@ int32_t ctgGetUdfInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const ch if (NULL == pTaskId) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - taosArrayPush(pTaskId, &pTask->taskId); + if (NULL == taosArrayPush(pTaskId, &pTask->taskId)) { + taosArrayDestroy(pTaskId); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif @@ -1089,7 +1184,7 @@ int32_t ctgGetUdfInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const ch }; SRpcMsg rpcRsp = {0}; - rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp); + CTG_ERR_RET(rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp)); CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)funcName)); @@ -1131,7 +1226,10 @@ int32_t ctgGetUserDbAuthFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const if (NULL == pTaskId) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - taosArrayPush(pTaskId, &pTask->taskId); + if (NULL == taosArrayPush(pTaskId, &pTask->taskId)) { + taosArrayDestroy(pTaskId); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif @@ -1144,7 +1242,7 @@ int32_t ctgGetUserDbAuthFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const }; SRpcMsg rpcRsp = {0}; - rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp); + CTG_ERR_RET(rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp)); CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)user)); @@ -1162,7 +1260,7 @@ int32_t ctgGetTbMetaFromMnodeImpl(SCatalog* pCtg, SRequestConnInfo* pConn, const int32_t msgLen = 0; int32_t reqType = TDMT_MND_TABLE_META; char tbFName[TSDB_TABLE_FNAME_LEN]; - sprintf(tbFName, "%s.%s", dbFName, tbName); + (void)sprintf(tbFName, "%s.%s", dbFName, tbName); void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; ctgDebug("try to get table meta from mnode, tbFName:%s", tbFName); @@ -1188,7 +1286,10 @@ int32_t ctgGetTbMetaFromMnodeImpl(SCatalog* pCtg, SRequestConnInfo* pConn, const if (NULL == pTaskId) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - taosArrayPush(pTaskId, &pTask->taskId); + if (NULL == taosArrayPush(pTaskId, &pTask->taskId)) { + taosArrayDestroy(pTaskId); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif @@ -1201,7 +1302,7 @@ int32_t ctgGetTbMetaFromMnodeImpl(SCatalog* pCtg, SRequestConnInfo* pConn, const }; SRpcMsg rpcRsp = {0}; - rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp); + CTG_ERR_RET(rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp)); CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, tbFName)); @@ -1213,7 +1314,7 @@ int32_t ctgGetTbMetaFromMnodeImpl(SCatalog* pCtg, SRequestConnInfo* pConn, const int32_t ctgGetTbMetaFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, STableMetaOutput* out, SCtgTaskReq* tReq) { char dbFName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(pTableName, dbFName); + (void)tNameGetFullDbName(pTableName, dbFName); return ctgGetTbMetaFromMnodeImpl(pCtg, pConn, dbFName, (char*)pTableName->tname, out, tReq); } @@ -1222,10 +1323,10 @@ int32_t ctgGetTbMetaFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SNa STableMetaOutput* out, SCtgTaskReq* tReq) { SCtgTask* pTask = tReq ? tReq->pTask : NULL; char dbFName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(pTableName, dbFName); + (void)tNameGetFullDbName(pTableName, dbFName); int32_t reqType = TDMT_VND_TABLE_META; char tbFName[TSDB_TABLE_FNAME_LEN]; - sprintf(tbFName, "%s.%s", dbFName, pTableName->tname); + (void)sprintf(tbFName, "%s.%s", dbFName, pTableName->tname); void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; SEp* pEp = &vgroupInfo->epSet.eps[vgroupInfo->epSet.inUse]; @@ -1261,12 +1362,15 @@ int32_t ctgGetTbMetaFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SNa #else SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx; char dbFName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(ctx->pName, dbFName); + (void)tNameGetFullDbName(ctx->pName, dbFName); SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); if (NULL == pTaskId) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - taosArrayPush(pTaskId, &pTask->taskId); + if (NULL == taosArrayPush(pTaskId, &pTask->taskId)) { + taosArrayDestroy(pTaskId); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } CTG_RET(ctgAsyncSendMsg(pCtg, &vConn, pTask->pJob, pTaskId, -1, NULL, dbFName, ctx->vgId, reqType, msg, msgLen)); #endif @@ -1279,7 +1383,7 @@ int32_t ctgGetTbMetaFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SNa }; SRpcMsg rpcRsp = {0}; - rpcSendRecv(pConn->pTrans, &vgroupInfo->epSet, &rpcMsg, &rpcRsp); + CTG_ERR_RET(rpcSendRecv(pConn->pTrans, &vgroupInfo->epSet, &rpcMsg, &rpcRsp)); CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, tbFName)); @@ -1294,10 +1398,10 @@ int32_t ctgGetTableCfgFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, const S int32_t msgLen = 0; int32_t reqType = TDMT_VND_TABLE_CFG; char tbFName[TSDB_TABLE_FNAME_LEN]; - tNameExtractFullName(pTableName, tbFName); + (void)tNameExtractFullName(pTableName, tbFName); void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; char dbFName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(pTableName, dbFName); + (void)tNameGetFullDbName(pTableName, dbFName); SBuildTableInput bInput = {.vgId = vgroupInfo->vgId, .dbFName = dbFName, .tbName = (char*)pTableName->tname}; SEp* pEp = &vgroupInfo->epSet.eps[vgroupInfo->epSet.inUse]; @@ -1325,12 +1429,15 @@ int32_t ctgGetTableCfgFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, const S #else SCtgTbCfgCtx* ctx = (SCtgTbCfgCtx*)pTask->taskCtx; char dbFName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(ctx->pName, dbFName); + (void)tNameGetFullDbName(ctx->pName, dbFName); SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); if (NULL == pTaskId) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - taosArrayPush(pTaskId, &pTask->taskId); + if (NULL == taosArrayPush(pTaskId, &pTask->taskId)) { + taosArrayDestroy(pTaskId); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } CTG_RET(ctgAsyncSendMsg(pCtg, &vConn, pTask->pJob, pTaskId, -1, NULL, dbFName, ctx->pVgInfo->vgId, reqType, msg, msgLen)); @@ -1344,7 +1451,7 @@ int32_t ctgGetTableCfgFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, const S }; SRpcMsg rpcRsp = {0}; - rpcSendRecv(pConn->pTrans, &vgroupInfo->epSet, &rpcMsg, &rpcRsp); + CTG_ERR_RET(rpcSendRecv(pConn->pTrans, &vgroupInfo->epSet, &rpcMsg, &rpcRsp)); CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)tbFName)); @@ -1359,10 +1466,10 @@ int32_t ctgGetTableCfgFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const S int32_t msgLen = 0; int32_t reqType = TDMT_MND_TABLE_CFG; char tbFName[TSDB_TABLE_FNAME_LEN]; - tNameExtractFullName(pTableName, tbFName); + (void)tNameExtractFullName(pTableName, tbFName); void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; char dbFName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(pTableName, dbFName); + (void)tNameGetFullDbName(pTableName, dbFName); SBuildTableInput bInput = {.vgId = 0, .dbFName = dbFName, .tbName = (char*)pTableName->tname}; ctgDebug("try to get table cfg from mnode, tbFName:%s", tbFName); @@ -1386,7 +1493,10 @@ int32_t ctgGetTableCfgFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const S if (NULL == pTaskId) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - taosArrayPush(pTaskId, &pTask->taskId); + if (NULL == taosArrayPush(pTaskId, &pTask->taskId)) { + taosArrayDestroy(pTaskId); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif @@ -1399,7 +1509,7 @@ int32_t ctgGetTableCfgFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const S }; SRpcMsg rpcRsp = {0}; - rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp); + CTG_ERR_RET(rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp)); CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)tbFName)); @@ -1435,7 +1545,10 @@ int32_t ctgGetSvrVerFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, char** ou if (NULL == pTaskId) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - taosArrayPush(pTaskId, &pTask->taskId); + if (NULL == taosArrayPush(pTaskId, &pTask->taskId)) { + taosArrayDestroy(pTaskId); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif @@ -1448,7 +1561,7 @@ int32_t ctgGetSvrVerFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, char** ou }; SRpcMsg rpcRsp = {0}; - rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp); + CTG_ERR_RET(rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp)); CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, NULL)); @@ -1465,7 +1578,7 @@ int32_t ctgGetViewInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SName* SCtgTask* pTask = tReq ? tReq->pTask : NULL; void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; char fullName[TSDB_TABLE_FNAME_LEN]; - tNameExtractFullName(pName, fullName); + (void)tNameExtractFullName(pName, fullName); ctgDebug("try to get view info from mnode, viewFName:%s", fullName); @@ -1490,7 +1603,10 @@ int32_t ctgGetViewInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SName* if (NULL == pTaskId) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - taosArrayPush(pTaskId, &pTask->taskId); + if (NULL == taosArrayPush(pTaskId, &pTask->taskId)) { + taosArrayDestroy(pTaskId); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif @@ -1503,7 +1619,7 @@ int32_t ctgGetViewInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SName* }; SRpcMsg rpcRsp = {0}; - rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp); + CTG_ERR_RET(rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp)); CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, fullName)); @@ -1519,7 +1635,7 @@ int32_t ctgGetTbTSMAFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SNa SCtgTask* pTask = tReq ? tReq->pTask : NULL; void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; char tbFName[TSDB_TABLE_FNAME_LEN]; - tNameExtractFullName(name, tbFName); + (void)tNameExtractFullName(name, tbFName); ctgDebug("try to get tb index from mnode, tbFName:%s", tbFName); @@ -1544,7 +1660,10 @@ int32_t ctgGetTbTSMAFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SNa if (NULL == pTaskId) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - taosArrayPush(pTaskId, &pTask->taskId); + if (NULL == taosArrayPush(pTaskId, &pTask->taskId)) { + taosArrayDestroy(pTaskId); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif @@ -1557,7 +1676,7 @@ int32_t ctgGetTbTSMAFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SNa }; SRpcMsg rpcRsp = {0}; - rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp); + CTG_ERR_RET(rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp)); CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)tbFName)); @@ -1573,7 +1692,7 @@ int32_t ctgGetStreamProgressFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, c int32_t msgLen = 0; int32_t reqType = TDMT_VND_GET_STREAM_PROGRESS; char tbFName[TSDB_TABLE_FNAME_LEN]; - tNameExtractFullName(pTbName, tbFName); + (void)tNameExtractFullName(pTbName, tbFName); SCtgTask* pTask = tReq ? tReq->pTask : NULL; void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont; @@ -1602,15 +1721,17 @@ int32_t ctgGetStreamProgressFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, c CTG_RET(ctgAddBatch(pCtg, vgroupInfo->vgId, &vConn, tReq, reqType, msg, msgLen)); #else char dbFName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(pTbName, dbFName); + (void)tNameGetFullDbName(pTbName, dbFName); SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); if (NULL == pTaskId) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - taosArrayPush(pTaskId, &pTask->taskId); + if (NULL == taosArrayPush(pTaskId, &pTask->taskId)) { + taosArrayDestroy(pTaskId); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } - CTG_RET( - ctgAsyncSendMsg(pCtg, &vConn, pTask->pJob, pTaskId, -1, NULL, dbFName, vgroupInfo->vgId, reqType, msg, msgLen)); + CTG_RET(ctgAsyncSendMsg(pCtg, &vConn, pTask->pJob, pTaskId, -1, NULL, dbFName, vgroupInfo->vgId, reqType, msg, msgLen)); #endif } @@ -1621,7 +1742,7 @@ int32_t ctgGetStreamProgressFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, c }; SRpcMsg rpcRsp = {0}; - rpcSendRecv(pConn->pTrans, &vgroupInfo->epSet, &rpcMsg, &rpcRsp); + CTG_ERR_RET(rpcSendRecv(pConn->pTrans, &vgroupInfo->epSet, &rpcMsg, &rpcRsp)); CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)tbFName)); diff --git a/source/libs/catalog/src/ctgRent.c b/source/libs/catalog/src/ctgRent.c index 67b85a7825..329aee1703 100755 --- a/source/libs/catalog/src/ctgRent.c +++ b/source/libs/catalog/src/ctgRent.c @@ -100,7 +100,7 @@ int32_t ctgMetaRentUpdate(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t si CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } - memcpy(orig, meta, size); + TAOS_MEMCPY(orig, meta, size); qDebug("meta in rent updated, id:0x%" PRIx64 ", slot idx:%d, type:%d", id, widx, mgmt->type); @@ -185,8 +185,12 @@ int32_t ctgMetaRentGetImpl(SCtgRentMgmt *mgmt, void **res, uint32_t *num, int32_ } void *meta = taosArrayGet(slot->meta, 0); + if (NULL == meta) { + qError("get the 0th meta in slot failed, total:%d", (int32_t)metaNum); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } - memcpy(*res, meta, msize); + TAOS_MEMCPY(*res, meta, msize); *num = (uint32_t)metaNum; @@ -227,13 +231,14 @@ void ctgRemoveStbRent(SCatalog *pCtg, SCtgDBCache *dbCache) { return; } + int32_t code = TSDB_CODE_SUCCESS; void *pIter = taosHashIterate(dbCache->stbCache, NULL); while (pIter) { uint64_t *suid = NULL; suid = taosHashGetKey(pIter, NULL); - if (TSDB_CODE_SUCCESS == - ctgMetaRentRemove(&pCtg->stbRent, *suid, ctgStbVersionSortCompare, ctgStbVersionSearchCompare)) { + code = ctgMetaRentRemove(&pCtg->stbRent, *suid, ctgStbVersionSortCompare, ctgStbVersionSearchCompare); + if (TSDB_CODE_SUCCESS == code) { ctgDebug("stb removed from rent, suid:0x%" PRIx64, *suid); } @@ -265,6 +270,7 @@ void ctgRemoveTSMARent(SCatalog *pCtg, SCtgDBCache *dbCache) { void* pIter = taosHashIterate(dbCache->tsmaCache, NULL); while (pIter) { SCtgTSMACache* pCtgCache = pIter; + CTG_LOCK(CTG_READ, &pCtgCache->tsmaLock); int32_t size = (pCtgCache && pCtgCache->pTsmas) ? pCtgCache->pTsmas->size : 0; for (int32_t i = 0; i < size; ++i) { @@ -274,6 +280,7 @@ void ctgRemoveTSMARent(SCatalog *pCtg, SCtgDBCache *dbCache) { } } CTG_UNLOCK(CTG_READ, &pCtgCache->tsmaLock); + pIter = taosHashIterate(dbCache->tsmaCache, pIter); } } @@ -325,8 +332,10 @@ int32_t ctgUpdateRentTSMAVersion(SCatalog *pCtg, char *dbFName, const STSMACache tstrncpy(tsmaRent.name, pTsmaInfo->name, TSDB_TABLE_NAME_LEN); tstrncpy(tsmaRent.dbFName, dbFName, TSDB_DB_FNAME_LEN); tstrncpy(tsmaRent.tbName, pTsmaInfo->tb, TSDB_TABLE_NAME_LEN); + CTG_ERR_RET(ctgMetaRentUpdate(&pCtg->tsmaRent, &tsmaRent, tsmaRent.tsmaId, sizeof(STSMAVersion), ctgTSMAVersionSortCompare, ctgTSMAVersionSearchCompare)); + ctgDebug("db %s, 0x%" PRIx64 " tsma %s, 0x%" PRIx64 "version %d updated to tsmaRent", dbFName, tsmaRent.dbId, pTsmaInfo->name, pTsmaInfo->tsmaId, pTsmaInfo->version); diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index eac716339b..c46ded17b8 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -107,6 +107,10 @@ char* ctgTaskTypeStr(CTG_TASK_TYPE type) { return "[get table tag]"; case CTG_TASK_GET_VIEW: return "[get view]"; + case CTG_TASK_GET_TB_TSMA: + return "[get table TSMA]"; + case CTG_TASK_GET_TSMA: + return "[get TSMA]"; default: return "unknown"; } @@ -443,10 +447,10 @@ void ctgClearHandleMeta(SCatalog* pCtg, int64_t *pClearedSize, int64_t *pCleardN continue; } - taosHashRemove(dbCache->tbCache, key, len); - cacheSize = - len + sizeof(SCtgTbCache) + ctgGetTbMetaCacheSize(pCache->pMeta) + ctgGetTbIndexCacheSize(pCache->pIndex); - atomic_sub_fetch_64(&dbCache->dbCacheSize, cacheSize); + (void)taosHashRemove(dbCache->tbCache, key, len); + + cacheSize = len + sizeof(SCtgTbCache) + ctgGetTbMetaCacheSize(pCache->pMeta) + ctgGetTbIndexCacheSize(pCache->pIndex); + (void)atomic_sub_fetch_64(&dbCache->dbCacheSize, cacheSize); *pClearedSize += cacheSize; (*pCleardNum)++; @@ -508,10 +512,10 @@ void ctgClearHandle(SCatalog* pCtg) { ctgFreeInstDbCache(pCtg->dbCache); ctgFreeInstUserCache(pCtg->userCache); - ctgMetaRentInit(&pCtg->dbRent, gCtgMgmt.cfg.dbRentSec, CTG_RENT_DB, sizeof(SDbCacheInfo)); - ctgMetaRentInit(&pCtg->stbRent, gCtgMgmt.cfg.stbRentSec, CTG_RENT_STABLE, sizeof(SSTableVersion)); - ctgMetaRentInit(&pCtg->viewRent, gCtgMgmt.cfg.viewRentSec, CTG_RENT_VIEW, sizeof(SViewVersion)); - ctgMetaRentInit(&pCtg->tsmaRent, gCtgMgmt.cfg.tsmaRentSec, CTG_RENT_TSMA, sizeof(STSMAVersion)); + (void)ctgMetaRentInit(&pCtg->dbRent, gCtgMgmt.cfg.dbRentSec, CTG_RENT_DB, sizeof(SDbCacheInfo)); + (void)ctgMetaRentInit(&pCtg->stbRent, gCtgMgmt.cfg.stbRentSec, CTG_RENT_STABLE, sizeof(SSTableVersion)); + (void)ctgMetaRentInit(&pCtg->viewRent, gCtgMgmt.cfg.viewRentSec, CTG_RENT_VIEW, sizeof(SViewVersion)); + (void)ctgMetaRentInit(&pCtg->tsmaRent, gCtgMgmt.cfg.tsmaRentSec, CTG_RENT_TSMA, sizeof(STSMAVersion)); pCtg->dbCache = taosHashInit(gCtgMgmt.cfg.maxDBCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); @@ -525,7 +529,7 @@ void ctgClearHandle(SCatalog* pCtg) { ctgError("taosHashInit %d user cache failed", gCtgMgmt.cfg.maxUserCacheNum); } - memset(pCtg->cacheStat.cacheNum, 0, sizeof(pCtg->cacheStat.cacheNum)); + TAOS_MEMSET(pCtg->cacheStat.cacheNum, 0, sizeof(pCtg->cacheStat.cacheNum)); CTG_STAT_RT_INC(numOfOpClearCache, 1); @@ -668,7 +672,7 @@ void ctgFreeSTableMetaOutput(STableMetaOutput* pOutput) { void ctgResetTbMetaTask(SCtgTask* pTask) { SCtgTbMetaCtx* taskCtx = (SCtgTbMetaCtx*)pTask->taskCtx; - memset(&taskCtx->tbInfo, 0, sizeof(taskCtx->tbInfo)); + TAOS_MEMSET(&taskCtx->tbInfo, 0, sizeof(taskCtx->tbInfo)); taskCtx->flag = CTG_FLAG_UNKNOWN_STB; if (pTask->msgCtx.lastOut) { @@ -1087,7 +1091,10 @@ int32_t ctgAddMsgCtx(SArray* pCtxs, int32_t reqType, void* out, char* target) { } } - taosArrayPush(pCtxs, &ctx); + if (NULL == taosArrayPush(pCtxs, &ctx)) { + ctgFreeMsgCtx(&ctx); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } return TSDB_CODE_SUCCESS; } @@ -1174,13 +1181,14 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SEpSet* pMgmtEps, SDBVgInfo* d int32_t vgNum = taosArrayGetSize(dbInfo->vgArray); char db[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(pTableName, db); + (void)tNameGetFullDbName(pTableName, db); if (IS_SYS_DBNAME(pTableName->dbname)) { pVgroup->vgId = MNODE_HANDLE; if (pMgmtEps) { - memcpy(&pVgroup->epSet, pMgmtEps, sizeof(pVgroup->epSet)); + TAOS_MEMCPY(&pVgroup->epSet, pMgmtEps, sizeof(pVgroup->epSet)); } + return TSDB_CODE_SUCCESS; } @@ -1191,7 +1199,7 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SEpSet* pMgmtEps, SDBVgInfo* d SVgroupInfo* vgInfo = NULL; char tbFullName[TSDB_TABLE_FNAME_LEN]; - tNameExtractFullName(pTableName, tbFullName); + (void)tNameExtractFullName(pTableName, tbFullName); uint32_t hashValue = taosGetTbHashVal(tbFullName, (uint32_t)strlen(tbFullName), dbInfo->hashMethod, dbInfo->hashPrefix, dbInfo->hashSuffix); @@ -1243,28 +1251,41 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SEpSet* pMgmgEpSet, SCtgTaskR SVgroupInfo mgmtInfo = {0}; mgmtInfo.vgId = MNODE_HANDLE; if (pMgmgEpSet) { - memcpy(&mgmtInfo.epSet, pMgmgEpSet, sizeof(mgmtInfo.epSet)); + TAOS_MEMCPY(&mgmtInfo.epSet, pMgmgEpSet, sizeof(mgmtInfo.epSet)); } + for (int32_t i = 0; i < tbNum; ++i) { vgInfo = taosMemoryMalloc(sizeof(SVgroupInfo)); if (NULL == vgInfo) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(vgInfo, &mgmtInfo, sizeof(mgmtInfo)); + TAOS_MEMCPY(vgInfo, &mgmtInfo, sizeof(mgmtInfo)); ctgDebug("Got tb hash vgroup, vgId:%d, epNum %d, current %s port %d", vgInfo->vgId, vgInfo->epSet.numOfEps, vgInfo->epSet.eps[vgInfo->epSet.inUse].fqdn, vgInfo->epSet.eps[vgInfo->epSet.inUse].port); if (update) { SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, tReq->msgIdx); + if (NULL == pFetch) { + ctgError("fail to get the %dth SCtgFetch, total:%d", tReq->msgIdx, (int32_t)taosArrayGetSize(pCtx->pFetchs)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } SMetaRes* pRes = taosArrayGet(pCtx->pResList, pFetch->resIdx + i); + if (NULL == pFetch) { + ctgError("fail to get the %dth SMetaRes, total:%d", pFetch->resIdx + i, (int32_t)taosArrayGetSize(pCtx->pResList)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + pRes->pRes = vgInfo; } else { res.pRes = vgInfo; - taosArrayPush(pCtx->pResList, &res); + if (NULL == taosArrayPush(pCtx->pResList, &res)) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } } } + return TSDB_CODE_SUCCESS; } @@ -1281,18 +1302,35 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SEpSet* pMgmgEpSet, SCtgTaskR CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - *vgInfo = *(SVgroupInfo*)taosArrayGet(dbInfo->vgArray, 0); + SVgroupInfo* pSrcVg = (SVgroupInfo*)taosArrayGet(dbInfo->vgArray, 0); + if (NULL == pSrcVg) { + ctgError("fail to get the 0th SVgroupInfo, total:%d", (int32_t)taosArrayGetSize(dbInfo->vgArray)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + TAOS_MEMCPY(vgInfo, pSrcVg, sizeof(*pSrcVg)); ctgDebug("Got tb hash vgroup, vgId:%d, epNum %d, current %s port %d", vgInfo->vgId, vgInfo->epSet.numOfEps, vgInfo->epSet.eps[vgInfo->epSet.inUse].fqdn, vgInfo->epSet.eps[vgInfo->epSet.inUse].port); if (update) { SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, tReq->msgIdx); + if (NULL == pFetch) { + ctgError("fail to get the %dth SCtgFetch, total:%d", tReq->msgIdx, (int32_t)taosArrayGetSize(pCtx->pFetchs)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } SMetaRes* pRes = taosArrayGet(pCtx->pResList, pFetch->resIdx + i); + if (NULL == pRes) { + ctgError("fail to get the %dth SMetaRes, total:%d", pFetch->resIdx + i, (int32_t)taosArrayGetSize(pCtx->pResList)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + pRes->pRes = vgInfo; } else { res.pRes = vgInfo; - taosArrayPush(pCtx->pResList, &res); + if (NULL == taosArrayPush(pCtx->pResList, &res)) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } } } @@ -1300,16 +1338,20 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SEpSet* pMgmgEpSet, SCtgTaskR } char tbFullName[TSDB_TABLE_FNAME_LEN]; - sprintf(tbFullName, "%s.", dbFName); + (void)sprintf(tbFullName, "%s.", dbFName); int32_t offset = strlen(tbFullName); SName* pName = NULL; int32_t tbNameLen = 0; for (int32_t i = 0; i < tbNum; ++i) { pName = taosArrayGet(pNames, i); + if (NULL == pName) { + ctgError("fail to get the %dth SName, total:%d", i, (int32_t)taosArrayGetSize(pNames)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } tbNameLen = offset + strlen(pName->tname); - strcpy(tbFullName + offset, pName->tname); + TAOS_STRCPY(tbFullName + offset, pName->tname); uint32_t hashValue = taosGetTbHashVal(tbFullName, (uint32_t)strlen(tbFullName), dbInfo->hashMethod, dbInfo->hashPrefix, dbInfo->hashSuffix); @@ -1334,11 +1376,22 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SEpSet* pMgmgEpSet, SCtgTaskR if (update) { SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, tReq->msgIdx); + if (NULL == pFetch) { + ctgError("fail to get the %dth SCtgFetch, total:%d", tReq->msgIdx, (int32_t)taosArrayGetSize(pCtx->pFetchs)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } SMetaRes* pRes = taosArrayGet(pCtx->pResList, pFetch->resIdx + i); + if (NULL == pRes) { + ctgError("fail to get the %dth SMetaRes, total:%d", pFetch->resIdx + i, (int32_t)taosArrayGetSize(pCtx->pResList)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + pRes->pRes = pNewVg; } else { res.pRes = pNewVg; - taosArrayPush(pCtx->pResList, &res); + if (NULL == taosArrayPush(pCtx->pResList, &res)) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } } } @@ -1351,7 +1404,6 @@ int32_t ctgGetVgIdsFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, char* dbFNam CTG_ERR_RET(ctgMakeVgArray(dbInfo)); int32_t vgNum = taosArrayGetSize(dbInfo->vgArray); - if (vgNum <= 0) { ctgError("db vgroup cache invalid, db:%s, vgroup number:%d", dbFName, vgNum); CTG_ERR_RET(TSDB_CODE_TSC_DB_NOT_SELECTED); @@ -1359,11 +1411,11 @@ int32_t ctgGetVgIdsFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, char* dbFNam SVgroupInfo* vgInfo = NULL; char tbFullName[TSDB_TABLE_FNAME_LEN]; - snprintf(tbFullName, sizeof(tbFullName), "%s.", dbFName); + (void)snprintf(tbFullName, sizeof(tbFullName), "%s.", dbFName); int32_t offset = strlen(tbFullName); for (int32_t i = 0; i < tbNum; ++i) { - snprintf(tbFullName + offset, sizeof(tbFullName) - offset, "%s", pTbs[i]); + (void)snprintf(tbFullName + offset, sizeof(tbFullName) - offset, "%s", pTbs[i]); uint32_t hashValue = taosGetTbHashVal(tbFullName, (uint32_t)strlen(tbFullName), dbInfo->hashMethod, dbInfo->hashPrefix, dbInfo->hashSuffix); @@ -1475,7 +1527,11 @@ int32_t ctgMakeVgArray(SDBVgInfo* dbInfo) { void* pIter = taosHashIterate(dbInfo->vgHash, NULL); while (pIter) { - taosArrayPush(dbInfo->vgArray, pIter); + if (NULL == taosArrayPush(dbInfo->vgArray, pIter)) { + taosHashCancelIterate(dbInfo->vgHash, pIter); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + pIter = taosHashIterate(dbInfo->vgHash, pIter); } @@ -1494,7 +1550,7 @@ int32_t ctgCloneVgInfo(SDBVgInfo* src, SDBVgInfo** dst) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(*dst, src, sizeof(SDBVgInfo)); + TAOS_MEMCPY(*dst, src, sizeof(SDBVgInfo)); size_t hashSize = taosHashGetSize(src->vgHash); (*dst)->vgHash = taosHashInit(hashSize, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); @@ -1522,6 +1578,11 @@ int32_t ctgCloneVgInfo(SDBVgInfo* src, SDBVgInfo** dst) { if (src->vgArray) { (*dst)->vgArray = taosArrayDup(src->vgArray, NULL); + if (NULL == (*dst)->vgArray) { + taosHashCleanup((*dst)->vgHash); + taosMemoryFreeClear(*dst); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } } return TSDB_CODE_SUCCESS; @@ -1534,7 +1595,7 @@ int32_t ctgCloneMetaOutput(STableMetaOutput* output, STableMetaOutput** pOutput) CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(*pOutput, output, sizeof(STableMetaOutput)); + TAOS_MEMCPY(*pOutput, output, sizeof(STableMetaOutput)); if (output->tbMeta) { int32_t metaSize = CTG_META_SIZE(output->tbMeta); @@ -1542,6 +1603,7 @@ int32_t ctgCloneMetaOutput(STableMetaOutput* output, STableMetaOutput** pOutput) if (useCompress(output->tbMeta->tableType) && (*pOutput)->tbMeta->schemaExt) { schemaExtSize = output->tbMeta->tableInfo.numOfColumns * sizeof(SSchemaExt); } + (*pOutput)->tbMeta = taosMemoryMalloc(metaSize + schemaExtSize); qDebug("tbMeta cloned, size:%d, p:%p", metaSize, (*pOutput)->tbMeta); if (NULL == (*pOutput)->tbMeta) { @@ -1550,10 +1612,10 @@ int32_t ctgCloneMetaOutput(STableMetaOutput* output, STableMetaOutput** pOutput) CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy((*pOutput)->tbMeta, output->tbMeta, metaSize); + TAOS_MEMCPY((*pOutput)->tbMeta, output->tbMeta, metaSize); if (useCompress(output->tbMeta->tableType) && (*pOutput)->tbMeta->schemaExt) { (*pOutput)->tbMeta->schemaExt = (SSchemaExt *)((char *)(*pOutput)->tbMeta + metaSize); - memcpy((*pOutput)->tbMeta->schemaExt, output->tbMeta->schemaExt, schemaExtSize); + TAOS_MEMCPY((*pOutput)->tbMeta->schemaExt, output->tbMeta->schemaExt, schemaExtSize); } else { (*pOutput)->tbMeta->schemaExt = NULL; } @@ -1576,8 +1638,18 @@ int32_t ctgCloneTableIndex(SArray* pIndex, SArray** pRes) { for (int32_t i = 0; i < num; ++i) { STableIndexInfo* pInfo = taosArrayGet(pIndex, i); + if (NULL == pInfo) { + qError("fail to get the %dth STableIndexInfo, total:%d", i, (int32_t)taosArrayGetSize(pIndex)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } pInfo = taosArrayPush(*pRes, pInfo); + if (NULL == pInfo) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } pInfo->expr = taosStrdup(pInfo->expr); + if (NULL == pInfo->expr) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } } return TSDB_CODE_SUCCESS; @@ -1588,6 +1660,9 @@ int32_t ctgUpdateSendTargetInfo(SMsgSendInfo* pMsgSendInfo, int32_t msgType, cha pMsgSendInfo->target.type = TARGET_TYPE_VNODE; pMsgSendInfo->target.vgId = vgId; pMsgSendInfo->target.dbFName = taosStrdup(dbFName); + if (NULL == pMsgSendInfo->target.dbFName) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } } else { pMsgSendInfo->target.type = TARGET_TYPE_MNODE; } @@ -1604,6 +1679,11 @@ int32_t ctgGetTablesReqNum(SArray* pList) { int32_t n = taosArrayGetSize(pList); for (int32_t i = 0; i < n; ++i) { STablesReq* pReq = taosArrayGet(pList, i); + if (NULL == pReq) { + qError("fail to get the %dth STablesReq, total:%d", i, (int32_t)taosArrayGetSize(pList)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + total += taosArrayGetSize(pReq->pTables); } @@ -1613,6 +1693,9 @@ int32_t ctgGetTablesReqNum(SArray* pList) { int32_t ctgAddFetch(SArray** pFetchs, int32_t dbIdx, int32_t tbIdx, int32_t* fetchIdx, int32_t resIdx, int32_t flag) { if (NULL == (*pFetchs)) { *pFetchs = taosArrayInit(CTG_DEFAULT_FETCH_NUM, sizeof(SCtgFetch)); + if (NULL == *pFetchs) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } } SCtgFetch fetch = {0}; @@ -1622,28 +1705,63 @@ int32_t ctgAddFetch(SArray** pFetchs, int32_t dbIdx, int32_t tbIdx, int32_t* fet fetch.resIdx = resIdx; fetch.flag = flag; - taosArrayPush(*pFetchs, &fetch); + if (NULL == taosArrayPush(*pFetchs, &fetch)) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } return TSDB_CODE_SUCCESS; } -SName* ctgGetFetchName(SArray* pNames, SCtgFetch* pFetch) { +int32_t ctgGetFetchName(SArray* pNames, SCtgFetch* pFetch, SName** ppName) { STablesReq* pReq = (STablesReq*)taosArrayGet(pNames, pFetch->dbIdx); - return (SName*)taosArrayGet(pReq->pTables, pFetch->tbIdx); + if (NULL == pReq) { + qError("fail to get the %dth tb in pTables, tbNum:%d", pFetch->tbIdx, (int32_t)taosArrayGetSize(pReq->pTables)); + return TSDB_CODE_CTG_INTERNAL_ERROR; + } + + *ppName = (SName*)taosArrayGet(pReq->pTables, pFetch->tbIdx); + if (NULL == *ppName) { + qError("fail to get the %dth tb in pTables, tbNum:%d", pFetch->tbIdx, (int32_t)taosArrayGetSize(pReq->pTables)); + return TSDB_CODE_CTG_INTERNAL_ERROR; + } + + return TSDB_CODE_SUCCESS; } -static void* ctgCloneDbVgroup(void* pSrc) { return taosArrayDup((const SArray*)pSrc, NULL); } +static int32_t ctgCloneDbVgroup(void* pSrc, void** ppDst) { +#if 0 + if (NULL == pSrc) { + *ppDst = NULL; + return TSDB_CODE_SUCCESS; + } + + *ppDst = taosArrayDup((const SArray*)pSrc, NULL); + return (*ppDst) ? TSDB_CODE_SUCCESS : TSDB_CODE_OUT_OF_MEMORY; +#else + return TSDB_CODE_CTG_INTERNAL_ERROR; +#endif +} static void ctgFreeDbVgroup(void* p) { taosArrayDestroy((SArray*)((SMetaRes*)p)->pRes); } -void* ctgCloneDbCfgInfo(void* pSrc) { +int32_t ctgCloneDbCfgInfo(void* pSrc, SDbCfgInfo** ppDst) { SDbCfgInfo* pDst = taosMemoryMalloc(sizeof(SDbCfgInfo)); if (NULL == pDst) { - return NULL; + return terrno; } - memcpy(pDst, pSrc, sizeof(SDbCfgInfo)); - pDst->pRetensions = taosArrayDup(((SDbCfgInfo *)pSrc)->pRetensions, NULL); - return pDst; + + TAOS_MEMCPY(pDst, pSrc, sizeof(SDbCfgInfo)); + if (((SDbCfgInfo *)pSrc)->pRetensions) { + pDst->pRetensions = taosArrayDup(((SDbCfgInfo *)pSrc)->pRetensions, NULL); + if (NULL == pDst->pRetensions) { + taosMemoryFree(pDst); + return terrno; + } + } + + *ppDst = pDst; + + return TSDB_CODE_SUCCESS; } static void ctgFreeDbCfgInfo(void* p) { @@ -1651,13 +1769,19 @@ static void ctgFreeDbCfgInfo(void* p) { freeDbCfgInfo(pDst); } -static void* ctgCloneDbInfo(void* pSrc) { +static int32_t ctgCloneDbInfo(void* pSrc, void** ppDst) { +#if 0 SDbInfo* pDst = taosMemoryMalloc(sizeof(SDbInfo)); if (NULL == pDst) { - return NULL; + return TSDB_CODE_OUT_OF_MEMORY; } - memcpy(pDst, pSrc, sizeof(SDbInfo)); - return pDst; + + TAOS_MEMCPY(pDst, pSrc, sizeof(SDbInfo)); + + return TSDB_CODE_SUCCESS; +#else + return TSDB_CODE_CTG_INTERNAL_ERROR; +#endif } static void ctgFreeDbInfo(void* p) { taosMemoryFree(((SMetaRes*)p)->pRes); } @@ -1690,55 +1814,83 @@ static void ctgFreeDbInfo(void* p) { taosMemoryFree(((SMetaRes*)p)->pRes); } static void ctgFreeTableMeta(void* p) { taosMemoryFree(((SMetaRes*)p)->pRes); } -static void* ctgCloneVgroupInfo(void* pSrc) { +static int32_t ctgCloneVgroupInfo(void* pSrc, void** ppDst) { +#if 0 SVgroupInfo* pDst = taosMemoryMalloc(sizeof(SVgroupInfo)); if (NULL == pDst) { return NULL; } memcpy(pDst, pSrc, sizeof(SVgroupInfo)); return pDst; +#else + return TSDB_CODE_CTG_INTERNAL_ERROR; +#endif } static void ctgFreeVgroupInfo(void* p) { taosMemoryFree(((SMetaRes*)p)->pRes); } -static void* ctgCloneTableIndexs(void* pSrc) { return taosArrayDup((const SArray*)pSrc, NULL); } +static int32_t ctgCloneTableIndexs(void* pSrc, void** ppDst) { +#if 0 + return taosArrayDup((const SArray*)pSrc, NULL); +#else + return TSDB_CODE_CTG_INTERNAL_ERROR; +#endif +} static void ctgFreeTableIndexs(void* p) { taosArrayDestroy((SArray*)((SMetaRes*)p)->pRes); } -static void* ctgCloneFuncInfo(void* pSrc) { +static int32_t ctgCloneFuncInfo(void* pSrc, void** ppDst) { +#if 0 SFuncInfo* pDst = taosMemoryMalloc(sizeof(SFuncInfo)); if (NULL == pDst) { return NULL; } memcpy(pDst, pSrc, sizeof(SFuncInfo)); return pDst; +#else + return TSDB_CODE_CTG_INTERNAL_ERROR; +#endif } static void ctgFreeFuncInfo(void* p) { taosMemoryFree(((SMetaRes*)p)->pRes); } -static void* ctgCloneIndexInfo(void* pSrc) { +static int32_t ctgCloneIndexInfo(void* pSrc) { +#if 0 SIndexInfo* pDst = taosMemoryMalloc(sizeof(SIndexInfo)); if (NULL == pDst) { return NULL; } memcpy(pDst, pSrc, sizeof(SIndexInfo)); return pDst; +#else + return TSDB_CODE_CTG_INTERNAL_ERROR; +#endif } static void ctgFreeIndexInfo(void* p) { taosMemoryFree(((SMetaRes*)p)->pRes); } -static void* ctgCloneUserAuth(void* pSrc) { +static int32_t ctgCloneUserAuth(void* pSrc) { +#if 0 bool* pDst = taosMemoryMalloc(sizeof(bool)); if (NULL == pDst) { return NULL; } *pDst = *(bool*)pSrc; return pDst; +#else + return TSDB_CODE_CTG_INTERNAL_ERROR; +#endif } static void ctgFreeUserAuth(void* p) { taosMemoryFree(((SMetaRes*)p)->pRes); } -static void* ctgCloneQnodeList(void* pSrc) { return taosArrayDup((const SArray*)pSrc, NULL); } +static int32_t ctgCloneQnodeList(void* pSrc) { +#if 0 + return taosArrayDup((const SArray*)pSrc, NULL); +#else + return TSDB_CODE_CTG_INTERNAL_ERROR; +#endif +} static void ctgFreeQnodeList(void* p) { taosArrayDestroy((SArray*)((SMetaRes*)p)->pRes); } @@ -1753,11 +1905,18 @@ static void ctgFreeQnodeList(void* p) { taosArrayDestroy((SArray*)((SMetaRes*)p) static void ctgFreeTableCfg(void* p) { taosMemoryFree(((SMetaRes*)p)->pRes); } -static void* ctgCloneDnodeList(void* pSrc) { return taosArrayDup((const SArray*)pSrc, NULL); } +static int32_t ctgCloneDnodeList(void* pSrc) { +#if 0 + return taosArrayDup((const SArray*)pSrc, NULL); +#else + return TSDB_CODE_CTG_INTERNAL_ERROR; +#endif +} static void ctgFreeDnodeList(void* p) { taosArrayDestroy((SArray*)((SMetaRes*)p)->pRes); } -static void* ctgCloneViewMeta(void* pSrc) { +static int32_t ctgCloneViewMeta(void* pSrc) { +#if 0 SViewMeta* pSrcMeta = pSrc; SViewMeta* pDst = taosMemoryMalloc(sizeof(SViewMeta)); if (NULL == pDst) { @@ -1771,6 +1930,9 @@ static void* ctgCloneViewMeta(void* pSrc) { } memcpy(pDst->pSchema, pSrcMeta->pSchema, pSrcMeta->numOfCols * sizeof(*pSrcMeta->pSchema)); return pDst; +#else + return TSDB_CODE_CTG_INTERNAL_ERROR; +#endif } static void ctgFreeViewMeta(void* p) { @@ -1798,8 +1960,8 @@ int32_t ctgChkSetTbAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) { char tbFName[TSDB_TABLE_FNAME_LEN]; char dbFName[TSDB_DB_FNAME_LEN]; - tNameExtractFullName(&req->pRawReq->tbName, tbFName); - tNameGetFullDbName(&req->pRawReq->tbName, dbFName); + (void)tNameExtractFullName(&req->pRawReq->tbName, tbFName); + (void)tNameGetFullDbName(&req->pRawReq->tbName, dbFName); while (true) { taosMemoryFreeClear(pMeta); @@ -1851,7 +2013,7 @@ int32_t ctgChkSetTbAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) { continue; } - sprintf(tbFName, "%s.%s", dbFName, stbName); + (void)sprintf(tbFName, "%s.%s", dbFName, stbName); continue; } @@ -1898,7 +2060,7 @@ int32_t ctgChkSetBasicAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res } char dbFName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(&pReq->tbName, dbFName); + (void)tNameGetFullDbName(&pReq->tbName, dbFName); // since that we add read/write previliges when create db, there is no need to check createdDbs #if 0 @@ -1982,9 +2144,9 @@ int32_t ctgChkSetViewAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) char viewFName[TSDB_VIEW_FNAME_LEN]; if (IS_SYS_DBNAME(req->pRawReq->tbName.dbname)) { - snprintf(viewFName, sizeof(viewFName), "%s.%s", req->pRawReq->tbName.dbname, req->pRawReq->tbName.tname); + (void)snprintf(viewFName, sizeof(viewFName), "%s.%s", req->pRawReq->tbName.dbname, req->pRawReq->tbName.tname); } else { - tNameExtractFullName(&req->pRawReq->tbName, viewFName); + (void)tNameExtractFullName(&req->pRawReq->tbName, viewFName); } int32_t len = strlen(viewFName) + 1; @@ -2358,7 +2520,7 @@ void ctgGetGlobalCacheStat(SCtgCacheStat* pStat) { pIter = taosHashIterate(gCtgMgmt.pCluster, pIter); } - memcpy(pStat, &gCtgMgmt.statInfo.cache, sizeof(gCtgMgmt.statInfo.cache)); + TAOS_MEMCPY(pStat, &gCtgMgmt.statInfo.cache, sizeof(gCtgMgmt.statInfo.cache)); } void ctgGetGlobalCacheSize(uint64_t *pSize) { @@ -2385,12 +2547,19 @@ int32_t ctgBuildViewNullRes(SCtgTask* pTask, SCtgViewsCtx* pCtx) { int32_t dbNum = taosArrayGetSize(pCtx->pNames); for (int32_t i = 0; i < dbNum; ++i) { STablesReq* pReq = taosArrayGet(pCtx->pNames, i); + if (NULL == pReq) { + qError("fail to get the %dth STablesReq, total:%d", i, (int32_t)taosArrayGetSize(pCtx->pNames)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + int32_t viewNum = taosArrayGetSize(pReq->pTables); ctgDebug("start to check views in db %s, viewNum %d", pReq->dbFName, viewNum); for (int32_t m = 0; m < viewNum; ++m) { - taosArrayPush(pCtx->pResList, &(SMetaData){0}); + if (NULL == taosArrayPush(pCtx->pResList, &(SMetaData){0})) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } } } @@ -2415,17 +2584,27 @@ int32_t dupViewMetaFromRsp(SViewMetaRsp* pRsp, SViewMeta* pViewMeta) { if (pViewMeta->pSchema == NULL) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - memcpy(pViewMeta->pSchema, pRsp->pSchema, pViewMeta->numOfCols * sizeof(SSchema)); + + TAOS_MEMCPY(pViewMeta->pSchema, pRsp->pSchema, pViewMeta->numOfCols * sizeof(SSchema)); return TSDB_CODE_SUCCESS; } uint64_t ctgGetTbTSMACacheSize(STableTSMAInfo* pTsmaInfo) { - if (!pTsmaInfo) return 0; + if (!pTsmaInfo) { + return 0; + } uint64_t size = sizeof(STableTSMAInfo); - if (pTsmaInfo->pFuncs) size += sizeof(STableTSMAFuncInfo) * pTsmaInfo->pFuncs->size; - if (pTsmaInfo->pTags) size += sizeof(SSchema) * pTsmaInfo->pTags->size; - if (pTsmaInfo->pUsedCols) size += sizeof(SSchema) * pTsmaInfo->pUsedCols->size; + if (pTsmaInfo->pFuncs) { + size += sizeof(STableTSMAFuncInfo) * pTsmaInfo->pFuncs->size; + } + if (pTsmaInfo->pTags) { + size += sizeof(SSchema) * pTsmaInfo->pTags->size; + } + if (pTsmaInfo->pUsedCols) { + size += sizeof(SSchema) * pTsmaInfo->pUsedCols->size; + } + return size; } @@ -2435,8 +2614,14 @@ bool hasOutOfDateTSMACache(SArray* pTsmas) { } for (int32_t i = 0; i < pTsmas->size; ++i) { STSMACache* pTsmaInfo = taosArrayGetP(pTsmas, i); - if (isCtgTSMACacheOutOfDate(pTsmaInfo)) return true; + if (NULL == pTsmaInfo) { + ASSERT(0); + } + if (isCtgTSMACacheOutOfDate(pTsmaInfo)) { + return true; + } } + return false; } @@ -2457,6 +2642,9 @@ int32_t ctgAddTSMAFetch(SArray** pFetchs, int32_t dbIdx, int32_t tbIdx, int32_t* CTG_TSMA_FETCH_TYPE fetchType, const SName* sourceTbName) { if (NULL == (*pFetchs)) { *pFetchs = taosArrayInit(CTG_DEFAULT_FETCH_NUM, sizeof(SCtgTSMAFetch)); + if (NULL == *pFetchs) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } } SCtgTSMAFetch fetch = {0}; @@ -2467,9 +2655,13 @@ int32_t ctgAddTSMAFetch(SArray** pFetchs, int32_t dbIdx, int32_t tbIdx, int32_t* fetch.flag = flag; fetch.fetchType = fetchType; - if (sourceTbName) fetch.tsmaSourceTbName = *sourceTbName; + if (sourceTbName) { + fetch.tsmaSourceTbName = *sourceTbName; + } - taosArrayPush(*pFetchs, &fetch); + if (NULL == taosArrayPush(*pFetchs, &fetch)) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } return TSDB_CODE_SUCCESS; } diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index f906e3ec3e..37860ac202 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -77,10 +77,10 @@ void schtInitLogFile() { const int32_t maxLogFileNum = 10; tsAsyncLog = 0; qDebugFlag = 159; - strcpy(tsLogDir, TD_LOG_DIR_PATH); + TAOS_STRCPY(tsLogDir, TD_LOG_DIR_PATH); if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { - printf("failed to open log file in directory:%s\n", tsLogDir); + (void)printf("failed to open log file in directory:%s\n", tsLogDir); } } @@ -628,7 +628,7 @@ void schtFreeQueryJob(int32_t freeThread) { schedulerFreeJob(&job, 0); if (freeThread) { if (++freeNum % schtTestPrintNum == 0) { - printf("FreeNum:%d\n", freeNum); + (void)printf("FreeNum:%d\n", freeNum); } } } @@ -667,7 +667,7 @@ void *schtRunJobThread(void *aa) { SQueryNodeLoad load = {0}; load.addr.epSet.numOfEps = 1; - strcpy(load.addr.epSet.eps[0].fqdn, "qnode0.ep"); + TAOS_STRCPY(load.addr.epSet.eps[0].fqdn, "qnode0.ep"); load.addr.epSet.eps[0].port = 6031; if (NULL == taosArrayPush(qnodeList, &load)) { assert(0); @@ -800,7 +800,7 @@ void *schtRunJobThread(void *aa) { schtFreeQueryDag(dag); if (++jobFinished % schtTestPrintNum == 0) { - printf("jobFinished:%d\n", jobFinished); + (void)printf("jobFinished:%d\n", jobFinished); } ++schtQueryId; @@ -834,9 +834,9 @@ TEST(queryTest, normalCase) { SQueryNodeLoad load = {0}; load.addr.epSet.numOfEps = 1; - strcpy(load.addr.epSet.eps[0].fqdn, "qnode0.ep"); + TAOS_STRCPY(load.addr.epSet.eps[0].fqdn, "qnode0.ep"); load.addr.epSet.eps[0].port = 6031; - taosArrayPush(qnodeList, &load); + assert(taosArrayPush(qnodeList, &load) != NULL); int32_t code = schedulerInit(); ASSERT_EQ(code, 0); @@ -872,7 +872,7 @@ TEST(queryTest, normalCase) { SDataBuf msg = {0}; void *rmsg = NULL; - schtBuildQueryRspMsg(&msg.len, &rmsg); + assert(0 == schtBuildQueryRspMsg(&msg.len, &rmsg)); msg.msgType = TDMT_SCH_QUERY_RSP; msg.pData = rmsg; @@ -888,7 +888,7 @@ TEST(queryTest, normalCase) { if (JOB_TASK_STATUS_EXEC == task->status) { SDataBuf msg = {0}; void *rmsg = NULL; - schtBuildQueryRspMsg(&msg.len, &rmsg); + assert(0 == schtBuildQueryRspMsg(&msg.len, &rmsg)); msg.msgType = TDMT_SCH_QUERY_RSP; msg.pData = rmsg; @@ -909,10 +909,10 @@ TEST(queryTest, normalCase) { } TdThreadAttr thattr; - taosThreadAttrInit(&thattr); + assert(0 == taosThreadAttrInit(&thattr)); TdThread thread1; - taosThreadCreate(&(thread1), &thattr, schtCreateFetchRspThread, &job); + assert(0 == taosThreadCreate(&(thread1), &thattr, schtCreateFetchRspThread, &job)); void *data = NULL; req.syncReq = true; @@ -926,13 +926,13 @@ TEST(queryTest, normalCase) { ASSERT_EQ(pRsp->numOfRows, 10); taosMemoryFreeClear(data); - schReleaseJob(job); + (void)schReleaseJob(job); schedulerDestroy(); schedulerFreeJob(&job, 0); - taosThreadJoin(thread1, NULL); + (void)taosThreadJoin(thread1, NULL); } TEST(queryTest, readyFirstCase) { @@ -948,9 +948,9 @@ TEST(queryTest, readyFirstCase) { SQueryNodeLoad load = {0}; load.addr.epSet.numOfEps = 1; - strcpy(load.addr.epSet.eps[0].fqdn, "qnode0.ep"); + TAOS_STRCPY(load.addr.epSet.eps[0].fqdn, "qnode0.ep"); load.addr.epSet.eps[0].port = 6031; - taosArrayPush(qnodeList, &load); + assert(NULL != taosArrayPush(qnodeList, &load)); int32_t code = schedulerInit(); ASSERT_EQ(code, 0); @@ -985,7 +985,7 @@ TEST(queryTest, readyFirstCase) { SDataBuf msg = {0}; void *rmsg = NULL; - schtBuildQueryRspMsg(&msg.len, &rmsg); + assert(0 == schtBuildQueryRspMsg(&msg.len, &rmsg)); msg.msgType = TDMT_SCH_QUERY_RSP; msg.pData = rmsg; @@ -1002,7 +1002,7 @@ TEST(queryTest, readyFirstCase) { if (JOB_TASK_STATUS_EXEC == task->status) { SDataBuf msg = {0}; void *rmsg = NULL; - schtBuildQueryRspMsg(&msg.len, &rmsg); + assert(0 == schtBuildQueryRspMsg(&msg.len, &rmsg)); msg.msgType = TDMT_SCH_QUERY_RSP; msg.pData = rmsg; @@ -1023,10 +1023,10 @@ TEST(queryTest, readyFirstCase) { } TdThreadAttr thattr; - taosThreadAttrInit(&thattr); + assert(0 == taosThreadAttrInit(&thattr)); TdThread thread1; - taosThreadCreate(&(thread1), &thattr, schtCreateFetchRspThread, &job); + assert(0 == taosThreadCreate(&(thread1), &thattr, schtCreateFetchRspThread, &job)); void *data = NULL; req.syncReq = true; @@ -1039,13 +1039,13 @@ TEST(queryTest, readyFirstCase) { ASSERT_EQ(pRsp->numOfRows, 10); taosMemoryFreeClear(data); - schReleaseJob(job); + (void)schReleaseJob(job); schedulerDestroy(); schedulerFreeJob(&job, 0); - taosThreadJoin(thread1, NULL); + (void)taosThreadJoin(thread1, NULL); } TEST(queryTest, flowCtrlCase) { @@ -1065,9 +1065,9 @@ TEST(queryTest, flowCtrlCase) { SQueryNodeLoad load = {0}; load.addr.epSet.numOfEps = 1; - strcpy(load.addr.epSet.eps[0].fqdn, "qnode0.ep"); + TAOS_STRCPY(load.addr.epSet.eps[0].fqdn, "qnode0.ep"); load.addr.epSet.eps[0].port = 6031; - taosArrayPush(qnodeList, &load); + assert(NULL != taosArrayPush(qnodeList, &load)); int32_t code = schedulerInit(); ASSERT_EQ(code, 0); @@ -1078,7 +1078,7 @@ TEST(queryTest, flowCtrlCase) { schtSetExecNode(); schtSetAsyncSendMsgToServer(); - initTaskQueue(); + assert(0 == initTaskQueue()); int32_t queryDone = 0; SRequestConnInfo conn = {0}; @@ -1106,7 +1106,7 @@ TEST(queryTest, flowCtrlCase) { if (JOB_TASK_STATUS_EXEC == task->status && 0 != task->lastMsgType) { SDataBuf msg = {0}; void *rmsg = NULL; - schtBuildQueryRspMsg(&msg.len, &rmsg); + assert(0 == schtBuildQueryRspMsg(&msg.len, &rmsg)); msg.msgType = TDMT_SCH_QUERY_RSP; msg.pData = rmsg; @@ -1120,10 +1120,10 @@ TEST(queryTest, flowCtrlCase) { } TdThreadAttr thattr; - taosThreadAttrInit(&thattr); + assert(0 == taosThreadAttrInit(&thattr)); TdThread thread1; - taosThreadCreate(&(thread1), &thattr, schtCreateFetchRspThread, &job); + assert(0 == taosThreadCreate(&(thread1), &thattr, schtCreateFetchRspThread, &job)); void *data = NULL; req.syncReq = true; @@ -1136,13 +1136,13 @@ TEST(queryTest, flowCtrlCase) { ASSERT_EQ(pRsp->numOfRows, 10); taosMemoryFreeClear(data); - schReleaseJob(job); + (void)schReleaseJob(job); schedulerDestroy(); schedulerFreeJob(&job, 0); - taosThreadJoin(thread1, NULL); + (void)taosThreadJoin(thread1, NULL); } TEST(insertTest, normalCase) { @@ -1158,9 +1158,9 @@ TEST(insertTest, normalCase) { SQueryNodeLoad load = {0}; load.addr.epSet.numOfEps = 1; - strcpy(load.addr.epSet.eps[0].fqdn, "qnode0.ep"); + TAOS_STRCPY(load.addr.epSet.eps[0].fqdn, "qnode0.ep"); load.addr.epSet.eps[0].port = 6031; - taosArrayPush(qnodeList, &load); + assert(NULL != taosArrayPush(qnodeList, &load)); int32_t code = schedulerInit(); ASSERT_EQ(code, 0); @@ -1171,12 +1171,12 @@ TEST(insertTest, normalCase) { schtSetAsyncSendMsgToServer(); TdThreadAttr thattr; - taosThreadAttrInit(&thattr); + assert(0 == taosThreadAttrInit(&thattr)); schtJobDone = false; TdThread thread1; - taosThreadCreate(&(thread1), &thattr, schtSendRsp, &insertJobRefId); + assert(0 == taosThreadCreate(&(thread1), &thattr, schtSendRsp, &insertJobRefId)); int32_t queryDone = 0; SRequestConnInfo conn = {0}; @@ -1204,17 +1204,17 @@ TEST(insertTest, normalCase) { schedulerDestroy(); - taosThreadJoin(thread1, NULL); + (void)taosThreadJoin(thread1, NULL); } TEST(multiThread, forceFree) { TdThreadAttr thattr; - taosThreadAttrInit(&thattr); + assert(0 == taosThreadAttrInit(&thattr)); TdThread thread1, thread2, thread3; - taosThreadCreate(&(thread1), &thattr, schtRunJobThread, NULL); + assert(0 == taosThreadCreate(&(thread1), &thattr, schtRunJobThread, NULL)); // taosThreadCreate(&(thread2), &thattr, schtFreeJobThread, NULL); - taosThreadCreate(&(thread3), &thattr, schtFetchRspThread, NULL); + assert(0 == taosThreadCreate(&(thread3), &thattr, schtFetchRspThread, NULL)); while (true) { if (schtTestDeadLoop) { @@ -1231,7 +1231,7 @@ TEST(multiThread, forceFree) { TEST(otherTest, otherCase) { // excpet test - schReleaseJob(0); + (void)schReleaseJob(0); schFreeRpcCtx(NULL); char* ep = NULL;