From 31f3bed34748a2a0172eb28b15fabfbf339b3787 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Sat, 11 Jun 2022 20:39:16 +0800 Subject: [PATCH 01/22] support index cache --- include/common/tmsg.h | 1 + source/common/src/tmsg.c | 2 + source/libs/catalog/inc/catalogInt.h | 60 +- source/libs/catalog/src/catalog.c | 38 +- source/libs/catalog/src/ctgAsync.c | 44 +- source/libs/catalog/src/ctgCache.c | 810 +++++++++++++--------- source/libs/catalog/src/ctgDbg.c | 8 +- source/libs/catalog/src/ctgRemote.c | 5 +- source/libs/catalog/src/ctgUtil.c | 88 ++- source/libs/catalog/test/catalogTests.cpp | 4 +- source/libs/qcom/src/querymsg.c | 10 +- source/libs/qworker/src/qworker.c | 1 - source/libs/scalar/src/filter.c | 5 + source/util/src/tarray.c | 3 + 14 files changed, 657 insertions(+), 422 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 48a677a686..7c37484ef6 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2513,6 +2513,7 @@ typedef struct { } STableIndexInfo; typedef struct { + int32_t version; SArray* pIndex; } STableIndexRsp; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index a7d4903696..3805a8b5b4 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -2437,6 +2437,7 @@ int32_t tSerializeSTableIndexRsp(void *buf, int32_t bufLen, const STableIndexRsp tEncoderInit(&encoder, buf, bufLen); if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI32(&encoder, pRsp->version) < 0) return -1; int32_t num = taosArrayGetSize(pRsp->pIndex); if (tEncodeI32(&encoder, num) < 0) return -1; if (num > 0) { @@ -2471,6 +2472,7 @@ int32_t tDeserializeSTableIndexRsp(void *buf, int32_t bufLen, STableIndexRsp *pR tDecoderInit(&decoder, buf, bufLen); if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI32(&decoder, &pRsp->version) < 0) return -1; int32_t num = 0; if (tDecodeI32(&decoder, &num) < 0) return -1; if (num > 0) { diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 1d56b6d5d4..ebd4d56aaf 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -57,6 +57,8 @@ enum { CTG_OP_DROP_TB_META, CTG_OP_UPDATE_USER, CTG_OP_UPDATE_VG_EPSET, + CTG_OP_UPDATE_TB_INDEX, + CTG_OP_DROP_TB_INDEX, CTG_OP_MAX }; @@ -128,19 +130,27 @@ typedef struct SCtgUserCtx { SUserAuthInfo user; } SCtgUserCtx; -typedef struct SCtgTbMetaCache { - SRWLatch stbLock; - SRWLatch metaLock; // RC between cache destroy and all other operations - SHashObj *metaCache; //key:tbname, value:STableMeta - SHashObj *stbCache; //key:suid, value:STableMeta* -} SCtgTbMetaCache; +typedef STableIndexRsp STableIndex; + +typedef struct SCtgTbCache { + SRWLatch metaLock; + STableMeta *pMeta; + SRWLatch indexLock; + STableIndex *pIndex; +} SCtgTbCache; + +typedef struct SCtgVgCache { + SRWLatch vgLock; + SDBVgInfo *vgInfo; +} SCtgVgCache; typedef struct SCtgDBCache { - SRWLatch vgLock; + SRWLatch dbLock; // RC between destroy tbCache/stbCache and all reads uint64_t dbId; int8_t deleted; - SDBVgInfo *vgInfo; - SCtgTbMetaCache tbCache; + SCtgVgCache vgCache; + SHashObj *tbCache; // key:tbname, value:SCtgTbCache + SHashObj *stbCache; // key:suid, value:STableMeta* } SCtgDBCache; typedef struct SCtgRentSlot { @@ -245,8 +255,10 @@ typedef struct SCtgCacheStat { uint64_t userNum; uint64_t vgHitNum; uint64_t vgMissNum; - uint64_t tblHitNum; - uint64_t tblMissNum; + uint64_t tbMetaHitNum; + uint64_t tbMetaMissNum; + uint64_t tbIndexHitNum; + uint64_t tbIndexMissNum; uint64_t userHitNum; uint64_t userMissNum; } SCtgCacheStat; @@ -268,10 +280,10 @@ typedef struct SCtgUpdateVgMsg { SDBVgInfo* dbInfo; } SCtgUpdateVgMsg; -typedef struct SCtgUpdateTblMsg { - SCatalog* pCtg; - STableMetaOutput* output; -} SCtgUpdateTblMsg; +typedef struct SCtgUpdateTbMetaMsg { + SCatalog* pCtg; + STableMetaOutput* pMeta; +} SCtgUpdateTbMetaMsg; typedef struct SCtgDropDBMsg { SCatalog* pCtg; @@ -305,6 +317,19 @@ typedef struct SCtgUpdateUserMsg { SGetUserAuthRsp userAuth; } SCtgUpdateUserMsg; +typedef struct SCtgUpdateTbIndexMsg { + SCatalog* pCtg; + char dbFName[TSDB_DB_FNAME_LEN]; + char tbName[TSDB_TABLE_NAME_LEN]; + STableIndex* pIndex; +} SCtgUpdateTbIndexMsg; + +typedef struct SCtgDropTbIndexMsg { + SCatalog* pCtg; + char dbFName[TSDB_DB_FNAME_LEN]; + char tbName[TSDB_TABLE_NAME_LEN]; +} SCtgDropTbIndexMsg; + typedef struct SCtgUpdateEpsetMsg { SCatalog* pCtg; char dbFName[TSDB_DB_FNAME_LEN]; @@ -465,8 +490,7 @@ int32_t ctgOpUpdateUser(SCtgCacheOperation *action); int32_t ctgOpUpdateEpset(SCtgCacheOperation *operation); int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache); void ctgReleaseDBCache(SCatalog *pCtg, SCtgDBCache *dbCache); -void ctgReleaseVgInfo(SCtgDBCache *dbCache); -int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache); +void ctgRUnlockVgInfo(SCtgDBCache *dbCache); int32_t ctgTbMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName, int32_t *exist); int32_t ctgReadTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta); int32_t ctgReadTbVerFromCache(SCatalog *pCtg, const SName *pTableName, int32_t *sver, int32_t *tver, int32_t *tbType, uint64_t *suid, char *stbName); @@ -479,6 +503,7 @@ int32_t ctgUpdateVgroupEnqueue(SCatalog* pCtg, const char *dbFName, int64_t dbId int32_t ctgUpdateTbMetaEnqueue(SCatalog* pCtg, STableMetaOutput *output, bool syncReq); int32_t ctgUpdateUserEnqueue(SCatalog* pCtg, SGetUserAuthRsp *pAuth, bool syncReq); int32_t ctgUpdateVgEpsetEnqueue(SCatalog* pCtg, char *dbFName, int32_t vgId, SEpSet* pEpSet); +int32_t ctgUpdateTbIndexEnqueue(SCatalog* pCtg, SName* pName, STableIndex *pIndex, bool syncOp); int32_t ctgMetaRentInit(SCtgRentMgmt *mgmt, uint32_t rentSec, int8_t type); int32_t ctgMetaRentAdd(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t size); int32_t ctgMetaRentGet(SCtgRentMgmt *mgmt, void **res, uint32_t *num, int32_t size); @@ -521,6 +546,7 @@ void ctgFreeSTableMetaOutput(STableMetaOutput* pOutput); int32_t ctgUpdateMsgCtx(SCtgMsgCtx* pCtx, int32_t reqType, void* out, char* target); char *ctgTaskTypeStr(CTG_TASK_TYPE type); int32_t ctgUpdateSendTargetInfo(SMsgSendInfo *pMsgSendInfo, int32_t msgType, SCtgTask* pTask); +int32_t ctgCloneTableIndex(SArray* pIndex, SArray** pRes); extern SCatalogMgmt gCtgMgmt; diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index f1a81d1112..bac13bf476 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -97,8 +97,7 @@ int32_t ctgRefreshDBVgInfo(SCatalog* pCtg, SRequestConnInfo *pConn, const char* if (NULL != dbCache) { input.dbId = dbCache->dbId; - ctgReleaseVgInfo(dbCache); - ctgReleaseDBCache(pCtg, dbCache); + ctgReleaseVgInfoToCache(pCtg, dbCache); } input.vgVersion = CTG_DEFAULT_INVALID_VERSION; @@ -383,6 +382,18 @@ _return: return TSDB_CODE_SUCCESS; } +int32_t ctgGetTbIndex(SCatalog* pCtg, SRequestConnInfo *pConn, const SName* pTableName, SArray** pRes) { + CTG_ERR_RET(ctgReadTbIndexFromCache(pCtg, pTableName, pRes)); + if (*pRes) { + return TSDB_CODE_SUCCESS; + } + + CTG_ERR_RET(ctgGetTbIndexFromMnode(pCtg, pConn, (SName*)pTableName, pRes, NULL)); + + return TSDB_CODE_SUCCESS; +} + + int32_t ctgGetTbDistVgInfo(SCatalog* pCtg, SRequestConnInfo *pConn, SName* pTableName, SArray** pVgList) { STableMeta *tbMeta = NULL; int32_t code = 0; @@ -443,7 +454,7 @@ int32_t ctgGetTbDistVgInfo(SCatalog* pCtg, SRequestConnInfo *pConn, SName* pTabl _return: if (dbCache) { - ctgReleaseVgInfo(dbCache); + ctgRUnlockVgInfo(dbCache); ctgReleaseDBCache(pCtg, dbCache); } @@ -633,8 +644,7 @@ int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* vers *dbId = dbCache->dbId; *tableNum = dbCache->vgInfo->numOfTable; - ctgReleaseVgInfo(dbCache); - ctgReleaseDBCache(pCtg, dbCache); + ctgReleaseVgInfoToCache(pCtg, dbCache); ctgDebug("Got db vgVersion from cache, dbFName:%s, vgVersion:%d", dbFName, *version); @@ -659,7 +669,7 @@ int32_t catalogGetDBVgInfo(SCatalog* pCtg, SRequestConnInfo *pConn, const char* SDBVgInfo *vgInfo = NULL; CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pConn, dbFName, &dbCache, &vgInfo)); if (dbCache) { - vgHash = dbCache->vgInfo->vgHash; + vgHash = dbCache->vgCache.vgInfo->vgHash; } else { vgHash = vgInfo->vgHash; } @@ -672,7 +682,7 @@ int32_t catalogGetDBVgInfo(SCatalog* pCtg, SRequestConnInfo *pConn, const char* _return: if (dbCache) { - ctgReleaseVgInfo(dbCache); + ctgRUnlockVgInfo(dbCache); ctgReleaseDBCache(pCtg, dbCache); } @@ -940,7 +950,7 @@ int32_t catalogGetTableHashVgroup(SCatalog *pCtg, SRequestConnInfo *pConn, const _return: if (dbCache) { - ctgReleaseVgInfo(dbCache); + ctgRUnlockVgInfo(dbCache); ctgReleaseDBCache(pCtg, dbCache); } @@ -1143,7 +1153,17 @@ int32_t catalogGetTableIndex(SCatalog* pCtg, SRequestConnInfo *pConn, const SNam CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - CTG_API_LEAVE(ctgGetTbIndexFromMnode(pCtg, pConn, (SName*)pTableName, pRes, NULL)); + int32_t code = 0; + CTG_ERR_JRET(ctgGetTbIndexFromMnode(pCtg, pConn, (SName*)pTableName, pRes, NULL)); + + SArray* pInfo = NULL; + CTG_ERR_JRET(ctgCloneTableIndex(*pRes, &pInfo)); + + CTG_ERR_JRET(ctgUpdateTbIndexEnqueue(pCtg, pTableName, pInfo, false)); + +_return: + + CTG_API_LEAVE(code); } diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index a96e4cec30..69eddd7ba6 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -344,6 +344,11 @@ int32_t ctgHandleForceUpdate(SCatalog* pCtg, SCtgJob *pJob, const SCatalogReq* p } } + for (int32_t i = 0; i < pJob->tbIndexNum; ++i) { + SName* name = taosArrayGet(pReq->pTableIndex, i); + ctgDropTbIndexEnqueue(pCtg, name, true); + } + return TSDB_CODE_SUCCESS; } @@ -524,7 +529,7 @@ int32_t ctgDumpTbIndexRes(SCtgTask* pTask) { } SMetaRes res = {.code = pTask->code, .pRes = pTask->res}; - taosArrayPush(pJob->jobRes.pTableHash, &res); + taosArrayPush(pJob->jobRes.pTableIndex, &res); return TSDB_CODE_SUCCESS; } @@ -687,8 +692,7 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf * ctx->vgId = vgInfo.vgId; CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, ctx->pName, &vgInfo, NULL, pTask)); - ctgReleaseVgInfo(dbCache); - ctgReleaseDBCache(pCtg, dbCache); + ctgReleaseVgInfoToCache(pCtg, dbCache); } else { SBuildUseDBInput input = {0}; @@ -786,7 +790,7 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf * _return: if (dbCache) { - ctgReleaseVgInfo(dbCache); + ctgRUnlockVgInfo(dbCache); ctgReleaseDBCache(pCtg, dbCache); } @@ -866,9 +870,16 @@ _return: int32_t ctgHandleGetTbIndexRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; - CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); + CTG_ERR_JRET(ctgProcessRspMsg(&pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); - TSWAP(pTask->res, pTask->msgCtx.out); + STableIndex* pOut = (STableIndex*)pTask->msgCtx.out; + SArray* pInfo = NULL; + CTG_ERR_JRET(ctgCloneTableIndex(pOut->pIndex, &pInfo)); + pTask->res = pInfo; + pTask->msgCtx.out = NULL; + + SCtgTbIndexCtx* ctx = pTask->taskCtx; + CTG_ERR_JRET(ctgUpdateTbIndexEnqueue(pTask->pJob->pCtg, ctx->pName, pOut, false)); _return: @@ -1024,8 +1035,7 @@ int32_t ctgAsyncRefreshTbMeta(SCtgTask *pTask) { _return: if (dbCache) { - ctgReleaseVgInfo(dbCache); - ctgReleaseDBCache(pCtg, dbCache); + ctgReleaseVgInfoToCache(pCtg, dbCache); } CTG_RET(code); @@ -1070,8 +1080,7 @@ int32_t ctgLaunchGetDbVgTask(SCtgTask *pTask) { _return: if (dbCache) { - ctgReleaseVgInfo(dbCache); - ctgReleaseDBCache(pCtg, dbCache); + ctgReleaseVgInfoToCache(pCtg, dbCache); } CTG_RET(code); @@ -1105,8 +1114,7 @@ int32_t ctgLaunchGetTbHashTask(SCtgTask *pTask) { _return: if (dbCache) { - ctgReleaseVgInfo(dbCache); - ctgReleaseDBCache(pCtg, dbCache); + ctgReleaseVgInfoToCache(pCtg, dbCache); } CTG_RET(code); @@ -1117,6 +1125,15 @@ int32_t ctgLaunchGetTbIndexTask(SCtgTask *pTask) { SCatalog* pCtg = pTask->pJob->pCtg; SRequestConnInfo* pConn = &pTask->pJob->conn; SCtgTbIndexCtx* pCtx = (SCtgTbIndexCtx*)pTask->taskCtx; + SArray* pRes = NULL; + + CTG_ERR_RET(ctgReadTbIndexFromCache(pCtg, pCtx->pName, &pRes)); + if (pRes) { + pTask->res = pRes; + + CTG_ERR_RET(ctgHandleTaskEnd(pTask, 0)); + return TSDB_CODE_SUCCESS; + } CTG_ERR_RET(ctgGetTbIndexFromMnode(pCtg, pConn, pCtx->pName, NULL, pTask)); return TSDB_CODE_SUCCESS; @@ -1167,8 +1184,7 @@ int32_t ctgLaunchGetDbInfoTask(SCtgTask *pTask) { _return: if (dbCache) { - ctgReleaseVgInfo(dbCache); - ctgReleaseDBCache(pCtg, dbCache); + ctgReleaseVgInfoToCache(pCtg, dbCache); } CTG_RET(code); diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index add44d6a16..2b56eb6b49 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -59,6 +59,11 @@ SCtgOperation gCtgCacheOperation[CTG_OP_MAX] = { CTG_OP_UPDATE_VG_EPSET, "update epset", ctgOpUpdateEpset + }, + { + CTG_OP_UPDATE_TB_INDEX, + "update tbIndex", + ctgOpUpdateTbIndex } }; @@ -66,11 +71,11 @@ SCtgOperation gCtgCacheOperation[CTG_OP_MAX] = { -int32_t ctgAcquireVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache, bool *inCache) { - CTG_LOCK(CTG_READ, &dbCache->vgLock); +int32_t ctgRLockVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache, bool *inCache) { + CTG_LOCK(CTG_READ, &dbCache->vgCache.vgLock); if (dbCache->deleted) { - CTG_UNLOCK(CTG_READ, &dbCache->vgLock); + CTG_UNLOCK(CTG_READ, &dbCache->vgCache.vgLock); ctgDebug("db is dropping, dbId:%"PRIx64, dbCache->dbId); @@ -79,8 +84,8 @@ int32_t ctgAcquireVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache, bool *inCache) { } - if (NULL == dbCache->vgInfo) { - CTG_UNLOCK(CTG_READ, &dbCache->vgLock); + if (NULL == dbCache->vgCache.vgInfo) { + CTG_UNLOCK(CTG_READ, &dbCache->vgCache.vgLock); *inCache = false; ctgDebug("db vgInfo is empty, dbId:%"PRIx64, dbCache->dbId); @@ -92,50 +97,47 @@ int32_t ctgAcquireVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache, bool *inCache) { return TSDB_CODE_SUCCESS; } -int32_t ctgWAcquireVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache) { - CTG_LOCK(CTG_WRITE, &dbCache->vgLock); +int32_t ctgWLockVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache) { + CTG_LOCK(CTG_WRITE, &dbCache->vgCache.vgLock); if (dbCache->deleted) { ctgDebug("db is dropping, dbId:%"PRIx64, dbCache->dbId); - CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); + CTG_UNLOCK(CTG_WRITE, &dbCache->vgCache.vgLock); CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); } return TSDB_CODE_SUCCESS; } +void ctgRUnlockVgInfo(SCtgDBCache *dbCache) { + CTG_UNLOCK(CTG_READ, &dbCache->vgCache.vgLock); +} + +void ctgWUnlockVgInfo(SCtgDBCache *dbCache) { + CTG_UNLOCK(CTG_WRITE, &dbCache->vgCache.vgLock); +} + void ctgReleaseDBCache(SCatalog *pCtg, SCtgDBCache *dbCache) { - taosHashRelease(pCtg->dbCache, dbCache); + CTG_UNLOCK(CTG_READ, &dbCache->dbLock); } -void ctgReleaseVgInfo(SCtgDBCache *dbCache) { - CTG_UNLOCK(CTG_READ, &dbCache->vgLock); -} - -void ctgWReleaseVgInfo(SCtgDBCache *dbCache) { - CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); -} - - int32_t ctgAcquireDBCacheImpl(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache, bool acquire) { char *p = strchr(dbFName, '.'); if (p && CTG_IS_SYS_DBNAME(p + 1)) { dbFName = p + 1; } - SCtgDBCache *dbCache = NULL; - if (acquire) { - dbCache = (SCtgDBCache *)taosHashAcquire(pCtg->dbCache, dbFName, strlen(dbFName)); - } else { - dbCache = (SCtgDBCache *)taosHashGet(pCtg->dbCache, dbFName, strlen(dbFName)); - } - + SCtgDBCache *dbCache = (SCtgDBCache *)taosHashGet(pCtg->dbCache, dbFName, strlen(dbFName)); if (NULL == dbCache) { *pCache = NULL; ctgDebug("db not in cache, dbFName:%s", dbFName); return TSDB_CODE_SUCCESS; } + if (acquire) { + CTG_LOCK(CTG_READ, &dbCache->dbLock); + } + if (dbCache->deleted) { if (acquire) { ctgReleaseDBCache(pCtg, dbCache); @@ -159,15 +161,35 @@ int32_t ctgGetDBCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache) CTG_RET(ctgAcquireDBCacheImpl(pCtg, dbFName, pCache, false)); } +void ctgReleaseVgInfoToCache(SCatalog* pCtg, SCtgDBCache *dbCache) { + ctgRUnlockVgInfo(dbCache); + ctgReleaseDBCache(pCtg, dbCache); +} + +void ctgReleaseTbMetaToCache(SCatalog* pCtg, SCtgDBCache *dbCache, SCtgTbCache* pCache) { + if (pCache) { + CTG_UNLOCK(CTG_READ, &pCache->metaLock); + taosHashRelease(dbCache->tbCache, pCache); + } + + if (dbCache) { + ctgReleaseDBCache(pCtg, dbCache); + } +} + +void ctgReleaseTbIndexToCache(SCatalog* pCtg, SCtgDBCache *dbCache, SCtgTbCache* pCache) { + if (pCache) { + CTG_UNLOCK(CTG_READ, &pCache->indexLock); + taosHashRelease(dbCache->tbCache, pCache); + } + + if (dbCache) { + ctgReleaseDBCache(pCtg, dbCache); + } +} int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache) { SCtgDBCache *dbCache = NULL; - - if (NULL == pCtg->dbCache) { - ctgDebug("empty db cache, dbFName:%s", dbFName); - goto _return; - } - ctgAcquireDBCache(pCtg, dbFName, &dbCache); if (NULL == dbCache) { ctgDebug("db %s not in cache", dbFName); @@ -175,7 +197,7 @@ int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char *dbFName, SCtgDBCac } bool inCache = false; - ctgAcquireVgInfo(pCtg, dbCache, &inCache); + ctgRLockVgInfo(pCtg, dbCache, &inCache); if (!inCache) { ctgDebug("vgInfo of db %s not in cache", dbFName); goto _return; @@ -202,54 +224,107 @@ _return: return TSDB_CODE_SUCCESS; } -int32_t ctgTbMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName, int32_t *exist) { - if (NULL == pCtg->dbCache) { - *exist = 0; - ctgWarn("empty db cache, dbFName:%s, tbName:%s", dbFName, tbName); - return TSDB_CODE_SUCCESS; - } - +int32_t ctgAcquireTbMetaFromCache(SCatalog* pCtg, char *dbFName, char* tbName, SCtgDBCache **pDb, SCtgTbCache** pTb) { SCtgDBCache *dbCache = NULL; ctgAcquireDBCache(pCtg, dbFName, &dbCache); if (NULL == dbCache) { - *exist = 0; - return TSDB_CODE_SUCCESS; + ctgDebug("db %s not in cache", dbFName); + goto _return; + } + + int32_t sz = 0; + SCtgTbCache* pCache = taosHashAcquire(dbCache->tbCache, tbName, strlen(tbName)); + if (NULL == pCache) { + ctgDebug("tb %s not in cache, dbFName:%s", tbName, dbFName); + goto _return; } - size_t sz = 0; - CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); - STableMeta *tbMeta = taosHashGet(dbCache->tbCache.metaCache, tbName, strlen(tbName)); - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - - if (NULL == tbMeta) { - ctgReleaseDBCache(pCtg, dbCache); - - *exist = 0; - ctgDebug("tbmeta not in cache, dbFName:%s, tbName:%s", dbFName, tbName); - return TSDB_CODE_SUCCESS; + CTG_LOCK(CTG_READ, &pCache->metaLock); + if (NULL == pCache->pMeta) { + ctgDebug("tb %s meta not in cache, dbFName:%s", tbName, dbFName); + goto _return; } - *exist = 1; + *pDb = dbCache; + *pTb = pCache; - ctgReleaseDBCache(pCtg, dbCache); + ctgDebug("tb %s meta got in cache, dbFName:%s", tbName, dbFName); - ctgDebug("tbmeta is in cache, dbFName:%s, tbName:%s", dbFName, tbName); + CTG_CACHE_STAT_INC(tbMetaHitNum, 1); + + return TSDB_CODE_SUCCESS; + +_return: + + ctgReleaseTbMetaToCache(pCtg, dbCache, pCache); + + CTG_CACHE_STAT_INC(tbMetaMissNum, 1); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgAcquireTbIndexFromCache(SCatalog* pCtg, char *dbFName, char* tbName, SCtgDBCache **pDb, SCtgTbCache** pTb) { + SCtgDBCache *dbCache = NULL; + ctgAcquireDBCache(pCtg, dbFName, &dbCache); + if (NULL == dbCache) { + ctgDebug("db %s not in cache", dbFName); + goto _return; + } + + int32_t sz = 0; + SCtgTbCache* pCache = taosHashAcquire(dbCache->tbCache, tbName, strlen(tbName)); + if (NULL == pCache) { + ctgDebug("tb %s not in cache, dbFName:%s", tbName, dbFName); + goto _return; + } + + CTG_LOCK(CTG_READ, &pCache->indexLock); + if (NULL == pCache->pIndex) { + ctgDebug("tb %s index not in cache, dbFName:%s", tbName, dbFName); + goto _return; + } + + *pDb = dbCache; + *pTb = pCache; + + ctgDebug("tb %s index got in cache, dbFName:%s", tbName, dbFName); + + CTG_CACHE_STAT_INC(tbIndexHitNum, 1); + + return TSDB_CODE_SUCCESS; + +_return: + + ctgReleaseTbIndexToCache(pCtg, dbCache, pCache); + + CTG_CACHE_STAT_INC(tbIndexMissNum, 1); return TSDB_CODE_SUCCESS; } +int32_t ctgTbMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName, int32_t *exist) { + SCtgDBCache *dbCache = NULL; + ctgAcquireTbMetaFromCache(pCtg, dbFName, tbName, &dbCache, &tbCache); + if (NULL == tbCache) { + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + + *exist = 0; + return TSDB_CODE_SUCCESS; + } + + *exist = 1; + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + + return TSDB_CODE_SUCCESS; +} + int32_t ctgReadTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta) { int32_t code = 0; SCtgDBCache *dbCache = NULL; - + SCtgTbCache *tbCache = NULL; *pTableMeta = NULL; - if (NULL == pCtg->dbCache) { - ctgDebug("empty tbmeta cache, tbName:%s", ctx->pName->tname); - return TSDB_CODE_SUCCESS; - } - char dbFName[TSDB_DB_FNAME_LEN] = {0}; if (CTG_FLAG_IS_SYS_DB(ctx->flag)) { strcpy(dbFName, ctx->pName->dbname); @@ -257,78 +332,53 @@ int32_t ctgReadTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta** tNameGetFullDbName(ctx->pName, dbFName); } - ctgAcquireDBCache(pCtg, dbFName, &dbCache); - if (NULL == dbCache) { - ctgDebug("db %d.%s not in cache", ctx->pName->acctId, ctx->pName->dbname); - return TSDB_CODE_SUCCESS; - } - - int32_t sz = 0; - CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); - taosHashGetDup_m(dbCache->tbCache.metaCache, ctx->pName->tname, strlen(ctx->pName->tname), (void **)pTableMeta, &sz); - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - - if (NULL == *pTableMeta) { - ctgReleaseDBCache(pCtg, dbCache); - ctgDebug("tbl not in cache, dbFName:%s, tbName:%s", dbFName, ctx->pName->tname); + ctgAcquireTbMetaFromCache(pCtg, dbFName, ctx->pName->tname, &dbCache, &tbCache); + if (NULL == tbCache) { + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); return TSDB_CODE_SUCCESS; } - STableMeta* tbMeta = *pTableMeta; + STableMeta* tbMeta = tbCache->pMeta; ctx->tbInfo.inCache = true; ctx->tbInfo.dbId = dbCache->dbId; ctx->tbInfo.suid = tbMeta->suid; ctx->tbInfo.tbType = tbMeta->tableType; if (tbMeta->tableType != TSDB_CHILD_TABLE) { - ctgReleaseDBCache(pCtg, dbCache); - ctgDebug("Got meta from cache, type:%d, dbFName:%s, tbName:%s", tbMeta->tableType, dbFName, ctx->pName->tname); - - CTG_CACHE_STAT_INC(tblHitNum, 1); + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + ctgDebug("Got tb %s meta from cache, type:%d, dbFName:%s", ctx->pName->tname, tbMeta->tableType, dbFName); return TSDB_CODE_SUCCESS; } - CTG_LOCK(CTG_READ, &dbCache->tbCache.stbLock); - - STableMeta **stbMeta = taosHashGet(dbCache->tbCache.stbCache, &tbMeta->suid, sizeof(tbMeta->suid)); + STableMeta **stbMeta = taosHashGet(dbCache->stbCache, &tbMeta->suid, sizeof(tbMeta->suid)); if (NULL == stbMeta || NULL == *stbMeta) { - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); ctgError("stb not in stbCache, suid:%"PRIx64, tbMeta->suid); goto _return; } if ((*stbMeta)->suid != tbMeta->suid) { - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); - ctgError("stable suid in stbCache mis-match, expected suid:%"PRIx64 ",actual suid:%"PRIx64, tbMeta->suid, (*stbMeta)->suid); + ctgError("stb suid %" PRIx64 " in stbCache mis-match, expected suid:%"PRIx64 , (*stbMeta)->suid, tbMeta->suid); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } int32_t metaSize = CTG_META_SIZE(*stbMeta); *pTableMeta = taosMemoryRealloc(*pTableMeta, metaSize); if (NULL == *pTableMeta) { - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); - ctgError("realloc size[%d] failed", metaSize); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } memcpy(&(*pTableMeta)->sversion, &(*stbMeta)->sversion, metaSize - sizeof(SCTableMeta)); - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); - ctgReleaseDBCache(pCtg, dbCache); - - CTG_CACHE_STAT_INC(tblHitNum, 1); - - ctgDebug("Got tbmeta from cache, dbFName:%s, tbName:%s", dbFName, ctx->pName->tname); + ctgDebug("Got tb %s meta from cache, dbFName:%s", ctx->pName->tname, dbFName); return TSDB_CODE_SUCCESS; _return: - ctgReleaseDBCache(pCtg, dbCache); + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); taosMemoryFreeClear(*pTableMeta); - - CTG_CACHE_STAT_INC(tblMissNum, 1); CTG_RET(code); } @@ -338,62 +388,42 @@ int32_t ctgReadTbVerFromCache(SCatalog *pCtg, const SName *pTableName, int32_t * *sver = -1; *tver = -1; - if (NULL == pCtg->dbCache) { - ctgDebug("empty tbmeta cache, tbName:%s", pTableName->tname); - return TSDB_CODE_SUCCESS; - } - SCtgDBCache *dbCache = NULL; + SCtgTbCache *tbCache = NULL; char dbFName[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(pTableName, dbFName); - ctgAcquireDBCache(pCtg, dbFName, &dbCache); - if (NULL == dbCache) { - ctgDebug("db %s not in cache", pTableName->tname); + ctgAcquireTbMetaFromCache(pCtg, dbFName, pTableName->tname, &dbCache, &tbCache); + if (NULL == tbCache) { + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); return TSDB_CODE_SUCCESS; } - CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); - STableMeta *tbMeta = taosHashGet(dbCache->tbCache.metaCache, pTableName->tname, strlen(pTableName->tname)); - if (tbMeta) { - *tbType = tbMeta->tableType; - *suid = tbMeta->suid; - if (*tbType != TSDB_CHILD_TABLE) { - *sver = tbMeta->sversion; - *tver = tbMeta->tversion; - } - } - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - - if (NULL == tbMeta) { - ctgReleaseDBCache(pCtg, dbCache); - return TSDB_CODE_SUCCESS; - } + STableMeta* tbMeta = tbCache->pMeta; + *tbType = tbMeta->tableType; + *suid = tbMeta->suid; if (*tbType != TSDB_CHILD_TABLE) { - ctgReleaseDBCache(pCtg, dbCache); - ctgDebug("Got sver %d tver %d from cache, type:%d, dbFName:%s, tbName:%s", *sver, *tver, *tbType, dbFName, pTableName->tname); + *sver = tbMeta->sversion; + *tver = tbMeta->tversion; + ctgDebug("Got tb %s ver from cache, dbFName:%s, tbType:%d, sver:%d, tver:%d, suid:%" PRIx64, + pTableName->tname, dbFName, *tbType, *sver, *tver, *suid); + + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); return TSDB_CODE_SUCCESS; } - ctgDebug("Got subtable meta from cache, dbFName:%s, tbName:%s, suid:%" PRIx64, dbFName, pTableName->tname, *suid); - - CTG_LOCK(CTG_READ, &dbCache->tbCache.stbLock); - - STableMeta **stbMeta = taosHashGet(dbCache->tbCache.stbCache, suid, sizeof(*suid)); + STableMeta **stbMeta = taosHashGet(dbCache->stbCache, suid, sizeof(*suid)); if (NULL == stbMeta || NULL == *stbMeta) { - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); - ctgReleaseDBCache(pCtg, dbCache); + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); ctgDebug("stb not in stbCache, suid:%" PRIx64, *suid); return TSDB_CODE_SUCCESS; } if ((*stbMeta)->suid != *suid) { - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); - ctgReleaseDBCache(pCtg, dbCache); - ctgError("stable suid in stbCache mis-match, expected suid:%" PRIx64 ",actual suid:%" PRIx64, *suid, - (*stbMeta)->suid); + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + ctgError("stb suid %" PRIx64 " in stbCache mis-match, expected suid:%" PRIx64 , (*stbMeta)->suid, *suid); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } @@ -406,52 +436,55 @@ int32_t ctgReadTbVerFromCache(SCatalog *pCtg, const SName *pTableName, int32_t * *sver = (*stbMeta)->sversion; *tver = (*stbMeta)->tversion; - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); - ctgReleaseDBCache(pCtg, dbCache); - - ctgDebug("Got sver %d tver %d from cache, type:%d, dbFName:%s, tbName:%s", *sver, *tver, *tbType, dbFName, pTableName->tname); + ctgDebug("Got tb %s sver %d tver %d from cache, type:%d, dbFName:%s", pTableName->tname, *sver, *tver, *tbType, dbFName); return TSDB_CODE_SUCCESS; } int32_t ctgReadTbTypeFromCache(SCatalog* pCtg, const char* dbFName, const char *tableName, int32_t *tbType) { - if (NULL == pCtg->dbCache) { - ctgWarn("empty db cache, dbFName:%s, tbName:%s", dbFName, tableName); - return TSDB_CODE_SUCCESS; - } - SCtgDBCache *dbCache = NULL; - ctgAcquireDBCache(pCtg, dbFName, &dbCache); - if (NULL == dbCache) { + SCtgTbCache *tbCache = NULL; + ctgAcquireTbMetaFromCache(pCtg, dbFName, tableName, &dbCache, &tbCache); + if (NULL == tbCache) { + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); return TSDB_CODE_SUCCESS; } - CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); - STableMeta *pTableMeta = (STableMeta *)taosHashAcquire(dbCache->tbCache.metaCache, tableName, strlen(tableName)); + *tbType = tbCache->pMeta->tableType; + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); - if (NULL == pTableMeta) { - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - ctgWarn("tbl not in cache, dbFName:%s, tbName:%s", dbFName, tableName); - ctgReleaseDBCache(pCtg, dbCache); - - return TSDB_CODE_SUCCESS; - } - - *tbType = atomic_load_8(&pTableMeta->tableType); - - taosHashRelease(dbCache->tbCache.metaCache, pTableMeta); - - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - - ctgReleaseDBCache(pCtg, dbCache); - - ctgDebug("Got tbtype from cache, dbFName:%s, tbName:%s, type:%d", dbFName, tableName, *tbType); + ctgDebug("Got tb %s tbType %d from cache, dbFName:%s", tableName, *tbType, dbFName); return TSDB_CODE_SUCCESS; } +int32_t ctgReadTbIndexFromCache(SCatalog* pCtg, const SName* pTableName, SArray** pRes) { + int32_t code = 0; + SCtgDBCache *dbCache = NULL; + SCtgTbCache *tbCache = NULL; + char dbFName[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(pTableName, dbFName); + + *pRes = NULL; + + ctgAcquireTbIndexFromCache(pCtg, dbFName, pTableName->tname, &dbCache, &tbCache); + if (NULL == tbCache) { + ctgReleaseTbIndexToCache(pCtg, dbCache, tbCache); + return TSDB_CODE_SUCCESS; + } + + CTG_ERR_JRET(ctgCloneTableIndex(tbCache->pIndex->pIndex, pRes)); + +_return: + + ctgReleaseTbIndexToCache(pCtg, dbCache, tbCache); + + CTG_RET(code); +} + int32_t ctgChkAuthFromCache(SCatalog* pCtg, const char* user, const char* dbFName, AUTH_TYPE type, bool *inCache, bool *pass) { if (NULL == pCtg->userCache) { ctgDebug("empty user auth cache, user:%s", user); @@ -718,9 +751,9 @@ int32_t ctgUpdateTbMetaEnqueue(SCatalog* pCtg, STableMetaOutput *output, bool sy op->opId = CTG_OP_UPDATE_TB_META; op->syncOp = syncOp; - SCtgUpdateTblMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateTblMsg)); + SCtgUpdateTbMetaMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateTbMetaMsg)); if (NULL == msg) { - ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateTblMsg)); + ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateTbMetaMsg)); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } @@ -730,7 +763,7 @@ int32_t ctgUpdateTbMetaEnqueue(SCatalog* pCtg, STableMetaOutput *output, bool sy } msg->pCtg = pCtg; - msg->output = output; + msg->pMeta = output; op->data = msg; @@ -805,6 +838,68 @@ _return: CTG_RET(code); } +int32_t ctgUpdateTbIndexEnqueue(SCatalog* pCtg, SName* pName, STableIndex *pIndex, bool syncOp) { + int32_t code = 0; + SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); + op->opId = CTG_OP_UPDATE_TB_INDEX; + op->syncOp = syncOp; + + SCtgUpdateTbIndexMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateTbIndexMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateTbIndexMsg)); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); + } + + msg->pCtg = pCtg; + msg->pIndex = pIndex; + tNameGetFullDbName(pName, msg->dbFName); + strcpy(msg->tbName, pName->tname); + + op->data = msg; + + CTG_ERR_JRET(ctgEnqueue(pCtg, op)); + + return TSDB_CODE_SUCCESS; + +_return: + + taosArrayDestroyEx(pIndex, tFreeSTableIndexInfo); + taosMemoryFreeClear(msg); + + CTG_RET(code); +} + +int32_t ctgDropTbIndexEnqueue(SCatalog* pCtg, SName* pName, bool syncOp) { + int32_t code = 0; + SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); + op->opId = CTG_OP_DROP_TB_INDEX; + op->syncOp = syncOp; + + SCtgDropTbIndexMsg *msg = taosMemoryMalloc(sizeof(SCtgDropTbIndexMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgDropTbIndexMsg)); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + msg->pCtg = pCtg; + tNameGetFullDbName(pName, msg->dbFName); + strcpy(msg->tbName, pName->tname); + + op->data = msg; + + CTG_ERR_JRET(ctgEnqueue(pCtg, op)); + + return TSDB_CODE_SUCCESS; + +_return: + + taosArrayDestroyEx(pIndex, tFreeSTableIndexInfo); + taosMemoryFreeClear(msg); + + CTG_RET(code); +} + + int32_t ctgMetaRentInit(SCtgRentMgmt *mgmt, uint32_t rentSec, int8_t type) { mgmt->slotRIdx = 0; mgmt->slotNum = rentSec / CTG_RENT_SLOT_SECOND; @@ -1006,14 +1101,14 @@ int32_t ctgAddNewDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId) { SCtgDBCache newDBCache = {0}; newDBCache.dbId = dbId; - newDBCache.tbCache.metaCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); - if (NULL == newDBCache.tbCache.metaCache) { + newDBCache.tbCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + if (NULL == newDBCache.tbCache) { ctgError("taosHashInit %d metaCache failed", gCtgMgmt.cfg.maxTblCacheNum); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } - newDBCache.tbCache.stbCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_ENTRY_LOCK); - if (NULL == newDBCache.tbCache.stbCache) { + newDBCache.stbCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_ENTRY_LOCK); + if (NULL == newDBCache.stbCache) { ctgError("taosHashInit %d stbCache failed", gCtgMgmt.cfg.maxTblCacheNum); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } @@ -1050,22 +1145,22 @@ _return: } -void ctgRemoveStbRent(SCatalog* pCtg, SCtgTbMetaCache *cache) { - CTG_LOCK(CTG_WRITE, &cache->stbLock); - if (cache->stbCache) { - void *pIter = taosHashIterate(cache->stbCache, NULL); - while (pIter) { - uint64_t *suid = NULL; - suid = taosHashGetKey(pIter, NULL); - - if (TSDB_CODE_SUCCESS == ctgMetaRentRemove(&pCtg->stbRent, *suid, ctgStbVersionSortCompare, ctgStbVersionSearchCompare)) { - ctgDebug("stb removed from rent, suid:%"PRIx64, *suid); - } - - pIter = taosHashIterate(cache->stbCache, pIter); - } +void ctgRemoveStbRent(SCatalog* pCtg, SCtgDBCache *dbCache) { + if (NULL == dbCache->stbCache) { + return; + } + + void *pIter = taosHashIterate(dbCache->stbCache, NULL); + while (pIter) { + uint64_t *suid = NULL; + suid = taosHashGetKey(pIter, NULL); + + if (TSDB_CODE_SUCCESS == ctgMetaRentRemove(&pCtg->stbRent, *suid, ctgStbVersionSortCompare, ctgStbVersionSearchCompare)) { + ctgDebug("stb removed from rent, suid:%"PRIx64, *suid); + } + + pIter = taosHashIterate(dbCache->stbCache, pIter); } - CTG_UNLOCK(CTG_WRITE, &cache->stbLock); } @@ -1074,15 +1169,16 @@ int32_t ctgRemoveDBFromCache(SCatalog* pCtg, SCtgDBCache *dbCache, const char* d ctgInfo("start to remove db from cache, dbFName:%s, dbId:%"PRIx64, dbFName, dbCache->dbId); + CTG_LOCK(CTG_WRITE, &dbCache->dbLock); + atomic_store_8(&dbCache->deleted, 1); - - ctgRemoveStbRent(pCtg, &dbCache->tbCache); - + ctgRemoveStbRent(pCtg, &dbCache); ctgFreeDbCache(dbCache); - CTG_ERR_RET(ctgMetaRentRemove(&pCtg->dbRent, dbCache->dbId, ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare)); - - ctgDebug("db removed from rent, dbFName:%s, dbId:%"PRIx64, dbFName, dbCache->dbId); + CTG_UNLOCK(CTG_WRITE, &dbCache->dbLock); + + CTG_ERR_RET(ctgMetaRentRemove(&pCtg->dbRent, dbId, ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare)); + ctgDebug("db removed from rent, dbFName:%s, dbId:%"PRIx64, dbFName, dbId); if (taosHashRemove(pCtg->dbCache, dbFName, strlen(dbFName))) { ctgInfo("taosHashRemove from dbCache failed, may be removed, dbFName:%s", dbFName); @@ -1090,7 +1186,6 @@ int32_t ctgRemoveDBFromCache(SCatalog* pCtg, SCtgDBCache *dbCache, const char* d } CTG_CACHE_STAT_DEC(dbNum, 1); - ctgInfo("db removed from cache, dbFName:%s, dbId:%"PRIx64, dbFName, dbId); return TSDB_CODE_SUCCESS; @@ -1138,120 +1233,54 @@ int32_t ctgGetAddDBCache(SCatalog* pCtg, const char *dbFName, uint64_t dbId, SCt return TSDB_CODE_SUCCESS; } - -int32_t ctgWriteDBVgInfoToCache(SCatalog* pCtg, const char* dbFName, uint64_t dbId, SDBVgInfo** pDbInfo) { - int32_t code = 0; - SDBVgInfo* dbInfo = *pDbInfo; - - if (NULL == dbInfo->vgHash) { - return TSDB_CODE_SUCCESS; - } - - if (dbInfo->vgVersion < 0 || taosHashGetSize(dbInfo->vgHash) <= 0) { - ctgError("invalid db vgInfo, dbFName:%s, vgHash:%p, vgVersion:%d, vgHashSize:%d", - dbFName, dbInfo->vgHash, dbInfo->vgVersion, taosHashGetSize(dbInfo->vgHash)); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } - - bool newAdded = false; - SDbVgVersion vgVersion = {.dbId = dbId, .vgVersion = dbInfo->vgVersion, .numOfTable = dbInfo->numOfTable}; - - SCtgDBCache *dbCache = NULL; - CTG_ERR_RET(ctgGetAddDBCache(pCtg, dbFName, dbId, &dbCache)); - if (NULL == dbCache) { - ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:%"PRIx64, dbFName, dbId); - CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); - } - - SDBVgInfo *vgInfo = NULL; - CTG_ERR_RET(ctgWAcquireVgInfo(pCtg, dbCache)); - - if (dbCache->vgInfo) { - if (dbInfo->vgVersion < dbCache->vgInfo->vgVersion) { - ctgDebug("db vgVersion is old, dbFName:%s, vgVersion:%d, currentVersion:%d", dbFName, dbInfo->vgVersion, dbCache->vgInfo->vgVersion); - ctgWReleaseVgInfo(dbCache); - - return TSDB_CODE_SUCCESS; - } - - if (dbInfo->vgVersion == dbCache->vgInfo->vgVersion && dbInfo->numOfTable == dbCache->vgInfo->numOfTable) { - ctgDebug("no new db vgVersion or numOfTable, dbFName:%s, vgVersion:%d, numOfTable:%d", dbFName, dbInfo->vgVersion, dbInfo->numOfTable); - ctgWReleaseVgInfo(dbCache); - - return TSDB_CODE_SUCCESS; - } - - ctgFreeVgInfo(dbCache->vgInfo); - } - - dbCache->vgInfo = dbInfo; - - *pDbInfo = NULL; - - ctgDebug("db vgInfo updated, dbFName:%s, vgVersion:%d, dbId:%"PRIx64, dbFName, vgVersion.vgVersion, vgVersion.dbId); - - ctgWReleaseVgInfo(dbCache); - - dbCache = NULL; - - strncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); - CTG_ERR_RET(ctgMetaRentUpdate(&pCtg->dbRent, &vgVersion, vgVersion.dbId, sizeof(SDbVgVersion), ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare)); - - CTG_RET(code); -} - - int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, uint64_t dbId, char *tbName, STableMeta *meta, int32_t metaSize) { - SCtgTbMetaCache *tbCache = &dbCache->tbCache; - - CTG_LOCK(CTG_READ, &tbCache->metaLock); - if (dbCache->deleted || NULL == tbCache->metaCache || NULL == tbCache->stbCache) { - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + if (NULL == dbCache->tbCache || NULL == dbCache->stbCache) { + taosMemoryFree(meta); ctgError("db is dropping, dbId:%"PRIx64, dbCache->dbId); CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); } + bool isStb = meta->tableType == TSDB_SUPER_TABLE; + SCtgTbCache* pCache = taosHashGet(dbCache->tbCache, tbName, strlen(tbName)); + STableMeta *orig = (pCache ? pCache->pMeta : NULL); int8_t origType = 0; uint64_t origSuid = 0; - bool isStb = meta->tableType == TSDB_SUPER_TABLE; - STableMeta *orig = taosHashGet(tbCache->metaCache, tbName, strlen(tbName)); + if (orig) { origType = orig->tableType; if (origType == meta->tableType && orig->uid == meta->uid && orig->sversion >= meta->sversion && orig->tversion >= meta->tversion) { - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + taosMemoryFree(meta); + ctgDebug("ignore table %s meta update", tbName); return TSDB_CODE_SUCCESS; } if (origType == TSDB_SUPER_TABLE) { - CTG_LOCK(CTG_WRITE, &tbCache->stbLock); - if (taosHashRemove(tbCache->stbCache, &orig->suid, sizeof(orig->suid))) { + if (taosHashRemove(dbCache->stbCache, &orig->suid, sizeof(orig->suid))) { ctgError("stb not exist in stbCache, dbFName:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid); } else { CTG_CACHE_STAT_DEC(stblNum, 1); + ctgDebug("stb removed from stbCache, dbFName:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid); } - CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); - - ctgDebug("stb removed from stbCache, dbFName:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid); ctgMetaRentRemove(&pCtg->stbRent, orig->suid, ctgStbVersionSortCompare, ctgStbVersionSearchCompare); - origSuid = orig->suid; } } - if (isStb) { - CTG_LOCK(CTG_WRITE, &tbCache->stbLock); - } - - if (taosHashPut(tbCache->metaCache, tbName, strlen(tbName), meta, metaSize) != 0) { - if (isStb) { - CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); + if (NULL == pCache) { + SCtgTbCache cache = {0}; + cache.pMeta = meta; + if (taosHashPut(dbCache->tbCache, tbName, strlen(tbName), &cache, sizeof(SCtgTbCache)) != 0) { + taosMemoryFree(meta); + ctgError("taosHashPut new tbCache failed, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); - ctgError("taosHashPut tbmeta to cache failed, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + pCache = taosHashGet(dbCache->tbCache, tbName, strlen(tbName)); + } else { + taosMemoryFree(pCache->pMeta); + pCache->pMeta = meta; } if (NULL == orig) { @@ -1262,23 +1291,15 @@ int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNam ctgdShowTableMeta(pCtg, tbName, meta); if (!isStb) { - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); return TSDB_CODE_SUCCESS; } - STableMeta *tbMeta = taosHashGet(tbCache->metaCache, tbName, strlen(tbName)); - if (taosHashPut(tbCache->stbCache, &meta->suid, sizeof(meta->suid), &tbMeta, POINTER_BYTES) != 0) { - CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); - ctgError("taosHashPut stable to stable cache failed, suid:%"PRIx64, meta->suid); + if (origSuid != meta->suid && taosHashPut(dbCache->stbCache, &meta->suid, sizeof(meta->suid), &pCache, POINTER_BYTES) != 0) { + ctgError("taosHashPut to stable cache failed, suid:%"PRIx64, meta->suid); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } CTG_CACHE_STAT_INC(stblNum, 1); - - CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); - - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); ctgDebug("stb updated to stbCache, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType); @@ -1290,6 +1311,43 @@ int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNam return TSDB_CODE_SUCCESS; } +int32_t ctgWriteTbIndexToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *tbName, STableIndex **index) { + if (NULL == dbCache->tbCache) { + taosMemoryFreeClear(*index); + ctgError("db is dropping, dbId:%"PRIx64, dbCache->dbId); + CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); + } + + STableIndex* pIndex = *index; + SCtgTbCache* pCache = taosHashGet(dbCache->tbCache, tbName, strlen(tbName)); + if (NULL == pCache) { + SCtgTbCache cache = {0}; + cache.pIndex = pIndex; + + if (taosHashPut(table->tbCache, tbName, strlen(tbName), &cache, sizeof(cache)) != 0) { + taosMemoryFreeClear(*index); + ctgError("taosHashPut new tbCache failed, tbName:%s", tbName); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + *index = NULL; + ctgDebug("table %s index updated to cache, ver:%d, num:%d", tbName, pIndex->version, taosArrayGetSize(pIndex->pIndex)); + return TSDB_CODE_SUCCESS; + } + + if (pCache->pIndex) { + taosArrayDestroyEx(pCache->pIndex->pIndex, tFreeSTableIndexInfo); + taosMemoryFreeClear(pCache->pIndex); + } + + pCache->pIndex = pIndex; + *index = NULL; + + ctgDebug("table %s index updated to cache, ver:%d, num:%d", tbName, pIndex->version, taosArrayGetSize(pIndex->pIndex)); + + return TSDB_CODE_SUCCESS; +} + int32_t ctgUpdateTbMetaToCache(SCatalog* pCtg, STableMetaOutput* pOut, bool syncReq) { STableMetaOutput* pOutput = NULL; int32_t code = 0; @@ -1305,12 +1363,66 @@ _return: CTG_RET(code); } - int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) { int32_t code = 0; SCtgUpdateVgMsg *msg = operation->data; + SDBVgInfo* dbInfo = msg->dbInfo; + char* dbFName = msg->dbFName; - CTG_ERR_JRET(ctgWriteDBVgInfoToCache(msg->pCtg, msg->dbFName, msg->dbId, &msg->dbInfo)); + if (NULL == dbInfo->vgHash) { + return TSDB_CODE_SUCCESS; + } + + if (dbInfo->vgVersion < 0 || taosHashGetSize(dbInfo->vgHash) <= 0) { + ctgError("invalid db vgInfo, dbFName:%s, vgHash:%p, vgVersion:%d, vgHashSize:%d", + dbFName, dbInfo->vgHash, dbInfo->vgVersion, taosHashGetSize(dbInfo->vgHash)); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + bool newAdded = false; + SDbVgVersion vgVersion = {.dbId = msg->dbId, .vgVersion = dbInfo->vgVersion, .numOfTable = dbInfo->numOfTable}; + + SCtgDBCache *dbCache = NULL; + CTG_ERR_RET(ctgGetAddDBCache(msg->pCtg, dbFName, msg->dbId, &dbCache)); + if (NULL == dbCache) { + ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:%"PRIx64, dbFName, msg->dbId); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + SCtgVgCache *vgCache = &dbCache->vgCache; + CTG_ERR_RET(ctgWLockVgInfo(msg->pCtg, dbCache)); + + if (vgCache->vgInfo) { + SDBVgInfo *vgInfo = vgCache->vgInfo; + + if (dbInfo->vgVersion < vgInfo->vgVersion) { + ctgDebug("db vgVer is old, dbFName:%s, vgVer:%d, curVer:%d", dbFName, dbInfo->vgVersion, vgInfo->vgVersion); + ctgWUnlockVgInfo(dbCache); + + return TSDB_CODE_SUCCESS; + } + + if (dbInfo->vgVersion == vgInfo->vgVersion && dbInfo->numOfTable == vgInfo->numOfTable) { + ctgDebug("no new db vgVer or numOfTable, dbFName:%s, vgVer:%d, numOfTable:%d", dbFName, dbInfo->vgVersion, dbInfo->numOfTable); + ctgWUnlockVgInfo(dbCache); + + return TSDB_CODE_SUCCESS; + } + + ctgFreeVgInfo(vgInfo); + } + + vgCache->vgInfo = dbInfo; + msg->dbInfo = NULL; + + ctgDebug("db vgInfo updated, dbFName:%s, vgVer:%d, dbId:%"PRIx64, dbFName, vgVersion.vgVersion, vgVersion.dbId); + + ctgWUnlockVgInfo(dbCache); + + dbCache = NULL; + + strncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); + CTG_ERR_RET(ctgMetaRentUpdate(&msg->pCtg->dbRent, &vgVersion, vgVersion.dbId, sizeof(SDbVgVersion), ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare)); _return: @@ -1356,14 +1468,14 @@ int32_t ctgOpDropDbVgroup(SCtgCacheOperation *operation) { goto _return; } - CTG_ERR_RET(ctgWAcquireVgInfo(pCtg, dbCache)); + CTG_ERR_RET(ctgWLockVgInfo(pCtg, dbCache)); - ctgFreeVgInfo(dbCache->vgInfo); - dbCache->vgInfo = NULL; + ctgFreeVgInfo(dbCache->vgCache.vgInfo); + dbCache->vgCache.vgInfo = NULL; ctgDebug("db vgInfo removed, dbFName:%s", msg->dbFName); - ctgWReleaseVgInfo(dbCache); + ctgWUnlockVgInfo(dbCache); _return: @@ -1375,42 +1487,47 @@ _return: int32_t ctgOpUpdateTbMeta(SCtgCacheOperation *operation) { int32_t code = 0; - SCtgUpdateTblMsg *msg = operation->data; + SCtgUpdateTbMetaMsg *msg = operation->data; SCatalog* pCtg = msg->pCtg; - STableMetaOutput* output = msg->output; + STableMetaOutput* pMeta = msg->pMeta; SCtgDBCache *dbCache = NULL; - if ((!CTG_IS_META_CTABLE(output->metaType)) && NULL == output->tbMeta) { - ctgError("no valid tbmeta got from meta rsp, dbFName:%s, tbName:%s", output->dbFName, output->tbName); + if ((!CTG_IS_META_CTABLE(pMeta->metaType)) && NULL == pMeta->tbMeta) { + ctgError("no valid tbmeta got from meta rsp, dbFName:%s, tbName:%s", pMeta->dbFName, pMeta->tbName); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } - if (CTG_IS_META_BOTH(output->metaType) && TSDB_SUPER_TABLE != output->tbMeta->tableType) { - ctgError("table type error, expected:%d, actual:%d", TSDB_SUPER_TABLE, output->tbMeta->tableType); + if (CTG_IS_META_BOTH(pMeta->metaType) && TSDB_SUPER_TABLE != pMeta->tbMeta->tableType) { + ctgError("table type error, expected:%d, actual:%d", TSDB_SUPER_TABLE, pMeta->tbMeta->tableType); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } - CTG_ERR_JRET(ctgGetAddDBCache(pCtg, output->dbFName, output->dbId, &dbCache)); + CTG_ERR_JRET(ctgGetAddDBCache(pCtg, pMeta->dbFName, pMeta->dbId, &dbCache)); if (NULL == dbCache) { - ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:%"PRIx64, output->dbFName, output->dbId); + ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:%" PRIx64, pMeta->dbFName, pMeta->dbId); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } - if (CTG_IS_META_TABLE(output->metaType) || CTG_IS_META_BOTH(output->metaType)) { - int32_t metaSize = CTG_META_SIZE(output->tbMeta); - - CTG_ERR_JRET(ctgWriteTbMetaToCache(pCtg, dbCache, output->dbFName, output->dbId, output->tbName, output->tbMeta, metaSize)); + if (CTG_IS_META_TABLE(pMeta->metaType) || CTG_IS_META_BOTH(pMeta->metaType)) { + int32_t metaSize = CTG_META_SIZE(pMeta->tbMeta); + CTG_ERR_JRET(ctgWriteTbMetaToCache(pCtg, dbCache, pMeta->dbFName, pMeta->dbId, pMeta->tbName, pMeta->tbMeta, metaSize)); + pMeta->tbMeta = NULL; } - if (CTG_IS_META_CTABLE(output->metaType) || CTG_IS_META_BOTH(output->metaType)) { - CTG_ERR_JRET(ctgWriteTbMetaToCache(pCtg, dbCache, output->dbFName, output->dbId, output->ctbName, (STableMeta *)&output->ctbMeta, sizeof(output->ctbMeta))); + if (CTG_IS_META_CTABLE(pMeta->metaType) || CTG_IS_META_BOTH(pMeta->metaType)) { + SCTableMeta* ctbMeta = taosMemoryMalloc(sizeof(SCTableMeta)); + if (NULL == ctbMeta) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + memcpy(ctbMeta, &pMeta->ctbMeta, sizeof(SCTableMeta)); + CTG_ERR_JRET(ctgWriteTbMetaToCache(pCtg, dbCache, pMeta->dbFName, pMeta->dbId, pMeta->ctbName, (STableMeta *)ctbMeta, sizeof(SCTableMeta))); } _return: - if (output) { - taosMemoryFreeClear(output->tbMeta); - taosMemoryFreeClear(output); + if (pMeta) { + taosMemoryFreeClear(pMeta->tbMeta); + taosMemoryFreeClear(pMeta); } taosMemoryFreeClear(msg); @@ -1435,22 +1552,17 @@ int32_t ctgOpDropStbMeta(SCtgCacheOperation *operation) { return TSDB_CODE_SUCCESS; } - CTG_LOCK(CTG_WRITE, &dbCache->tbCache.stbLock); - if (taosHashRemove(dbCache->tbCache.stbCache, &msg->suid, sizeof(msg->suid))) { + if (taosHashRemove(dbCache->stbCache, &msg->suid, sizeof(msg->suid))) { ctgDebug("stb not exist in stbCache, may be removed, dbFName:%s, stb:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); } else { CTG_CACHE_STAT_DEC(stblNum, 1); } - CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); - if (taosHashRemove(dbCache->tbCache.metaCache, msg->stbName, strlen(msg->stbName))) { + if (taosHashRemove(dbCache->tbCache, msg->stbName, strlen(msg->stbName))) { ctgError("stb not exist in cache, dbFName:%s, stb:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); } else { CTG_CACHE_STAT_DEC(tblNum, 1); } - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - - CTG_UNLOCK(CTG_WRITE, &dbCache->tbCache.stbLock); ctgInfo("stb removed from cache, dbFName:%s, stbName:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); @@ -1477,21 +1589,18 @@ int32_t ctgOpDropTbMeta(SCtgCacheOperation *operation) { } if (dbCache->dbId != msg->dbId) { - ctgDebug("dbId already modified, dbFName:%s, current:%"PRIx64", dbId:%"PRIx64", tbName:%s", msg->dbFName, dbCache->dbId, msg->dbId, msg->tbName); + ctgDebug("dbId %" PRIx64 " not match with curId %"PRIx64", dbFName:%s, tbName:%s"msg->dbId, dbCache->dbId, msg->dbFName, msg->tbName); return TSDB_CODE_SUCCESS; } - CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); - if (taosHashRemove(dbCache->tbCache.metaCache, msg->tbName, strlen(msg->tbName))) { - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - ctgError("stb not exist in cache, dbFName:%s, tbName:%s", msg->dbFName, msg->tbName); - CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + if (taosHashRemove(dbCache->tbCache, msg->tbName, strlen(msg->tbName))) { + ctgError("tb %s not exist in cache, dbFName:%s", msg->tbName, msg->dbFName); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } else { CTG_CACHE_STAT_DEC(tblNum, 1); } - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - ctgInfo("table removed from cache, dbFName:%s, tbName:%s", msg->dbFName, msg->tbName); + ctgDebug("table %s removed from cache, dbFName:%s", msg->tbName, msg->dbFName); _return: @@ -1553,7 +1662,6 @@ int32_t ctgOpUpdateUser(SCtgCacheOperation *operation) { _return: - taosHashCleanup(msg->userAuth.createdDbs); taosHashCleanup(msg->userAuth.readDbs); taosHashCleanup(msg->userAuth.writeDbs); @@ -1569,24 +1677,22 @@ int32_t ctgOpUpdateEpset(SCtgCacheOperation *operation) { SCatalog* pCtg = msg->pCtg; SCtgDBCache *dbCache = NULL; - CTG_ERR_RET(ctgAcquireDBCache(pCtg, msg->dbFName, &dbCache)); + CTG_ERR_JRET(ctgGetDBCache(pCtg, msg->dbFName, &dbCache)); if (NULL == dbCache) { ctgDebug("db %s not exist, ignore epset update", msg->dbFName); goto _return; } - SDBVgInfo *vgInfo = NULL; - CTG_ERR_RET(ctgWAcquireVgInfo(pCtg, dbCache)); - - if (NULL == dbCache->vgInfo) { - ctgWReleaseVgInfo(dbCache); + CTG_ERR_JRET(ctgWLockVgInfo(pCtg, dbCache)); + + SDBVgInfo *vgInfo = dbCache->vgCache.vgInfo; + if (NULL == vgInfo) { ctgDebug("vgroup in db %s not cached, ignore epset update", msg->dbFName); goto _return; } - SVgroupInfo* pInfo = taosHashGet(dbCache->vgInfo->vgHash, &msg->vgId, sizeof(msg->vgId)); + SVgroupInfo* pInfo = taosHashGet(vgInfo->vgHash, &msg->vgId, sizeof(msg->vgId)); if (NULL == pInfo) { - ctgWReleaseVgInfo(dbCache); ctgDebug("no vgroup %d in db %s, ignore epset update", msg->vgId, msg->dbFName); goto _return; } @@ -1599,12 +1705,10 @@ int32_t ctgOpUpdateEpset(SCtgCacheOperation *operation) { pInfo->epSet = msg->epSet; - ctgWReleaseVgInfo(dbCache); - _return: if (dbCache) { - ctgReleaseDBCache(msg->pCtg, dbCache); + ctgWUnlockVgInfo(dbCache); } taosMemoryFreeClear(msg); @@ -1612,6 +1716,32 @@ _return: CTG_RET(code); } +int32_t ctgOpUpdateTbIndex(SCtgCacheOperation *operation) { + int32_t code = 0; + SCtgUpdateTbIndexMsg *msg = operation->data; + SCatalog* pCtg = msg->pCtg; + STableIndex* pIndex = msg->pIndex; + SCtgDBCache *dbCache = NULL; + + CTG_ERR_JRET(ctgGetAddDBCache(pCtg, msg->dbFName, 0, &dbCache)); + if (NULL == dbCache) { + CTG_ERR_JRET(code); + } + + CTG_ERR_JRET(ctgWriteTbIndexToCache(pCtg, dbCache, msg->tbName, &pIndex)); + +_return: + + if (pIndex) { + taosArrayDestroyEx(pIndex->pIndex, tFreeSTableIndexInfo); + taosMemoryFreeClear(pIndex); + } + + taosMemoryFreeClear(msg); + + CTG_RET(code); +} + void ctgUpdateThreadUnexpectedStopped(void) { if (CTG_IS_LOCKED(&gCtgMgmt.lock) > 0) CTG_UNLOCK(CTG_READ, &gCtgMgmt.lock); diff --git a/source/libs/catalog/src/ctgDbg.c b/source/libs/catalog/src/ctgDbg.c index d70a176631..52f29e3acb 100644 --- a/source/libs/catalog/src/ctgDbg.c +++ b/source/libs/catalog/src/ctgDbg.c @@ -266,11 +266,11 @@ int32_t ctgdGetStatNum(char *option, void *res) { } int32_t ctgdGetTbMetaNum(SCtgDBCache *dbCache) { - return dbCache->tbCache.metaCache ? (int32_t)taosHashGetSize(dbCache->tbCache.metaCache) : 0; + return dbCache->table.tbCache ? (int32_t)taosHashGetSize(dbCache->table.tbCache) : 0; } int32_t ctgdGetStbNum(SCtgDBCache *dbCache) { - return dbCache->tbCache.stbCache ? (int32_t)taosHashGetSize(dbCache->tbCache.stbCache) : 0; + return dbCache->table.stbCache ? (int32_t)taosHashGetSize(dbCache->table.stbCache) : 0; } int32_t ctgdGetRentNum(SCtgRentMgmt *rent) { @@ -363,8 +363,8 @@ void ctgdShowDBCache(SCatalog* pCtg, SHashObj *dbHash) { dbFName = taosHashGetKey(pIter, &len); - int32_t metaNum = dbCache->tbCache.metaCache ? taosHashGetSize(dbCache->tbCache.metaCache) : 0; - int32_t stbNum = dbCache->tbCache.stbCache ? taosHashGetSize(dbCache->tbCache.stbCache) : 0; + int32_t metaNum = dbCache->table.tbCache ? taosHashGetSize(dbCache->table.tbCache) : 0; + int32_t stbNum = dbCache->table.stbCache ? taosHashGetSize(dbCache->table.stbCache) : 0; int32_t vgVersion = CTG_DEFAULT_INVALID_VERSION; int32_t hashMethod = -1; int32_t vgNum = 0; diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c index bfd5057c69..c2ebd9e210 100644 --- a/source/libs/catalog/src/ctgRemote.c +++ b/source/libs/catalog/src/ctgRemote.c @@ -448,10 +448,7 @@ int32_t ctgGetTbIndexFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SName *n } if (pTask) { - void* pOut = taosMemoryCalloc(1, POINTER_BYTES); - if (NULL == pOut) { - CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); - } + void* pOut = NULL; CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, (char*)tbFName)); CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask, reqType, msg, msgLen)); diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index a792c555ee..7ab9a6ab9c 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -110,25 +110,39 @@ void ctgFreeMetaRent(SCtgRentMgmt *mgmt) { taosMemoryFreeClear(mgmt->slots); } - -void ctgFreeTbMetaCache(SCtgTbMetaCache *cache) { - CTG_LOCK(CTG_WRITE, &cache->stbLock); - if (cache->stbCache) { - int32_t stblNum = taosHashGetSize(cache->stbCache); - taosHashCleanup(cache->stbCache); - cache->stbCache = NULL; - CTG_CACHE_STAT_DEC(stblNum, stblNum); +void ctgFreeStbMetaCache(SCtgDBCache *dbCache) { + if (NULL == dbCache->stbCache) { + return; } - CTG_UNLOCK(CTG_WRITE, &cache->stbLock); - CTG_LOCK(CTG_WRITE, &cache->metaLock); - if (cache->metaCache) { - int32_t tblNum = taosHashGetSize(cache->metaCache); - taosHashCleanup(cache->metaCache); - cache->metaCache = NULL; - CTG_CACHE_STAT_DEC(tblNum, tblNum); + int32_t stblNum = taosHashGetSize(dbCache->stbCache); + taosHashCleanup(dbCache->stbCache); + dbCache->stbCache = NULL; + CTG_CACHE_STAT_DEC(stblNum, stblNum); +} + +void ctgFreeTbCacheImpl(SCtgTbCache *pCache) { + taosMemoryFreeClear(pCache->pMeta); + if (pCache->pIndex) { + taosArrayDestroyEx(pCache->pIndex->pIndex, tFreeSTableIndexInfo); + taosMemoryFreeClear(pCache->pIndex); } - CTG_UNLOCK(CTG_WRITE, &cache->metaLock); +} + +void ctgFreeTbCache(SCtgDBCache *dbCache) { + if (NULL == dbCache->tbCache) { + return; + } + + int32_t tblNum = taosHashGetSize(dbCache->tbCache); + SCtgTbCache *pCache = taosHashIterate(dbCache->tbCache, NULL); + while (NULL != pCache) { + ctgFreeTbCacheImpl(pCache); + pCache = taosHashIterate(dbCache->tbCache, pCache); + } + taosHashCleanup(dbCache->tbCache); + dbCache->tbCache = NULL; + CTG_CACHE_STAT_DEC(tblNum, tblNum); } void ctgFreeVgInfo(SDBVgInfo *vgInfo) { @@ -144,16 +158,18 @@ void ctgFreeVgInfo(SDBVgInfo *vgInfo) { taosMemoryFreeClear(vgInfo); } +void ctgFreeVgInfoCache(SCtgDBCache *dbCache) { + ctgFreeVgInfo(dbCache->vgCache.vgInfo); +} + void ctgFreeDbCache(SCtgDBCache *dbCache) { if (NULL == dbCache) { return; } - CTG_LOCK(CTG_WRITE, &dbCache->vgLock); - ctgFreeVgInfo (dbCache->vgInfo); - CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); - - ctgFreeTbMetaCache(&dbCache->tbCache); + ctgFreeVgInfoCache(dbCache); + ctgFreeStbMetaCache(dbCache); + ctgFreeTbCache(dbCache); } @@ -167,16 +183,13 @@ void ctgFreeHandle(SCatalog* pCtg) { void *pIter = taosHashIterate(pCtg->dbCache, NULL); while (pIter) { SCtgDBCache *dbCache = pIter; - atomic_store_8(&dbCache->deleted, 1); - ctgFreeDbCache(dbCache); pIter = taosHashIterate(pCtg->dbCache, pIter); } taosHashCleanup(pCtg->dbCache); - CTG_CACHE_STAT_DEC(dbNum, dbNum); } @@ -186,14 +199,12 @@ void ctgFreeHandle(SCatalog* pCtg) { void *pIter = taosHashIterate(pCtg->userCache, NULL); while (pIter) { SCtgUserAuth *userCache = pIter; - ctgFreeSCtgUserAuth(userCache); pIter = taosHashIterate(pCtg->userCache, pIter); } taosHashCleanup(pCtg->userCache); - CTG_CACHE_STAT_DEC(userNum, userNum); } @@ -252,9 +263,9 @@ void ctgFreeMsgCtx(SCtgMsgCtx* pCtx) { break; } case TDMT_MND_GET_TABLE_INDEX: { - SArray** pOut = (SArray**)pCtx->out; + STableIndex* pOut = (STableIndex*)pCtx->out; if (pOut) { - taosArrayDestroyEx(*pOut, tFreeSTableIndexInfo); + taosArrayDestroyEx(pOut->pIndex, tFreeSTableIndexInfo); taosMemoryFreeClear(pCtx->out); } break; @@ -640,6 +651,27 @@ int32_t ctgCloneMetaOutput(STableMetaOutput *output, STableMetaOutput **pOutput) return TSDB_CODE_SUCCESS; } +int32_t ctgCloneTableIndex(SArray* pIndex, SArray** pRes) { + if (NULL == pIndex) { + *pRes = NULL; + return TSDB_CODE_SUCCESS; + } + + int32_t num = taosArrayGetSize(pIndex); + *pRes = taosArrayInit(num, sizeof(STableIndexInfo)); + if (NULL == *pRes) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + for (int32_t i = 0; i < num; ++i) { + STableIndexInfo *pInfo = taosArrayGet(pIndex, i); + taosArrayPush(*pRes, pInfo); + } + + return TSDB_CODE_SUCCESS; +} + + int32_t ctgUpdateSendTargetInfo(SMsgSendInfo *pMsgSendInfo, int32_t msgType, SCtgTask* pTask) { if (msgType == TDMT_VND_TABLE_META) { SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx; diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index 04dbc26774..254525f78f 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -895,9 +895,9 @@ void *ctgTestSetCtableMetaThread(void *param) { output = (STableMetaOutput *)taosMemoryMalloc(sizeof(STableMetaOutput)); ctgTestBuildCTableMetaOutput(output); - SCtgUpdateTblMsg *msg = (SCtgUpdateTblMsg *)taosMemoryMalloc(sizeof(SCtgUpdateTblMsg)); + SCtgUpdateTbMetaMsg *msg = (SCtgUpdateTbMetaMsg *)taosMemoryMalloc(sizeof(SCtgUpdateTbMetaMsg)); msg->pCtg = pCtg; - msg->output = output; + msg->pMeta = output; operation.data = msg; code = ctgOpUpdateTbMeta(&operation); diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 10d0f4a44a..edc468a6e2 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -484,13 +484,17 @@ int32_t queryProcessGetTbIndexRsp(void *output, char *msg, int32_t msgSize) { return TSDB_CODE_TSC_INVALID_INPUT; } - STableIndexRsp out = {0}; - if (tDeserializeSTableIndexRsp(msg, msgSize, &out) != 0) { + STableIndexRsp *out = taosMemoryCalloc(1, sizeof(STableIndexRsp)); + if (NULL == out) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + if (tDeserializeSTableIndexRsp(msg, msgSize, out) != 0) { qError("tDeserializeSTableIndexRsp failed, msgSize:%d", msgSize); return TSDB_CODE_INVALID_MSG; } - *(void **)output = out.pIndex; + *(void **)output = out; return TSDB_CODE_SUCCESS; } diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index ae0eda5449..451607e7d0 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -837,7 +837,6 @@ void qwProcessHbTimerEvent(void *param, void *tmrId) { SQWorker *mgmt = qwAcquire(refId); if (NULL == mgmt) { QW_DLOG("qwAcquire %" PRIx64 "failed", refId); - taosMemoryFree(param); return; } diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 6704bfdde3..a446aa7bdb 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -1476,6 +1476,11 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) for (uint32_t i = 0; i < info->fields[FLD_TYPE_VALUE].num; ++i) { SFilterField *field = &info->fields[FLD_TYPE_VALUE].fields[i]; if (field->desc) { + if (QUERY_NODE_VALUE != nodeType(field->desc)) { + qDebug("VAL%d => [type:not value node][val:NIL]", i); //TODO + continue; + } + SValueNode *var = (SValueNode *)field->desc; SDataType *dType = &var->node.resType; if (dType->type == TSDB_DATA_TYPE_VALUE_ARRAY) { diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index a34b332b9d..2357f760d1 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -458,6 +458,9 @@ static void taosArrayInsertSort(SArray* pArray, __ext_compar_fn_t fn, const void } SArray* taosArrayDeepCopy(const SArray* pSrc, FCopy deepCopy) { + if (NULL == pSrc) { + return NULL; + } ASSERT(pSrc->elemSize == sizeof(void*)); SArray* pArray = taosArrayInit(pSrc->size, sizeof(void*)); for (int32_t i = 0; i < pSrc->size; i++) { From 17f6275015745c9b348a74c7b390964be34df239 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 13 Jun 2022 10:38:45 +0800 Subject: [PATCH 02/22] add sma cache --- include/common/tmsg.h | 16 +++---- include/libs/catalog/catalog.h | 7 ++-- source/client/src/clientHb.c | 19 +++++---- source/common/src/tmsg.c | 8 ++-- source/dnode/mnode/impl/inc/mndStb.h | 2 +- source/dnode/mnode/impl/src/mndProfile.c | 2 +- source/dnode/mnode/impl/src/mndSma.c | 17 ++++++-- source/dnode/mnode/impl/src/mndStb.c | 51 +++++++++++++++++------ source/libs/catalog/inc/catalogInt.h | 2 +- source/libs/catalog/src/catalog.c | 4 +- source/libs/catalog/src/ctgCache.c | 40 ++++++++++++++---- source/libs/catalog/src/ctgUtil.c | 8 ++-- source/libs/catalog/test/catalogTests.cpp | 8 ++-- 13 files changed, 124 insertions(+), 60 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 7c37484ef6..ed35e46c98 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1136,12 +1136,13 @@ int32_t tDeserializeSTableMetaRsp(void* buf, int32_t bufLen, STableMetaRsp* pRsp void tFreeSTableMetaRsp(STableMetaRsp* pRsp); typedef struct { - SArray* pArray; // Array of STableMetaRsp -} STableMetaBatchRsp; + SArray* pArray; // Array of STableMetaRsp + STableIndexRsp* pIndexRsp; +} SSTbHbRsp; -int32_t tSerializeSTableMetaBatchRsp(void* buf, int32_t bufLen, STableMetaBatchRsp* pRsp); -int32_t tDeserializeSTableMetaBatchRsp(void* buf, int32_t bufLen, STableMetaBatchRsp* pRsp); -void tFreeSTableMetaBatchRsp(STableMetaBatchRsp* pRsp); +int32_t tSerializeSSTbHbRsp(void* buf, int32_t bufLen, SSTbHbRsp* pRsp); +int32_t tDeserializeSSTbHbRsp(void* buf, int32_t bufLen, SSTbHbRsp* pRsp); +void tFreeSSTbHbRsp(SSTbHbRsp* pRsp); typedef struct { int32_t numOfTables; @@ -2513,8 +2514,9 @@ typedef struct { } STableIndexInfo; typedef struct { - int32_t version; - SArray* pIndex; + uint64_t suid; + int32_t version; + SArray* pIndex; } STableIndexRsp; int32_t tSerializeSTableIndexRsp(void* buf, int32_t bufLen, const STableIndexRsp* pRsp); diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 89fdeac7b9..80b677dbd2 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -98,14 +98,15 @@ typedef struct SCatalogCfg { uint32_t stbRentSec; } SCatalogCfg; -typedef struct SSTableMetaVersion { +typedef struct SSTableVersion { char dbFName[TSDB_DB_FNAME_LEN]; char stbName[TSDB_TABLE_NAME_LEN]; uint64_t dbId; uint64_t suid; int16_t sversion; int16_t tversion; -} SSTableMetaVersion; + int32_t smaVer; +} SSTableVersion; typedef struct SDbVgVersion { char dbFName[TSDB_DB_FNAME_LEN]; @@ -269,7 +270,7 @@ int32_t catalogAsyncGetAllMeta(SCatalog* pCtg, SRequestConnInfo* pConn, uint64_t int32_t catalogGetQnodeList(SCatalog* pCatalog, SRequestConnInfo* pConn, SArray* pQnodeList); -int32_t catalogGetExpiredSTables(SCatalog* pCatalog, SSTableMetaVersion **stables, uint32_t *num); +int32_t catalogGetExpiredSTables(SCatalog* pCatalog, SSTableVersion **stables, uint32_t *num); int32_t catalogGetExpiredDBs(SCatalog* pCatalog, SDbVgVersion **dbs, uint32_t *num); diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 9df0e0c1c8..1f6b00466c 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -99,15 +99,15 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) { int32_t code = 0; - STableMetaBatchRsp batchMetaRsp = {0}; - if (tDeserializeSTableMetaBatchRsp(value, valueLen, &batchMetaRsp) != 0) { + SSTbHbRsp hbRsp = {0}; + if (tDeserializeSSTbHbRsp(value, valueLen, &hbRsp) != 0) { terrno = TSDB_CODE_INVALID_MSG; return -1; } - int32_t numOfBatchs = taosArrayGetSize(batchMetaRsp.pArray); + int32_t numOfBatchs = taosArrayGetSize(hbRsp.pArray); for (int32_t i = 0; i < numOfBatchs; ++i) { - STableMetaRsp *rsp = taosArrayGet(batchMetaRsp.pArray, i); + STableMetaRsp *rsp = taosArrayGet(hbRsp.pArray, i); if (rsp->numOfColumns < 0) { tscDebug("hb remove stb, db:%s, stb:%s", rsp->dbFName, rsp->stbName); @@ -116,7 +116,7 @@ static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalo tscDebug("hb update stb, db:%s, stb:%s", rsp->dbFName, rsp->stbName); if (rsp->pSchemas[0].colId != PRIMARYKEY_TIMESTAMP_COL_ID) { tscError("invalid colId[%" PRIi16 "] for the first column in table meta rsp msg", rsp->pSchemas[0].colId); - tFreeSTableMetaBatchRsp(&batchMetaRsp); + tFreeSSTbHbRsp(&hbRsp); return TSDB_CODE_TSC_INVALID_VALUE; } @@ -124,7 +124,7 @@ static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalo } } - tFreeSTableMetaBatchRsp(&batchMetaRsp); + tFreeSSTbHbRsp(&hbRsp); return TSDB_CODE_SUCCESS; } @@ -455,7 +455,7 @@ int32_t hbGetExpiredDBInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SCl } int32_t hbGetExpiredStbInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SClientHbReq *req) { - SSTableMetaVersion *stbs = NULL; + SSTableVersion *stbs = NULL; uint32_t stbNum = 0; int32_t code = 0; @@ -469,15 +469,16 @@ int32_t hbGetExpiredStbInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SC } for (int32_t i = 0; i < stbNum; ++i) { - SSTableMetaVersion *stb = &stbs[i]; + SSTableVersion *stb = &stbs[i]; stb->suid = htobe64(stb->suid); stb->sversion = htons(stb->sversion); stb->tversion = htons(stb->tversion); + stb->smaVer = htonl(stb->smaVer); } SKv kv = { .key = HEARTBEAT_KEY_STBINFO, - .valueLen = sizeof(SSTableMetaVersion) * stbNum, + .valueLen = sizeof(SSTableVersion) * stbNum, .value = stbs, }; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 3805a8b5b4..138aaef3da 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -2437,6 +2437,7 @@ int32_t tSerializeSTableIndexRsp(void *buf, int32_t bufLen, const STableIndexRsp tEncoderInit(&encoder, buf, bufLen); if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeU64(&encoder, pRsp->suid) < 0) return -1; if (tEncodeI32(&encoder, pRsp->version) < 0) return -1; int32_t num = taosArrayGetSize(pRsp->pIndex); if (tEncodeI32(&encoder, num) < 0) return -1; @@ -2472,6 +2473,7 @@ int32_t tDeserializeSTableIndexRsp(void *buf, int32_t bufLen, STableIndexRsp *pR tDecoderInit(&decoder, buf, bufLen); if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeU64(&decoder, &pRsp->suid) < 0) return -1; if (tDecodeI32(&decoder, &pRsp->version) < 0) return -1; int32_t num = 0; if (tDecodeI32(&decoder, &num) < 0) return -1; @@ -2632,7 +2634,7 @@ int32_t tSerializeSTableMetaRsp(void *buf, int32_t bufLen, STableMetaRsp *pRsp) return tlen; } -int32_t tSerializeSTableMetaBatchRsp(void *buf, int32_t bufLen, STableMetaBatchRsp *pRsp) { +int32_t tSerializeSSTbHbRsp(void *buf, int32_t bufLen, SSTbHbRsp *pRsp) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -2663,7 +2665,7 @@ int32_t tDeserializeSTableMetaRsp(void *buf, int32_t bufLen, STableMetaRsp *pRsp return 0; } -int32_t tDeserializeSTableMetaBatchRsp(void *buf, int32_t bufLen, STableMetaBatchRsp *pRsp) { +int32_t tDeserializeSSTbHbRsp(void *buf, int32_t bufLen, SSTbHbRsp *pRsp) { SDecoder decoder = {0}; tDecoderInit(&decoder, buf, bufLen); @@ -2691,7 +2693,7 @@ int32_t tDeserializeSTableMetaBatchRsp(void *buf, int32_t bufLen, STableMetaBatc void tFreeSTableMetaRsp(STableMetaRsp *pRsp) { taosMemoryFreeClear(pRsp->pSchemas); } -void tFreeSTableMetaBatchRsp(STableMetaBatchRsp *pRsp) { +void tFreeSSTbHbRsp(SSTbHbRsp *pRsp) { int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); for (int32_t i = 0; i < numOfBatch; ++i) { STableMetaRsp *pMetaRsp = taosArrayGet(pRsp->pArray, i); diff --git a/source/dnode/mnode/impl/inc/mndStb.h b/source/dnode/mnode/impl/inc/mndStb.h index 28d3215b98..d3de4e3b3f 100644 --- a/source/dnode/mnode/impl/inc/mndStb.h +++ b/source/dnode/mnode/impl/inc/mndStb.h @@ -27,7 +27,7 @@ void mndCleanupStb(SMnode *pMnode); SStbObj *mndAcquireStb(SMnode *pMnode, char *stbName); void mndReleaseStb(SMnode *pMnode, SStbObj *pStb); SSdbRaw *mndStbActionEncode(SStbObj *pStb); -int32_t mndValidateStbInfo(SMnode *pMnode, SSTableMetaVersion *pStbs, int32_t numOfStbs, void **ppRsp, +int32_t mndValidateStbInfo(SMnode *pMnode, SSTableVersion *pStbs, int32_t numOfStbs, void **ppRsp, int32_t *pRspLen); int32_t mndGetNumOfStbs(SMnode *pMnode, char *dbName, int32_t *pNumOfStbs); diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index bacdf2f366..4cfcf389aa 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -433,7 +433,7 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb case HEARTBEAT_KEY_STBINFO: { void *rspMsg = NULL; int32_t rspLen = 0; - mndValidateStbInfo(pMnode, kv->value, kv->valueLen / sizeof(SSTableMetaVersion), &rspMsg, &rspLen); + mndValidateStbInfo(pMnode, kv->value, kv->valueLen / sizeof(SSTableVersion), &rspMsg, &rspLen); if (rspMsg && rspLen > 0) { SKv kv1 = {.key = HEARTBEAT_KEY_STBINFO, .valueLen = rspLen, .value = rspMsg}; taosArrayPush(hbRsp.info, &kv1); diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index cde36eac58..edb8a29ac0 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -872,18 +872,29 @@ static int32_t mndGetSma(SMnode *pMnode, SUserIndexReq *indexReq, SUserIndexRsp return code; } -static int32_t mndGetTableSma(SMnode *pMnode, STableIndexReq *indexReq, STableIndexRsp *rsp, bool *exist) { +static int32_t mndGetTableSma(SMnode *pMnode, char *tbFName, STableIndexRsp *rsp, bool *exist) { int32_t code = 0; SSmaObj *pSma = NULL; SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; STableIndexInfo info; + SStbObj* pStb = mndAcquireStb(pMnode, tbFName); + if (NULL == pStb) { + *exist = false; + return TSDB_CODE_SUCCESS; + } + + rsp->suid = pStb->uid; + rsp->version = pStb->smaVer; + mndReleaseStb(pMnode, pStb); + + while (1) { pIter = sdbFetch(pSdb, SDB_SMA, pIter, (void **)&pSma); if (pIter == NULL) break; - if (pSma->stb[0] != indexReq->tbFName[0] || strcmp(pSma->stb, indexReq->tbFName)) { + if (pSma->stb[0] != tbFName[0] || strcmp(pSma->stb, tbFName)) { continue; } @@ -995,7 +1006,7 @@ static int32_t mndProcessGetTbSmaReq(SRpcMsg *pReq) { goto _OVER; } - code = mndGetTableSma(pMnode, &indexReq, &rsp, &exist); + code = mndGetTableSma(pMnode, indexReq.tbFName, &rsp, &exist); if (code) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 9981dc8530..4b5b9e8a7b 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -1271,7 +1271,7 @@ static int32_t mndBuildStbSchemaImp(SDbObj *pDb, SStbObj *pStb, const char *tbNa return 0; } -static int32_t mndBuildStbSchema(SMnode *pMnode, const char *dbFName, const char *tbName, STableMetaRsp *pRsp) { +static int32_t mndBuildStbSchema(SMnode *pMnode, const char *dbFName, const char *tbName, STableMetaRsp *pRsp, int32_t *smaVer) { char tbFName[TSDB_TABLE_FNAME_LEN] = {0}; snprintf(tbFName, sizeof(tbFName), "%s.%s", dbFName, tbName); @@ -1288,6 +1288,10 @@ static int32_t mndBuildStbSchema(SMnode *pMnode, const char *dbFName, const char return -1; } + if (smaVer) { + *smaVer = pStb->smaVer; + } + int32_t code = mndBuildStbSchemaImp(pDb, pStb, tbName, pRsp); mndReleaseDb(pMnode, pDb); mndReleaseStb(pMnode, pStb); @@ -1667,51 +1671,72 @@ _OVER: return code; } -int32_t mndValidateStbInfo(SMnode *pMnode, SSTableMetaVersion *pStbVersions, int32_t numOfStbs, void **ppRsp, +int32_t mndValidateStbInfo(SMnode *pMnode, SSTableVersion *pStbVersions, int32_t numOfStbs, void **ppRsp, int32_t *pRspLen) { - STableMetaBatchRsp batchMetaRsp = {0}; - batchMetaRsp.pArray = taosArrayInit(numOfStbs, sizeof(STableMetaRsp)); - if (batchMetaRsp.pArray == NULL) { + SSTbHbRsp hbRsp = {0}; + hbRsp.pArray = taosArrayInit(numOfStbs, sizeof(STableMetaRsp)); + if (hbRsp.pArray == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } + hbRsp.pIndexRsp = taosMemoryCalloc(1, sizeof(STableIndexRsp)); + if (NULL == hbRsp.pIndexRsp) { + taosArrayDestroy(hbRsp.pArray); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + for (int32_t i = 0; i < numOfStbs; ++i) { - SSTableMetaVersion *pStbVersion = &pStbVersions[i]; + SSTableVersion *pStbVersion = &pStbVersions[i]; pStbVersion->suid = be64toh(pStbVersion->suid); pStbVersion->sversion = ntohs(pStbVersion->sversion); pStbVersion->tversion = ntohs(pStbVersion->tversion); + pStbVersion->smaVer = ntohl(pStbVersion->smaVer); STableMetaRsp metaRsp = {0}; + int32_t smaVer = 0; mDebug("stb:%s.%s, start to retrieve meta", pStbVersion->dbFName, pStbVersion->stbName); - if (mndBuildStbSchema(pMnode, pStbVersion->dbFName, pStbVersion->stbName, &metaRsp) != 0) { + if (mndBuildStbSchema(pMnode, pStbVersion->dbFName, pStbVersion->stbName, &metaRsp, &smaVer) != 0) { metaRsp.numOfColumns = -1; metaRsp.suid = pStbVersion->suid; + taosArrayPush(hbRsp.pArray, &metaRsp); + continue; } if (pStbVersion->sversion != metaRsp.sversion || pStbVersion->tversion != metaRsp.tversion) { - taosArrayPush(batchMetaRsp.pArray, &metaRsp); + taosArrayPush(hbRsp.pArray, &metaRsp); } else { tFreeSTableMetaRsp(&metaRsp); } + + if (pStbVersion->smaVer != smaVer) { + bool exist = false; + char tbFName[TSDB_TABLE_FNAME_LEN]; + sprintf(tbFName, "%s.%s", pStbVersion->dbFName, pStbVersion->stbName); + int32_t code = mndGetTableSma(pMnode, tbFName, hbRsp.pIndexRsp, &exist); + if (code || !exist) { + taosMemoryFreeClear(hbRsp.pIndexRsp); + } + } } - int32_t rspLen = tSerializeSTableMetaBatchRsp(NULL, 0, &batchMetaRsp); + int32_t rspLen = tSerializeSSTbHbRsp(NULL, 0, &hbRsp); if (rspLen < 0) { - tFreeSTableMetaBatchRsp(&batchMetaRsp); + tFreeSSTbHbRsp(&hbRsp); terrno = TSDB_CODE_INVALID_MSG; return -1; } void *pRsp = taosMemoryMalloc(rspLen); if (pRsp == NULL) { - tFreeSTableMetaBatchRsp(&batchMetaRsp); + tFreeSSTbHbRsp(&hbRsp); terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - tSerializeSTableMetaBatchRsp(pRsp, rspLen, &batchMetaRsp); - tFreeSTableMetaBatchRsp(&batchMetaRsp); + tSerializeSSTbHbRsp(pRsp, rspLen, &hbRsp); + tFreeSSTbHbRsp(&hbRsp); *ppRsp = pRsp; *pRspLen = rspLen; return 0; diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index ebd4d56aaf..82bb151d86 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -156,7 +156,7 @@ typedef struct SCtgDBCache { typedef struct SCtgRentSlot { SRWLatch lock; bool needSort; - SArray *meta; // element is SDbVgVersion or SSTableMetaVersion + SArray *meta; // element is SDbVgVersion or SSTableVersion } SCtgRentSlot; typedef struct SCtgRentMgmt { diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index bac13bf476..d01fa00878 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -1074,14 +1074,14 @@ _return: CTG_API_LEAVE(TSDB_CODE_SUCCESS); } -int32_t catalogGetExpiredSTables(SCatalog* pCtg, SSTableMetaVersion **stables, uint32_t *num) { +int32_t catalogGetExpiredSTables(SCatalog* pCtg, SSTableVersion **stables, uint32_t *num) { CTG_API_ENTER(); if (NULL == pCtg || NULL == stables || NULL == num) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - CTG_API_LEAVE(ctgMetaRentGet(&pCtg->stbRent, (void **)stables, num, sizeof(SSTableMetaVersion))); + CTG_API_LEAVE(ctgMetaRentGet(&pCtg->stbRent, (void **)stables, num, sizeof(SSTableVersion))); } int32_t catalogGetExpiredDBs(SCatalog* pCtg, SDbVgVersion **dbs, uint32_t *num) { diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 2b56eb6b49..49714ac2de 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -970,7 +970,7 @@ int32_t ctgMetaRentUpdate(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t si void *orig = taosArraySearch(slot->meta, &id, searchCompare, TD_EQ); if (NULL == orig) { - qError("meta not found in slot, id:%"PRIx64", slot idx:%d, type:%d, size:%d", id, widx, mgmt->type, (int32_t)taosArrayGetSize(slot->meta)); + qDebug("meta not found in slot, id:%"PRIx64", slot idx:%d, type:%d, size:%d", id, widx, mgmt->type, (int32_t)taosArrayGetSize(slot->meta)); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } @@ -983,7 +983,7 @@ _return: CTG_UNLOCK(CTG_WRITE, &slot->lock); if (code) { - qWarn("meta in rent update failed, will try to add it, code:%x, id:%"PRIx64", slot idx:%d, type:%d", code, id, widx, mgmt->type); + qDebug("meta in rent update failed, will try to add it, code:%x, id:%"PRIx64", slot idx:%d, type:%d", code, id, widx, mgmt->type); CTG_RET(ctgMetaRentAdd(mgmt, meta, id, size)); } @@ -1233,6 +1233,27 @@ int32_t ctgGetAddDBCache(SCatalog* pCtg, const char *dbFName, uint64_t dbId, SCt return TSDB_CODE_SUCCESS; } +int32_t ctgUpdateRentStbVersion(SCatalog *pCtg, char* dbFName, char* tbName, uint64_t dbId, uint64_t suid, SCtgTbCache* pCache) { + SSTableVersion metaRent = {.dbId = dbId, .suid = suid}; + if (pCache->pMeta) { + metaRent.sversion = pCache->pMeta->sversion; + metaRent.tversion = pCache->pMeta->tversion; + } + + if (pCache->pIndex) { + metaRent.smaVer = pCache->pIndex->version; + } + + strcpy(metaRent.dbFName, dbFName); + strcpy(metaRent.stbName, tbName); + + CTG_ERR_RET(ctgMetaRentUpdate(&pCtg->stbRent, &metaRent, metaRent.suid, sizeof(SSTableVersion), ctgStbVersionSortCompare, ctgStbVersionSearchCompare)); + + ctgDebug("db %s,%" PRIx64 " stb %s,%" PRIx64 " sver %d tver %d smaVer %d updated to stbRent", + dbFName, dbId, tbName, suid, metaRent.sversion, metaRent.tversion, metaRent.smaVer); +} + + int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, uint64_t dbId, char *tbName, STableMeta *meta, int32_t metaSize) { if (NULL == dbCache->tbCache || NULL == dbCache->stbCache) { taosMemoryFree(meta); @@ -1263,7 +1284,6 @@ int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNam ctgDebug("stb removed from stbCache, dbFName:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid); } - ctgMetaRentRemove(&pCtg->stbRent, orig->suid, ctgStbVersionSortCompare, ctgStbVersionSearchCompare); origSuid = orig->suid; } } @@ -1303,15 +1323,12 @@ int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNam ctgDebug("stb updated to stbCache, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType); - SSTableMetaVersion metaRent = {.dbId = dbId, .suid = meta->suid, .sversion = meta->sversion, .tversion = meta->tversion}; - strcpy(metaRent.dbFName, dbFName); - strcpy(metaRent.stbName, tbName); - CTG_ERR_RET(ctgMetaRentAdd(&pCtg->stbRent, &metaRent, metaRent.suid, sizeof(SSTableMetaVersion))); + CTG_ERR_RET(ctgUpdateRentStbVersion(pCtg, dbFName, tbName, dbId, meta->suid, pCache)); return TSDB_CODE_SUCCESS; } -int32_t ctgWriteTbIndexToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *tbName, STableIndex **index) { +int32_t ctgWriteTbIndexToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char* dbFName, char *tbName, STableIndex **index) { if (NULL == dbCache->tbCache) { taosMemoryFreeClear(*index); ctgError("db is dropping, dbId:%"PRIx64, dbCache->dbId); @@ -1332,6 +1349,9 @@ int32_t ctgWriteTbIndexToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *tbNam *index = NULL; ctgDebug("table %s index updated to cache, ver:%d, num:%d", tbName, pIndex->version, taosArrayGetSize(pIndex->pIndex)); + + CTG_ERR_RET(ctgUpdateRentStbVersion(pCtg, dbFName, tbName, dbCache->dbId, pIndex->suid, pCache)); + return TSDB_CODE_SUCCESS; } @@ -1344,6 +1364,8 @@ int32_t ctgWriteTbIndexToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *tbNam *index = NULL; ctgDebug("table %s index updated to cache, ver:%d, num:%d", tbName, pIndex->version, taosArrayGetSize(pIndex->pIndex)); + + CTG_ERR_RET(ctgUpdateRentStbVersion(pCtg, dbFName, tbName, dbCache->dbId, pIndex->suid, pCache)); return TSDB_CODE_SUCCESS; } @@ -1728,7 +1750,7 @@ int32_t ctgOpUpdateTbIndex(SCtgCacheOperation *operation) { CTG_ERR_JRET(code); } - CTG_ERR_JRET(ctgWriteTbIndexToCache(pCtg, dbCache, msg->tbName, &pIndex)); + CTG_ERR_JRET(ctgWriteTbIndexToCache(pCtg, dbCache, msg->dbFName, msg->tbName, &pIndex)); _return: diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index 7ab9a6ab9c..f0ed60a689 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -546,9 +546,9 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog *pCtg, SDBVgInfo *dbInfo, const SName } int32_t ctgStbVersionSearchCompare(const void* key1, const void* key2) { - if (*(uint64_t *)key1 < ((SSTableMetaVersion*)key2)->suid) { + if (*(uint64_t *)key1 < ((SSTableVersion*)key2)->suid) { return -1; - } else if (*(uint64_t *)key1 > ((SSTableMetaVersion*)key2)->suid) { + } else if (*(uint64_t *)key1 > ((SSTableVersion*)key2)->suid) { return 1; } else { return 0; @@ -566,9 +566,9 @@ int32_t ctgDbVgVersionSearchCompare(const void* key1, const void* key2) { } int32_t ctgStbVersionSortCompare(const void* key1, const void* key2) { - if (((SSTableMetaVersion*)key1)->suid < ((SSTableMetaVersion*)key2)->suid) { + if (((SSTableVersion*)key1)->suid < ((SSTableVersion*)key2)->suid) { return -1; - } else if (((SSTableMetaVersion*)key1)->suid > ((SSTableMetaVersion*)key2)->suid) { + } else if (((SSTableVersion*)key1)->suid > ((SSTableVersion*)key2)->suid) { return 1; } else { return 0; diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index 254525f78f..aab341544f 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -989,7 +989,7 @@ TEST(tableMeta, normalTable) { ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); SDbVgVersion *dbs = NULL; - SSTableMetaVersion *stb = NULL; + SSTableVersion *stb = NULL; uint32_t dbNum = 0, stbNum = 0, allDbNum = 0, allStbNum = 0; int32_t i = 0; while (i < 5) { @@ -1098,7 +1098,7 @@ TEST(tableMeta, childTableCase) { ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); SDbVgVersion *dbs = NULL; - SSTableMetaVersion *stb = NULL; + SSTableVersion *stb = NULL; uint32_t dbNum = 0, stbNum = 0, allDbNum = 0, allStbNum = 0; int32_t i = 0; while (i < 5) { @@ -1220,7 +1220,7 @@ TEST(tableMeta, superTableCase) { ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); SDbVgVersion *dbs = NULL; - SSTableMetaVersion *stb = NULL; + SSTableVersion *stb = NULL; uint32_t dbNum = 0, stbNum = 0, allDbNum = 0, allStbNum = 0; int32_t i = 0; while (i < 5) { @@ -2299,7 +2299,7 @@ TEST(rentTest, allRent) { SArray *vgList = NULL; ctgTestStop = false; SDbVgVersion *dbs = NULL; - SSTableMetaVersion *stable = NULL; + SSTableVersion *stable = NULL; uint32_t num = 0; ctgTestInitLogFile(); From 803363571704761d603b96ce8098192e887eb502 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 13 Jun 2022 11:29:53 +0800 Subject: [PATCH 03/22] other: add debug logs --- source/common/src/trow.c | 17 +++ source/dnode/vnode/src/sma/smaTimeRange.c | 178 ++++++++++++++++++++++ source/dnode/vnode/src/vnd/vnodeSvr.c | 6 +- 3 files changed, 200 insertions(+), 1 deletion(-) create mode 100644 source/dnode/vnode/src/sma/smaTimeRange.c diff --git a/source/common/src/trow.c b/source/common/src/trow.c index c8a28d7f28..fb4413478d 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -523,8 +523,24 @@ static int32_t tdAppendTpRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols SCellVal sVal = {0}; if (pRowCol->colId == pDataCol->colId) { if (tdGetTpRowValOfCol(&sVal, pRow, pBitmap, pRowCol->type, pRowCol->offset - sizeof(TSKEY), rcol - 1) < 0) { + ASSERT(0); return terrno; } + + if (pRowCol->colId == 2) { + ASSERT(sVal.valType == 0); + int32_t val = *(int32_t *)sVal.val; + ASSERT(val == 7); + } else if (pRowCol->colId == 3) { + ASSERT(sVal.valType == 0); + int64_t val = *(int64_t *)sVal.val; + ASSERT(val == 77777); + } else if (pRowCol->colId == 4) { + ASSERT(sVal.valType == 0); + int16_t val = *(int16_t *)sVal.val; + ASSERT(val == 777); + } + tdAppendValToDataCol(pDataCol, sVal.valType, sVal.val, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode, isMerge); ++dcol; @@ -535,6 +551,7 @@ static int32_t tdAppendTpRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode, isMerge); ++dcol; + ASSERT(0); } } #if 0 diff --git a/source/dnode/vnode/src/sma/smaTimeRange.c b/source/dnode/vnode/src/sma/smaTimeRange.c new file mode 100644 index 0000000000..bca5b1543e --- /dev/null +++ b/source/dnode/vnode/src/sma/smaTimeRange.c @@ -0,0 +1,178 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "sma.h" +#include "tsdb.h" + +#define SMA_STORAGE_MINUTES_MAX 86400 +#define SMA_STORAGE_MINUTES_DAY 1440 +#define SMA_STORAGE_SPLIT_FACTOR 14400 // least records in tsma file + +/** + * @brief Judge the tsma file split days + * + * @param pCfg + * @param pCont + * @param contLen + * @param days unit is minute + * @return int32_t + */ +int32_t tdProcessTSmaGetDaysImpl(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days) { + SDecoder coder = {0}; + tDecoderInit(&coder, pCont, contLen); + + STSma tsma = {0}; + if (tDecodeSVCreateTSmaReq(&coder, &tsma) < 0) { + terrno = TSDB_CODE_MSG_DECODE_ERROR; + goto _err; + } + STsdbCfg *pTsdbCfg = &pCfg->tsdbCfg; + int64_t sInterval = convertTimeFromPrecisionToUnit(tsma.interval, pTsdbCfg->precision, TIME_UNIT_SECOND); + if (sInterval <= 0) { + *days = pTsdbCfg->days; + return 0; + } + int64_t records = pTsdbCfg->days * 60 / sInterval; + if (records >= SMA_STORAGE_SPLIT_FACTOR) { + *days = pTsdbCfg->days; + } else { + int64_t mInterval = convertTimeFromPrecisionToUnit(tsma.interval, pTsdbCfg->precision, TIME_UNIT_MINUTE); + int64_t daysPerFile = mInterval * SMA_STORAGE_MINUTES_DAY * 2; + + if (daysPerFile > SMA_STORAGE_MINUTES_MAX) { + *days = SMA_STORAGE_MINUTES_MAX; + } else { + *days = (int32_t)daysPerFile; + } + + if (*days < pTsdbCfg->days) { + *days = pTsdbCfg->days; + } + } + tDecoderClear(&coder); + return 0; +_err: + tDecoderClear(&coder); + return -1; +} + +/** + * @brief create tsma meta and result stable + * + * @param pSma + * @param version + * @param pMsg + * @return int32_t + */ +int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *pMsg) { + SSmaCfg *pCfg = (SSmaCfg *)pMsg; + + if (TD_VID(pSma->pVnode) == pCfg->dstVgId) { + // create tsma meta in dstVgId + if (metaCreateTSma(SMA_META(pSma), version, pCfg) < 0) { + return -1; + } + + // create stable to save tsma result in dstVgId + SVCreateStbReq pReq = {0}; + pReq.name = pCfg->dstTbName; + pReq.suid = pCfg->dstTbUid; + pReq.schemaRow = pCfg->schemaRow; + pReq.schemaTag = pCfg->schemaTag; + + if (metaCreateSTable(SMA_META(pSma), version, &pReq) < 0) { + return -1; + } + } + + return 0; +} + +/** + * @brief Insert/Update Time-range-wise SMA data. + * + * @param pSma + * @param msg + * @return int32_t + */ +int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) { + const SArray *pDataBlocks = (const SArray *)msg; + // TODO: destroy SSDataBlocks(msg) + if (!pDataBlocks) { + terrno = TSDB_CODE_TSMA_INVALID_PTR; + smaWarn("vgId:%d, insert tsma data failed since pDataBlocks is NULL", SMA_VID(pSma)); + return terrno; + } + + if (taosArrayGetSize(pDataBlocks) <= 0) { + terrno = TSDB_CODE_TSMA_INVALID_PARA; + smaWarn("vgId:%d, insert tsma data failed since pDataBlocks is empty", SMA_VID(pSma)); + return TSDB_CODE_FAILED; + } + + if (tdCheckAndInitSmaEnv(pSma, TSDB_SMA_TYPE_TIME_RANGE) != 0) { + terrno = TSDB_CODE_TSMA_INIT_FAILED; + return TSDB_CODE_FAILED; + } + + SSmaEnv *pEnv = SMA_TSMA_ENV(pSma); + SSmaStat *pStat = NULL; + SSmaStatItem *pItem = NULL; + + if (!pEnv || !(pStat = SMA_ENV_STAT(pEnv))) { + terrno = TSDB_CODE_TSMA_INVALID_STAT; + return TSDB_CODE_FAILED; + } + + tdRefSmaStat(pSma, pStat); + pItem = &pStat->tsmaStatItem; + + ASSERT(pItem); + + if (!pItem->pTSma) { + // cache smaMeta + STSma *pTSma = metaGetSmaInfoByIndex(SMA_META(pSma), indexUid); + if (!pTSma) { + terrno = TSDB_CODE_TSMA_NO_INDEX_IN_META; + smaWarn("vgId:%d, tsma insert for smaIndex %" PRIi64 " since %s", SMA_VID(pSma), indexUid, tstrerror(terrno)); + return TSDB_CODE_FAILED; + } + pItem->pTSma = pTSma; + } + + STSma *pTSma = pItem->pTSma; + + ASSERT(pTSma->indexUid == indexUid); + + SMetaReader mr = {0}; + + const char *dbName = "testDb"; + if (metaGetTableEntryByName(&mr, dbName) != 0) { + smaDebug("vgId:%d, tsma no table testTb exists for smaIndex %" PRIi64 " since %s", SMA_VID(pSma), indexUid, tstrerror(terrno)); + SVCreateStbReq pReq = {0}; + pReq.name = dbName; + pReq.suid = pTSma->dstTbUid; + pReq.schemaRow = pCfg->schemaRow; + pReq.schemaTag = pCfg->schemaTag; + } + + SSubmitReq *pSubmitReq = NULL; + buildSubmitReqFromDataBlock(&pSubmitReq, (const SArray *)msg, NULL, pItem->pTSma->dstVgId, + pItem->pTSma->dstTbUid); + + tdUnRefSmaStat(pSma, pStat); + + return TSDB_CODE_SUCCESS; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index ab2efa4791..9bb0834547 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -15,6 +15,8 @@ #include "vnd.h" +int32_t gForceCommit = 1; + static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessAlterStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessDropStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); @@ -182,7 +184,8 @@ int32_t vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp } // commit if need - if (vnodeShouldCommit(pVnode)) { + if ((gForceCommit == 1) || vnodeShouldCommit(pVnode)) { + gForceCommit = 0; vInfo("vgId:%d, commit at version %" PRId64, TD_VID(pVnode), version); // commit current change vnodeCommit(pVnode); @@ -719,6 +722,7 @@ static int32_t vnodeDebugPrintSubmitMsg(SVnode *pVnode, SSubmitReq *pMsg, const } static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { + gForceCommit = 1; SSubmitReq *pSubmitReq = (SSubmitReq *)pReq; SSubmitRsp submitRsp = {0}; SSubmitMsgIter msgIter = {0}; From c0964e03bee2cf02445dd797a630544cbf9f2ed1 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 13 Jun 2022 12:51:21 +0800 Subject: [PATCH 04/22] other: add debug logs --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 90093f2510..40a23447b7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -1348,6 +1348,24 @@ static int tsdbCommitMemData(SCommitH *pCommith, SCommitIter *pIter, TSKEY keyLi isLast = true; } + SDataCols *pCols = pCommith->pDataCols; + SDataCol pCol_1 = pCols->cols[1]; + SDataCol pCol_2 = pCols->cols[2]; + SDataCol pCol_3 = pCols->cols[3]; + SCellVal sVal = {0}; + tdGetColDataOfRow(&sVal, &pCol_1, 0, pCols->bitmapMode); + ASSERT(sVal.valType == 0); + int32_t val1 = *(int32_t *)sVal.val; + ASSERT(val1 == 7); + tdGetColDataOfRow(&sVal, &pCol_2, 0, pCols->bitmapMode); + ASSERT(sVal.valType == 0); + int64_t val2 = *(int64_t *)sVal.val; + ASSERT(val2 == 77777); + tdGetColDataOfRow(&sVal, &pCol_3, 0, pCols->bitmapMode); + ASSERT(sVal.valType == 0); + int16_t val3 = *(int16_t *)sVal.val; + ASSERT(val3 == 777); + if (tsdbWriteBlock(pCommith, pDFile, pCommith->pDataCols, &block, isLast, true) < 0) return -1; if (tsdbCommitAddBlock(pCommith, &block, NULL, 0) < 0) { From eff9f9be4db8c0b5081c0d6b14a87e0c30128a35 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 13 Jun 2022 14:06:11 +0800 Subject: [PATCH 05/22] add sma update --- include/common/tmsg.h | 6 +- source/client/src/clientHb.c | 16 ++++- source/common/src/tmsg.c | 102 +++++++++++++++++++++++---- source/dnode/mnode/impl/src/mndSma.c | 2 + source/dnode/mnode/impl/src/mndStb.c | 24 ++++--- source/libs/catalog/inc/catalogInt.h | 4 +- source/libs/catalog/src/catalog.c | 54 ++++++++++++-- source/libs/catalog/src/ctgAsync.c | 3 +- source/libs/catalog/src/ctgCache.c | 16 ++--- source/libs/catalog/src/ctgRemote.c | 8 ++- source/libs/qcom/src/querymsg.c | 8 +-- 11 files changed, 189 insertions(+), 54 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index ed35e46c98..8a2b2a31f2 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1136,8 +1136,8 @@ int32_t tDeserializeSTableMetaRsp(void* buf, int32_t bufLen, STableMetaRsp* pRsp void tFreeSTableMetaRsp(STableMetaRsp* pRsp); typedef struct { - SArray* pArray; // Array of STableMetaRsp - STableIndexRsp* pIndexRsp; + SArray* pMetaRsp; // Array of STableMetaRsp + SArray* pIndexRsp; // Array of STableIndexRsp; } SSTbHbRsp; int32_t tSerializeSSTbHbRsp(void* buf, int32_t bufLen, SSTbHbRsp* pRsp); @@ -2514,6 +2514,8 @@ typedef struct { } STableIndexInfo; typedef struct { + char tbName[TSDB_TABLE_NAME_LEN]; + char dbFName[TSDB_DB_FNAME_LEN]; uint64_t suid; int32_t version; SArray* pIndex; diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 1f6b00466c..e111883381 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -105,9 +105,9 @@ static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalo return -1; } - int32_t numOfBatchs = taosArrayGetSize(hbRsp.pArray); - for (int32_t i = 0; i < numOfBatchs; ++i) { - STableMetaRsp *rsp = taosArrayGet(hbRsp.pArray, i); + int32_t numOfMeta = taosArrayGetSize(hbRsp.pMetaRsp); + for (int32_t i = 0; i < numOfMeta; ++i) { + STableMetaRsp *rsp = taosArrayGet(hbRsp.pMetaRsp, i); if (rsp->numOfColumns < 0) { tscDebug("hb remove stb, db:%s, stb:%s", rsp->dbFName, rsp->stbName); @@ -124,6 +124,16 @@ static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalo } } + int32_t numOfIndex = taosArrayGetSize(hbRsp.pIndexRsp); + for (int32_t i = 0; i < numOfIndex; ++i) { + STableIndexRsp *rsp = taosArrayGet(hbRsp.pIndexRsp, i); + + catalogUpdateTableIndex(pCatalog, rsp); + } + + taosArrayDestroy(hbRsp.pIndexRsp); + hbRsp.pIndexRsp = NULL; + tFreeSSTbHbRsp(&hbRsp); return TSDB_CODE_SUCCESS; } diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 138aaef3da..5d0cc2b28d 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -2437,6 +2437,8 @@ int32_t tSerializeSTableIndexRsp(void *buf, int32_t bufLen, const STableIndexRsp tEncoderInit(&encoder, buf, bufLen); if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pRsp->tbName) < 0) return -1; + if (tEncodeCStr(&encoder, pRsp->dbFName) < 0) return -1; if (tEncodeU64(&encoder, pRsp->suid) < 0) return -1; if (tEncodeI32(&encoder, pRsp->version) < 0) return -1; int32_t num = taosArrayGetSize(pRsp->pIndex); @@ -2473,6 +2475,8 @@ int32_t tDeserializeSTableIndexRsp(void *buf, int32_t bufLen, STableIndexRsp *pR tDecoderInit(&decoder, buf, bufLen); if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeCStrTo(&decoder, pRsp->tbName) < 0) return -1; + if (tDecodeCStrTo(&decoder, pRsp->dbFName) < 0) return -1; if (tDecodeU64(&decoder, &pRsp->suid) < 0) return -1; if (tDecodeI32(&decoder, &pRsp->version) < 0) return -1; int32_t num = 0; @@ -2640,12 +2644,30 @@ int32_t tSerializeSSTbHbRsp(void *buf, int32_t bufLen, SSTbHbRsp *pRsp) { if (tStartEncode(&encoder) < 0) return -1; - int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); - if (tEncodeI32(&encoder, numOfBatch) < 0) return -1; - for (int32_t i = 0; i < numOfBatch; ++i) { - STableMetaRsp *pMetaRsp = taosArrayGet(pRsp->pArray, i); + int32_t numOfMeta = taosArrayGetSize(pRsp->pMetaRsp); + if (tEncodeI32(&encoder, numOfMeta) < 0) return -1; + for (int32_t i = 0; i < numOfMeta; ++i) { + STableMetaRsp *pMetaRsp = taosArrayGet(pRsp->pMetaRsp, i); if (tEncodeSTableMetaRsp(&encoder, pMetaRsp) < 0) return -1; } + + int32_t numOfIndex = taosArrayGetSize(pRsp->pIndexRsp); + for (int32_t i = 0; i < numOfMeta; ++i) { + STableIndexRsp *pIndexRsp = taosArrayGet(pRsp->pIndexRsp, i); + if (tEncodeCStr(&encoder, pIndexRsp->tbName) < 0) return -1; + if (tEncodeCStr(&encoder, pIndexRsp->dbFName) < 0) return -1; + if (tEncodeU64(&encoder, pIndexRsp->suid) < 0) return -1; + if (tEncodeI32(&encoder, pIndexRsp->version) < 0) return -1; + int32_t num = taosArrayGetSize(pIndexRsp->pIndex); + if (tEncodeI32(&encoder, num) < 0) return -1; + if (num > 0) { + for (int32_t i = 0; i < num; ++i) { + STableIndexInfo *pInfo = (STableIndexInfo *)taosArrayGet(pIndexRsp->pIndex, i); + if (tSerializeSTableIndexInfo(&encoder, pInfo) < 0) return -1; + } + } + } + tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -2671,20 +2693,53 @@ int32_t tDeserializeSSTbHbRsp(void *buf, int32_t bufLen, SSTbHbRsp *pRsp) { if (tStartDecode(&decoder) < 0) return -1; - int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); - if (tDecodeI32(&decoder, &numOfBatch) < 0) return -1; + int32_t numOfMeta = taosArrayGetSize(pRsp->pMetaRsp); + if (tDecodeI32(&decoder, &numOfMeta) < 0) return -1; - pRsp->pArray = taosArrayInit(numOfBatch, sizeof(STableMetaRsp)); - if (pRsp->pArray == NULL) { + pRsp->pMetaRsp = taosArrayInit(numOfMeta, sizeof(STableMetaRsp)); + if (pRsp->pMetaRsp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - for (int32_t i = 0; i < numOfBatch; ++i) { + for (int32_t i = 0; i < numOfMeta; ++i) { STableMetaRsp tableMetaRsp = {0}; if (tDecodeSTableMetaRsp(&decoder, &tableMetaRsp) < 0) return -1; - taosArrayPush(pRsp->pArray, &tableMetaRsp); + taosArrayPush(pRsp->pMetaRsp, &tableMetaRsp); } + + int32_t numOfIndex = taosArrayGetSize(pRsp->pIndexRsp); + if (tDecodeI32(&decoder, &numOfIndex) < 0) return -1; + + pRsp->pIndexRsp = taosArrayInit(numOfIndex, sizeof(STableIndexRsp)); + if (pRsp->pIndexRsp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + for (int32_t i = 0; i < numOfIndex; ++i) { + STableIndexRsp tableIndexRsp = {0}; + if (tDecodeCStrTo(&decoder, tableIndexRsp.tbName) < 0) return -1; + if (tDecodeCStrTo(&decoder, tableIndexRsp.dbFName) < 0) return -1; + if (tDecodeU64(&decoder, &tableIndexRsp.suid) < 0) return -1; + if (tDecodeI32(&decoder, &tableIndexRsp.version) < 0) return -1; + int32_t num = 0; + if (tDecodeI32(&decoder, &num) < 0) return -1; + if (num > 0) { + tableIndexRsp.pIndex = taosArrayInit(num, sizeof(STableIndexInfo)); + if (NULL == tableIndexRsp.pIndex) return -1; + STableIndexInfo info; + for (int32_t i = 0; i < num; ++i) { + if (tDeserializeSTableIndexInfo(&decoder, &info) < 0) return -1; + if (NULL == taosArrayPush(tableIndexRsp.pIndex, &info)) { + taosMemoryFree(info.expr); + return -1; + } + } + } + taosArrayPush(pRsp->pIndexRsp, &tableIndexRsp); + } + tEndDecode(&decoder); tDecoderClear(&decoder); @@ -2693,14 +2748,33 @@ int32_t tDeserializeSSTbHbRsp(void *buf, int32_t bufLen, SSTbHbRsp *pRsp) { void tFreeSTableMetaRsp(STableMetaRsp *pRsp) { taosMemoryFreeClear(pRsp->pSchemas); } +void tFreeSTableIndexRsp(void *info) { + if (NULL == info) { + return; + } + + STableIndexRsp *pInfo = (STableIndexRsp *)info; + + taosArrayDestroyEx(pInfo->pIndex, tFreeSTableIndexInfo); +} + + void tFreeSSTbHbRsp(SSTbHbRsp *pRsp) { - int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); - for (int32_t i = 0; i < numOfBatch; ++i) { - STableMetaRsp *pMetaRsp = taosArrayGet(pRsp->pArray, i); + int32_t numOfMeta = taosArrayGetSize(pRsp->pMetaRsp); + for (int32_t i = 0; i < numOfMeta; ++i) { + STableMetaRsp *pMetaRsp = taosArrayGet(pRsp->pMetaRsp, i); tFreeSTableMetaRsp(pMetaRsp); } - taosArrayDestroy(pRsp->pArray); + taosArrayDestroy(pRsp->pMetaRsp); + + int32_t numOfIndex = taosArrayGetSize(pRsp->pIndexRsp); + for (int32_t i = 0; i < numOfIndex; ++i) { + STableIndexRsp *pIndexRsp = taosArrayGet(pRsp->pIndexRsp, i); + tFreeSTableIndexRsp(pIndexRsp); + } + + taosArrayDestroy(pRsp->pIndexRsp); } int32_t tSerializeSShowRsp(void *buf, int32_t bufLen, SShowRsp *pRsp) { diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index edb8a29ac0..395e2b5255 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -885,6 +885,8 @@ static int32_t mndGetTableSma(SMnode *pMnode, char *tbFName, STableIndexRsp *rsp return TSDB_CODE_SUCCESS; } + strcpy(rsp->dbFName, pStb->db); + strcpy(rsp->tbName, pStb->name); rsp->suid = pStb->uid; rsp->version = pStb->smaVer; mndReleaseStb(pMnode, pStb); diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 4b5b9e8a7b..9db0ab053f 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -1674,15 +1674,15 @@ _OVER: int32_t mndValidateStbInfo(SMnode *pMnode, SSTableVersion *pStbVersions, int32_t numOfStbs, void **ppRsp, int32_t *pRspLen) { SSTbHbRsp hbRsp = {0}; - hbRsp.pArray = taosArrayInit(numOfStbs, sizeof(STableMetaRsp)); - if (hbRsp.pArray == NULL) { + hbRsp.pMetaRsp = taosArrayInit(numOfStbs, sizeof(STableMetaRsp)); + if (hbRsp.pMetaRsp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - hbRsp.pIndexRsp = taosMemoryCalloc(1, sizeof(STableIndexRsp)); + hbRsp.pIndexRsp = taosArrayInit(numOfStbs, sizeof(STableIndexRsp)); if (NULL == hbRsp.pIndexRsp) { - taosArrayDestroy(hbRsp.pArray); + taosArrayDestroy(hbRsp.pMetaRsp); terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -1700,12 +1700,12 @@ int32_t mndValidateStbInfo(SMnode *pMnode, SSTableVersion *pStbVersions, int32_t if (mndBuildStbSchema(pMnode, pStbVersion->dbFName, pStbVersion->stbName, &metaRsp, &smaVer) != 0) { metaRsp.numOfColumns = -1; metaRsp.suid = pStbVersion->suid; - taosArrayPush(hbRsp.pArray, &metaRsp); + taosArrayPush(hbRsp.pMetaRsp, &metaRsp); continue; } if (pStbVersion->sversion != metaRsp.sversion || pStbVersion->tversion != metaRsp.tversion) { - taosArrayPush(hbRsp.pArray, &metaRsp); + taosArrayPush(hbRsp.pMetaRsp, &metaRsp); } else { tFreeSTableMetaRsp(&metaRsp); } @@ -1713,11 +1713,19 @@ int32_t mndValidateStbInfo(SMnode *pMnode, SSTableVersion *pStbVersions, int32_t if (pStbVersion->smaVer != smaVer) { bool exist = false; char tbFName[TSDB_TABLE_FNAME_LEN]; + STableIndexRsp indexRsp = {0}; sprintf(tbFName, "%s.%s", pStbVersion->dbFName, pStbVersion->stbName); - int32_t code = mndGetTableSma(pMnode, tbFName, hbRsp.pIndexRsp, &exist); + int32_t code = mndGetTableSma(pMnode, tbFName, &indexRsp, &exist); if (code || !exist) { - taosMemoryFreeClear(hbRsp.pIndexRsp); + indexRsp.suid = pStbVersion->suid; + indexRsp.version = -1; + indexRsp.pIndex = NULL; } + + strcpy(indexRsp->dbFName, pStbVersion->dbFName); + strcpy(indexRsp->tbName, pStbVersion->stbName); + + taosArrayPush(hbRsp.pIndexRsp, &indexRsp); } } diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 82bb151d86..11ae5b3770 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -319,8 +319,6 @@ typedef struct SCtgUpdateUserMsg { typedef struct SCtgUpdateTbIndexMsg { SCatalog* pCtg; - char dbFName[TSDB_DB_FNAME_LEN]; - char tbName[TSDB_TABLE_NAME_LEN]; STableIndex* pIndex; } SCtgUpdateTbIndexMsg; @@ -503,7 +501,7 @@ int32_t ctgUpdateVgroupEnqueue(SCatalog* pCtg, const char *dbFName, int64_t dbId int32_t ctgUpdateTbMetaEnqueue(SCatalog* pCtg, STableMetaOutput *output, bool syncReq); int32_t ctgUpdateUserEnqueue(SCatalog* pCtg, SGetUserAuthRsp *pAuth, bool syncReq); int32_t ctgUpdateVgEpsetEnqueue(SCatalog* pCtg, char *dbFName, int32_t vgId, SEpSet* pEpSet); -int32_t ctgUpdateTbIndexEnqueue(SCatalog* pCtg, SName* pName, STableIndex *pIndex, bool syncOp); +int32_t ctgUpdateTbIndexEnqueue(SCatalog* pCtg, STableIndex *pIndex, bool syncOp); int32_t ctgMetaRentInit(SCtgRentMgmt *mgmt, uint32_t rentSec, int8_t type); int32_t ctgMetaRentAdd(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t size); int32_t ctgMetaRentGet(SCtgRentMgmt *mgmt, void **res, uint32_t *num, int32_t size); diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index d01fa00878..de74dc1c68 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -751,6 +751,30 @@ _return: CTG_API_LEAVE(code); } +int32_t catalogUpdateTableIndex(SCatalog* pCtg, STableIndexRsp *pRsp) { + CTG_API_ENTER(); + + int32_t code = 0; + + if (NULL == pCtg || NULL == pRsp) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + + STableIndex* pIndex = taosMemoryCalloc(1, sizeof(STableIndex)); + if (NULL == pIndex) { + CTG_API_LEAVE(TSDB_CODE_OUT_OF_MEMORY); + } + + memcpy(pIndex, pRsp, sizeof(STableIndex)); + + CTG_ERR_JRET(ctgUpdateTbIndexEnqueue(pCtg, &pIndex, false)); + +_return: + + CTG_API_LEAVE(code); +} + + int32_t catalogRemoveTableMeta(SCatalog* pCtg, SName* pTableName) { CTG_API_ENTER(); @@ -1153,16 +1177,36 @@ int32_t catalogGetTableIndex(SCatalog* pCtg, SRequestConnInfo *pConn, const SNam CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - int32_t code = 0; - CTG_ERR_JRET(ctgGetTbIndexFromMnode(pCtg, pConn, (SName*)pTableName, pRes, NULL)); + CTG_ERR_RET(ctgReadTbIndexFromCache(pCtg, pTableName, pRes)); + if (*pRes) { + CTG_API_LEAVE(TSDB_CODE_SUCCESS); + } - SArray* pInfo = NULL; - CTG_ERR_JRET(ctgCloneTableIndex(*pRes, &pInfo)); + STableIndex *pIndex = taosMemoryCalloc(1, sizeof(STableIndex)); + if (NULL == pIndex) { + CTG_API_LEAVE(TSDB_CODE_OUT_OF_MEMORY); + } - CTG_ERR_JRET(ctgUpdateTbIndexEnqueue(pCtg, pTableName, pInfo, false)); + int32_t code = 0; + CTG_ERR_JRET(ctgGetTbIndexFromMnode(pCtg, pConn, (SName*)pTableName, pIndex, NULL)); + + SArray* pInfo = NULL; + CTG_ERR_JRET(ctgCloneTableIndex(pIndex->pIndex, &pInfo)); + *pRes = pInfo; + + CTG_ERR_JRET(ctgUpdateTbIndexEnqueue(pCtg, pTableName, &pIndex, false)); + + CTG_API_LEAVE(code); + _return: + tFreeSTableIndexRsp(pIndex); + taosMemoryFree(pIndex); + + taosArrayDestroyEx(*pRes, tFreeSTableIndexInfo); + *pRes = NULL; + CTG_API_LEAVE(code); } diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 69eddd7ba6..36e5b33521 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -876,10 +876,9 @@ int32_t ctgHandleGetTbIndexRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf SArray* pInfo = NULL; CTG_ERR_JRET(ctgCloneTableIndex(pOut->pIndex, &pInfo)); pTask->res = pInfo; - pTask->msgCtx.out = NULL; SCtgTbIndexCtx* ctx = pTask->taskCtx; - CTG_ERR_JRET(ctgUpdateTbIndexEnqueue(pTask->pJob->pCtg, ctx->pName, pOut, false)); + CTG_ERR_JRET(ctgUpdateTbIndexEnqueue(pTask->pJob->pCtg, (STableIndex**)&pTask->msgCtx.out, false)); _return: diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 49714ac2de..f2502f5943 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -838,7 +838,7 @@ _return: CTG_RET(code); } -int32_t ctgUpdateTbIndexEnqueue(SCatalog* pCtg, SName* pName, STableIndex *pIndex, bool syncOp) { +int32_t ctgUpdateTbIndexEnqueue(SCatalog* pCtg, STableIndex **pIndex, bool syncOp) { int32_t code = 0; SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); op->opId = CTG_OP_UPDATE_TB_INDEX; @@ -851,19 +851,19 @@ int32_t ctgUpdateTbIndexEnqueue(SCatalog* pCtg, SName* pName, STableIndex *pInde } msg->pCtg = pCtg; - msg->pIndex = pIndex; - tNameGetFullDbName(pName, msg->dbFName); - strcpy(msg->tbName, pName->tname); + msg->pIndex = *pIndex; op->data = msg; CTG_ERR_JRET(ctgEnqueue(pCtg, op)); - + + *pIndex = NULL; return TSDB_CODE_SUCCESS; _return: - taosArrayDestroyEx(pIndex, tFreeSTableIndexInfo); + taosArrayDestroyEx(*pIndex, tFreeSTableIndexInfo); + taosMemoryFreeClear(*pIndex); taosMemoryFreeClear(msg); CTG_RET(code); @@ -1745,12 +1745,12 @@ int32_t ctgOpUpdateTbIndex(SCtgCacheOperation *operation) { STableIndex* pIndex = msg->pIndex; SCtgDBCache *dbCache = NULL; - CTG_ERR_JRET(ctgGetAddDBCache(pCtg, msg->dbFName, 0, &dbCache)); + CTG_ERR_JRET(ctgGetAddDBCache(pCtg, pIndex->dbFName, 0, &dbCache)); if (NULL == dbCache) { CTG_ERR_JRET(code); } - CTG_ERR_JRET(ctgWriteTbIndexToCache(pCtg, dbCache, msg->dbFName, msg->tbName, &pIndex)); + CTG_ERR_JRET(ctgWriteTbIndexToCache(pCtg, dbCache, pIndex->dbFName, pIndex->tbName, &pIndex)); _return: diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c index c2ebd9e210..178025704f 100644 --- a/source/libs/catalog/src/ctgRemote.c +++ b/source/libs/catalog/src/ctgRemote.c @@ -431,7 +431,7 @@ int32_t ctgGetIndexInfoFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const return TSDB_CODE_SUCCESS; } -int32_t ctgGetTbIndexFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SName *name, SArray** out, SCtgTask* pTask) { +int32_t ctgGetTbIndexFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SName *name, STableIndex* out, SCtgTask* pTask) { char *msg = NULL; int32_t msgLen = 0; int32_t reqType = TDMT_MND_GET_TABLE_INDEX; @@ -448,7 +448,11 @@ int32_t ctgGetTbIndexFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SName *n } if (pTask) { - void* pOut = NULL; + void* pOut = taosMemoryCalloc(1, sizeof(STableIndex)); + if (NULL == pOut) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, (char*)tbFName)); CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask, reqType, msg, msgLen)); diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index edc468a6e2..b89c220519 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -484,17 +484,11 @@ int32_t queryProcessGetTbIndexRsp(void *output, char *msg, int32_t msgSize) { return TSDB_CODE_TSC_INVALID_INPUT; } - STableIndexRsp *out = taosMemoryCalloc(1, sizeof(STableIndexRsp)); - if (NULL == out) { - return TSDB_CODE_OUT_OF_MEMORY; - } - + STableIndexRsp *out = (STableIndexRsp*)output; if (tDeserializeSTableIndexRsp(msg, msgSize, out) != 0) { qError("tDeserializeSTableIndexRsp failed, msgSize:%d", msgSize); return TSDB_CODE_INVALID_MSG; } - - *(void **)output = out; return TSDB_CODE_SUCCESS; } From eb6b87b3e3609e9d3820a2707126a96d444e9613 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 13 Jun 2022 14:15:05 +0800 Subject: [PATCH 06/22] fix: invalid read/write --- source/libs/index/src/indexFilter.c | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index 6197edd474..1cc2679b06 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -97,7 +97,14 @@ static int32_t sifGetOperParamNum(EOperatorType ty) { } return 2; } -static int32_t sifValidateColumn(SColumnNode *cn) { +static int32_t sifValidOp(EOperatorType ty) { + if ((ty >= OP_TYPE_ADD && ty <= OP_TYPE_BIT_OR) || (ty == OP_TYPE_IN || ty == OP_TYPE_NOT_IN) || + (ty == OP_TYPE_LIKE || ty == OP_TYPE_NOT_LIKE || ty == OP_TYPE_MATCH || ty == OP_TYPE_NMATCH)) { + return -1; + } + return 0; +} +static int32_t sifValidColumn(SColumnNode *cn) { // add more check if (cn == NULL) { return TSDB_CODE_QRY_INVALID_INPUT; @@ -197,7 +204,7 @@ static int32_t sifInitParam(SNode *node, SIFParam *param, SIFCtx *ctx) { case QUERY_NODE_COLUMN: { SColumnNode *cn = (SColumnNode *)node; /*only support tag column*/ - SIF_ERR_RET(sifValidateColumn(cn)); + SIF_ERR_RET(sifValidColumn(cn)); param->colId = cn->colId; param->colValType = cn->node.resType.type; @@ -505,6 +512,11 @@ static int32_t sifGetOperFn(int32_t funcId, sif_func_t *func, SIdxFltStatus *sta static int32_t sifExecOper(SOperatorNode *node, SIFCtx *ctx, SIFParam *output) { int32_t code = 0; + if (sifValidOp(node->opType) < 0) { + output->status = SFLT_NOT_INDEX; + return code; + } + int32_t nParam = sifGetOperParamNum(node->opType); if (nParam <= 1) { output->status = SFLT_NOT_INDEX; @@ -516,6 +528,12 @@ static int32_t sifExecOper(SOperatorNode *node, SIFCtx *ctx, SIFParam *output) { SIFParam *params = NULL; SIF_ERR_RET(sifInitOperParams(¶ms, node, ctx)); + + if (params[0].status == SFLT_NOT_INDEX || (nParam > 1 && params[1].status == SFLT_NOT_INDEX)) { + output->status = SFLT_NOT_INDEX; + return code; + } + // ugly code, refactor later output->arg = ctx->arg; sif_func_t operFn = sifNullFunc; From 7a78d81232317e63e6ee80408bfd3ba984ea967c Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 13 Jun 2022 15:36:28 +0800 Subject: [PATCH 07/22] fix compile issue --- include/common/tmsg.h | 1 + include/libs/catalog/catalog.h | 2 + source/common/src/tmsg.c | 1 - source/dnode/mnode/impl/inc/mndSma.h | 1 + source/dnode/mnode/impl/src/mndSma.c | 2 +- source/dnode/mnode/impl/src/mndStb.c | 9 +-- source/libs/catalog/inc/catalogInt.h | 14 +++-- source/libs/catalog/src/catalog.c | 64 ++++++++++----------- source/libs/catalog/src/ctgAsync.c | 12 ++-- source/libs/catalog/src/ctgCache.c | 85 +++++++++++++++++++++------- source/libs/catalog/src/ctgDbg.c | 18 +++--- source/libs/catalog/src/ctgUtil.c | 10 ++++ 12 files changed, 142 insertions(+), 77 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 8a2b2a31f2..b4f60c2761 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1134,6 +1134,7 @@ void tFreeSMAlterStbRsp(SMAlterStbRsp* pRsp); int32_t tSerializeSTableMetaRsp(void* buf, int32_t bufLen, STableMetaRsp* pRsp); int32_t tDeserializeSTableMetaRsp(void* buf, int32_t bufLen, STableMetaRsp* pRsp); void tFreeSTableMetaRsp(STableMetaRsp* pRsp); +void tFreeSTableIndexRsp(void *info); typedef struct { SArray* pMetaRsp; // Array of STableMetaRsp diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 80b677dbd2..dba4ac39d2 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -282,6 +282,8 @@ int32_t catalogGetIndexMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const char* int32_t catalogGetTableIndex(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SArray** pRes); +int32_t catalogUpdateTableIndex(SCatalog* pCtg, STableIndexRsp *pRsp); + int32_t catalogGetUdfInfo(SCatalog* pCtg, SRequestConnInfo* pConn, const char* funcName, SFuncInfo* pInfo); int32_t catalogChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, const char* user, const char* dbFName, AUTH_TYPE type, bool *pass); diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 5d0cc2b28d..63153d68b4 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -2758,7 +2758,6 @@ void tFreeSTableIndexRsp(void *info) { taosArrayDestroyEx(pInfo->pIndex, tFreeSTableIndexInfo); } - void tFreeSSTbHbRsp(SSTbHbRsp *pRsp) { int32_t numOfMeta = taosArrayGetSize(pRsp->pMetaRsp); for (int32_t i = 0; i < numOfMeta; ++i) { diff --git a/source/dnode/mnode/impl/inc/mndSma.h b/source/dnode/mnode/impl/inc/mndSma.h index 4a80f619d3..63530b403d 100644 --- a/source/dnode/mnode/impl/inc/mndSma.h +++ b/source/dnode/mnode/impl/inc/mndSma.h @@ -26,6 +26,7 @@ int32_t mndInitSma(SMnode *pMnode); void mndCleanupSma(SMnode *pMnode); SSmaObj *mndAcquireSma(SMnode *pMnode, char *smaName); void mndReleaseSma(SMnode *pMnode, SSmaObj *pSma); +int32_t mndGetTableSma(SMnode *pMnode, char *tbFName, STableIndexRsp *rsp, bool *exist); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 395e2b5255..9c28763082 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -872,7 +872,7 @@ static int32_t mndGetSma(SMnode *pMnode, SUserIndexReq *indexReq, SUserIndexRsp return code; } -static int32_t mndGetTableSma(SMnode *pMnode, char *tbFName, STableIndexRsp *rsp, bool *exist) { +int32_t mndGetTableSma(SMnode *pMnode, char *tbFName, STableIndexRsp *rsp, bool *exist) { int32_t code = 0; SSmaObj *pSma = NULL; SSdb *pSdb = pMnode->pSdb; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 9db0ab053f..8d936e68ce 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -27,6 +27,7 @@ #include "mndTrans.h" #include "mndUser.h" #include "mndVgroup.h" +#include "mndSma.h" #include "tname.h" #define STB_VER_NUMBER 1 @@ -1638,7 +1639,7 @@ static int32_t mndProcessTableMetaReq(SRpcMsg *pReq) { } } else { mDebug("stb:%s.%s, start to retrieve meta", infoReq.dbFName, infoReq.tbName); - if (mndBuildStbSchema(pMnode, infoReq.dbFName, infoReq.tbName, &metaRsp) != 0) { + if (mndBuildStbSchema(pMnode, infoReq.dbFName, infoReq.tbName, &metaRsp, NULL) != 0) { goto _OVER; } } @@ -1710,7 +1711,7 @@ int32_t mndValidateStbInfo(SMnode *pMnode, SSTableVersion *pStbVersions, int32_t tFreeSTableMetaRsp(&metaRsp); } - if (pStbVersion->smaVer != smaVer) { + if (pStbVersion->smaVer && pStbVersion->smaVer != smaVer) { bool exist = false; char tbFName[TSDB_TABLE_FNAME_LEN]; STableIndexRsp indexRsp = {0}; @@ -1722,8 +1723,8 @@ int32_t mndValidateStbInfo(SMnode *pMnode, SSTableVersion *pStbVersions, int32_t indexRsp.pIndex = NULL; } - strcpy(indexRsp->dbFName, pStbVersion->dbFName); - strcpy(indexRsp->tbName, pStbVersion->stbName); + strcpy(indexRsp.dbFName, pStbVersion->dbFName); + strcpy(indexRsp.tbName, pStbVersion->stbName); taosArrayPush(hbRsp.pIndexRsp, &indexRsp); } diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 11ae5b3770..df4b9b6ea6 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -491,8 +491,8 @@ void ctgReleaseDBCache(SCatalog *pCtg, SCtgDBCache *dbCache); void ctgRUnlockVgInfo(SCtgDBCache *dbCache); int32_t ctgTbMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName, int32_t *exist); int32_t ctgReadTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta); -int32_t ctgReadTbVerFromCache(SCatalog *pCtg, const SName *pTableName, int32_t *sver, int32_t *tver, int32_t *tbType, uint64_t *suid, char *stbName); -int32_t ctgChkAuthFromCache(SCatalog* pCtg, const char* user, const char* dbFName, AUTH_TYPE type, bool *inCache, bool *pass); +int32_t ctgReadTbVerFromCache(SCatalog *pCtg, SName *pTableName, int32_t *sver, int32_t *tver, int32_t *tbType, uint64_t *suid, char *stbName); +int32_t ctgChkAuthFromCache(SCatalog* pCtg, char* user, char* dbFName, AUTH_TYPE type, bool *inCache, bool *pass); int32_t ctgDropDbCacheEnqueue(SCatalog* pCtg, const char *dbFName, int64_t dbId); int32_t ctgDropDbVgroupEnqueue(SCatalog* pCtg, const char *dbFName, bool syncReq); int32_t ctgDropStbMetaEnqueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, const char *stbName, uint64_t suid, bool syncReq); @@ -501,13 +501,18 @@ int32_t ctgUpdateVgroupEnqueue(SCatalog* pCtg, const char *dbFName, int64_t dbId int32_t ctgUpdateTbMetaEnqueue(SCatalog* pCtg, STableMetaOutput *output, bool syncReq); int32_t ctgUpdateUserEnqueue(SCatalog* pCtg, SGetUserAuthRsp *pAuth, bool syncReq); int32_t ctgUpdateVgEpsetEnqueue(SCatalog* pCtg, char *dbFName, int32_t vgId, SEpSet* pEpSet); -int32_t ctgUpdateTbIndexEnqueue(SCatalog* pCtg, STableIndex *pIndex, bool syncOp); +int32_t ctgUpdateTbIndexEnqueue(SCatalog* pCtg, STableIndex **pIndex, bool syncOp); int32_t ctgMetaRentInit(SCtgRentMgmt *mgmt, uint32_t rentSec, int8_t type); int32_t ctgMetaRentAdd(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t size); int32_t ctgMetaRentGet(SCtgRentMgmt *mgmt, void **res, uint32_t *num, int32_t size); int32_t ctgUpdateTbMetaToCache(SCatalog* pCtg, STableMetaOutput* pOut, bool syncReq); int32_t ctgStartUpdateThread(); int32_t ctgRelaunchGetTbMetaTask(SCtgTask *pTask); +void ctgReleaseVgInfoToCache(SCatalog* pCtg, SCtgDBCache *dbCache); +int32_t ctgReadTbIndexFromCache(SCatalog* pCtg, SName* pTableName, SArray** pRes); +int32_t ctgDropTbIndexEnqueue(SCatalog* pCtg, SName* pName, bool syncOp); +int32_t ctgOpDropTbIndex(SCtgCacheOperation *operation); +int32_t ctgOpUpdateTbIndex(SCtgCacheOperation *operation); @@ -516,7 +521,7 @@ int32_t ctgGetDBVgInfoFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SBuildU int32_t ctgGetQnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SArray *out, SCtgTask* pTask); int32_t ctgGetDBCfgFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const char *dbFName, SDbCfgInfo *out, SCtgTask* pTask); int32_t ctgGetIndexInfoFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const char *indexName, SIndexInfo *out, SCtgTask* pTask); -int32_t ctgGetTbIndexFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SName* name, SArray** out, SCtgTask* pTask); +int32_t ctgGetTbIndexFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SName *name, STableIndex* out, SCtgTask* pTask); int32_t ctgGetUdfInfoFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const char *funcName, SFuncInfo *out, SCtgTask* pTask); int32_t ctgGetUserDbAuthFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const char *user, SGetUserAuthRsp *out, SCtgTask* pTask); int32_t ctgGetTbMetaFromMnodeImpl(SCatalog* pCtg, SRequestConnInfo *pConn, char *dbFName, char* tbName, STableMetaOutput* out, SCtgTask* pTask); @@ -545,6 +550,7 @@ int32_t ctgUpdateMsgCtx(SCtgMsgCtx* pCtx, int32_t reqType, void* out, char* targ char *ctgTaskTypeStr(CTG_TASK_TYPE type); int32_t ctgUpdateSendTargetInfo(SMsgSendInfo *pMsgSendInfo, int32_t msgType, SCtgTask* pTask); int32_t ctgCloneTableIndex(SArray* pIndex, SArray** pRes); +void ctgFreeSTableIndex(void *info); extern SCatalogMgmt gCtgMgmt; diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index de74dc1c68..2ef7771c11 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -350,7 +350,7 @@ int32_t ctgChkAuth(SCatalog* pCtg, SRequestConnInfo *pConn, const char* user, co *pass = false; - CTG_ERR_RET(ctgChkAuthFromCache(pCtg, user, dbFName, type, &inCache, pass)); + CTG_ERR_RET(ctgChkAuthFromCache(pCtg, (char*)user, (char*)dbFName, type, &inCache, pass)); if (inCache) { return TSDB_CODE_SUCCESS; @@ -382,15 +382,38 @@ _return: return TSDB_CODE_SUCCESS; } -int32_t ctgGetTbIndex(SCatalog* pCtg, SRequestConnInfo *pConn, const SName* pTableName, SArray** pRes) { +int32_t ctgGetTbIndex(SCatalog* pCtg, SRequestConnInfo *pConn, SName* pTableName, SArray** pRes) { CTG_ERR_RET(ctgReadTbIndexFromCache(pCtg, pTableName, pRes)); if (*pRes) { return TSDB_CODE_SUCCESS; } - CTG_ERR_RET(ctgGetTbIndexFromMnode(pCtg, pConn, (SName*)pTableName, pRes, NULL)); + STableIndex *pIndex = taosMemoryCalloc(1, sizeof(STableIndex)); + if (NULL == pIndex) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + int32_t code = 0; + CTG_ERR_JRET(ctgGetTbIndexFromMnode(pCtg, pConn, (SName*)pTableName, pIndex, NULL)); + + SArray* pInfo = NULL; + CTG_ERR_JRET(ctgCloneTableIndex(pIndex->pIndex, &pInfo)); + + *pRes = pInfo; + + CTG_ERR_JRET(ctgUpdateTbIndexEnqueue(pCtg, &pIndex, false)); return TSDB_CODE_SUCCESS; + +_return: + + tFreeSTableIndexRsp(pIndex); + taosMemoryFree(pIndex); + + taosArrayDestroyEx(*pRes, tFreeSTableIndexInfo); + *pRes = NULL; + + CTG_RET(code); } @@ -416,7 +439,7 @@ int32_t ctgGetTbDistVgInfo(SCatalog* pCtg, SRequestConnInfo *pConn, SName* pTabl CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pConn, db, &dbCache, &vgInfo)); if (dbCache) { - vgHash = dbCache->vgInfo->vgHash; + vgHash = dbCache->vgCache.vgInfo->vgHash; } else { vgHash = vgInfo->vgHash; } @@ -640,9 +663,9 @@ int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* vers CTG_API_LEAVE(TSDB_CODE_SUCCESS); } - *version = dbCache->vgInfo->vgVersion; + *version = dbCache->vgCache.vgInfo->vgVersion; *dbId = dbCache->dbId; - *tableNum = dbCache->vgInfo->numOfTable; + *tableNum = dbCache->vgCache.vgInfo->numOfTable; ctgReleaseVgInfoToCache(pCtg, dbCache); @@ -969,7 +992,7 @@ int32_t catalogGetTableHashVgroup(SCatalog *pCtg, SRequestConnInfo *pConn, const SDBVgInfo *vgInfo = NULL; CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pConn, db, &dbCache, &vgInfo)); - CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, vgInfo ? vgInfo : dbCache->vgInfo, pTableName, pVgroup)); + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, vgInfo ? vgInfo : dbCache->vgCache.vgInfo, pTableName, pVgroup)); _return: @@ -1177,36 +1200,11 @@ int32_t catalogGetTableIndex(SCatalog* pCtg, SRequestConnInfo *pConn, const SNam CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - CTG_ERR_RET(ctgReadTbIndexFromCache(pCtg, pTableName, pRes)); - if (*pRes) { - CTG_API_LEAVE(TSDB_CODE_SUCCESS); - } - - STableIndex *pIndex = taosMemoryCalloc(1, sizeof(STableIndex)); - if (NULL == pIndex) { - CTG_API_LEAVE(TSDB_CODE_OUT_OF_MEMORY); - } - int32_t code = 0; - CTG_ERR_JRET(ctgGetTbIndexFromMnode(pCtg, pConn, (SName*)pTableName, pIndex, NULL)); - - SArray* pInfo = NULL; - CTG_ERR_JRET(ctgCloneTableIndex(pIndex->pIndex, &pInfo)); - - *pRes = pInfo; - - CTG_ERR_JRET(ctgUpdateTbIndexEnqueue(pCtg, pTableName, &pIndex, false)); - - CTG_API_LEAVE(code); + CTG_ERR_JRET(ctgGetTbIndex(pCtg, pConn, (SName*)pTableName, pRes)); _return: - tFreeSTableIndexRsp(pIndex); - taosMemoryFree(pIndex); - - taosArrayDestroyEx(*pRes, tFreeSTableIndexInfo); - *pRes = NULL; - CTG_API_LEAVE(code); } diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 36e5b33521..b893201c53 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -685,7 +685,7 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf * CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache)); if (NULL != dbCache) { SVgroupInfo vgInfo = {0}; - CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgInfo, ctx->pName, &vgInfo)); + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgCache.vgInfo, ctx->pName, &vgInfo)); ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(ctx->pName), ctx->flag); @@ -1016,7 +1016,7 @@ int32_t ctgAsyncRefreshTbMeta(SCtgTask *pTask) { CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache)); if (dbCache) { SVgroupInfo vgInfo = {0}; - CTG_ERR_RET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgInfo, ctx->pName, &vgInfo)); + CTG_ERR_RET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgCache.vgInfo, ctx->pName, &vgInfo)); ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(ctx->pName), ctx->flag); @@ -1064,7 +1064,7 @@ int32_t ctgLaunchGetDbVgTask(SCtgTask *pTask) { CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, pCtx->dbFName, &dbCache)); if (NULL != dbCache) { - CTG_ERR_JRET(ctgGenerateVgList(pCtg, dbCache->vgInfo->vgHash, (SArray**)&pTask->res)); + CTG_ERR_JRET(ctgGenerateVgList(pCtg, dbCache->vgCache.vgInfo->vgHash, (SArray**)&pTask->res)); CTG_ERR_JRET(ctgHandleTaskEnd(pTask, 0)); } else { @@ -1098,7 +1098,7 @@ int32_t ctgLaunchGetTbHashTask(SCtgTask *pTask) { if (NULL == pTask->res) { CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } - CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgInfo, pCtx->pName, (SVgroupInfo*)pTask->res)); + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgCache.vgInfo, pCtx->pName, (SVgroupInfo*)pTask->res)); CTG_ERR_JRET(ctgHandleTaskEnd(pTask, 0)); } else { @@ -1171,9 +1171,9 @@ int32_t ctgLaunchGetDbInfoTask(SCtgTask *pTask) { SDbInfo* pInfo = (SDbInfo*)pTask->res; CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, pCtx->dbFName, &dbCache)); if (NULL != dbCache) { - pInfo->vgVer = dbCache->vgInfo->vgVersion; + pInfo->vgVer = dbCache->vgCache.vgInfo->vgVersion; pInfo->dbId = dbCache->dbId; - pInfo->tbNum = dbCache->vgInfo->numOfTable; + pInfo->tbNum = dbCache->vgCache.vgInfo->numOfTable; } else { pInfo->vgVer = CTG_DEFAULT_INVALID_VERSION; } diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index f2502f5943..1fa61f1c59 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -64,8 +64,12 @@ SCtgOperation gCtgCacheOperation[CTG_OP_MAX] = { CTG_OP_UPDATE_TB_INDEX, "update tbIndex", ctgOpUpdateTbIndex - } - + }, + { + CTG_OP_DROP_TB_INDEX, + "drop tbIndex", + ctgOpDropTbIndex + } }; @@ -305,6 +309,7 @@ _return: int32_t ctgTbMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName, int32_t *exist) { SCtgDBCache *dbCache = NULL; + SCtgTbCache *tbCache = NULL; ctgAcquireTbMetaFromCache(pCtg, dbFName, tbName, &dbCache, &tbCache); if (NULL == tbCache) { ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); @@ -383,7 +388,7 @@ _return: CTG_RET(code); } -int32_t ctgReadTbVerFromCache(SCatalog *pCtg, const SName *pTableName, int32_t *sver, int32_t *tver, int32_t *tbType, uint64_t *suid, +int32_t ctgReadTbVerFromCache(SCatalog *pCtg, SName *pTableName, int32_t *sver, int32_t *tver, int32_t *tbType, uint64_t *suid, char *stbName) { *sver = -1; *tver = -1; @@ -444,10 +449,10 @@ int32_t ctgReadTbVerFromCache(SCatalog *pCtg, const SName *pTableName, int32_t * } -int32_t ctgReadTbTypeFromCache(SCatalog* pCtg, const char* dbFName, const char *tableName, int32_t *tbType) { +int32_t ctgReadTbTypeFromCache(SCatalog* pCtg, char* dbFName, char *tableName, int32_t *tbType) { SCtgDBCache *dbCache = NULL; SCtgTbCache *tbCache = NULL; - ctgAcquireTbMetaFromCache(pCtg, dbFName, tableName, &dbCache, &tbCache); + CTG_ERR_RET(ctgAcquireTbMetaFromCache(pCtg, dbFName, tableName, &dbCache, &tbCache)); if (NULL == tbCache) { ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); return TSDB_CODE_SUCCESS; @@ -461,7 +466,7 @@ int32_t ctgReadTbTypeFromCache(SCatalog* pCtg, const char* dbFName, const char * return TSDB_CODE_SUCCESS; } -int32_t ctgReadTbIndexFromCache(SCatalog* pCtg, const SName* pTableName, SArray** pRes) { +int32_t ctgReadTbIndexFromCache(SCatalog* pCtg, SName* pTableName, SArray** pRes) { int32_t code = 0; SCtgDBCache *dbCache = NULL; SCtgTbCache *tbCache = NULL; @@ -485,7 +490,7 @@ _return: CTG_RET(code); } -int32_t ctgChkAuthFromCache(SCatalog* pCtg, const char* user, const char* dbFName, AUTH_TYPE type, bool *inCache, bool *pass) { +int32_t ctgChkAuthFromCache(SCatalog* pCtg, char* user, char* dbFName, AUTH_TYPE type, bool *inCache, bool *pass) { if (NULL == pCtg->userCache) { ctgDebug("empty user auth cache, user:%s", user); goto _return; @@ -862,7 +867,7 @@ int32_t ctgUpdateTbIndexEnqueue(SCatalog* pCtg, STableIndex **pIndex, bool syncO _return: - taosArrayDestroyEx(*pIndex, tFreeSTableIndexInfo); + taosArrayDestroyEx((*pIndex)->pIndex, tFreeSTableIndexInfo); taosMemoryFreeClear(*pIndex); taosMemoryFreeClear(msg); @@ -893,7 +898,6 @@ int32_t ctgDropTbIndexEnqueue(SCatalog* pCtg, SName* pName, bool syncOp) { _return: - taosArrayDestroyEx(pIndex, tFreeSTableIndexInfo); taosMemoryFreeClear(msg); CTG_RET(code); @@ -1172,7 +1176,7 @@ int32_t ctgRemoveDBFromCache(SCatalog* pCtg, SCtgDBCache *dbCache, const char* d CTG_LOCK(CTG_WRITE, &dbCache->dbLock); atomic_store_8(&dbCache->deleted, 1); - ctgRemoveStbRent(pCtg, &dbCache); + ctgRemoveStbRent(pCtg, dbCache); ctgFreeDbCache(dbCache); CTG_UNLOCK(CTG_WRITE, &dbCache->dbLock); @@ -1251,6 +1255,8 @@ int32_t ctgUpdateRentStbVersion(SCatalog *pCtg, char* dbFName, char* tbName, uin ctgDebug("db %s,%" PRIx64 " stb %s,%" PRIx64 " sver %d tver %d smaVer %d updated to stbRent", dbFName, dbId, tbName, suid, metaRent.sversion, metaRent.tversion, metaRent.smaVer); + + return TSDB_CODE_SUCCESS; } @@ -1330,32 +1336,40 @@ int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNam int32_t ctgWriteTbIndexToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char* dbFName, char *tbName, STableIndex **index) { if (NULL == dbCache->tbCache) { + ctgFreeSTableIndex(*index); taosMemoryFreeClear(*index); ctgError("db is dropping, dbId:%"PRIx64, dbCache->dbId); CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); } STableIndex* pIndex = *index; + uint64_t suid = pIndex->suid; SCtgTbCache* pCache = taosHashGet(dbCache->tbCache, tbName, strlen(tbName)); if (NULL == pCache) { SCtgTbCache cache = {0}; cache.pIndex = pIndex; - if (taosHashPut(table->tbCache, tbName, strlen(tbName), &cache, sizeof(cache)) != 0) { + if (taosHashPut(dbCache->tbCache, tbName, strlen(tbName), &cache, sizeof(cache)) != 0) { + ctgFreeSTableIndex(*index); taosMemoryFreeClear(*index); ctgError("taosHashPut new tbCache failed, tbName:%s", tbName); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } *index = NULL; - ctgDebug("table %s index updated to cache, ver:%d, num:%d", tbName, pIndex->version, taosArrayGetSize(pIndex->pIndex)); - - CTG_ERR_RET(ctgUpdateRentStbVersion(pCtg, dbFName, tbName, dbCache->dbId, pIndex->suid, pCache)); + ctgDebug("table %s index updated to cache, ver:%d, num:%d", tbName, pIndex->version, (int32_t)taosArrayGetSize(pIndex->pIndex)); + if (suid) { + CTG_ERR_RET(ctgUpdateRentStbVersion(pCtg, dbFName, tbName, dbCache->dbId, pIndex->suid, pCache)); + } + return TSDB_CODE_SUCCESS; } if (pCache->pIndex) { + if (0 == suid) { + suid = pCache->pIndex->suid; + } taosArrayDestroyEx(pCache->pIndex->pIndex, tFreeSTableIndexInfo); taosMemoryFreeClear(pCache->pIndex); } @@ -1363,9 +1377,11 @@ int32_t ctgWriteTbIndexToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char* dbFNa pCache->pIndex = pIndex; *index = NULL; - ctgDebug("table %s index updated to cache, ver:%d, num:%d", tbName, pIndex->version, taosArrayGetSize(pIndex->pIndex)); + ctgDebug("table %s index updated to cache, ver:%d, num:%d", tbName, pIndex->version, (int32_t)taosArrayGetSize(pIndex->pIndex)); - CTG_ERR_RET(ctgUpdateRentStbVersion(pCtg, dbFName, tbName, dbCache->dbId, pIndex->suid, pCache)); + if (suid) { + CTG_ERR_RET(ctgUpdateRentStbVersion(pCtg, dbFName, tbName, dbCache->dbId, suid, pCache)); + } return TSDB_CODE_SUCCESS; } @@ -1390,6 +1406,7 @@ int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) { SCtgUpdateVgMsg *msg = operation->data; SDBVgInfo* dbInfo = msg->dbInfo; char* dbFName = msg->dbFName; + SCatalog* pCtg = msg->pCtg; if (NULL == dbInfo->vgHash) { return TSDB_CODE_SUCCESS; @@ -1611,7 +1628,7 @@ int32_t ctgOpDropTbMeta(SCtgCacheOperation *operation) { } if (dbCache->dbId != msg->dbId) { - ctgDebug("dbId %" PRIx64 " not match with curId %"PRIx64", dbFName:%s, tbName:%s"msg->dbId, dbCache->dbId, msg->dbFName, msg->tbName); + ctgDebug("dbId %" PRIx64 " not match with curId %"PRIx64", dbFName:%s, tbName:%s", msg->dbId, dbCache->dbId, msg->dbFName, msg->tbName); return TSDB_CODE_SUCCESS; } @@ -1746,9 +1763,39 @@ int32_t ctgOpUpdateTbIndex(SCtgCacheOperation *operation) { SCtgDBCache *dbCache = NULL; CTG_ERR_JRET(ctgGetAddDBCache(pCtg, pIndex->dbFName, 0, &dbCache)); - if (NULL == dbCache) { - CTG_ERR_JRET(code); + + CTG_ERR_JRET(ctgWriteTbIndexToCache(pCtg, dbCache, pIndex->dbFName, pIndex->tbName, &pIndex)); + +_return: + + if (pIndex) { + taosArrayDestroyEx(pIndex->pIndex, tFreeSTableIndexInfo); + taosMemoryFreeClear(pIndex); } + + taosMemoryFreeClear(msg); + + CTG_RET(code); +} + +int32_t ctgOpDropTbIndex(SCtgCacheOperation *operation) { + int32_t code = 0; + SCtgDropTbIndexMsg *msg = operation->data; + SCatalog* pCtg = msg->pCtg; + SCtgDBCache *dbCache = NULL; + + CTG_ERR_JRET(ctgGetDBCache(pCtg, msg->dbFName, &dbCache)); + if (NULL == dbCache) { + return TSDB_CODE_SUCCESS; + } + + STableIndex* pIndex = taosMemoryCalloc(1, sizeof(STableIndex)); + if (NULL == pIndex) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + strcpy(pIndex->tbName, msg->tbName); + strcpy(pIndex->dbFName, msg->dbFName); + pIndex->version = -1; CTG_ERR_JRET(ctgWriteTbIndexToCache(pCtg, dbCache, pIndex->dbFName, pIndex->tbName, &pIndex)); diff --git a/source/libs/catalog/src/ctgDbg.c b/source/libs/catalog/src/ctgDbg.c index 52f29e3acb..2403dc3c5a 100644 --- a/source/libs/catalog/src/ctgDbg.c +++ b/source/libs/catalog/src/ctgDbg.c @@ -266,11 +266,11 @@ int32_t ctgdGetStatNum(char *option, void *res) { } int32_t ctgdGetTbMetaNum(SCtgDBCache *dbCache) { - return dbCache->table.tbCache ? (int32_t)taosHashGetSize(dbCache->table.tbCache) : 0; + return dbCache->tbCache ? (int32_t)taosHashGetSize(dbCache->tbCache) : 0; } int32_t ctgdGetStbNum(SCtgDBCache *dbCache) { - return dbCache->table.stbCache ? (int32_t)taosHashGetSize(dbCache->table.stbCache) : 0; + return dbCache->stbCache ? (int32_t)taosHashGetSize(dbCache->stbCache) : 0; } int32_t ctgdGetRentNum(SCtgRentMgmt *rent) { @@ -363,17 +363,17 @@ void ctgdShowDBCache(SCatalog* pCtg, SHashObj *dbHash) { dbFName = taosHashGetKey(pIter, &len); - int32_t metaNum = dbCache->table.tbCache ? taosHashGetSize(dbCache->table.tbCache) : 0; - int32_t stbNum = dbCache->table.stbCache ? taosHashGetSize(dbCache->table.stbCache) : 0; + int32_t metaNum = dbCache->tbCache ? taosHashGetSize(dbCache->tbCache) : 0; + int32_t stbNum = dbCache->stbCache ? taosHashGetSize(dbCache->stbCache) : 0; int32_t vgVersion = CTG_DEFAULT_INVALID_VERSION; int32_t hashMethod = -1; int32_t vgNum = 0; - if (dbCache->vgInfo) { - vgVersion = dbCache->vgInfo->vgVersion; - hashMethod = dbCache->vgInfo->hashMethod; - if (dbCache->vgInfo->vgHash) { - vgNum = taosHashGetSize(dbCache->vgInfo->vgHash); + if (dbCache->vgCache.vgInfo) { + vgVersion = dbCache->vgCache.vgInfo->vgVersion; + hashMethod = dbCache->vgCache.vgInfo->hashMethod; + if (dbCache->vgCache.vgInfo->vgHash) { + vgNum = taosHashGetSize(dbCache->vgCache.vgInfo->vgHash); } } diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index f0ed60a689..1cb2b99a67 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -44,6 +44,16 @@ char *ctgTaskTypeStr(CTG_TASK_TYPE type) { } } +void ctgFreeSTableIndex(void *info) { + if (NULL == info) { + return; + } + + STableIndex *pInfo = (STableIndex *)info; + + taosArrayDestroyEx(pInfo->pIndex, tFreeSTableIndexInfo); +} + void ctgFreeSMetaData(SMetaData* pData) { taosArrayDestroy(pData->pTableMeta); pData->pTableMeta = NULL; From bb89ceb6bd0bc4ee03d2c8ae1f047194ad581e06 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 13 Jun 2022 18:46:02 +0800 Subject: [PATCH 08/22] change stb hash value to stbName --- source/client/inc/clientInt.h | 2 +- source/libs/catalog/src/ctgCache.c | 127 ++++++++++++++++++++++++----- source/libs/catalog/src/ctgDbg.c | 2 +- 3 files changed, 108 insertions(+), 23 deletions(-) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 6aa83e9575..d72aa39e4c 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -45,7 +45,7 @@ extern "C" { #define ERROR_MSG_BUF_DEFAULT_SIZE 512 #define HEARTBEAT_INTERVAL 1500 // ms -#define SYNC_ON_TOP_OF_ASYNC 1 +#define SYNC_ON_TOP_OF_ASYNC 0 enum { RES_TYPE__QUERY = 1, diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 1fa61f1c59..b938dff932 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -230,6 +230,7 @@ _return: int32_t ctgAcquireTbMetaFromCache(SCatalog* pCtg, char *dbFName, char* tbName, SCtgDBCache **pDb, SCtgTbCache** pTb) { SCtgDBCache *dbCache = NULL; + SCtgTbCache* pCache = NULL; ctgAcquireDBCache(pCtg, dbFName, &dbCache); if (NULL == dbCache) { ctgDebug("db %s not in cache", dbFName); @@ -237,7 +238,7 @@ int32_t ctgAcquireTbMetaFromCache(SCatalog* pCtg, char *dbFName, char* tbName, S } int32_t sz = 0; - SCtgTbCache* pCache = taosHashAcquire(dbCache->tbCache, tbName, strlen(tbName)); + pCache = taosHashAcquire(dbCache->tbCache, tbName, strlen(tbName)); if (NULL == pCache) { ctgDebug("tb %s not in cache, dbFName:%s", tbName, dbFName); goto _return; @@ -267,6 +268,54 @@ _return: return TSDB_CODE_SUCCESS; } +int32_t ctgAcquireStbMetaFromCache(SCatalog* pCtg, char *dbFName, uint64_t suid, SCtgDBCache **pDb, SCtgTbCache** pTb) { + SCtgDBCache* dbCache = NULL; + SCtgTbCache* pCache = NULL; + ctgAcquireDBCache(pCtg, dbFName, &dbCache); + if (NULL == dbCache) { + ctgDebug("db %s not in cache", dbFName); + goto _return; + } + + int32_t sz = 0; + char* stName = taosHashAcquire(dbCache->stbCache, &suid, sizeof(suid)); + if (NULL == stName) { + ctgDebug("stb %" PRIx64 " not in cache, dbFName:%s", suid, dbFName); + goto _return; + } + + pCache = taosHashAcquire(dbCache->tbCache, stName, strlen(stName)); + if (NULL == pCache) { + ctgDebug("stb %" PRIx64 " name %s not in cache, dbFName:%s", suid, stName, dbFName); + taosHashRelease(dbCache->stbCache, stName); + goto _return; + } + + CTG_LOCK(CTG_READ, &pCache->metaLock); + if (NULL == pCache->pMeta) { + ctgDebug("stb %" PRIx64 " meta not in cache, dbFName:%s", suid, dbFName); + goto _return; + } + + *pDb = dbCache; + *pTb = pCache; + + ctgDebug("stb %" PRIx64 " meta got in cache, dbFName:%s", suid, dbFName); + + CTG_CACHE_STAT_INC(tbMetaHitNum, 1); + + return TSDB_CODE_SUCCESS; + +_return: + + ctgReleaseTbMetaToCache(pCtg, dbCache, pCache); + + CTG_CACHE_STAT_INC(tbMetaMissNum, 1); + + return TSDB_CODE_SUCCESS; +} + + int32_t ctgAcquireTbIndexFromCache(SCatalog* pCtg, char *dbFName, char* tbName, SCtgDBCache **pDb, SCtgTbCache** pTb) { SCtgDBCache *dbCache = NULL; ctgAcquireDBCache(pCtg, dbFName, &dbCache); @@ -348,31 +397,61 @@ int32_t ctgReadTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta** ctx->tbInfo.dbId = dbCache->dbId; ctx->tbInfo.suid = tbMeta->suid; ctx->tbInfo.tbType = tbMeta->tableType; - + if (tbMeta->tableType != TSDB_CHILD_TABLE) { + int32_t metaSize = CTG_META_SIZE(tbMeta); + *pTableMeta = taosMemoryCalloc(1, metaSize); + if (NULL == *pTableMeta) { + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + + memcpy(*pTableMeta, tbMeta, metaSize); + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); ctgDebug("Got tb %s meta from cache, type:%d, dbFName:%s", ctx->pName->tname, tbMeta->tableType, dbFName); return TSDB_CODE_SUCCESS; } + + // PROCESS FOR CHILD TABLE - STableMeta **stbMeta = taosHashGet(dbCache->stbCache, &tbMeta->suid, sizeof(tbMeta->suid)); - if (NULL == stbMeta || NULL == *stbMeta) { - ctgError("stb not in stbCache, suid:%"PRIx64, tbMeta->suid); - goto _return; + int32_t metaSize = sizeof(SCTableMeta); + *pTableMeta = taosMemoryCalloc(1, metaSize); + if (NULL == *pTableMeta) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } - if ((*stbMeta)->suid != tbMeta->suid) { - ctgError("stb suid %" PRIx64 " in stbCache mis-match, expected suid:%"PRIx64 , (*stbMeta)->suid, tbMeta->suid); + memcpy(*pTableMeta, tbMeta, metaSize); + + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + ctgDebug("Got ctb %s meta from cache, will continue to get its stb meta, type:%d, dbFName:%s", + ctx->pName->tname, ctx->tbInfo.tbType, dbFName); + + dbCache = NULL; + tbCache = NULL; + ctgAcquireStbMetaFromCache(pCtg, dbFName, ctx->tbInfo.suid, &dbCache, &tbCache); + if (NULL == tbCache) { + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + taosMemoryFreeClear(*pTableMeta); + ctgDebug("stb %" PRIx64 " meta not in cache", ctx->tbInfo.suid); + return TSDB_CODE_SUCCESS; + } + + STableMeta* stbMeta = tbCache->pMeta; + if (stbMeta->suid != ctx->tbInfo.suid) { + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + ctgError("stb suid %" PRIx64 " in stbCache mis-match, expected suid:%"PRIx64 , stbMeta->suid, ctx->tbInfo.suid); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } - int32_t metaSize = CTG_META_SIZE(*stbMeta); + metaSize = CTG_META_SIZE(stbMeta); *pTableMeta = taosMemoryRealloc(*pTableMeta, metaSize); if (NULL == *pTableMeta) { + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } - memcpy(&(*pTableMeta)->sversion, &(*stbMeta)->sversion, metaSize - sizeof(SCTableMeta)); + memcpy(&(*pTableMeta)->sversion, &stbMeta->sversion, metaSize - sizeof(SCTableMeta)); ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); @@ -419,27 +498,33 @@ int32_t ctgReadTbVerFromCache(SCatalog *pCtg, SName *pTableName, int32_t *sver, return TSDB_CODE_SUCCESS; } - STableMeta **stbMeta = taosHashGet(dbCache->stbCache, suid, sizeof(*suid)); - if (NULL == stbMeta || NULL == *stbMeta) { + // PROCESS FOR CHILD TABLE + + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + ctgDebug("Got ctb %s ver from cache, will continue to get its stb ver, dbFName:%s", pTableName->tname, dbFName); + + ctgAcquireStbMetaFromCache(pCtg, dbFName, *suid, &dbCache, &tbCache); + if (NULL == tbCache) { ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); - ctgDebug("stb not in stbCache, suid:%" PRIx64, *suid); + ctgDebug("stb %" PRIx64 " meta not in cache", *suid); return TSDB_CODE_SUCCESS; } - - if ((*stbMeta)->suid != *suid) { + + STableMeta* stbMeta = tbCache->pMeta; + if (stbMeta->suid != *suid) { ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); - ctgError("stb suid %" PRIx64 " in stbCache mis-match, expected suid:%" PRIx64 , (*stbMeta)->suid, *suid); + ctgError("stb suid %" PRIx64 " in stbCache mis-match, expected suid:%" PRIx64 , stbMeta->suid, *suid); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } size_t nameLen = 0; - char *name = taosHashGetKey(*stbMeta, &nameLen); + char *name = taosHashGetKey(tbCache, &nameLen); strncpy(stbName, name, nameLen); stbName[nameLen] = 0; - *sver = (*stbMeta)->sversion; - *tver = (*stbMeta)->tversion; + *sver = stbMeta->sversion; + *tver = stbMeta->tversion; ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); @@ -1320,14 +1405,14 @@ int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNam return TSDB_CODE_SUCCESS; } - if (origSuid != meta->suid && taosHashPut(dbCache->stbCache, &meta->suid, sizeof(meta->suid), &pCache, POINTER_BYTES) != 0) { + if (origSuid != meta->suid && taosHashPut(dbCache->stbCache, &meta->suid, sizeof(meta->suid), tbName, strlen(tbName) + 1) != 0) { ctgError("taosHashPut to stable cache failed, suid:%"PRIx64, meta->suid); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } CTG_CACHE_STAT_INC(stblNum, 1); - ctgDebug("stb updated to stbCache, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType); + ctgDebug("stb %" PRIx64 " updated to cache, dbFName:%s, tbName:%s, tbType:%d", meta->suid, dbFName, tbName, meta->tableType); CTG_ERR_RET(ctgUpdateRentStbVersion(pCtg, dbFName, tbName, dbId, meta->suid, pCache)); diff --git a/source/libs/catalog/src/ctgDbg.c b/source/libs/catalog/src/ctgDbg.c index 2403dc3c5a..cd61abf6cc 100644 --- a/source/libs/catalog/src/ctgDbg.c +++ b/source/libs/catalog/src/ctgDbg.c @@ -19,7 +19,7 @@ #include "catalogInt.h" extern SCatalogMgmt gCtgMgmt; -SCtgDebug gCTGDebug = {.apiEnable = true}; +SCtgDebug gCTGDebug = {.apiEnable = true, .lockEnable = true}; void ctgdUserCallback(SMetaData* pResult, void* param, int32_t code) { ASSERT(*(int32_t*)param == 1); From d1b149a9f87c134ee47cc5b81d9c43e688ae43ea Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 13 Jun 2022 19:35:56 +0800 Subject: [PATCH 09/22] fix: adjust the initial array size of fs --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 2 ++ source/dnode/vnode/src/tsdb/tsdbFS.c | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 40a23447b7..4dae9bdeab 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -110,6 +110,8 @@ int32_t tsdbBegin(STsdb *pTsdb) { } int32_t tsdbCommit(STsdb *pTsdb) { + if (!pTsdb) return 0; + int32_t code = 0; SCommitH commith = {0}; SDFileSet *pSet = NULL; diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index f1941a3bad..055b6c62de 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -20,6 +20,7 @@ extern const char *TSDB_LEVEL_DNAME[]; typedef enum { TSDB_TXN_TEMP_FILE = 0, TSDB_TXN_CURR_FILE } TSDB_TXN_FILE_T; static const char *tsdbTxnFname[] = {"current.t", "current"}; #define TSDB_MAX_FSETS(keep, days) ((keep) / (days) + 3) +#define TSDB_MAX_INIT_FSETS (365000) static int tsdbComparFidFSet(const void *arg1, const void *arg2); static void tsdbResetFSStatus(SFSStatus *pStatus); @@ -210,6 +211,10 @@ STsdbFS *tsdbNewFS(const STsdbKeepCfg *pCfg) { return NULL; } + if (maxFSet > TSDB_MAX_INIT_FSETS) { + maxFSet = TSDB_MAX_INIT_FSETS; + } + pfs->cstatus = tsdbNewFSStatus(maxFSet); if (pfs->cstatus == NULL) { tsdbFreeFS(pfs); From 1227d6aed222be463c9b83b2c7a27f4e7682ed28 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 13 Jun 2022 19:39:40 +0800 Subject: [PATCH 10/22] other: adjust debug logs --- source/common/src/trow.c | 15 --------------- source/dnode/vnode/src/tsdb/tsdbCommit.c | 18 ------------------ source/dnode/vnode/src/vnd/vnodeSvr.c | 6 +----- 3 files changed, 1 insertion(+), 38 deletions(-) diff --git a/source/common/src/trow.c b/source/common/src/trow.c index fb4413478d..b1ca406a4a 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -527,20 +527,6 @@ static int32_t tdAppendTpRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols return terrno; } - if (pRowCol->colId == 2) { - ASSERT(sVal.valType == 0); - int32_t val = *(int32_t *)sVal.val; - ASSERT(val == 7); - } else if (pRowCol->colId == 3) { - ASSERT(sVal.valType == 0); - int64_t val = *(int64_t *)sVal.val; - ASSERT(val == 77777); - } else if (pRowCol->colId == 4) { - ASSERT(sVal.valType == 0); - int16_t val = *(int16_t *)sVal.val; - ASSERT(val == 777); - } - tdAppendValToDataCol(pDataCol, sVal.valType, sVal.val, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode, isMerge); ++dcol; @@ -551,7 +537,6 @@ static int32_t tdAppendTpRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode, isMerge); ++dcol; - ASSERT(0); } } #if 0 diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 4dae9bdeab..fe89321ae9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -1350,24 +1350,6 @@ static int tsdbCommitMemData(SCommitH *pCommith, SCommitIter *pIter, TSKEY keyLi isLast = true; } - SDataCols *pCols = pCommith->pDataCols; - SDataCol pCol_1 = pCols->cols[1]; - SDataCol pCol_2 = pCols->cols[2]; - SDataCol pCol_3 = pCols->cols[3]; - SCellVal sVal = {0}; - tdGetColDataOfRow(&sVal, &pCol_1, 0, pCols->bitmapMode); - ASSERT(sVal.valType == 0); - int32_t val1 = *(int32_t *)sVal.val; - ASSERT(val1 == 7); - tdGetColDataOfRow(&sVal, &pCol_2, 0, pCols->bitmapMode); - ASSERT(sVal.valType == 0); - int64_t val2 = *(int64_t *)sVal.val; - ASSERT(val2 == 77777); - tdGetColDataOfRow(&sVal, &pCol_3, 0, pCols->bitmapMode); - ASSERT(sVal.valType == 0); - int16_t val3 = *(int16_t *)sVal.val; - ASSERT(val3 == 777); - if (tsdbWriteBlock(pCommith, pDFile, pCommith->pDataCols, &block, isLast, true) < 0) return -1; if (tsdbCommitAddBlock(pCommith, &block, NULL, 0) < 0) { diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 9bb0834547..ab2efa4791 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -15,8 +15,6 @@ #include "vnd.h" -int32_t gForceCommit = 1; - static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessAlterStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessDropStbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); @@ -184,8 +182,7 @@ int32_t vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp } // commit if need - if ((gForceCommit == 1) || vnodeShouldCommit(pVnode)) { - gForceCommit = 0; + if (vnodeShouldCommit(pVnode)) { vInfo("vgId:%d, commit at version %" PRId64, TD_VID(pVnode), version); // commit current change vnodeCommit(pVnode); @@ -722,7 +719,6 @@ static int32_t vnodeDebugPrintSubmitMsg(SVnode *pVnode, SSubmitReq *pMsg, const } static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { - gForceCommit = 1; SSubmitReq *pSubmitReq = (SSubmitReq *)pReq; SSubmitRsp submitRsp = {0}; SSubmitMsgIter msgIter = {0}; From 8fbf5f3dba20d9089251c3783712c4eb29ae5c7d Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 13 Jun 2022 19:40:39 +0800 Subject: [PATCH 11/22] other: adjust debug logs --- source/common/src/trow.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/source/common/src/trow.c b/source/common/src/trow.c index b1ca406a4a..c8a28d7f28 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -523,10 +523,8 @@ static int32_t tdAppendTpRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols SCellVal sVal = {0}; if (pRowCol->colId == pDataCol->colId) { if (tdGetTpRowValOfCol(&sVal, pRow, pBitmap, pRowCol->type, pRowCol->offset - sizeof(TSKEY), rcol - 1) < 0) { - ASSERT(0); return terrno; } - tdAppendValToDataCol(pDataCol, sVal.valType, sVal.val, pCols->numOfRows, pCols->maxPoints, pCols->bitmapMode, isMerge); ++dcol; From b83197398eeee6e6f8d7b8506701a1ee1c675ba1 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 13 Jun 2022 19:47:14 +0800 Subject: [PATCH 12/22] fix msg decode issue --- source/common/src/tmsg.c | 18 ++++++++---------- source/dnode/mnode/impl/src/mndSma.c | 5 +++-- source/dnode/mnode/impl/src/mndStb.c | 6 ++++++ source/libs/catalog/src/ctgCache.c | 2 +- 4 files changed, 18 insertions(+), 13 deletions(-) diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 127d4ac273..31b8eb71bf 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -2652,7 +2652,8 @@ int32_t tSerializeSSTbHbRsp(void *buf, int32_t bufLen, SSTbHbRsp *pRsp) { } int32_t numOfIndex = taosArrayGetSize(pRsp->pIndexRsp); - for (int32_t i = 0; i < numOfMeta; ++i) { + if (tEncodeI32(&encoder, numOfIndex) < 0) return -1; + for (int32_t i = 0; i < numOfIndex; ++i) { STableIndexRsp *pIndexRsp = taosArrayGet(pRsp->pIndexRsp, i); if (tEncodeCStr(&encoder, pIndexRsp->tbName) < 0) return -1; if (tEncodeCStr(&encoder, pIndexRsp->dbFName) < 0) return -1; @@ -2660,12 +2661,10 @@ int32_t tSerializeSSTbHbRsp(void *buf, int32_t bufLen, SSTbHbRsp *pRsp) { if (tEncodeI32(&encoder, pIndexRsp->version) < 0) return -1; int32_t num = taosArrayGetSize(pIndexRsp->pIndex); if (tEncodeI32(&encoder, num) < 0) return -1; - if (num > 0) { - for (int32_t i = 0; i < num; ++i) { - STableIndexInfo *pInfo = (STableIndexInfo *)taosArrayGet(pIndexRsp->pIndex, i); - if (tSerializeSTableIndexInfo(&encoder, pInfo) < 0) return -1; - } - } + for (int32_t i = 0; i < num; ++i) { + STableIndexInfo *pInfo = (STableIndexInfo *)taosArrayGet(pIndexRsp->pIndex, i); + if (tSerializeSTableIndexInfo(&encoder, pInfo) < 0) return -1; + } } tEndEncode(&encoder); @@ -2693,9 +2692,8 @@ int32_t tDeserializeSSTbHbRsp(void *buf, int32_t bufLen, SSTbHbRsp *pRsp) { if (tStartDecode(&decoder) < 0) return -1; - int32_t numOfMeta = taosArrayGetSize(pRsp->pMetaRsp); + int32_t numOfMeta = 0; if (tDecodeI32(&decoder, &numOfMeta) < 0) return -1; - pRsp->pMetaRsp = taosArrayInit(numOfMeta, sizeof(STableMetaRsp)); if (pRsp->pMetaRsp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -2708,7 +2706,7 @@ int32_t tDeserializeSSTbHbRsp(void *buf, int32_t bufLen, SSTbHbRsp *pRsp) { taosArrayPush(pRsp->pMetaRsp, &tableMetaRsp); } - int32_t numOfIndex = taosArrayGetSize(pRsp->pIndexRsp); + int32_t numOfIndex = 0; if (tDecodeI32(&decoder, &numOfIndex) < 0) return -1; pRsp->pIndexRsp = taosArrayInit(numOfIndex, sizeof(STableIndexRsp)); diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 9cb7991d33..3f0d20348b 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -532,6 +532,7 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea SStreamObj streamObj = {0}; tstrncpy(streamObj.name, pCreate->name, TSDB_STREAM_FNAME_LEN); tstrncpy(streamObj.sourceDb, pDb->name, TSDB_DB_FNAME_LEN); + tstrncpy(streamObj.targetDb, streamObj.sourceDb, TSDB_DB_FNAME_LEN); streamObj.createTime = taosGetTimestampMs(); streamObj.updateTime = streamObj.createTime; streamObj.uid = mndGenerateUid(pCreate->name, strlen(pCreate->name)); @@ -913,7 +914,7 @@ int32_t mndGetTableSma(SMnode *pMnode, char *tbFName, STableIndexRsp *rsp, bool } strcpy(rsp->dbFName, pStb->db); - strcpy(rsp->tbName, pStb->name); + strcpy(rsp->tbName, pStb->name + strlen(pStb->db) + 1); rsp->suid = pStb->uid; rsp->version = pStb->smaVer; mndReleaseStb(pMnode, pStb); @@ -1127,4 +1128,4 @@ static int32_t mndRetrieveSma(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc static void mndCancelGetNextSma(SMnode *pMnode, void *pIter) { SSdb *pSdb = pMnode->pSdb; sdbCancelFetch(pSdb, pIter); -} \ No newline at end of file +} diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index e099359d47..6f89c97f83 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -1715,6 +1715,12 @@ int32_t mndValidateStbInfo(SMnode *pMnode, SSTableVersion *pStbVersions, int32_t bool exist = false; char tbFName[TSDB_TABLE_FNAME_LEN]; STableIndexRsp indexRsp = {0}; + indexRsp.pIndex = taosArrayInit(10, sizeof(STableIndexInfo)); + if (NULL == indexRsp.pIndex) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + sprintf(tbFName, "%s.%s", pStbVersion->dbFName, pStbVersion->stbName); int32_t code = mndGetTableSma(pMnode, tbFName, &indexRsp, &exist); if (code || !exist) { diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index b938dff932..724b54d250 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -1445,7 +1445,7 @@ int32_t ctgWriteTbIndexToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char* dbFNa ctgDebug("table %s index updated to cache, ver:%d, num:%d", tbName, pIndex->version, (int32_t)taosArrayGetSize(pIndex->pIndex)); if (suid) { - CTG_ERR_RET(ctgUpdateRentStbVersion(pCtg, dbFName, tbName, dbCache->dbId, pIndex->suid, pCache)); + CTG_ERR_RET(ctgUpdateRentStbVersion(pCtg, dbFName, tbName, dbCache->dbId, pIndex->suid, &cache)); } return TSDB_CODE_SUCCESS; From 2488382dd6de0f5c45a7c7df99dee4019cd6c730 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 13 Jun 2022 20:49:54 +0800 Subject: [PATCH 13/22] fix: avoid invalid value --- source/libs/index/src/indexFilter.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index 1cc2679b06..90aafb1097 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -183,6 +183,7 @@ static int32_t sifInitJsonParam(SNode *node, SIFParam *param, SIFCtx *ctx) { memcpy(param->colName, r->literal, strlen(r->literal)); // sprintf(param->colName, "%s_%s", l->colName, r->literal); param->colValType = r->typeData; + param->status = SFLT_COARSE_INDEX; return 0; // memcpy(param->colName, l->colName, sizeof(l->colName)); } @@ -254,11 +255,13 @@ static int32_t sifInitOperParams(SIFParam **params, SOperatorNode *node, SIFCtx return code; } SIFParam *paramList = taosMemoryCalloc(nParam, sizeof(SIFParam)); + if (NULL == paramList) { SIF_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - if (nodeType(node->pLeft) == QUERY_NODE_OPERATOR) { + if (nodeType(node->pLeft) == QUERY_NODE_OPERATOR && + (((SOperatorNode *)(node->pLeft))->opType == OP_TYPE_JSON_GET_VALUE)) { SNode *interNode = (node->pLeft); SIF_ERR_JRET(sifInitJsonParam(interNode, ¶mList[0], ctx)); if (nParam > 1) { @@ -525,8 +528,8 @@ static int32_t sifExecOper(SOperatorNode *node, SIFCtx *ctx, SIFParam *output) { if (node->opType == OP_TYPE_JSON_GET_VALUE) { return code; } - SIFParam *params = NULL; + SIFParam *params = NULL; SIF_ERR_RET(sifInitOperParams(¶ms, node, ctx)); if (params[0].status == SFLT_NOT_INDEX || (nParam > 1 && params[1].status == SFLT_NOT_INDEX)) { From 0111549155f08d1c03bd04674c9bda5290a09a4f Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 14 Jun 2022 08:41:52 +0800 Subject: [PATCH 14/22] index not exist as success --- source/libs/catalog/src/catalog.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 6aa86aa4ba..a3576f8738 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -392,8 +392,12 @@ int32_t ctgGetTbIndex(SCatalog* pCtg, SRequestConnInfo *pConn, SName* pTableName CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - int32_t code = 0; - CTG_ERR_JRET(ctgGetTbIndexFromMnode(pCtg, pConn, (SName*)pTableName, pIndex, NULL)); + int32_t code = ctgGetTbIndexFromMnode(pCtg, pConn, (SName*)pTableName, pIndex, NULL); + if (TSDB_CODE_MND_DB_INDEX_NOT_EXIST == code) { + code = 0; + goto _return; + } + CTG_ERR_JRET(code); SArray* pInfo = NULL; CTG_ERR_JRET(ctgCloneTableIndex(pIndex->pIndex, &pInfo)); From e8ab28bee17e379634764012ecc091b67f3245ed Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 14 Jun 2022 09:02:57 +0800 Subject: [PATCH 15/22] fix release crash issue --- source/libs/catalog/src/ctgCache.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 724b54d250..62890b8326 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -311,6 +311,9 @@ _return: ctgReleaseTbMetaToCache(pCtg, dbCache, pCache); CTG_CACHE_STAT_INC(tbMetaMissNum, 1); + + *pDb = NULL; + *pTb = NULL; return TSDB_CODE_SUCCESS; } @@ -427,8 +430,6 @@ int32_t ctgReadTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta** ctgDebug("Got ctb %s meta from cache, will continue to get its stb meta, type:%d, dbFName:%s", ctx->pName->tname, ctx->tbInfo.tbType, dbFName); - dbCache = NULL; - tbCache = NULL; ctgAcquireStbMetaFromCache(pCtg, dbFName, ctx->tbInfo.suid, &dbCache, &tbCache); if (NULL == tbCache) { ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); From 979420801eb0acda4cb3449b82a6332eadf9c87e Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 14 Jun 2022 13:07:27 +0800 Subject: [PATCH 16/22] fix(sync) if FpEqMsg return error, do not enqueue msg --- source/libs/sync/src/syncMain.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index f3faa60d34..7934087ff3 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1658,7 +1658,13 @@ static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) { syncTimeout2RpcMsg(pSyncMsg, &rpcMsg); syncRpcMsgLog2((char*)"==syncNodeEqHeartbeatTimer==", &rpcMsg); if (pSyncNode->FpEqMsg != NULL) { - pSyncNode->FpEqMsg(pSyncNode->msgcb, &rpcMsg); + int32_t code = pSyncNode->FpEqMsg(pSyncNode->msgcb, &rpcMsg); + if (code != 0) { + sError("vgId:%d sync enqueue timer msg error, code:%d", pSyncNode->vgId, code); + rpcFreeCont(rpcMsg.pCont); + return; + } + } else { sTrace("syncNodeEqHeartbeatTimer pSyncNode->FpEqMsg is NULL"); } From a2039d8bb2dda45763804be5a267ebb1cd0d2693 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 14 Jun 2022 13:16:49 +0800 Subject: [PATCH 17/22] fix(sync) if FpEqMsg return error, do not enqueue msg --- source/libs/sync/src/syncMain.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 7934087ff3..e7f11b9a7d 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1670,8 +1670,12 @@ static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) { } syncTimeoutDestroy(pSyncMsg); + if (gSyncEnv != NULL) { taosTmrReset(syncNodeEqHeartbeatTimer, pSyncNode->heartbeatTimerMS, pSyncNode, gSyncEnv->pTimerManager, &pSyncNode->pHeartbeatTimer); + } else { + sError("sync env is already stop"); + } } else { sTrace("==syncNodeEqHeartbeatTimer== heartbeatTimerLogicClock:%" PRIu64 ", heartbeatTimerLogicClockUser:%" PRIu64 "", From 110bf979044f8aa012ad57a93bfbd07049c23506 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 14 Jun 2022 13:22:24 +0800 Subject: [PATCH 18/22] fix: mnode not start --- source/dnode/mgmt/node_mgmt/src/dmEnv.c | 2 +- tests/script/jenkins/basic.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c index 498af36786..119d521827 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c +++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c @@ -124,7 +124,7 @@ static int32_t dmProcessCreateNodeReq(EDndNodeType ntype, SRpcMsg *pMsg) { } else { dInfo("node:%s, has been created", pWrapper->name); code = dmOpenNode(pWrapper); - if (code != 0) { + if (code == 0) { code = dmStartNode(pWrapper); } pWrapper->deployed = true; diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 1c694bf7fb..db8a055362 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -58,7 +58,7 @@ # ---- mnode ./test.sh -f tsim/mnode/basic1.sim ./test.sh -f tsim/mnode/basic2.sim -#./test.sh -f tsim/mnode/basic3.sim +./test.sh -f tsim/mnode/basic3.sim ./test.sh -f tsim/mnode/basic4.sim ./test.sh -f tsim/mnode/basic5.sim From 545345de7b7e67dce5c3135dfc5424619dd334c1 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 14 Jun 2022 13:23:47 +0800 Subject: [PATCH 19/22] remove debug flag --- source/client/inc/clientInt.h | 2 +- source/libs/catalog/src/ctgDbg.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index d72aa39e4c..6aa83e9575 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -45,7 +45,7 @@ extern "C" { #define ERROR_MSG_BUF_DEFAULT_SIZE 512 #define HEARTBEAT_INTERVAL 1500 // ms -#define SYNC_ON_TOP_OF_ASYNC 0 +#define SYNC_ON_TOP_OF_ASYNC 1 enum { RES_TYPE__QUERY = 1, diff --git a/source/libs/catalog/src/ctgDbg.c b/source/libs/catalog/src/ctgDbg.c index cd61abf6cc..9de1ea22be 100644 --- a/source/libs/catalog/src/ctgDbg.c +++ b/source/libs/catalog/src/ctgDbg.c @@ -19,7 +19,7 @@ #include "catalogInt.h" extern SCatalogMgmt gCtgMgmt; -SCtgDebug gCTGDebug = {.apiEnable = true, .lockEnable = true}; +SCtgDebug gCTGDebug = {0}; void ctgdUserCallback(SMetaData* pResult, void* param, int32_t code) { ASSERT(*(int32_t*)param == 1); From 0237c68e272da34a38439b5c2707a69e2ece361a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 14 Jun 2022 13:44:13 +0800 Subject: [PATCH 20/22] fix: invalid read/write --- source/dnode/vnode/src/meta/metaQuery.c | 11 ++++ tests/script/tsim/stable/tag_filter.sim | 75 +++++++++++++++++++++++++ 2 files changed, 86 insertions(+) diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 1d8082084f..a6339125c4 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -663,12 +663,23 @@ int32_t metaFilteTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { void * entryKey = NULL, *entryVal = NULL; int32_t nEntryKey, nEntryVal; + bool first = true; while (1) { valid = tdbTbcGet(pCursor->pCur, (const void **)&entryKey, &nEntryKey, (const void **)&entryVal, &nEntryVal); if (valid < 0) { break; } STagIdxKey *p = entryKey; + if (p->type != pCursor->type) { + if (first) { + valid = param->reverse ? tdbTbcMoveToPrev(pCursor->pCur) : tdbTbcMoveToNext(pCursor->pCur); + if (valid < 0) break; + continue; + } else { + break; + } + } + first = false; if (p != NULL) { int32_t cmp = (*param->filterFunc)(p->data, pKey->data, pKey->type); if (cmp == 0) { diff --git a/tests/script/tsim/stable/tag_filter.sim b/tests/script/tsim/stable/tag_filter.sim index 04b8f21de9..1f400eb803 100644 --- a/tests/script/tsim/stable/tag_filter.sim +++ b/tests/script/tsim/stable/tag_filter.sim @@ -73,5 +73,80 @@ if $rows != 6 then endi +print ========== prepare stbBin and ctbBin +sql create table db.stbBin (ts timestamp, c1 int, c2 binary(4)) tags(t1 binary(16)) + +sql create table db.ctbBin using db.stbBin tags("a") +sql insert into db.ctbBin values(now, 1, "2") + +sql create table db.ctbBin1 using db.stbBin tags("b") +sql insert into db.ctbBin1 values(now, 2, "2") + +sql create table db.ctbBin2 using db.stbBin tags("c") +sql insert into db.ctbBin2 values(now, 3, "2") + +sql create table db.ctbBin3 using db.stbBin tags("d") +sql insert into db.ctbBin3 values(now, 4, "2") + +sql select * from db.stbBin where t1 = "a" +if $rows != 1 then + return -1 +endi + + +sql select * from db.stbBin where t1 < "a" +if $rows != 0 then + return -=1 +endi + +sql select * from db.stbBin where t1 < "b" +if $rows != 1 then + return -1 +endi + + +sql select * from db.stbBin where t1 between "a" and "e" +if $rows != 4 then + return -1 +endi + + +print ========== prepare stbNc and ctbNc +sql create table db.stbNc (ts timestamp, c1 int, c2 binary(4)) tags(t1 nchar(16)) + + +sql create table db.ctbNc using db.stbNc tags("a") +sql insert into db.ctbNc values(now, 1, "2") + +sql create table db.ctbNc1 using db.stbNc tags("b") +sql insert into db.ctbNc1 values(now, 2, "2") + +sql create table db.ctbNc2 using db.stbNc tags("c") +sql insert into db.ctbNc2 values(now, 3, "2") + +sql create table db.ctbNc3 using db.stbNc tags("d") +sql insert into db.ctbNc3 values(now, 4, "2") + +sql select * from db.stbNc where t1 = "a" +if $rows != 1 then + return -1 +endi + + +sql select * from db.stbNc where t1 < "a" +if $rows != 0 then + return -=1 +endi + +sql select * from db.stbNc where t1 < "b" +if $rows != 1 then + return -1 +endi + + +sql select * from db.stbNc where t1 between "a" and "e" +if $rows != 4 then + return -1 +endi system sh/exec.sh -n dnode1 -s stop -x SIGINT From ad11eef979b82ad517e8154e9ba7e8006fff9f51 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 14 Jun 2022 13:52:07 +0800 Subject: [PATCH 21/22] enh: show db status such as creating and dropping --- source/dnode/mnode/impl/src/mndDb.c | 25 ++++++++++++++----------- source/dnode/mnode/impl/src/mndMnode.c | 2 +- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index b4237466d4..cfe363fdf0 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -1366,7 +1366,7 @@ char *buildRetension(SArray *pRetension) { } static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, int32_t rows, int64_t numOfTables, - bool sysDb) { + bool sysDb, ESdbStatus objStatus) { int32_t cols = 0; int32_t bytes = pShow->pMeta->pSchemas[cols].bytes; @@ -1379,6 +1379,8 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in } char *status = "ready"; + if (objStatus == SDB_STATUS_CREATING) status = "creating"; + if (objStatus == SDB_STATUS_DROPPING) status = "dropping"; char statusB[24] = {0}; STR_WITH_SIZE_TO_VARSTR(statusB, status, strlen(status)); @@ -1503,8 +1505,8 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, rows, (const char *)statusB, false); -// pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); -// colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.schemaless, false); + // pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + // colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.schemaless, false); char *p = buildRetension(pDb->cfg.pRetensions); @@ -1548,29 +1550,30 @@ static bool mndGetTablesOfDbFp(SMnode *pMnode, void *pObj, void *p1, void *p2, v } static int32_t mndRetrieveDbs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rowsCapacity) { - SMnode *pMnode = pReq->info.node; - SSdb *pSdb = pMnode->pSdb; - int32_t numOfRows = 0; - SDbObj *pDb = NULL; + SMnode *pMnode = pReq->info.node; + SSdb *pSdb = pMnode->pSdb; + int32_t numOfRows = 0; + SDbObj *pDb = NULL; + ESdbStatus objStatus = 0; // Append the information_schema database into the result. if (!pShow->sysDbRsp) { SDbObj infoschemaDb = {0}; setInformationSchemaDbCfg(&infoschemaDb); - dumpDbInfoData(pBlock, &infoschemaDb, pShow, numOfRows, 14, true); + dumpDbInfoData(pBlock, &infoschemaDb, pShow, numOfRows, 14, true, 0); numOfRows += 1; SDbObj perfschemaDb = {0}; setPerfSchemaDbCfg(&perfschemaDb); - dumpDbInfoData(pBlock, &perfschemaDb, pShow, numOfRows, 3, true); + dumpDbInfoData(pBlock, &perfschemaDb, pShow, numOfRows, 3, true, 0); numOfRows += 1; pShow->sysDbRsp = true; } while (numOfRows < rowsCapacity) { - pShow->pIter = sdbFetch(pSdb, SDB_DB, pShow->pIter, (void **)&pDb); + pShow->pIter = sdbFetchAll(pSdb, SDB_DB, pShow->pIter, (void **)&pDb, &objStatus); if (pShow->pIter == NULL) { break; } @@ -1578,7 +1581,7 @@ static int32_t mndRetrieveDbs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc int32_t numOfTables = 0; sdbTraverse(pSdb, SDB_VGROUP, mndGetTablesOfDbFp, &numOfTables, NULL, NULL); - dumpDbInfoData(pBlock, pDb, pShow, numOfRows, numOfTables, false); + dumpDbInfoData(pBlock, pDb, pShow, numOfRows, numOfTables, false, objStatus); numOfRows++; sdbRelease(pSdb, pDb); } diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 95a2c3d935..f9749a969d 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -659,7 +659,7 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB int32_t numOfRows = 0; int32_t cols = 0; SMnodeObj *pObj = NULL; - ESdbStatus objStatus; + ESdbStatus objStatus = 0; char *pWrite; int64_t curMs = taosGetTimestampMs(); From 8dea8dbbebeca2cf8b61d739d3cadbfe7b8c6b7f Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Tue, 14 Jun 2022 13:40:48 +0800 Subject: [PATCH 22/22] docs: move delete data from developer guide to sql reference guide --- .../05-delete-data.mdx => 12-taos-sql/08-delete-data.mdx} | 0 docs/en/12-taos-sql/{07-function.md => 10-function.md} | 0 docs/en/12-taos-sql/{08-interval.md => 12-interval.md} | 0 docs/en/12-taos-sql/{09-limit.md => 14-limit.md} | 0 docs/en/12-taos-sql/{10-json.md => 16-json.md} | 0 docs/en/12-taos-sql/{11-escape.md => 18-escape.md} | 0 docs/en/12-taos-sql/{12-keywords.md => 20-keywords.md} | 0 .../05-delete-data.mdx => 12-taos-sql/08-delete-data.mdx} | 0 docs/zh/12-taos-sql/{07-function.md => 10-function.md} | 0 docs/zh/12-taos-sql/{08-interval.md => 12-interval.md} | 0 docs/zh/12-taos-sql/{09-limit.md => 14-limit.md} | 0 docs/zh/12-taos-sql/{10-json.md => 16-json.md} | 0 docs/zh/12-taos-sql/{11-escape.md => 18-escape.md} | 0 docs/zh/12-taos-sql/{12-keywords.md => 20-keywords.md} | 0 14 files changed, 0 insertions(+), 0 deletions(-) rename docs/en/{07-develop/05-delete-data.mdx => 12-taos-sql/08-delete-data.mdx} (100%) rename docs/en/12-taos-sql/{07-function.md => 10-function.md} (100%) rename docs/en/12-taos-sql/{08-interval.md => 12-interval.md} (100%) rename docs/en/12-taos-sql/{09-limit.md => 14-limit.md} (100%) rename docs/en/12-taos-sql/{10-json.md => 16-json.md} (100%) rename docs/en/12-taos-sql/{11-escape.md => 18-escape.md} (100%) rename docs/en/12-taos-sql/{12-keywords.md => 20-keywords.md} (100%) rename docs/zh/{07-develop/05-delete-data.mdx => 12-taos-sql/08-delete-data.mdx} (100%) rename docs/zh/12-taos-sql/{07-function.md => 10-function.md} (100%) rename docs/zh/12-taos-sql/{08-interval.md => 12-interval.md} (100%) rename docs/zh/12-taos-sql/{09-limit.md => 14-limit.md} (100%) rename docs/zh/12-taos-sql/{10-json.md => 16-json.md} (100%) rename docs/zh/12-taos-sql/{11-escape.md => 18-escape.md} (100%) rename docs/zh/12-taos-sql/{12-keywords.md => 20-keywords.md} (100%) diff --git a/docs/en/07-develop/05-delete-data.mdx b/docs/en/12-taos-sql/08-delete-data.mdx similarity index 100% rename from docs/en/07-develop/05-delete-data.mdx rename to docs/en/12-taos-sql/08-delete-data.mdx diff --git a/docs/en/12-taos-sql/07-function.md b/docs/en/12-taos-sql/10-function.md similarity index 100% rename from docs/en/12-taos-sql/07-function.md rename to docs/en/12-taos-sql/10-function.md diff --git a/docs/en/12-taos-sql/08-interval.md b/docs/en/12-taos-sql/12-interval.md similarity index 100% rename from docs/en/12-taos-sql/08-interval.md rename to docs/en/12-taos-sql/12-interval.md diff --git a/docs/en/12-taos-sql/09-limit.md b/docs/en/12-taos-sql/14-limit.md similarity index 100% rename from docs/en/12-taos-sql/09-limit.md rename to docs/en/12-taos-sql/14-limit.md diff --git a/docs/en/12-taos-sql/10-json.md b/docs/en/12-taos-sql/16-json.md similarity index 100% rename from docs/en/12-taos-sql/10-json.md rename to docs/en/12-taos-sql/16-json.md diff --git a/docs/en/12-taos-sql/11-escape.md b/docs/en/12-taos-sql/18-escape.md similarity index 100% rename from docs/en/12-taos-sql/11-escape.md rename to docs/en/12-taos-sql/18-escape.md diff --git a/docs/en/12-taos-sql/12-keywords.md b/docs/en/12-taos-sql/20-keywords.md similarity index 100% rename from docs/en/12-taos-sql/12-keywords.md rename to docs/en/12-taos-sql/20-keywords.md diff --git a/docs/zh/07-develop/05-delete-data.mdx b/docs/zh/12-taos-sql/08-delete-data.mdx similarity index 100% rename from docs/zh/07-develop/05-delete-data.mdx rename to docs/zh/12-taos-sql/08-delete-data.mdx diff --git a/docs/zh/12-taos-sql/07-function.md b/docs/zh/12-taos-sql/10-function.md similarity index 100% rename from docs/zh/12-taos-sql/07-function.md rename to docs/zh/12-taos-sql/10-function.md diff --git a/docs/zh/12-taos-sql/08-interval.md b/docs/zh/12-taos-sql/12-interval.md similarity index 100% rename from docs/zh/12-taos-sql/08-interval.md rename to docs/zh/12-taos-sql/12-interval.md diff --git a/docs/zh/12-taos-sql/09-limit.md b/docs/zh/12-taos-sql/14-limit.md similarity index 100% rename from docs/zh/12-taos-sql/09-limit.md rename to docs/zh/12-taos-sql/14-limit.md diff --git a/docs/zh/12-taos-sql/10-json.md b/docs/zh/12-taos-sql/16-json.md similarity index 100% rename from docs/zh/12-taos-sql/10-json.md rename to docs/zh/12-taos-sql/16-json.md diff --git a/docs/zh/12-taos-sql/11-escape.md b/docs/zh/12-taos-sql/18-escape.md similarity index 100% rename from docs/zh/12-taos-sql/11-escape.md rename to docs/zh/12-taos-sql/18-escape.md diff --git a/docs/zh/12-taos-sql/12-keywords.md b/docs/zh/12-taos-sql/20-keywords.md similarity index 100% rename from docs/zh/12-taos-sql/12-keywords.md rename to docs/zh/12-taos-sql/20-keywords.md