diff --git a/docs/en/07-develop/05-delete-data.mdx b/docs/en/12-taos-sql/08-delete-data.mdx similarity index 100% rename from docs/en/07-develop/05-delete-data.mdx rename to docs/en/12-taos-sql/08-delete-data.mdx diff --git a/docs/en/12-taos-sql/07-function.md b/docs/en/12-taos-sql/10-function.md similarity index 100% rename from docs/en/12-taos-sql/07-function.md rename to docs/en/12-taos-sql/10-function.md diff --git a/docs/en/12-taos-sql/08-interval.md b/docs/en/12-taos-sql/12-interval.md similarity index 100% rename from docs/en/12-taos-sql/08-interval.md rename to docs/en/12-taos-sql/12-interval.md diff --git a/docs/en/12-taos-sql/09-limit.md b/docs/en/12-taos-sql/14-limit.md similarity index 100% rename from docs/en/12-taos-sql/09-limit.md rename to docs/en/12-taos-sql/14-limit.md diff --git a/docs/en/12-taos-sql/10-json.md b/docs/en/12-taos-sql/16-json.md similarity index 100% rename from docs/en/12-taos-sql/10-json.md rename to docs/en/12-taos-sql/16-json.md diff --git a/docs/en/12-taos-sql/11-escape.md b/docs/en/12-taos-sql/18-escape.md similarity index 100% rename from docs/en/12-taos-sql/11-escape.md rename to docs/en/12-taos-sql/18-escape.md diff --git a/docs/en/12-taos-sql/12-keywords.md b/docs/en/12-taos-sql/20-keywords.md similarity index 100% rename from docs/en/12-taos-sql/12-keywords.md rename to docs/en/12-taos-sql/20-keywords.md diff --git a/docs/zh/07-develop/05-delete-data.mdx b/docs/zh/12-taos-sql/08-delete-data.mdx similarity index 100% rename from docs/zh/07-develop/05-delete-data.mdx rename to docs/zh/12-taos-sql/08-delete-data.mdx diff --git a/docs/zh/12-taos-sql/07-function.md b/docs/zh/12-taos-sql/10-function.md similarity index 100% rename from docs/zh/12-taos-sql/07-function.md rename to docs/zh/12-taos-sql/10-function.md diff --git a/docs/zh/12-taos-sql/08-interval.md b/docs/zh/12-taos-sql/12-interval.md similarity index 100% rename from docs/zh/12-taos-sql/08-interval.md rename to docs/zh/12-taos-sql/12-interval.md diff --git a/docs/zh/12-taos-sql/09-limit.md b/docs/zh/12-taos-sql/14-limit.md similarity index 100% rename from docs/zh/12-taos-sql/09-limit.md rename to docs/zh/12-taos-sql/14-limit.md diff --git a/docs/zh/12-taos-sql/10-json.md b/docs/zh/12-taos-sql/16-json.md similarity index 100% rename from docs/zh/12-taos-sql/10-json.md rename to docs/zh/12-taos-sql/16-json.md diff --git a/docs/zh/12-taos-sql/11-escape.md b/docs/zh/12-taos-sql/18-escape.md similarity index 100% rename from docs/zh/12-taos-sql/11-escape.md rename to docs/zh/12-taos-sql/18-escape.md diff --git a/docs/zh/12-taos-sql/12-keywords.md b/docs/zh/12-taos-sql/20-keywords.md similarity index 100% rename from docs/zh/12-taos-sql/12-keywords.md rename to docs/zh/12-taos-sql/20-keywords.md diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 935e2f6626..4e7dfb4ccd 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1134,14 +1134,16 @@ void tFreeSMAlterStbRsp(SMAlterStbRsp* pRsp); int32_t tSerializeSTableMetaRsp(void* buf, int32_t bufLen, STableMetaRsp* pRsp); int32_t tDeserializeSTableMetaRsp(void* buf, int32_t bufLen, STableMetaRsp* pRsp); void tFreeSTableMetaRsp(STableMetaRsp* pRsp); +void tFreeSTableIndexRsp(void *info); typedef struct { - SArray* pArray; // Array of STableMetaRsp -} STableMetaBatchRsp; + SArray* pMetaRsp; // Array of STableMetaRsp + SArray* pIndexRsp; // Array of STableIndexRsp; +} SSTbHbRsp; -int32_t tSerializeSTableMetaBatchRsp(void* buf, int32_t bufLen, STableMetaBatchRsp* pRsp); -int32_t tDeserializeSTableMetaBatchRsp(void* buf, int32_t bufLen, STableMetaBatchRsp* pRsp); -void tFreeSTableMetaBatchRsp(STableMetaBatchRsp* pRsp); +int32_t tSerializeSSTbHbRsp(void* buf, int32_t bufLen, SSTbHbRsp* pRsp); +int32_t tDeserializeSSTbHbRsp(void* buf, int32_t bufLen, SSTbHbRsp* pRsp); +void tFreeSSTbHbRsp(SSTbHbRsp* pRsp); typedef struct { int32_t numOfTables; @@ -2502,7 +2504,11 @@ typedef struct { } STableIndexInfo; typedef struct { - SArray* pIndex; + char tbName[TSDB_TABLE_NAME_LEN]; + char dbFName[TSDB_DB_FNAME_LEN]; + uint64_t suid; + int32_t version; + SArray* pIndex; } STableIndexRsp; int32_t tSerializeSTableIndexRsp(void* buf, int32_t bufLen, const STableIndexRsp* pRsp); diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 9e573b7ed1..f3600a509e 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -98,14 +98,15 @@ typedef struct SCatalogCfg { uint32_t stbRentSec; } SCatalogCfg; -typedef struct SSTableMetaVersion { +typedef struct SSTableVersion { char dbFName[TSDB_DB_FNAME_LEN]; char stbName[TSDB_TABLE_NAME_LEN]; uint64_t dbId; uint64_t suid; int16_t sversion; - int16_t tversion; -} SSTableMetaVersion; + int16_t tversion; + int32_t smaVer; +} SSTableVersion; typedef struct SDbVgVersion { char dbFName[TSDB_DB_FNAME_LEN]; @@ -267,7 +268,7 @@ int32_t catalogAsyncGetAllMeta(SCatalog* pCtg, SRequestConnInfo* pConn, uint64_t int32_t catalogGetQnodeList(SCatalog* pCatalog, SRequestConnInfo* pConn, SArray* pQnodeList); -int32_t catalogGetExpiredSTables(SCatalog* pCatalog, SSTableMetaVersion** stables, uint32_t* num); +int32_t catalogGetExpiredSTables(SCatalog* pCatalog, SSTableVersion **stables, uint32_t *num); int32_t catalogGetExpiredDBs(SCatalog* pCatalog, SDbVgVersion** dbs, uint32_t* num); @@ -279,6 +280,8 @@ int32_t catalogGetIndexMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const char* int32_t catalogGetTableIndex(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SArray** pRes); +int32_t catalogUpdateTableIndex(SCatalog* pCtg, STableIndexRsp *pRsp); + int32_t catalogGetUdfInfo(SCatalog* pCtg, SRequestConnInfo* pConn, const char* funcName, SFuncInfo* pInfo); int32_t catalogChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, const char* user, const char* dbFName, AUTH_TYPE type, bool *pass); diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 2412b32df2..5db27ae438 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -99,15 +99,15 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) { int32_t code = 0; - STableMetaBatchRsp batchMetaRsp = {0}; - if (tDeserializeSTableMetaBatchRsp(value, valueLen, &batchMetaRsp) != 0) { + SSTbHbRsp hbRsp = {0}; + if (tDeserializeSSTbHbRsp(value, valueLen, &hbRsp) != 0) { terrno = TSDB_CODE_INVALID_MSG; return -1; } - int32_t numOfBatchs = taosArrayGetSize(batchMetaRsp.pArray); - for (int32_t i = 0; i < numOfBatchs; ++i) { - STableMetaRsp *rsp = taosArrayGet(batchMetaRsp.pArray, i); + int32_t numOfMeta = taosArrayGetSize(hbRsp.pMetaRsp); + for (int32_t i = 0; i < numOfMeta; ++i) { + STableMetaRsp *rsp = taosArrayGet(hbRsp.pMetaRsp, i); if (rsp->numOfColumns < 0) { tscDebug("hb remove stb, db:%s, stb:%s", rsp->dbFName, rsp->stbName); @@ -116,7 +116,7 @@ static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalo tscDebug("hb update stb, db:%s, stb:%s", rsp->dbFName, rsp->stbName); if (rsp->pSchemas[0].colId != PRIMARYKEY_TIMESTAMP_COL_ID) { tscError("invalid colId[%" PRIi16 "] for the first column in table meta rsp msg", rsp->pSchemas[0].colId); - tFreeSTableMetaBatchRsp(&batchMetaRsp); + tFreeSSTbHbRsp(&hbRsp); return TSDB_CODE_TSC_INVALID_VALUE; } @@ -124,7 +124,17 @@ static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalo } } - tFreeSTableMetaBatchRsp(&batchMetaRsp); + int32_t numOfIndex = taosArrayGetSize(hbRsp.pIndexRsp); + for (int32_t i = 0; i < numOfIndex; ++i) { + STableIndexRsp *rsp = taosArrayGet(hbRsp.pIndexRsp, i); + + catalogUpdateTableIndex(pCatalog, rsp); + } + + taosArrayDestroy(hbRsp.pIndexRsp); + hbRsp.pIndexRsp = NULL; + + tFreeSSTbHbRsp(&hbRsp); return TSDB_CODE_SUCCESS; } @@ -455,7 +465,7 @@ int32_t hbGetExpiredDBInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SCl } int32_t hbGetExpiredStbInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SClientHbReq *req) { - SSTableMetaVersion *stbs = NULL; + SSTableVersion *stbs = NULL; uint32_t stbNum = 0; int32_t code = 0; @@ -469,15 +479,16 @@ int32_t hbGetExpiredStbInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SC } for (int32_t i = 0; i < stbNum; ++i) { - SSTableMetaVersion *stb = &stbs[i]; + SSTableVersion *stb = &stbs[i]; stb->suid = htobe64(stb->suid); stb->sversion = htons(stb->sversion); stb->tversion = htons(stb->tversion); + stb->smaVer = htonl(stb->smaVer); } SKv kv = { .key = HEARTBEAT_KEY_STBINFO, - .valueLen = sizeof(SSTableMetaVersion) * stbNum, + .valueLen = sizeof(SSTableVersion) * stbNum, .value = stbs, }; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 2ba2253780..526da5fe0b 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -2438,6 +2438,10 @@ int32_t tSerializeSTableIndexRsp(void *buf, int32_t bufLen, const STableIndexRsp tEncoderInit(&encoder, buf, bufLen); if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pRsp->tbName) < 0) return -1; + if (tEncodeCStr(&encoder, pRsp->dbFName) < 0) return -1; + if (tEncodeU64(&encoder, pRsp->suid) < 0) return -1; + if (tEncodeI32(&encoder, pRsp->version) < 0) return -1; int32_t num = taosArrayGetSize(pRsp->pIndex); if (tEncodeI32(&encoder, num) < 0) return -1; if (num > 0) { @@ -2472,6 +2476,10 @@ int32_t tDeserializeSTableIndexRsp(void *buf, int32_t bufLen, STableIndexRsp *pR tDecoderInit(&decoder, buf, bufLen); if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeCStrTo(&decoder, pRsp->tbName) < 0) return -1; + if (tDecodeCStrTo(&decoder, pRsp->dbFName) < 0) return -1; + if (tDecodeU64(&decoder, &pRsp->suid) < 0) return -1; + if (tDecodeI32(&decoder, &pRsp->version) < 0) return -1; int32_t num = 0; if (tDecodeI32(&decoder, &num) < 0) return -1; if (num > 0) { @@ -2631,18 +2639,35 @@ int32_t tSerializeSTableMetaRsp(void *buf, int32_t bufLen, STableMetaRsp *pRsp) return tlen; } -int32_t tSerializeSTableMetaBatchRsp(void *buf, int32_t bufLen, STableMetaBatchRsp *pRsp) { +int32_t tSerializeSSTbHbRsp(void *buf, int32_t bufLen, SSTbHbRsp *pRsp) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); if (tStartEncode(&encoder) < 0) return -1; - int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); - if (tEncodeI32(&encoder, numOfBatch) < 0) return -1; - for (int32_t i = 0; i < numOfBatch; ++i) { - STableMetaRsp *pMetaRsp = taosArrayGet(pRsp->pArray, i); + int32_t numOfMeta = taosArrayGetSize(pRsp->pMetaRsp); + if (tEncodeI32(&encoder, numOfMeta) < 0) return -1; + for (int32_t i = 0; i < numOfMeta; ++i) { + STableMetaRsp *pMetaRsp = taosArrayGet(pRsp->pMetaRsp, i); if (tEncodeSTableMetaRsp(&encoder, pMetaRsp) < 0) return -1; } + + int32_t numOfIndex = taosArrayGetSize(pRsp->pIndexRsp); + if (tEncodeI32(&encoder, numOfIndex) < 0) return -1; + for (int32_t i = 0; i < numOfIndex; ++i) { + STableIndexRsp *pIndexRsp = taosArrayGet(pRsp->pIndexRsp, i); + if (tEncodeCStr(&encoder, pIndexRsp->tbName) < 0) return -1; + if (tEncodeCStr(&encoder, pIndexRsp->dbFName) < 0) return -1; + if (tEncodeU64(&encoder, pIndexRsp->suid) < 0) return -1; + if (tEncodeI32(&encoder, pIndexRsp->version) < 0) return -1; + int32_t num = taosArrayGetSize(pIndexRsp->pIndex); + if (tEncodeI32(&encoder, num) < 0) return -1; + for (int32_t i = 0; i < num; ++i) { + STableIndexInfo *pInfo = (STableIndexInfo *)taosArrayGet(pIndexRsp->pIndex, i); + if (tSerializeSTableIndexInfo(&encoder, pInfo) < 0) return -1; + } + } + tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -2662,26 +2687,58 @@ int32_t tDeserializeSTableMetaRsp(void *buf, int32_t bufLen, STableMetaRsp *pRsp return 0; } -int32_t tDeserializeSTableMetaBatchRsp(void *buf, int32_t bufLen, STableMetaBatchRsp *pRsp) { +int32_t tDeserializeSSTbHbRsp(void *buf, int32_t bufLen, SSTbHbRsp *pRsp) { SDecoder decoder = {0}; tDecoderInit(&decoder, buf, bufLen); if (tStartDecode(&decoder) < 0) return -1; - int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); - if (tDecodeI32(&decoder, &numOfBatch) < 0) return -1; - - pRsp->pArray = taosArrayInit(numOfBatch, sizeof(STableMetaRsp)); - if (pRsp->pArray == NULL) { + int32_t numOfMeta = 0; + if (tDecodeI32(&decoder, &numOfMeta) < 0) return -1; + pRsp->pMetaRsp = taosArrayInit(numOfMeta, sizeof(STableMetaRsp)); + if (pRsp->pMetaRsp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - for (int32_t i = 0; i < numOfBatch; ++i) { + for (int32_t i = 0; i < numOfMeta; ++i) { STableMetaRsp tableMetaRsp = {0}; if (tDecodeSTableMetaRsp(&decoder, &tableMetaRsp) < 0) return -1; - taosArrayPush(pRsp->pArray, &tableMetaRsp); + taosArrayPush(pRsp->pMetaRsp, &tableMetaRsp); } + + int32_t numOfIndex = 0; + if (tDecodeI32(&decoder, &numOfIndex) < 0) return -1; + + pRsp->pIndexRsp = taosArrayInit(numOfIndex, sizeof(STableIndexRsp)); + if (pRsp->pIndexRsp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + for (int32_t i = 0; i < numOfIndex; ++i) { + STableIndexRsp tableIndexRsp = {0}; + if (tDecodeCStrTo(&decoder, tableIndexRsp.tbName) < 0) return -1; + if (tDecodeCStrTo(&decoder, tableIndexRsp.dbFName) < 0) return -1; + if (tDecodeU64(&decoder, &tableIndexRsp.suid) < 0) return -1; + if (tDecodeI32(&decoder, &tableIndexRsp.version) < 0) return -1; + int32_t num = 0; + if (tDecodeI32(&decoder, &num) < 0) return -1; + if (num > 0) { + tableIndexRsp.pIndex = taosArrayInit(num, sizeof(STableIndexInfo)); + if (NULL == tableIndexRsp.pIndex) return -1; + STableIndexInfo info; + for (int32_t i = 0; i < num; ++i) { + if (tDeserializeSTableIndexInfo(&decoder, &info) < 0) return -1; + if (NULL == taosArrayPush(tableIndexRsp.pIndex, &info)) { + taosMemoryFree(info.expr); + return -1; + } + } + } + taosArrayPush(pRsp->pIndexRsp, &tableIndexRsp); + } + tEndDecode(&decoder); tDecoderClear(&decoder); @@ -2690,14 +2747,32 @@ int32_t tDeserializeSTableMetaBatchRsp(void *buf, int32_t bufLen, STableMetaBatc void tFreeSTableMetaRsp(STableMetaRsp *pRsp) { taosMemoryFreeClear(pRsp->pSchemas); } -void tFreeSTableMetaBatchRsp(STableMetaBatchRsp *pRsp) { - int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); - for (int32_t i = 0; i < numOfBatch; ++i) { - STableMetaRsp *pMetaRsp = taosArrayGet(pRsp->pArray, i); +void tFreeSTableIndexRsp(void *info) { + if (NULL == info) { + return; + } + + STableIndexRsp *pInfo = (STableIndexRsp *)info; + + taosArrayDestroyEx(pInfo->pIndex, tFreeSTableIndexInfo); +} + +void tFreeSSTbHbRsp(SSTbHbRsp *pRsp) { + int32_t numOfMeta = taosArrayGetSize(pRsp->pMetaRsp); + for (int32_t i = 0; i < numOfMeta; ++i) { + STableMetaRsp *pMetaRsp = taosArrayGet(pRsp->pMetaRsp, i); tFreeSTableMetaRsp(pMetaRsp); } - taosArrayDestroy(pRsp->pArray); + taosArrayDestroy(pRsp->pMetaRsp); + + int32_t numOfIndex = taosArrayGetSize(pRsp->pIndexRsp); + for (int32_t i = 0; i < numOfIndex; ++i) { + STableIndexRsp *pIndexRsp = taosArrayGet(pRsp->pIndexRsp, i); + tFreeSTableIndexRsp(pIndexRsp); + } + + taosArrayDestroy(pRsp->pIndexRsp); } int32_t tSerializeSShowRsp(void *buf, int32_t bufLen, SShowRsp *pRsp) { diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c index 498af36786..119d521827 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c +++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c @@ -124,7 +124,7 @@ static int32_t dmProcessCreateNodeReq(EDndNodeType ntype, SRpcMsg *pMsg) { } else { dInfo("node:%s, has been created", pWrapper->name); code = dmOpenNode(pWrapper); - if (code != 0) { + if (code == 0) { code = dmStartNode(pWrapper); } pWrapper->deployed = true; diff --git a/source/dnode/mnode/impl/inc/mndSma.h b/source/dnode/mnode/impl/inc/mndSma.h index 4a80f619d3..63530b403d 100644 --- a/source/dnode/mnode/impl/inc/mndSma.h +++ b/source/dnode/mnode/impl/inc/mndSma.h @@ -26,6 +26,7 @@ int32_t mndInitSma(SMnode *pMnode); void mndCleanupSma(SMnode *pMnode); SSmaObj *mndAcquireSma(SMnode *pMnode, char *smaName); void mndReleaseSma(SMnode *pMnode, SSmaObj *pSma); +int32_t mndGetTableSma(SMnode *pMnode, char *tbFName, STableIndexRsp *rsp, bool *exist); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndStb.h b/source/dnode/mnode/impl/inc/mndStb.h index 28d3215b98..d3de4e3b3f 100644 --- a/source/dnode/mnode/impl/inc/mndStb.h +++ b/source/dnode/mnode/impl/inc/mndStb.h @@ -27,7 +27,7 @@ void mndCleanupStb(SMnode *pMnode); SStbObj *mndAcquireStb(SMnode *pMnode, char *stbName); void mndReleaseStb(SMnode *pMnode, SStbObj *pStb); SSdbRaw *mndStbActionEncode(SStbObj *pStb); -int32_t mndValidateStbInfo(SMnode *pMnode, SSTableMetaVersion *pStbs, int32_t numOfStbs, void **ppRsp, +int32_t mndValidateStbInfo(SMnode *pMnode, SSTableVersion *pStbs, int32_t numOfStbs, void **ppRsp, int32_t *pRspLen); int32_t mndGetNumOfStbs(SMnode *pMnode, char *dbName, int32_t *pNumOfStbs); diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index b4237466d4..cfe363fdf0 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -1366,7 +1366,7 @@ char *buildRetension(SArray *pRetension) { } static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, int32_t rows, int64_t numOfTables, - bool sysDb) { + bool sysDb, ESdbStatus objStatus) { int32_t cols = 0; int32_t bytes = pShow->pMeta->pSchemas[cols].bytes; @@ -1379,6 +1379,8 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in } char *status = "ready"; + if (objStatus == SDB_STATUS_CREATING) status = "creating"; + if (objStatus == SDB_STATUS_DROPPING) status = "dropping"; char statusB[24] = {0}; STR_WITH_SIZE_TO_VARSTR(statusB, status, strlen(status)); @@ -1503,8 +1505,8 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, rows, (const char *)statusB, false); -// pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); -// colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.schemaless, false); + // pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + // colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.schemaless, false); char *p = buildRetension(pDb->cfg.pRetensions); @@ -1548,29 +1550,30 @@ static bool mndGetTablesOfDbFp(SMnode *pMnode, void *pObj, void *p1, void *p2, v } static int32_t mndRetrieveDbs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rowsCapacity) { - SMnode *pMnode = pReq->info.node; - SSdb *pSdb = pMnode->pSdb; - int32_t numOfRows = 0; - SDbObj *pDb = NULL; + SMnode *pMnode = pReq->info.node; + SSdb *pSdb = pMnode->pSdb; + int32_t numOfRows = 0; + SDbObj *pDb = NULL; + ESdbStatus objStatus = 0; // Append the information_schema database into the result. if (!pShow->sysDbRsp) { SDbObj infoschemaDb = {0}; setInformationSchemaDbCfg(&infoschemaDb); - dumpDbInfoData(pBlock, &infoschemaDb, pShow, numOfRows, 14, true); + dumpDbInfoData(pBlock, &infoschemaDb, pShow, numOfRows, 14, true, 0); numOfRows += 1; SDbObj perfschemaDb = {0}; setPerfSchemaDbCfg(&perfschemaDb); - dumpDbInfoData(pBlock, &perfschemaDb, pShow, numOfRows, 3, true); + dumpDbInfoData(pBlock, &perfschemaDb, pShow, numOfRows, 3, true, 0); numOfRows += 1; pShow->sysDbRsp = true; } while (numOfRows < rowsCapacity) { - pShow->pIter = sdbFetch(pSdb, SDB_DB, pShow->pIter, (void **)&pDb); + pShow->pIter = sdbFetchAll(pSdb, SDB_DB, pShow->pIter, (void **)&pDb, &objStatus); if (pShow->pIter == NULL) { break; } @@ -1578,7 +1581,7 @@ static int32_t mndRetrieveDbs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc int32_t numOfTables = 0; sdbTraverse(pSdb, SDB_VGROUP, mndGetTablesOfDbFp, &numOfTables, NULL, NULL); - dumpDbInfoData(pBlock, pDb, pShow, numOfRows, numOfTables, false); + dumpDbInfoData(pBlock, pDb, pShow, numOfRows, numOfTables, false, objStatus); numOfRows++; sdbRelease(pSdb, pDb); } diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 95a2c3d935..f9749a969d 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -659,7 +659,7 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB int32_t numOfRows = 0; int32_t cols = 0; SMnodeObj *pObj = NULL; - ESdbStatus objStatus; + ESdbStatus objStatus = 0; char *pWrite; int64_t curMs = taosGetTimestampMs(); diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index bacdf2f366..4cfcf389aa 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -433,7 +433,7 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb case HEARTBEAT_KEY_STBINFO: { void *rspMsg = NULL; int32_t rspLen = 0; - mndValidateStbInfo(pMnode, kv->value, kv->valueLen / sizeof(SSTableMetaVersion), &rspMsg, &rspLen); + mndValidateStbInfo(pMnode, kv->value, kv->valueLen / sizeof(SSTableVersion), &rspMsg, &rspLen); if (rspMsg && rspLen > 0) { SKv kv1 = {.key = HEARTBEAT_KEY_STBINFO, .valueLen = rspLen, .value = rspMsg}; taosArrayPush(hbRsp.info, &kv1); diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index c19b558f19..3f0d20348b 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -532,6 +532,7 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea SStreamObj streamObj = {0}; tstrncpy(streamObj.name, pCreate->name, TSDB_STREAM_FNAME_LEN); tstrncpy(streamObj.sourceDb, pDb->name, TSDB_DB_FNAME_LEN); + tstrncpy(streamObj.targetDb, streamObj.sourceDb, TSDB_DB_FNAME_LEN); streamObj.createTime = taosGetTimestampMs(); streamObj.updateTime = streamObj.createTime; streamObj.uid = mndGenerateUid(pCreate->name, strlen(pCreate->name)); @@ -899,18 +900,31 @@ static int32_t mndGetSma(SMnode *pMnode, SUserIndexReq *indexReq, SUserIndexRsp return code; } -static int32_t mndGetTableSma(SMnode *pMnode, STableIndexReq *indexReq, STableIndexRsp *rsp, bool *exist) { +int32_t mndGetTableSma(SMnode *pMnode, char *tbFName, STableIndexRsp *rsp, bool *exist) { int32_t code = 0; SSmaObj *pSma = NULL; SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; STableIndexInfo info; + SStbObj* pStb = mndAcquireStb(pMnode, tbFName); + if (NULL == pStb) { + *exist = false; + return TSDB_CODE_SUCCESS; + } + + strcpy(rsp->dbFName, pStb->db); + strcpy(rsp->tbName, pStb->name + strlen(pStb->db) + 1); + rsp->suid = pStb->uid; + rsp->version = pStb->smaVer; + mndReleaseStb(pMnode, pStb); + + while (1) { pIter = sdbFetch(pSdb, SDB_SMA, pIter, (void **)&pSma); if (pIter == NULL) break; - if (pSma->stb[0] != indexReq->tbFName[0] || strcmp(pSma->stb, indexReq->tbFName)) { + if (pSma->stb[0] != tbFName[0] || strcmp(pSma->stb, tbFName)) { continue; } @@ -1022,7 +1036,7 @@ static int32_t mndProcessGetTbSmaReq(SRpcMsg *pReq) { goto _OVER; } - code = mndGetTableSma(pMnode, &indexReq, &rsp, &exist); + code = mndGetTableSma(pMnode, indexReq.tbFName, &rsp, &exist); if (code) { goto _OVER; } @@ -1114,4 +1128,4 @@ static int32_t mndRetrieveSma(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc static void mndCancelGetNextSma(SMnode *pMnode, void *pIter) { SSdb *pSdb = pMnode->pSdb; sdbCancelFetch(pSdb, pIter); -} \ No newline at end of file +} diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index ad0c913fb8..6f89c97f83 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -27,6 +27,7 @@ #include "mndTrans.h" #include "mndUser.h" #include "mndVgroup.h" +#include "mndSma.h" #include "tname.h" #define STB_VER_NUMBER 1 @@ -1271,7 +1272,7 @@ static int32_t mndBuildStbSchemaImp(SDbObj *pDb, SStbObj *pStb, const char *tbNa return 0; } -static int32_t mndBuildStbSchema(SMnode *pMnode, const char *dbFName, const char *tbName, STableMetaRsp *pRsp) { +static int32_t mndBuildStbSchema(SMnode *pMnode, const char *dbFName, const char *tbName, STableMetaRsp *pRsp, int32_t *smaVer) { char tbFName[TSDB_TABLE_FNAME_LEN] = {0}; snprintf(tbFName, sizeof(tbFName), "%s.%s", dbFName, tbName); @@ -1288,6 +1289,10 @@ static int32_t mndBuildStbSchema(SMnode *pMnode, const char *dbFName, const char return -1; } + if (smaVer) { + *smaVer = pStb->smaVer; + } + int32_t code = mndBuildStbSchemaImp(pDb, pStb, tbName, pRsp); mndReleaseDb(pMnode, pDb); mndReleaseStb(pMnode, pStb); @@ -1634,7 +1639,7 @@ static int32_t mndProcessTableMetaReq(SRpcMsg *pReq) { } } else { mDebug("stb:%s.%s, start to retrieve meta", infoReq.dbFName, infoReq.tbName); - if (mndBuildStbSchema(pMnode, infoReq.dbFName, infoReq.tbName, &metaRsp) != 0) { + if (mndBuildStbSchema(pMnode, infoReq.dbFName, infoReq.tbName, &metaRsp, NULL) != 0) { goto _OVER; } } @@ -1667,51 +1672,86 @@ _OVER: return code; } -int32_t mndValidateStbInfo(SMnode *pMnode, SSTableMetaVersion *pStbVersions, int32_t numOfStbs, void **ppRsp, +int32_t mndValidateStbInfo(SMnode *pMnode, SSTableVersion *pStbVersions, int32_t numOfStbs, void **ppRsp, int32_t *pRspLen) { - STableMetaBatchRsp batchMetaRsp = {0}; - batchMetaRsp.pArray = taosArrayInit(numOfStbs, sizeof(STableMetaRsp)); - if (batchMetaRsp.pArray == NULL) { + SSTbHbRsp hbRsp = {0}; + hbRsp.pMetaRsp = taosArrayInit(numOfStbs, sizeof(STableMetaRsp)); + if (hbRsp.pMetaRsp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } + hbRsp.pIndexRsp = taosArrayInit(numOfStbs, sizeof(STableIndexRsp)); + if (NULL == hbRsp.pIndexRsp) { + taosArrayDestroy(hbRsp.pMetaRsp); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + for (int32_t i = 0; i < numOfStbs; ++i) { - SSTableMetaVersion *pStbVersion = &pStbVersions[i]; + SSTableVersion *pStbVersion = &pStbVersions[i]; pStbVersion->suid = be64toh(pStbVersion->suid); pStbVersion->sversion = ntohs(pStbVersion->sversion); pStbVersion->tversion = ntohs(pStbVersion->tversion); + pStbVersion->smaVer = ntohl(pStbVersion->smaVer); STableMetaRsp metaRsp = {0}; + int32_t smaVer = 0; mDebug("stb:%s.%s, start to retrieve meta", pStbVersion->dbFName, pStbVersion->stbName); - if (mndBuildStbSchema(pMnode, pStbVersion->dbFName, pStbVersion->stbName, &metaRsp) != 0) { + if (mndBuildStbSchema(pMnode, pStbVersion->dbFName, pStbVersion->stbName, &metaRsp, &smaVer) != 0) { metaRsp.numOfColumns = -1; metaRsp.suid = pStbVersion->suid; + taosArrayPush(hbRsp.pMetaRsp, &metaRsp); + continue; } if (pStbVersion->sversion != metaRsp.sversion || pStbVersion->tversion != metaRsp.tversion) { - taosArrayPush(batchMetaRsp.pArray, &metaRsp); + taosArrayPush(hbRsp.pMetaRsp, &metaRsp); } else { tFreeSTableMetaRsp(&metaRsp); } + + if (pStbVersion->smaVer && pStbVersion->smaVer != smaVer) { + bool exist = false; + char tbFName[TSDB_TABLE_FNAME_LEN]; + STableIndexRsp indexRsp = {0}; + indexRsp.pIndex = taosArrayInit(10, sizeof(STableIndexInfo)); + if (NULL == indexRsp.pIndex) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + sprintf(tbFName, "%s.%s", pStbVersion->dbFName, pStbVersion->stbName); + int32_t code = mndGetTableSma(pMnode, tbFName, &indexRsp, &exist); + if (code || !exist) { + indexRsp.suid = pStbVersion->suid; + indexRsp.version = -1; + indexRsp.pIndex = NULL; + } + + strcpy(indexRsp.dbFName, pStbVersion->dbFName); + strcpy(indexRsp.tbName, pStbVersion->stbName); + + taosArrayPush(hbRsp.pIndexRsp, &indexRsp); + } } - int32_t rspLen = tSerializeSTableMetaBatchRsp(NULL, 0, &batchMetaRsp); + int32_t rspLen = tSerializeSSTbHbRsp(NULL, 0, &hbRsp); if (rspLen < 0) { - tFreeSTableMetaBatchRsp(&batchMetaRsp); + tFreeSSTbHbRsp(&hbRsp); terrno = TSDB_CODE_INVALID_MSG; return -1; } void *pRsp = taosMemoryMalloc(rspLen); if (pRsp == NULL) { - tFreeSTableMetaBatchRsp(&batchMetaRsp); + tFreeSSTbHbRsp(&hbRsp); terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - tSerializeSTableMetaBatchRsp(pRsp, rspLen, &batchMetaRsp); - tFreeSTableMetaBatchRsp(&batchMetaRsp); + tSerializeSSTbHbRsp(pRsp, rspLen, &hbRsp); + tFreeSSTbHbRsp(&hbRsp); *ppRsp = pRsp; *pRspLen = rspLen; return 0; diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 1d8082084f..a6339125c4 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -663,12 +663,23 @@ int32_t metaFilteTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { void * entryKey = NULL, *entryVal = NULL; int32_t nEntryKey, nEntryVal; + bool first = true; while (1) { valid = tdbTbcGet(pCursor->pCur, (const void **)&entryKey, &nEntryKey, (const void **)&entryVal, &nEntryVal); if (valid < 0) { break; } STagIdxKey *p = entryKey; + if (p->type != pCursor->type) { + if (first) { + valid = param->reverse ? tdbTbcMoveToPrev(pCursor->pCur) : tdbTbcMoveToNext(pCursor->pCur); + if (valid < 0) break; + continue; + } else { + break; + } + } + first = false; if (p != NULL) { int32_t cmp = (*param->filterFunc)(p->data, pKey->data, pKey->type); if (cmp == 0) { diff --git a/source/dnode/vnode/src/sma/smaTimeRange.c b/source/dnode/vnode/src/sma/smaTimeRange.c new file mode 100644 index 0000000000..bca5b1543e --- /dev/null +++ b/source/dnode/vnode/src/sma/smaTimeRange.c @@ -0,0 +1,178 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "sma.h" +#include "tsdb.h" + +#define SMA_STORAGE_MINUTES_MAX 86400 +#define SMA_STORAGE_MINUTES_DAY 1440 +#define SMA_STORAGE_SPLIT_FACTOR 14400 // least records in tsma file + +/** + * @brief Judge the tsma file split days + * + * @param pCfg + * @param pCont + * @param contLen + * @param days unit is minute + * @return int32_t + */ +int32_t tdProcessTSmaGetDaysImpl(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days) { + SDecoder coder = {0}; + tDecoderInit(&coder, pCont, contLen); + + STSma tsma = {0}; + if (tDecodeSVCreateTSmaReq(&coder, &tsma) < 0) { + terrno = TSDB_CODE_MSG_DECODE_ERROR; + goto _err; + } + STsdbCfg *pTsdbCfg = &pCfg->tsdbCfg; + int64_t sInterval = convertTimeFromPrecisionToUnit(tsma.interval, pTsdbCfg->precision, TIME_UNIT_SECOND); + if (sInterval <= 0) { + *days = pTsdbCfg->days; + return 0; + } + int64_t records = pTsdbCfg->days * 60 / sInterval; + if (records >= SMA_STORAGE_SPLIT_FACTOR) { + *days = pTsdbCfg->days; + } else { + int64_t mInterval = convertTimeFromPrecisionToUnit(tsma.interval, pTsdbCfg->precision, TIME_UNIT_MINUTE); + int64_t daysPerFile = mInterval * SMA_STORAGE_MINUTES_DAY * 2; + + if (daysPerFile > SMA_STORAGE_MINUTES_MAX) { + *days = SMA_STORAGE_MINUTES_MAX; + } else { + *days = (int32_t)daysPerFile; + } + + if (*days < pTsdbCfg->days) { + *days = pTsdbCfg->days; + } + } + tDecoderClear(&coder); + return 0; +_err: + tDecoderClear(&coder); + return -1; +} + +/** + * @brief create tsma meta and result stable + * + * @param pSma + * @param version + * @param pMsg + * @return int32_t + */ +int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *pMsg) { + SSmaCfg *pCfg = (SSmaCfg *)pMsg; + + if (TD_VID(pSma->pVnode) == pCfg->dstVgId) { + // create tsma meta in dstVgId + if (metaCreateTSma(SMA_META(pSma), version, pCfg) < 0) { + return -1; + } + + // create stable to save tsma result in dstVgId + SVCreateStbReq pReq = {0}; + pReq.name = pCfg->dstTbName; + pReq.suid = pCfg->dstTbUid; + pReq.schemaRow = pCfg->schemaRow; + pReq.schemaTag = pCfg->schemaTag; + + if (metaCreateSTable(SMA_META(pSma), version, &pReq) < 0) { + return -1; + } + } + + return 0; +} + +/** + * @brief Insert/Update Time-range-wise SMA data. + * + * @param pSma + * @param msg + * @return int32_t + */ +int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) { + const SArray *pDataBlocks = (const SArray *)msg; + // TODO: destroy SSDataBlocks(msg) + if (!pDataBlocks) { + terrno = TSDB_CODE_TSMA_INVALID_PTR; + smaWarn("vgId:%d, insert tsma data failed since pDataBlocks is NULL", SMA_VID(pSma)); + return terrno; + } + + if (taosArrayGetSize(pDataBlocks) <= 0) { + terrno = TSDB_CODE_TSMA_INVALID_PARA; + smaWarn("vgId:%d, insert tsma data failed since pDataBlocks is empty", SMA_VID(pSma)); + return TSDB_CODE_FAILED; + } + + if (tdCheckAndInitSmaEnv(pSma, TSDB_SMA_TYPE_TIME_RANGE) != 0) { + terrno = TSDB_CODE_TSMA_INIT_FAILED; + return TSDB_CODE_FAILED; + } + + SSmaEnv *pEnv = SMA_TSMA_ENV(pSma); + SSmaStat *pStat = NULL; + SSmaStatItem *pItem = NULL; + + if (!pEnv || !(pStat = SMA_ENV_STAT(pEnv))) { + terrno = TSDB_CODE_TSMA_INVALID_STAT; + return TSDB_CODE_FAILED; + } + + tdRefSmaStat(pSma, pStat); + pItem = &pStat->tsmaStatItem; + + ASSERT(pItem); + + if (!pItem->pTSma) { + // cache smaMeta + STSma *pTSma = metaGetSmaInfoByIndex(SMA_META(pSma), indexUid); + if (!pTSma) { + terrno = TSDB_CODE_TSMA_NO_INDEX_IN_META; + smaWarn("vgId:%d, tsma insert for smaIndex %" PRIi64 " since %s", SMA_VID(pSma), indexUid, tstrerror(terrno)); + return TSDB_CODE_FAILED; + } + pItem->pTSma = pTSma; + } + + STSma *pTSma = pItem->pTSma; + + ASSERT(pTSma->indexUid == indexUid); + + SMetaReader mr = {0}; + + const char *dbName = "testDb"; + if (metaGetTableEntryByName(&mr, dbName) != 0) { + smaDebug("vgId:%d, tsma no table testTb exists for smaIndex %" PRIi64 " since %s", SMA_VID(pSma), indexUid, tstrerror(terrno)); + SVCreateStbReq pReq = {0}; + pReq.name = dbName; + pReq.suid = pTSma->dstTbUid; + pReq.schemaRow = pCfg->schemaRow; + pReq.schemaTag = pCfg->schemaTag; + } + + SSubmitReq *pSubmitReq = NULL; + buildSubmitReqFromDataBlock(&pSubmitReq, (const SArray *)msg, NULL, pItem->pTSma->dstVgId, + pItem->pTSma->dstTbUid); + + tdUnRefSmaStat(pSma, pStat); + + return TSDB_CODE_SUCCESS; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index bc126650c3..268d87183e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -78,6 +78,8 @@ _err: } int32_t tsdbCommit(STsdb *pTsdb) { + if (!pTsdb) return 0; + int32_t code = 0; SCommitter commith; SMemTable *pMemTable = pTsdb->mem; diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 4d12155609..f0402c4b53 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -55,6 +55,7 @@ extern const char *TSDB_LEVEL_DNAME[]; typedef enum { TSDB_TXN_TEMP_FILE = 0, TSDB_TXN_CURR_FILE } TSDB_TXN_FILE_T; static const char *tsdbTxnFname[] = {"current.t", "current"}; #define TSDB_MAX_FSETS(keep, days) ((keep) / (days) + 3) +#define TSDB_MAX_INIT_FSETS (365000) static int tsdbComparFidFSet(const void *arg1, const void *arg2); static void tsdbResetFSStatus(SFSStatus *pStatus); @@ -245,6 +246,10 @@ STsdbFS *tsdbNewFS(const STsdbKeepCfg *pCfg) { return NULL; } + if (maxFSet > TSDB_MAX_INIT_FSETS) { + maxFSet = TSDB_MAX_INIT_FSETS; + } + pfs->cstatus = tsdbNewFSStatus(maxFSet); if (pfs->cstatus == NULL) { tsdbFreeFS(pfs); diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 1d56b6d5d4..df4b9b6ea6 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -57,6 +57,8 @@ enum { CTG_OP_DROP_TB_META, CTG_OP_UPDATE_USER, CTG_OP_UPDATE_VG_EPSET, + CTG_OP_UPDATE_TB_INDEX, + CTG_OP_DROP_TB_INDEX, CTG_OP_MAX }; @@ -128,25 +130,33 @@ typedef struct SCtgUserCtx { SUserAuthInfo user; } SCtgUserCtx; -typedef struct SCtgTbMetaCache { - SRWLatch stbLock; - SRWLatch metaLock; // RC between cache destroy and all other operations - SHashObj *metaCache; //key:tbname, value:STableMeta - SHashObj *stbCache; //key:suid, value:STableMeta* -} SCtgTbMetaCache; +typedef STableIndexRsp STableIndex; + +typedef struct SCtgTbCache { + SRWLatch metaLock; + STableMeta *pMeta; + SRWLatch indexLock; + STableIndex *pIndex; +} SCtgTbCache; + +typedef struct SCtgVgCache { + SRWLatch vgLock; + SDBVgInfo *vgInfo; +} SCtgVgCache; typedef struct SCtgDBCache { - SRWLatch vgLock; + SRWLatch dbLock; // RC between destroy tbCache/stbCache and all reads uint64_t dbId; int8_t deleted; - SDBVgInfo *vgInfo; - SCtgTbMetaCache tbCache; + SCtgVgCache vgCache; + SHashObj *tbCache; // key:tbname, value:SCtgTbCache + SHashObj *stbCache; // key:suid, value:STableMeta* } SCtgDBCache; typedef struct SCtgRentSlot { SRWLatch lock; bool needSort; - SArray *meta; // element is SDbVgVersion or SSTableMetaVersion + SArray *meta; // element is SDbVgVersion or SSTableVersion } SCtgRentSlot; typedef struct SCtgRentMgmt { @@ -245,8 +255,10 @@ typedef struct SCtgCacheStat { uint64_t userNum; uint64_t vgHitNum; uint64_t vgMissNum; - uint64_t tblHitNum; - uint64_t tblMissNum; + uint64_t tbMetaHitNum; + uint64_t tbMetaMissNum; + uint64_t tbIndexHitNum; + uint64_t tbIndexMissNum; uint64_t userHitNum; uint64_t userMissNum; } SCtgCacheStat; @@ -268,10 +280,10 @@ typedef struct SCtgUpdateVgMsg { SDBVgInfo* dbInfo; } SCtgUpdateVgMsg; -typedef struct SCtgUpdateTblMsg { - SCatalog* pCtg; - STableMetaOutput* output; -} SCtgUpdateTblMsg; +typedef struct SCtgUpdateTbMetaMsg { + SCatalog* pCtg; + STableMetaOutput* pMeta; +} SCtgUpdateTbMetaMsg; typedef struct SCtgDropDBMsg { SCatalog* pCtg; @@ -305,6 +317,17 @@ typedef struct SCtgUpdateUserMsg { SGetUserAuthRsp userAuth; } SCtgUpdateUserMsg; +typedef struct SCtgUpdateTbIndexMsg { + SCatalog* pCtg; + STableIndex* pIndex; +} SCtgUpdateTbIndexMsg; + +typedef struct SCtgDropTbIndexMsg { + SCatalog* pCtg; + char dbFName[TSDB_DB_FNAME_LEN]; + char tbName[TSDB_TABLE_NAME_LEN]; +} SCtgDropTbIndexMsg; + typedef struct SCtgUpdateEpsetMsg { SCatalog* pCtg; char dbFName[TSDB_DB_FNAME_LEN]; @@ -465,12 +488,11 @@ int32_t ctgOpUpdateUser(SCtgCacheOperation *action); int32_t ctgOpUpdateEpset(SCtgCacheOperation *operation); int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache); void ctgReleaseDBCache(SCatalog *pCtg, SCtgDBCache *dbCache); -void ctgReleaseVgInfo(SCtgDBCache *dbCache); -int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache); +void ctgRUnlockVgInfo(SCtgDBCache *dbCache); int32_t ctgTbMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName, int32_t *exist); int32_t ctgReadTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta); -int32_t ctgReadTbVerFromCache(SCatalog *pCtg, const SName *pTableName, int32_t *sver, int32_t *tver, int32_t *tbType, uint64_t *suid, char *stbName); -int32_t ctgChkAuthFromCache(SCatalog* pCtg, const char* user, const char* dbFName, AUTH_TYPE type, bool *inCache, bool *pass); +int32_t ctgReadTbVerFromCache(SCatalog *pCtg, SName *pTableName, int32_t *sver, int32_t *tver, int32_t *tbType, uint64_t *suid, char *stbName); +int32_t ctgChkAuthFromCache(SCatalog* pCtg, char* user, char* dbFName, AUTH_TYPE type, bool *inCache, bool *pass); int32_t ctgDropDbCacheEnqueue(SCatalog* pCtg, const char *dbFName, int64_t dbId); int32_t ctgDropDbVgroupEnqueue(SCatalog* pCtg, const char *dbFName, bool syncReq); int32_t ctgDropStbMetaEnqueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, const char *stbName, uint64_t suid, bool syncReq); @@ -479,12 +501,18 @@ int32_t ctgUpdateVgroupEnqueue(SCatalog* pCtg, const char *dbFName, int64_t dbId int32_t ctgUpdateTbMetaEnqueue(SCatalog* pCtg, STableMetaOutput *output, bool syncReq); int32_t ctgUpdateUserEnqueue(SCatalog* pCtg, SGetUserAuthRsp *pAuth, bool syncReq); int32_t ctgUpdateVgEpsetEnqueue(SCatalog* pCtg, char *dbFName, int32_t vgId, SEpSet* pEpSet); +int32_t ctgUpdateTbIndexEnqueue(SCatalog* pCtg, STableIndex **pIndex, bool syncOp); int32_t ctgMetaRentInit(SCtgRentMgmt *mgmt, uint32_t rentSec, int8_t type); int32_t ctgMetaRentAdd(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t size); int32_t ctgMetaRentGet(SCtgRentMgmt *mgmt, void **res, uint32_t *num, int32_t size); int32_t ctgUpdateTbMetaToCache(SCatalog* pCtg, STableMetaOutput* pOut, bool syncReq); int32_t ctgStartUpdateThread(); int32_t ctgRelaunchGetTbMetaTask(SCtgTask *pTask); +void ctgReleaseVgInfoToCache(SCatalog* pCtg, SCtgDBCache *dbCache); +int32_t ctgReadTbIndexFromCache(SCatalog* pCtg, SName* pTableName, SArray** pRes); +int32_t ctgDropTbIndexEnqueue(SCatalog* pCtg, SName* pName, bool syncOp); +int32_t ctgOpDropTbIndex(SCtgCacheOperation *operation); +int32_t ctgOpUpdateTbIndex(SCtgCacheOperation *operation); @@ -493,7 +521,7 @@ int32_t ctgGetDBVgInfoFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SBuildU int32_t ctgGetQnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SArray *out, SCtgTask* pTask); int32_t ctgGetDBCfgFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const char *dbFName, SDbCfgInfo *out, SCtgTask* pTask); int32_t ctgGetIndexInfoFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const char *indexName, SIndexInfo *out, SCtgTask* pTask); -int32_t ctgGetTbIndexFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SName* name, SArray** out, SCtgTask* pTask); +int32_t ctgGetTbIndexFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SName *name, STableIndex* out, SCtgTask* pTask); int32_t ctgGetUdfInfoFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const char *funcName, SFuncInfo *out, SCtgTask* pTask); int32_t ctgGetUserDbAuthFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const char *user, SGetUserAuthRsp *out, SCtgTask* pTask); int32_t ctgGetTbMetaFromMnodeImpl(SCatalog* pCtg, SRequestConnInfo *pConn, char *dbFName, char* tbName, STableMetaOutput* out, SCtgTask* pTask); @@ -521,6 +549,8 @@ void ctgFreeSTableMetaOutput(STableMetaOutput* pOutput); int32_t ctgUpdateMsgCtx(SCtgMsgCtx* pCtx, int32_t reqType, void* out, char* target); char *ctgTaskTypeStr(CTG_TASK_TYPE type); int32_t ctgUpdateSendTargetInfo(SMsgSendInfo *pMsgSendInfo, int32_t msgType, SCtgTask* pTask); +int32_t ctgCloneTableIndex(SArray* pIndex, SArray** pRes); +void ctgFreeSTableIndex(void *info); extern SCatalogMgmt gCtgMgmt; diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index bc94de89ea..a3576f8738 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -96,8 +96,7 @@ int32_t ctgRefreshDBVgInfo(SCatalog* pCtg, SRequestConnInfo *pConn, const char* if (NULL != dbCache) { input.dbId = dbCache->dbId; - ctgReleaseVgInfo(dbCache); - ctgReleaseDBCache(pCtg, dbCache); + ctgReleaseVgInfoToCache(pCtg, dbCache); } input.vgVersion = CTG_DEFAULT_INVALID_VERSION; @@ -349,8 +348,8 @@ int32_t ctgChkAuth(SCatalog* pCtg, SRequestConnInfo *pConn, const char* user, co int32_t code = 0; *pass = false; - - CTG_ERR_RET(ctgChkAuthFromCache(pCtg, user, dbFName, type, &inCache, pass)); + + CTG_ERR_RET(ctgChkAuthFromCache(pCtg, (char*)user, (char*)dbFName, type, &inCache, pass)); if (inCache) { return TSDB_CODE_SUCCESS; @@ -382,6 +381,45 @@ _return: return TSDB_CODE_SUCCESS; } +int32_t ctgGetTbIndex(SCatalog* pCtg, SRequestConnInfo *pConn, SName* pTableName, SArray** pRes) { + CTG_ERR_RET(ctgReadTbIndexFromCache(pCtg, pTableName, pRes)); + if (*pRes) { + return TSDB_CODE_SUCCESS; + } + + STableIndex *pIndex = taosMemoryCalloc(1, sizeof(STableIndex)); + if (NULL == pIndex) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + int32_t code = ctgGetTbIndexFromMnode(pCtg, pConn, (SName*)pTableName, pIndex, NULL); + if (TSDB_CODE_MND_DB_INDEX_NOT_EXIST == code) { + code = 0; + goto _return; + } + CTG_ERR_JRET(code); + + SArray* pInfo = NULL; + CTG_ERR_JRET(ctgCloneTableIndex(pIndex->pIndex, &pInfo)); + + *pRes = pInfo; + + CTG_ERR_JRET(ctgUpdateTbIndexEnqueue(pCtg, &pIndex, false)); + + return TSDB_CODE_SUCCESS; + +_return: + + tFreeSTableIndexRsp(pIndex); + taosMemoryFree(pIndex); + + taosArrayDestroyEx(*pRes, tFreeSTableIndexInfo); + *pRes = NULL; + + CTG_RET(code); +} + + int32_t ctgGetTbDistVgInfo(SCatalog* pCtg, SRequestConnInfo *pConn, SName* pTableName, SArray** pVgList) { STableMeta *tbMeta = NULL; int32_t code = 0; @@ -404,7 +442,7 @@ int32_t ctgGetTbDistVgInfo(SCatalog* pCtg, SRequestConnInfo *pConn, SName* pTabl CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pConn, db, &dbCache, &vgInfo)); if (dbCache) { - vgHash = dbCache->vgInfo->vgHash; + vgHash = dbCache->vgCache.vgInfo->vgHash; } else { vgHash = vgInfo->vgHash; } @@ -442,7 +480,7 @@ int32_t ctgGetTbDistVgInfo(SCatalog* pCtg, SRequestConnInfo *pConn, SName* pTabl _return: if (dbCache) { - ctgReleaseVgInfo(dbCache); + ctgRUnlockVgInfo(dbCache); ctgReleaseDBCache(pCtg, dbCache); } @@ -631,12 +669,11 @@ int32_t catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* vers CTG_API_LEAVE(TSDB_CODE_SUCCESS); } - *version = dbCache->vgInfo->vgVersion; + *version = dbCache->vgCache.vgInfo->vgVersion; *dbId = dbCache->dbId; - *tableNum = dbCache->vgInfo->numOfTable; + *tableNum = dbCache->vgCache.vgInfo->numOfTable; - ctgReleaseVgInfo(dbCache); - ctgReleaseDBCache(pCtg, dbCache); + ctgReleaseVgInfoToCache(pCtg, dbCache); ctgDebug("Got db vgVersion from cache, dbFName:%s, vgVersion:%d", dbFName, *version); @@ -661,7 +698,7 @@ int32_t catalogGetDBVgInfo(SCatalog* pCtg, SRequestConnInfo *pConn, const char* SDBVgInfo *vgInfo = NULL; CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pConn, dbFName, &dbCache, &vgInfo)); if (dbCache) { - vgHash = dbCache->vgInfo->vgHash; + vgHash = dbCache->vgCache.vgInfo->vgHash; } else { vgHash = vgInfo->vgHash; } @@ -674,7 +711,7 @@ int32_t catalogGetDBVgInfo(SCatalog* pCtg, SRequestConnInfo *pConn, const char* _return: if (dbCache) { - ctgReleaseVgInfo(dbCache); + ctgRUnlockVgInfo(dbCache); ctgReleaseDBCache(pCtg, dbCache); } @@ -741,6 +778,30 @@ _return: CTG_API_LEAVE(code); } +int32_t catalogUpdateTableIndex(SCatalog* pCtg, STableIndexRsp *pRsp) { + CTG_API_ENTER(); + + int32_t code = 0; + + if (NULL == pCtg || NULL == pRsp) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + + STableIndex* pIndex = taosMemoryCalloc(1, sizeof(STableIndex)); + if (NULL == pIndex) { + CTG_API_LEAVE(TSDB_CODE_OUT_OF_MEMORY); + } + + memcpy(pIndex, pRsp, sizeof(STableIndex)); + + CTG_ERR_JRET(ctgUpdateTbIndexEnqueue(pCtg, &pIndex, false)); + +_return: + + CTG_API_LEAVE(code); +} + + int32_t catalogRemoveTableMeta(SCatalog* pCtg, SName* pTableName) { CTG_API_ENTER(); @@ -932,12 +993,12 @@ int32_t catalogGetTableHashVgroup(SCatalog *pCtg, SRequestConnInfo *pConn, const SDBVgInfo *vgInfo = NULL; CTG_ERR_JRET(ctgGetDBVgInfo(pCtg, pConn, db, &dbCache, &vgInfo)); - CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, vgInfo ? vgInfo : dbCache->vgInfo, pTableName, pVgroup)); + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, vgInfo ? vgInfo : dbCache->vgCache.vgInfo, pTableName, pVgroup)); _return: if (dbCache) { - ctgReleaseVgInfo(dbCache); + ctgRUnlockVgInfo(dbCache); ctgReleaseDBCache(pCtg, dbCache); } @@ -1060,14 +1121,14 @@ _return: CTG_API_LEAVE(TSDB_CODE_SUCCESS); } -int32_t catalogGetExpiredSTables(SCatalog* pCtg, SSTableMetaVersion** stables, uint32_t* num) { +int32_t catalogGetExpiredSTables(SCatalog* pCtg, SSTableVersion **stables, uint32_t *num) { CTG_API_ENTER(); if (NULL == pCtg || NULL == stables || NULL == num) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - CTG_API_LEAVE(ctgMetaRentGet(&pCtg->stbRent, (void**)stables, num, sizeof(SSTableMetaVersion))); + CTG_API_LEAVE(ctgMetaRentGet(&pCtg->stbRent, (void **)stables, num, sizeof(SSTableVersion))); } int32_t catalogGetExpiredDBs(SCatalog* pCtg, SDbVgVersion** dbs, uint32_t* num) { @@ -1138,7 +1199,12 @@ int32_t catalogGetTableIndex(SCatalog* pCtg, SRequestConnInfo *pConn, const SNam CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - CTG_API_LEAVE(ctgGetTbIndexFromMnode(pCtg, pConn, (SName*)pTableName, pRes, NULL)); + int32_t code = 0; + CTG_ERR_JRET(ctgGetTbIndex(pCtg, pConn, (SName*)pTableName, pRes)); + +_return: + + CTG_API_LEAVE(code); } int32_t catalogGetUdfInfo(SCatalog* pCtg, SRequestConnInfo *pConn, const char* funcName, SFuncInfo* pInfo) { diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index aff04fff06..159c120dde 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -344,6 +344,11 @@ int32_t ctgHandleForceUpdate(SCatalog* pCtg, SCtgJob *pJob, const SCatalogReq* p } } + for (int32_t i = 0; i < pJob->tbIndexNum; ++i) { + SName* name = taosArrayGet(pReq->pTableIndex, i); + ctgDropTbIndexEnqueue(pCtg, name, true); + } + return TSDB_CODE_SUCCESS; } @@ -680,15 +685,14 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf * CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache)); if (NULL != dbCache) { SVgroupInfo vgInfo = {0}; - CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgInfo, ctx->pName, &vgInfo)); + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgCache.vgInfo, ctx->pName, &vgInfo)); ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(ctx->pName), ctx->flag); ctx->vgId = vgInfo.vgId; CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, ctx->pName, &vgInfo, NULL, pTask)); - ctgReleaseVgInfo(dbCache); - ctgReleaseDBCache(pCtg, dbCache); + ctgReleaseVgInfoToCache(pCtg, dbCache); } else { SBuildUseDBInput input = {0}; @@ -786,7 +790,7 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf * _return: if (dbCache) { - ctgReleaseVgInfo(dbCache); + ctgRUnlockVgInfo(dbCache); ctgReleaseDBCache(pCtg, dbCache); } @@ -866,9 +870,15 @@ _return: int32_t ctgHandleGetTbIndexRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; - CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); + CTG_ERR_JRET(ctgProcessRspMsg(&pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); - TSWAP(pTask->res, pTask->msgCtx.out); + STableIndex* pOut = (STableIndex*)pTask->msgCtx.out; + SArray* pInfo = NULL; + CTG_ERR_JRET(ctgCloneTableIndex(pOut->pIndex, &pInfo)); + pTask->res = pInfo; + + SCtgTbIndexCtx* ctx = pTask->taskCtx; + CTG_ERR_JRET(ctgUpdateTbIndexEnqueue(pTask->pJob->pCtg, (STableIndex**)&pTask->msgCtx.out, false)); _return: if (TSDB_CODE_MND_DB_INDEX_NOT_EXIST == code) { @@ -1008,7 +1018,7 @@ int32_t ctgAsyncRefreshTbMeta(SCtgTask *pTask) { CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache)); if (dbCache) { SVgroupInfo vgInfo = {0}; - CTG_ERR_RET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgInfo, ctx->pName, &vgInfo)); + CTG_ERR_RET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgCache.vgInfo, ctx->pName, &vgInfo)); ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(ctx->pName), ctx->flag); @@ -1026,8 +1036,7 @@ int32_t ctgAsyncRefreshTbMeta(SCtgTask *pTask) { _return: if (dbCache) { - ctgReleaseVgInfo(dbCache); - ctgReleaseDBCache(pCtg, dbCache); + ctgReleaseVgInfoToCache(pCtg, dbCache); } CTG_RET(code); @@ -1057,7 +1066,7 @@ int32_t ctgLaunchGetDbVgTask(SCtgTask *pTask) { CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, pCtx->dbFName, &dbCache)); if (NULL != dbCache) { - CTG_ERR_JRET(ctgGenerateVgList(pCtg, dbCache->vgInfo->vgHash, (SArray**)&pTask->res)); + CTG_ERR_JRET(ctgGenerateVgList(pCtg, dbCache->vgCache.vgInfo->vgHash, (SArray**)&pTask->res)); CTG_ERR_JRET(ctgHandleTaskEnd(pTask, 0)); } else { @@ -1072,8 +1081,7 @@ int32_t ctgLaunchGetDbVgTask(SCtgTask *pTask) { _return: if (dbCache) { - ctgReleaseVgInfo(dbCache); - ctgReleaseDBCache(pCtg, dbCache); + ctgReleaseVgInfoToCache(pCtg, dbCache); } CTG_RET(code); @@ -1092,7 +1100,7 @@ int32_t ctgLaunchGetTbHashTask(SCtgTask *pTask) { if (NULL == pTask->res) { CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } - CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgInfo, pCtx->pName, (SVgroupInfo*)pTask->res)); + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgCache.vgInfo, pCtx->pName, (SVgroupInfo*)pTask->res)); CTG_ERR_JRET(ctgHandleTaskEnd(pTask, 0)); } else { @@ -1107,8 +1115,7 @@ int32_t ctgLaunchGetTbHashTask(SCtgTask *pTask) { _return: if (dbCache) { - ctgReleaseVgInfo(dbCache); - ctgReleaseDBCache(pCtg, dbCache); + ctgReleaseVgInfoToCache(pCtg, dbCache); } CTG_RET(code); @@ -1119,6 +1126,15 @@ int32_t ctgLaunchGetTbIndexTask(SCtgTask *pTask) { SCatalog* pCtg = pTask->pJob->pCtg; SRequestConnInfo* pConn = &pTask->pJob->conn; SCtgTbIndexCtx* pCtx = (SCtgTbIndexCtx*)pTask->taskCtx; + SArray* pRes = NULL; + + CTG_ERR_RET(ctgReadTbIndexFromCache(pCtg, pCtx->pName, &pRes)); + if (pRes) { + pTask->res = pRes; + + CTG_ERR_RET(ctgHandleTaskEnd(pTask, 0)); + return TSDB_CODE_SUCCESS; + } CTG_ERR_RET(ctgGetTbIndexFromMnode(pCtg, pConn, pCtx->pName, NULL, pTask)); return TSDB_CODE_SUCCESS; @@ -1157,9 +1173,9 @@ int32_t ctgLaunchGetDbInfoTask(SCtgTask *pTask) { SDbInfo* pInfo = (SDbInfo*)pTask->res; CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, pCtx->dbFName, &dbCache)); if (NULL != dbCache) { - pInfo->vgVer = dbCache->vgInfo->vgVersion; + pInfo->vgVer = dbCache->vgCache.vgInfo->vgVersion; pInfo->dbId = dbCache->dbId; - pInfo->tbNum = dbCache->vgInfo->numOfTable; + pInfo->tbNum = dbCache->vgCache.vgInfo->numOfTable; } else { pInfo->vgVer = CTG_DEFAULT_INVALID_VERSION; } @@ -1169,8 +1185,7 @@ int32_t ctgLaunchGetDbInfoTask(SCtgTask *pTask) { _return: if (dbCache) { - ctgReleaseVgInfo(dbCache); - ctgReleaseDBCache(pCtg, dbCache); + ctgReleaseVgInfoToCache(pCtg, dbCache); } CTG_RET(code); diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index add44d6a16..62890b8326 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -59,18 +59,27 @@ SCtgOperation gCtgCacheOperation[CTG_OP_MAX] = { CTG_OP_UPDATE_VG_EPSET, "update epset", ctgOpUpdateEpset - } - + }, + { + CTG_OP_UPDATE_TB_INDEX, + "update tbIndex", + ctgOpUpdateTbIndex + }, + { + CTG_OP_DROP_TB_INDEX, + "drop tbIndex", + ctgOpDropTbIndex + } }; -int32_t ctgAcquireVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache, bool *inCache) { - CTG_LOCK(CTG_READ, &dbCache->vgLock); +int32_t ctgRLockVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache, bool *inCache) { + CTG_LOCK(CTG_READ, &dbCache->vgCache.vgLock); if (dbCache->deleted) { - CTG_UNLOCK(CTG_READ, &dbCache->vgLock); + CTG_UNLOCK(CTG_READ, &dbCache->vgCache.vgLock); ctgDebug("db is dropping, dbId:%"PRIx64, dbCache->dbId); @@ -79,8 +88,8 @@ int32_t ctgAcquireVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache, bool *inCache) { } - if (NULL == dbCache->vgInfo) { - CTG_UNLOCK(CTG_READ, &dbCache->vgLock); + if (NULL == dbCache->vgCache.vgInfo) { + CTG_UNLOCK(CTG_READ, &dbCache->vgCache.vgLock); *inCache = false; ctgDebug("db vgInfo is empty, dbId:%"PRIx64, dbCache->dbId); @@ -92,50 +101,47 @@ int32_t ctgAcquireVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache, bool *inCache) { return TSDB_CODE_SUCCESS; } -int32_t ctgWAcquireVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache) { - CTG_LOCK(CTG_WRITE, &dbCache->vgLock); +int32_t ctgWLockVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache) { + CTG_LOCK(CTG_WRITE, &dbCache->vgCache.vgLock); if (dbCache->deleted) { ctgDebug("db is dropping, dbId:%"PRIx64, dbCache->dbId); - CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); + CTG_UNLOCK(CTG_WRITE, &dbCache->vgCache.vgLock); CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); } return TSDB_CODE_SUCCESS; } +void ctgRUnlockVgInfo(SCtgDBCache *dbCache) { + CTG_UNLOCK(CTG_READ, &dbCache->vgCache.vgLock); +} + +void ctgWUnlockVgInfo(SCtgDBCache *dbCache) { + CTG_UNLOCK(CTG_WRITE, &dbCache->vgCache.vgLock); +} + void ctgReleaseDBCache(SCatalog *pCtg, SCtgDBCache *dbCache) { - taosHashRelease(pCtg->dbCache, dbCache); + CTG_UNLOCK(CTG_READ, &dbCache->dbLock); } -void ctgReleaseVgInfo(SCtgDBCache *dbCache) { - CTG_UNLOCK(CTG_READ, &dbCache->vgLock); -} - -void ctgWReleaseVgInfo(SCtgDBCache *dbCache) { - CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); -} - - int32_t ctgAcquireDBCacheImpl(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache, bool acquire) { char *p = strchr(dbFName, '.'); if (p && CTG_IS_SYS_DBNAME(p + 1)) { dbFName = p + 1; } - SCtgDBCache *dbCache = NULL; - if (acquire) { - dbCache = (SCtgDBCache *)taosHashAcquire(pCtg->dbCache, dbFName, strlen(dbFName)); - } else { - dbCache = (SCtgDBCache *)taosHashGet(pCtg->dbCache, dbFName, strlen(dbFName)); - } - + SCtgDBCache *dbCache = (SCtgDBCache *)taosHashGet(pCtg->dbCache, dbFName, strlen(dbFName)); if (NULL == dbCache) { *pCache = NULL; ctgDebug("db not in cache, dbFName:%s", dbFName); return TSDB_CODE_SUCCESS; } + if (acquire) { + CTG_LOCK(CTG_READ, &dbCache->dbLock); + } + if (dbCache->deleted) { if (acquire) { ctgReleaseDBCache(pCtg, dbCache); @@ -159,15 +165,35 @@ int32_t ctgGetDBCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache) CTG_RET(ctgAcquireDBCacheImpl(pCtg, dbFName, pCache, false)); } +void ctgReleaseVgInfoToCache(SCatalog* pCtg, SCtgDBCache *dbCache) { + ctgRUnlockVgInfo(dbCache); + ctgReleaseDBCache(pCtg, dbCache); +} + +void ctgReleaseTbMetaToCache(SCatalog* pCtg, SCtgDBCache *dbCache, SCtgTbCache* pCache) { + if (pCache) { + CTG_UNLOCK(CTG_READ, &pCache->metaLock); + taosHashRelease(dbCache->tbCache, pCache); + } + + if (dbCache) { + ctgReleaseDBCache(pCtg, dbCache); + } +} + +void ctgReleaseTbIndexToCache(SCatalog* pCtg, SCtgDBCache *dbCache, SCtgTbCache* pCache) { + if (pCache) { + CTG_UNLOCK(CTG_READ, &pCache->indexLock); + taosHashRelease(dbCache->tbCache, pCache); + } + + if (dbCache) { + ctgReleaseDBCache(pCtg, dbCache); + } +} int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache) { SCtgDBCache *dbCache = NULL; - - if (NULL == pCtg->dbCache) { - ctgDebug("empty db cache, dbFName:%s", dbFName); - goto _return; - } - ctgAcquireDBCache(pCtg, dbFName, &dbCache); if (NULL == dbCache) { ctgDebug("db %s not in cache", dbFName); @@ -175,7 +201,7 @@ int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char *dbFName, SCtgDBCac } bool inCache = false; - ctgAcquireVgInfo(pCtg, dbCache, &inCache); + ctgRLockVgInfo(pCtg, dbCache, &inCache); if (!inCache) { ctgDebug("vgInfo of db %s not in cache", dbFName); goto _return; @@ -202,54 +228,160 @@ _return: return TSDB_CODE_SUCCESS; } -int32_t ctgTbMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName, int32_t *exist) { - if (NULL == pCtg->dbCache) { - *exist = 0; - ctgWarn("empty db cache, dbFName:%s, tbName:%s", dbFName, tbName); - return TSDB_CODE_SUCCESS; - } - +int32_t ctgAcquireTbMetaFromCache(SCatalog* pCtg, char *dbFName, char* tbName, SCtgDBCache **pDb, SCtgTbCache** pTb) { SCtgDBCache *dbCache = NULL; + SCtgTbCache* pCache = NULL; ctgAcquireDBCache(pCtg, dbFName, &dbCache); if (NULL == dbCache) { - *exist = 0; - return TSDB_CODE_SUCCESS; + ctgDebug("db %s not in cache", dbFName); + goto _return; + } + + int32_t sz = 0; + pCache = taosHashAcquire(dbCache->tbCache, tbName, strlen(tbName)); + if (NULL == pCache) { + ctgDebug("tb %s not in cache, dbFName:%s", tbName, dbFName); + goto _return; } - size_t sz = 0; - CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); - STableMeta *tbMeta = taosHashGet(dbCache->tbCache.metaCache, tbName, strlen(tbName)); - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - - if (NULL == tbMeta) { - ctgReleaseDBCache(pCtg, dbCache); - - *exist = 0; - ctgDebug("tbmeta not in cache, dbFName:%s, tbName:%s", dbFName, tbName); - return TSDB_CODE_SUCCESS; + CTG_LOCK(CTG_READ, &pCache->metaLock); + if (NULL == pCache->pMeta) { + ctgDebug("tb %s meta not in cache, dbFName:%s", tbName, dbFName); + goto _return; } - *exist = 1; + *pDb = dbCache; + *pTb = pCache; - ctgReleaseDBCache(pCtg, dbCache); + ctgDebug("tb %s meta got in cache, dbFName:%s", tbName, dbFName); - ctgDebug("tbmeta is in cache, dbFName:%s, tbName:%s", dbFName, tbName); + CTG_CACHE_STAT_INC(tbMetaHitNum, 1); + + return TSDB_CODE_SUCCESS; + +_return: + + ctgReleaseTbMetaToCache(pCtg, dbCache, pCache); + + CTG_CACHE_STAT_INC(tbMetaMissNum, 1); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgAcquireStbMetaFromCache(SCatalog* pCtg, char *dbFName, uint64_t suid, SCtgDBCache **pDb, SCtgTbCache** pTb) { + SCtgDBCache* dbCache = NULL; + SCtgTbCache* pCache = NULL; + ctgAcquireDBCache(pCtg, dbFName, &dbCache); + if (NULL == dbCache) { + ctgDebug("db %s not in cache", dbFName); + goto _return; + } + + int32_t sz = 0; + char* stName = taosHashAcquire(dbCache->stbCache, &suid, sizeof(suid)); + if (NULL == stName) { + ctgDebug("stb %" PRIx64 " not in cache, dbFName:%s", suid, dbFName); + goto _return; + } + + pCache = taosHashAcquire(dbCache->tbCache, stName, strlen(stName)); + if (NULL == pCache) { + ctgDebug("stb %" PRIx64 " name %s not in cache, dbFName:%s", suid, stName, dbFName); + taosHashRelease(dbCache->stbCache, stName); + goto _return; + } + + CTG_LOCK(CTG_READ, &pCache->metaLock); + if (NULL == pCache->pMeta) { + ctgDebug("stb %" PRIx64 " meta not in cache, dbFName:%s", suid, dbFName); + goto _return; + } + + *pDb = dbCache; + *pTb = pCache; + + ctgDebug("stb %" PRIx64 " meta got in cache, dbFName:%s", suid, dbFName); + + CTG_CACHE_STAT_INC(tbMetaHitNum, 1); + + return TSDB_CODE_SUCCESS; + +_return: + + ctgReleaseTbMetaToCache(pCtg, dbCache, pCache); + + CTG_CACHE_STAT_INC(tbMetaMissNum, 1); + + *pDb = NULL; + *pTb = NULL; return TSDB_CODE_SUCCESS; } +int32_t ctgAcquireTbIndexFromCache(SCatalog* pCtg, char *dbFName, char* tbName, SCtgDBCache **pDb, SCtgTbCache** pTb) { + SCtgDBCache *dbCache = NULL; + ctgAcquireDBCache(pCtg, dbFName, &dbCache); + if (NULL == dbCache) { + ctgDebug("db %s not in cache", dbFName); + goto _return; + } + + int32_t sz = 0; + SCtgTbCache* pCache = taosHashAcquire(dbCache->tbCache, tbName, strlen(tbName)); + if (NULL == pCache) { + ctgDebug("tb %s not in cache, dbFName:%s", tbName, dbFName); + goto _return; + } + + CTG_LOCK(CTG_READ, &pCache->indexLock); + if (NULL == pCache->pIndex) { + ctgDebug("tb %s index not in cache, dbFName:%s", tbName, dbFName); + goto _return; + } + + *pDb = dbCache; + *pTb = pCache; + + ctgDebug("tb %s index got in cache, dbFName:%s", tbName, dbFName); + + CTG_CACHE_STAT_INC(tbIndexHitNum, 1); + + return TSDB_CODE_SUCCESS; + +_return: + + ctgReleaseTbIndexToCache(pCtg, dbCache, pCache); + + CTG_CACHE_STAT_INC(tbIndexMissNum, 1); + + return TSDB_CODE_SUCCESS; +} + + +int32_t ctgTbMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName, int32_t *exist) { + SCtgDBCache *dbCache = NULL; + SCtgTbCache *tbCache = NULL; + ctgAcquireTbMetaFromCache(pCtg, dbFName, tbName, &dbCache, &tbCache); + if (NULL == tbCache) { + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + + *exist = 0; + return TSDB_CODE_SUCCESS; + } + + *exist = 1; + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + + return TSDB_CODE_SUCCESS; +} + int32_t ctgReadTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta) { int32_t code = 0; SCtgDBCache *dbCache = NULL; - + SCtgTbCache *tbCache = NULL; *pTableMeta = NULL; - if (NULL == pCtg->dbCache) { - ctgDebug("empty tbmeta cache, tbName:%s", ctx->pName->tname); - return TSDB_CODE_SUCCESS; - } - char dbFName[TSDB_DB_FNAME_LEN] = {0}; if (CTG_FLAG_IS_SYS_DB(ctx->flag)) { strcpy(dbFName, ctx->pName->dbname); @@ -257,202 +389,194 @@ int32_t ctgReadTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta** tNameGetFullDbName(ctx->pName, dbFName); } - ctgAcquireDBCache(pCtg, dbFName, &dbCache); - if (NULL == dbCache) { - ctgDebug("db %d.%s not in cache", ctx->pName->acctId, ctx->pName->dbname); - return TSDB_CODE_SUCCESS; - } - - int32_t sz = 0; - CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); - taosHashGetDup_m(dbCache->tbCache.metaCache, ctx->pName->tname, strlen(ctx->pName->tname), (void **)pTableMeta, &sz); - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - - if (NULL == *pTableMeta) { - ctgReleaseDBCache(pCtg, dbCache); - ctgDebug("tbl not in cache, dbFName:%s, tbName:%s", dbFName, ctx->pName->tname); + ctgAcquireTbMetaFromCache(pCtg, dbFName, ctx->pName->tname, &dbCache, &tbCache); + if (NULL == tbCache) { + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); return TSDB_CODE_SUCCESS; } - STableMeta* tbMeta = *pTableMeta; + STableMeta* tbMeta = tbCache->pMeta; ctx->tbInfo.inCache = true; ctx->tbInfo.dbId = dbCache->dbId; ctx->tbInfo.suid = tbMeta->suid; ctx->tbInfo.tbType = tbMeta->tableType; - + if (tbMeta->tableType != TSDB_CHILD_TABLE) { - ctgReleaseDBCache(pCtg, dbCache); - ctgDebug("Got meta from cache, type:%d, dbFName:%s, tbName:%s", tbMeta->tableType, dbFName, ctx->pName->tname); + int32_t metaSize = CTG_META_SIZE(tbMeta); + *pTableMeta = taosMemoryCalloc(1, metaSize); + if (NULL == *pTableMeta) { + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } - CTG_CACHE_STAT_INC(tblHitNum, 1); + memcpy(*pTableMeta, tbMeta, metaSize); + + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + ctgDebug("Got tb %s meta from cache, type:%d, dbFName:%s", ctx->pName->tname, tbMeta->tableType, dbFName); + return TSDB_CODE_SUCCESS; + } + + // PROCESS FOR CHILD TABLE + + int32_t metaSize = sizeof(SCTableMeta); + *pTableMeta = taosMemoryCalloc(1, metaSize); + if (NULL == *pTableMeta) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + + memcpy(*pTableMeta, tbMeta, metaSize); + + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + ctgDebug("Got ctb %s meta from cache, will continue to get its stb meta, type:%d, dbFName:%s", + ctx->pName->tname, ctx->tbInfo.tbType, dbFName); + + ctgAcquireStbMetaFromCache(pCtg, dbFName, ctx->tbInfo.suid, &dbCache, &tbCache); + if (NULL == tbCache) { + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + taosMemoryFreeClear(*pTableMeta); + ctgDebug("stb %" PRIx64 " meta not in cache", ctx->tbInfo.suid); return TSDB_CODE_SUCCESS; } - CTG_LOCK(CTG_READ, &dbCache->tbCache.stbLock); - - STableMeta **stbMeta = taosHashGet(dbCache->tbCache.stbCache, &tbMeta->suid, sizeof(tbMeta->suid)); - if (NULL == stbMeta || NULL == *stbMeta) { - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); - ctgError("stb not in stbCache, suid:%"PRIx64, tbMeta->suid); - goto _return; - } - - if ((*stbMeta)->suid != tbMeta->suid) { - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); - ctgError("stable suid in stbCache mis-match, expected suid:%"PRIx64 ",actual suid:%"PRIx64, tbMeta->suid, (*stbMeta)->suid); + STableMeta* stbMeta = tbCache->pMeta; + if (stbMeta->suid != ctx->tbInfo.suid) { + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + ctgError("stb suid %" PRIx64 " in stbCache mis-match, expected suid:%"PRIx64 , stbMeta->suid, ctx->tbInfo.suid); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } - int32_t metaSize = CTG_META_SIZE(*stbMeta); + metaSize = CTG_META_SIZE(stbMeta); *pTableMeta = taosMemoryRealloc(*pTableMeta, metaSize); if (NULL == *pTableMeta) { - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); - ctgError("realloc size[%d] failed", metaSize); + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } - memcpy(&(*pTableMeta)->sversion, &(*stbMeta)->sversion, metaSize - sizeof(SCTableMeta)); + memcpy(&(*pTableMeta)->sversion, &stbMeta->sversion, metaSize - sizeof(SCTableMeta)); - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); - ctgReleaseDBCache(pCtg, dbCache); - - CTG_CACHE_STAT_INC(tblHitNum, 1); - - ctgDebug("Got tbmeta from cache, dbFName:%s, tbName:%s", dbFName, ctx->pName->tname); + ctgDebug("Got tb %s meta from cache, dbFName:%s", ctx->pName->tname, dbFName); return TSDB_CODE_SUCCESS; _return: - ctgReleaseDBCache(pCtg, dbCache); + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); taosMemoryFreeClear(*pTableMeta); - - CTG_CACHE_STAT_INC(tblMissNum, 1); CTG_RET(code); } -int32_t ctgReadTbVerFromCache(SCatalog *pCtg, const SName *pTableName, int32_t *sver, int32_t *tver, int32_t *tbType, uint64_t *suid, +int32_t ctgReadTbVerFromCache(SCatalog *pCtg, SName *pTableName, int32_t *sver, int32_t *tver, int32_t *tbType, uint64_t *suid, char *stbName) { *sver = -1; *tver = -1; - if (NULL == pCtg->dbCache) { - ctgDebug("empty tbmeta cache, tbName:%s", pTableName->tname); - return TSDB_CODE_SUCCESS; - } - SCtgDBCache *dbCache = NULL; + SCtgTbCache *tbCache = NULL; char dbFName[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(pTableName, dbFName); - ctgAcquireDBCache(pCtg, dbFName, &dbCache); - if (NULL == dbCache) { - ctgDebug("db %s not in cache", pTableName->tname); + ctgAcquireTbMetaFromCache(pCtg, dbFName, pTableName->tname, &dbCache, &tbCache); + if (NULL == tbCache) { + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); return TSDB_CODE_SUCCESS; } - CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); - STableMeta *tbMeta = taosHashGet(dbCache->tbCache.metaCache, pTableName->tname, strlen(pTableName->tname)); - if (tbMeta) { - *tbType = tbMeta->tableType; - *suid = tbMeta->suid; - if (*tbType != TSDB_CHILD_TABLE) { - *sver = tbMeta->sversion; - *tver = tbMeta->tversion; - } - } - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - - if (NULL == tbMeta) { - ctgReleaseDBCache(pCtg, dbCache); - return TSDB_CODE_SUCCESS; - } + STableMeta* tbMeta = tbCache->pMeta; + *tbType = tbMeta->tableType; + *suid = tbMeta->suid; if (*tbType != TSDB_CHILD_TABLE) { - ctgReleaseDBCache(pCtg, dbCache); - ctgDebug("Got sver %d tver %d from cache, type:%d, dbFName:%s, tbName:%s", *sver, *tver, *tbType, dbFName, pTableName->tname); + *sver = tbMeta->sversion; + *tver = tbMeta->tversion; + ctgDebug("Got tb %s ver from cache, dbFName:%s, tbType:%d, sver:%d, tver:%d, suid:%" PRIx64, + pTableName->tname, dbFName, *tbType, *sver, *tver, *suid); + + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); return TSDB_CODE_SUCCESS; } - ctgDebug("Got subtable meta from cache, dbFName:%s, tbName:%s, suid:%" PRIx64, dbFName, pTableName->tname, *suid); - - CTG_LOCK(CTG_READ, &dbCache->tbCache.stbLock); - - STableMeta **stbMeta = taosHashGet(dbCache->tbCache.stbCache, suid, sizeof(*suid)); - if (NULL == stbMeta || NULL == *stbMeta) { - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); - ctgReleaseDBCache(pCtg, dbCache); - ctgDebug("stb not in stbCache, suid:%" PRIx64, *suid); + // PROCESS FOR CHILD TABLE + + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + ctgDebug("Got ctb %s ver from cache, will continue to get its stb ver, dbFName:%s", pTableName->tname, dbFName); + + ctgAcquireStbMetaFromCache(pCtg, dbFName, *suid, &dbCache, &tbCache); + if (NULL == tbCache) { + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + ctgDebug("stb %" PRIx64 " meta not in cache", *suid); return TSDB_CODE_SUCCESS; } - - if ((*stbMeta)->suid != *suid) { - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); - ctgReleaseDBCache(pCtg, dbCache); - ctgError("stable suid in stbCache mis-match, expected suid:%" PRIx64 ",actual suid:%" PRIx64, *suid, - (*stbMeta)->suid); + + STableMeta* stbMeta = tbCache->pMeta; + if (stbMeta->suid != *suid) { + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + ctgError("stb suid %" PRIx64 " in stbCache mis-match, expected suid:%" PRIx64 , stbMeta->suid, *suid); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } size_t nameLen = 0; - char *name = taosHashGetKey(*stbMeta, &nameLen); + char *name = taosHashGetKey(tbCache, &nameLen); strncpy(stbName, name, nameLen); stbName[nameLen] = 0; - *sver = (*stbMeta)->sversion; - *tver = (*stbMeta)->tversion; + *sver = stbMeta->sversion; + *tver = stbMeta->tversion; - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); - ctgReleaseDBCache(pCtg, dbCache); - - ctgDebug("Got sver %d tver %d from cache, type:%d, dbFName:%s, tbName:%s", *sver, *tver, *tbType, dbFName, pTableName->tname); + ctgDebug("Got tb %s sver %d tver %d from cache, type:%d, dbFName:%s", pTableName->tname, *sver, *tver, *tbType, dbFName); return TSDB_CODE_SUCCESS; } -int32_t ctgReadTbTypeFromCache(SCatalog* pCtg, const char* dbFName, const char *tableName, int32_t *tbType) { - if (NULL == pCtg->dbCache) { - ctgWarn("empty db cache, dbFName:%s, tbName:%s", dbFName, tableName); - return TSDB_CODE_SUCCESS; - } - +int32_t ctgReadTbTypeFromCache(SCatalog* pCtg, char* dbFName, char *tableName, int32_t *tbType) { SCtgDBCache *dbCache = NULL; - ctgAcquireDBCache(pCtg, dbFName, &dbCache); - if (NULL == dbCache) { + SCtgTbCache *tbCache = NULL; + CTG_ERR_RET(ctgAcquireTbMetaFromCache(pCtg, dbFName, tableName, &dbCache, &tbCache)); + if (NULL == tbCache) { + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); return TSDB_CODE_SUCCESS; } - CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); - STableMeta *pTableMeta = (STableMeta *)taosHashAcquire(dbCache->tbCache.metaCache, tableName, strlen(tableName)); + *tbType = tbCache->pMeta->tableType; + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); - if (NULL == pTableMeta) { - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - ctgWarn("tbl not in cache, dbFName:%s, tbName:%s", dbFName, tableName); - ctgReleaseDBCache(pCtg, dbCache); - - return TSDB_CODE_SUCCESS; - } - - *tbType = atomic_load_8(&pTableMeta->tableType); - - taosHashRelease(dbCache->tbCache.metaCache, pTableMeta); - - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - - ctgReleaseDBCache(pCtg, dbCache); - - ctgDebug("Got tbtype from cache, dbFName:%s, tbName:%s, type:%d", dbFName, tableName, *tbType); + ctgDebug("Got tb %s tbType %d from cache, dbFName:%s", tableName, *tbType, dbFName); return TSDB_CODE_SUCCESS; } -int32_t ctgChkAuthFromCache(SCatalog* pCtg, const char* user, const char* dbFName, AUTH_TYPE type, bool *inCache, bool *pass) { +int32_t ctgReadTbIndexFromCache(SCatalog* pCtg, SName* pTableName, SArray** pRes) { + int32_t code = 0; + SCtgDBCache *dbCache = NULL; + SCtgTbCache *tbCache = NULL; + char dbFName[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(pTableName, dbFName); + + *pRes = NULL; + + ctgAcquireTbIndexFromCache(pCtg, dbFName, pTableName->tname, &dbCache, &tbCache); + if (NULL == tbCache) { + ctgReleaseTbIndexToCache(pCtg, dbCache, tbCache); + return TSDB_CODE_SUCCESS; + } + + CTG_ERR_JRET(ctgCloneTableIndex(tbCache->pIndex->pIndex, pRes)); + +_return: + + ctgReleaseTbIndexToCache(pCtg, dbCache, tbCache); + + CTG_RET(code); +} + +int32_t ctgChkAuthFromCache(SCatalog* pCtg, char* user, char* dbFName, AUTH_TYPE type, bool *inCache, bool *pass) { if (NULL == pCtg->userCache) { ctgDebug("empty user auth cache, user:%s", user); goto _return; @@ -718,9 +842,9 @@ int32_t ctgUpdateTbMetaEnqueue(SCatalog* pCtg, STableMetaOutput *output, bool sy op->opId = CTG_OP_UPDATE_TB_META; op->syncOp = syncOp; - SCtgUpdateTblMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateTblMsg)); + SCtgUpdateTbMetaMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateTbMetaMsg)); if (NULL == msg) { - ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateTblMsg)); + ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateTbMetaMsg)); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } @@ -730,7 +854,7 @@ int32_t ctgUpdateTbMetaEnqueue(SCatalog* pCtg, STableMetaOutput *output, bool sy } msg->pCtg = pCtg; - msg->output = output; + msg->pMeta = output; op->data = msg; @@ -805,6 +929,67 @@ _return: CTG_RET(code); } +int32_t ctgUpdateTbIndexEnqueue(SCatalog* pCtg, STableIndex **pIndex, bool syncOp) { + int32_t code = 0; + SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); + op->opId = CTG_OP_UPDATE_TB_INDEX; + op->syncOp = syncOp; + + SCtgUpdateTbIndexMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateTbIndexMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateTbIndexMsg)); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); + } + + msg->pCtg = pCtg; + msg->pIndex = *pIndex; + + op->data = msg; + + CTG_ERR_JRET(ctgEnqueue(pCtg, op)); + + *pIndex = NULL; + return TSDB_CODE_SUCCESS; + +_return: + + taosArrayDestroyEx((*pIndex)->pIndex, tFreeSTableIndexInfo); + taosMemoryFreeClear(*pIndex); + taosMemoryFreeClear(msg); + + CTG_RET(code); +} + +int32_t ctgDropTbIndexEnqueue(SCatalog* pCtg, SName* pName, bool syncOp) { + int32_t code = 0; + SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); + op->opId = CTG_OP_DROP_TB_INDEX; + op->syncOp = syncOp; + + SCtgDropTbIndexMsg *msg = taosMemoryMalloc(sizeof(SCtgDropTbIndexMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgDropTbIndexMsg)); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + msg->pCtg = pCtg; + tNameGetFullDbName(pName, msg->dbFName); + strcpy(msg->tbName, pName->tname); + + op->data = msg; + + CTG_ERR_JRET(ctgEnqueue(pCtg, op)); + + return TSDB_CODE_SUCCESS; + +_return: + + taosMemoryFreeClear(msg); + + CTG_RET(code); +} + + int32_t ctgMetaRentInit(SCtgRentMgmt *mgmt, uint32_t rentSec, int8_t type) { mgmt->slotRIdx = 0; mgmt->slotNum = rentSec / CTG_RENT_SLOT_SECOND; @@ -875,7 +1060,7 @@ int32_t ctgMetaRentUpdate(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t si void *orig = taosArraySearch(slot->meta, &id, searchCompare, TD_EQ); if (NULL == orig) { - qError("meta not found in slot, id:%"PRIx64", slot idx:%d, type:%d, size:%d", id, widx, mgmt->type, (int32_t)taosArrayGetSize(slot->meta)); + qDebug("meta not found in slot, id:%"PRIx64", slot idx:%d, type:%d, size:%d", id, widx, mgmt->type, (int32_t)taosArrayGetSize(slot->meta)); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } @@ -888,7 +1073,7 @@ _return: CTG_UNLOCK(CTG_WRITE, &slot->lock); if (code) { - qWarn("meta in rent update failed, will try to add it, code:%x, id:%"PRIx64", slot idx:%d, type:%d", code, id, widx, mgmt->type); + qDebug("meta in rent update failed, will try to add it, code:%x, id:%"PRIx64", slot idx:%d, type:%d", code, id, widx, mgmt->type); CTG_RET(ctgMetaRentAdd(mgmt, meta, id, size)); } @@ -1006,14 +1191,14 @@ int32_t ctgAddNewDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId) { SCtgDBCache newDBCache = {0}; newDBCache.dbId = dbId; - newDBCache.tbCache.metaCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); - if (NULL == newDBCache.tbCache.metaCache) { + newDBCache.tbCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + if (NULL == newDBCache.tbCache) { ctgError("taosHashInit %d metaCache failed", gCtgMgmt.cfg.maxTblCacheNum); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } - newDBCache.tbCache.stbCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_ENTRY_LOCK); - if (NULL == newDBCache.tbCache.stbCache) { + newDBCache.stbCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_ENTRY_LOCK); + if (NULL == newDBCache.stbCache) { ctgError("taosHashInit %d stbCache failed", gCtgMgmt.cfg.maxTblCacheNum); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } @@ -1050,22 +1235,22 @@ _return: } -void ctgRemoveStbRent(SCatalog* pCtg, SCtgTbMetaCache *cache) { - CTG_LOCK(CTG_WRITE, &cache->stbLock); - if (cache->stbCache) { - void *pIter = taosHashIterate(cache->stbCache, NULL); - while (pIter) { - uint64_t *suid = NULL; - suid = taosHashGetKey(pIter, NULL); - - if (TSDB_CODE_SUCCESS == ctgMetaRentRemove(&pCtg->stbRent, *suid, ctgStbVersionSortCompare, ctgStbVersionSearchCompare)) { - ctgDebug("stb removed from rent, suid:%"PRIx64, *suid); - } - - pIter = taosHashIterate(cache->stbCache, pIter); - } +void ctgRemoveStbRent(SCatalog* pCtg, SCtgDBCache *dbCache) { + if (NULL == dbCache->stbCache) { + return; + } + + void *pIter = taosHashIterate(dbCache->stbCache, NULL); + while (pIter) { + uint64_t *suid = NULL; + suid = taosHashGetKey(pIter, NULL); + + if (TSDB_CODE_SUCCESS == ctgMetaRentRemove(&pCtg->stbRent, *suid, ctgStbVersionSortCompare, ctgStbVersionSearchCompare)) { + ctgDebug("stb removed from rent, suid:%"PRIx64, *suid); + } + + pIter = taosHashIterate(dbCache->stbCache, pIter); } - CTG_UNLOCK(CTG_WRITE, &cache->stbLock); } @@ -1074,15 +1259,16 @@ int32_t ctgRemoveDBFromCache(SCatalog* pCtg, SCtgDBCache *dbCache, const char* d ctgInfo("start to remove db from cache, dbFName:%s, dbId:%"PRIx64, dbFName, dbCache->dbId); + CTG_LOCK(CTG_WRITE, &dbCache->dbLock); + atomic_store_8(&dbCache->deleted, 1); - - ctgRemoveStbRent(pCtg, &dbCache->tbCache); - + ctgRemoveStbRent(pCtg, dbCache); ctgFreeDbCache(dbCache); - CTG_ERR_RET(ctgMetaRentRemove(&pCtg->dbRent, dbCache->dbId, ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare)); - - ctgDebug("db removed from rent, dbFName:%s, dbId:%"PRIx64, dbFName, dbCache->dbId); + CTG_UNLOCK(CTG_WRITE, &dbCache->dbLock); + + CTG_ERR_RET(ctgMetaRentRemove(&pCtg->dbRent, dbId, ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare)); + ctgDebug("db removed from rent, dbFName:%s, dbId:%"PRIx64, dbFName, dbId); if (taosHashRemove(pCtg->dbCache, dbFName, strlen(dbFName))) { ctgInfo("taosHashRemove from dbCache failed, may be removed, dbFName:%s", dbFName); @@ -1090,7 +1276,6 @@ int32_t ctgRemoveDBFromCache(SCatalog* pCtg, SCtgDBCache *dbCache, const char* d } CTG_CACHE_STAT_DEC(dbNum, 1); - ctgInfo("db removed from cache, dbFName:%s, dbId:%"PRIx64, dbFName, dbId); return TSDB_CODE_SUCCESS; @@ -1138,120 +1323,76 @@ int32_t ctgGetAddDBCache(SCatalog* pCtg, const char *dbFName, uint64_t dbId, SCt return TSDB_CODE_SUCCESS; } +int32_t ctgUpdateRentStbVersion(SCatalog *pCtg, char* dbFName, char* tbName, uint64_t dbId, uint64_t suid, SCtgTbCache* pCache) { + SSTableVersion metaRent = {.dbId = dbId, .suid = suid}; + if (pCache->pMeta) { + metaRent.sversion = pCache->pMeta->sversion; + metaRent.tversion = pCache->pMeta->tversion; + } -int32_t ctgWriteDBVgInfoToCache(SCatalog* pCtg, const char* dbFName, uint64_t dbId, SDBVgInfo** pDbInfo) { - int32_t code = 0; - SDBVgInfo* dbInfo = *pDbInfo; - - if (NULL == dbInfo->vgHash) { - return TSDB_CODE_SUCCESS; + if (pCache->pIndex) { + metaRent.smaVer = pCache->pIndex->version; } - if (dbInfo->vgVersion < 0 || taosHashGetSize(dbInfo->vgHash) <= 0) { - ctgError("invalid db vgInfo, dbFName:%s, vgHash:%p, vgVersion:%d, vgHashSize:%d", - dbFName, dbInfo->vgHash, dbInfo->vgVersion, taosHashGetSize(dbInfo->vgHash)); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); - } - - bool newAdded = false; - SDbVgVersion vgVersion = {.dbId = dbId, .vgVersion = dbInfo->vgVersion, .numOfTable = dbInfo->numOfTable}; - - SCtgDBCache *dbCache = NULL; - CTG_ERR_RET(ctgGetAddDBCache(pCtg, dbFName, dbId, &dbCache)); - if (NULL == dbCache) { - ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:%"PRIx64, dbFName, dbId); - CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); - } - - SDBVgInfo *vgInfo = NULL; - CTG_ERR_RET(ctgWAcquireVgInfo(pCtg, dbCache)); + strcpy(metaRent.dbFName, dbFName); + strcpy(metaRent.stbName, tbName); - if (dbCache->vgInfo) { - if (dbInfo->vgVersion < dbCache->vgInfo->vgVersion) { - ctgDebug("db vgVersion is old, dbFName:%s, vgVersion:%d, currentVersion:%d", dbFName, dbInfo->vgVersion, dbCache->vgInfo->vgVersion); - ctgWReleaseVgInfo(dbCache); - - return TSDB_CODE_SUCCESS; - } + CTG_ERR_RET(ctgMetaRentUpdate(&pCtg->stbRent, &metaRent, metaRent.suid, sizeof(SSTableVersion), ctgStbVersionSortCompare, ctgStbVersionSearchCompare)); - if (dbInfo->vgVersion == dbCache->vgInfo->vgVersion && dbInfo->numOfTable == dbCache->vgInfo->numOfTable) { - ctgDebug("no new db vgVersion or numOfTable, dbFName:%s, vgVersion:%d, numOfTable:%d", dbFName, dbInfo->vgVersion, dbInfo->numOfTable); - ctgWReleaseVgInfo(dbCache); - - return TSDB_CODE_SUCCESS; - } + ctgDebug("db %s,%" PRIx64 " stb %s,%" PRIx64 " sver %d tver %d smaVer %d updated to stbRent", + dbFName, dbId, tbName, suid, metaRent.sversion, metaRent.tversion, metaRent.smaVer); - ctgFreeVgInfo(dbCache->vgInfo); - } - - dbCache->vgInfo = dbInfo; - - *pDbInfo = NULL; - - ctgDebug("db vgInfo updated, dbFName:%s, vgVersion:%d, dbId:%"PRIx64, dbFName, vgVersion.vgVersion, vgVersion.dbId); - - ctgWReleaseVgInfo(dbCache); - - dbCache = NULL; - - strncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); - CTG_ERR_RET(ctgMetaRentUpdate(&pCtg->dbRent, &vgVersion, vgVersion.dbId, sizeof(SDbVgVersion), ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare)); - - CTG_RET(code); + return TSDB_CODE_SUCCESS; } int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, uint64_t dbId, char *tbName, STableMeta *meta, int32_t metaSize) { - SCtgTbMetaCache *tbCache = &dbCache->tbCache; - - CTG_LOCK(CTG_READ, &tbCache->metaLock); - if (dbCache->deleted || NULL == tbCache->metaCache || NULL == tbCache->stbCache) { - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + if (NULL == dbCache->tbCache || NULL == dbCache->stbCache) { + taosMemoryFree(meta); ctgError("db is dropping, dbId:%"PRIx64, dbCache->dbId); CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); } + bool isStb = meta->tableType == TSDB_SUPER_TABLE; + SCtgTbCache* pCache = taosHashGet(dbCache->tbCache, tbName, strlen(tbName)); + STableMeta *orig = (pCache ? pCache->pMeta : NULL); int8_t origType = 0; uint64_t origSuid = 0; - bool isStb = meta->tableType == TSDB_SUPER_TABLE; - STableMeta *orig = taosHashGet(tbCache->metaCache, tbName, strlen(tbName)); + if (orig) { origType = orig->tableType; if (origType == meta->tableType && orig->uid == meta->uid && orig->sversion >= meta->sversion && orig->tversion >= meta->tversion) { - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); + taosMemoryFree(meta); + ctgDebug("ignore table %s meta update", tbName); return TSDB_CODE_SUCCESS; } if (origType == TSDB_SUPER_TABLE) { - CTG_LOCK(CTG_WRITE, &tbCache->stbLock); - if (taosHashRemove(tbCache->stbCache, &orig->suid, sizeof(orig->suid))) { + if (taosHashRemove(dbCache->stbCache, &orig->suid, sizeof(orig->suid))) { ctgError("stb not exist in stbCache, dbFName:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid); } else { CTG_CACHE_STAT_DEC(stblNum, 1); + ctgDebug("stb removed from stbCache, dbFName:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid); } - CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); - - ctgDebug("stb removed from stbCache, dbFName:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid); - ctgMetaRentRemove(&pCtg->stbRent, orig->suid, ctgStbVersionSortCompare, ctgStbVersionSearchCompare); - origSuid = orig->suid; } } - if (isStb) { - CTG_LOCK(CTG_WRITE, &tbCache->stbLock); - } - - if (taosHashPut(tbCache->metaCache, tbName, strlen(tbName), meta, metaSize) != 0) { - if (isStb) { - CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); + if (NULL == pCache) { + SCtgTbCache cache = {0}; + cache.pMeta = meta; + if (taosHashPut(dbCache->tbCache, tbName, strlen(tbName), &cache, sizeof(SCtgTbCache)) != 0) { + taosMemoryFree(meta); + ctgError("taosHashPut new tbCache failed, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); - ctgError("taosHashPut tbmeta to cache failed, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + pCache = taosHashGet(dbCache->tbCache, tbName, strlen(tbName)); + } else { + taosMemoryFree(pCache->pMeta); + pCache->pMeta = meta; } if (NULL == orig) { @@ -1262,30 +1403,71 @@ int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNam ctgdShowTableMeta(pCtg, tbName, meta); if (!isStb) { - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); return TSDB_CODE_SUCCESS; } - STableMeta *tbMeta = taosHashGet(tbCache->metaCache, tbName, strlen(tbName)); - if (taosHashPut(tbCache->stbCache, &meta->suid, sizeof(meta->suid), &tbMeta, POINTER_BYTES) != 0) { - CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); - ctgError("taosHashPut stable to stable cache failed, suid:%"PRIx64, meta->suid); + if (origSuid != meta->suid && taosHashPut(dbCache->stbCache, &meta->suid, sizeof(meta->suid), tbName, strlen(tbName) + 1) != 0) { + ctgError("taosHashPut to stable cache failed, suid:%"PRIx64, meta->suid); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } CTG_CACHE_STAT_INC(stblNum, 1); + + ctgDebug("stb %" PRIx64 " updated to cache, dbFName:%s, tbName:%s, tbType:%d", meta->suid, dbFName, tbName, meta->tableType); + + CTG_ERR_RET(ctgUpdateRentStbVersion(pCtg, dbFName, tbName, dbId, meta->suid, pCache)); - CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock); + return TSDB_CODE_SUCCESS; +} - CTG_UNLOCK(CTG_READ, &tbCache->metaLock); +int32_t ctgWriteTbIndexToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char* dbFName, char *tbName, STableIndex **index) { + if (NULL == dbCache->tbCache) { + ctgFreeSTableIndex(*index); + taosMemoryFreeClear(*index); + ctgError("db is dropping, dbId:%"PRIx64, dbCache->dbId); + CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); + } - ctgDebug("stb updated to stbCache, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType); + STableIndex* pIndex = *index; + uint64_t suid = pIndex->suid; + SCtgTbCache* pCache = taosHashGet(dbCache->tbCache, tbName, strlen(tbName)); + if (NULL == pCache) { + SCtgTbCache cache = {0}; + cache.pIndex = pIndex; + + if (taosHashPut(dbCache->tbCache, tbName, strlen(tbName), &cache, sizeof(cache)) != 0) { + ctgFreeSTableIndex(*index); + taosMemoryFreeClear(*index); + ctgError("taosHashPut new tbCache failed, tbName:%s", tbName); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } - SSTableMetaVersion metaRent = {.dbId = dbId, .suid = meta->suid, .sversion = meta->sversion, .tversion = meta->tversion}; - strcpy(metaRent.dbFName, dbFName); - strcpy(metaRent.stbName, tbName); - CTG_ERR_RET(ctgMetaRentAdd(&pCtg->stbRent, &metaRent, metaRent.suid, sizeof(SSTableMetaVersion))); + *index = NULL; + ctgDebug("table %s index updated to cache, ver:%d, num:%d", tbName, pIndex->version, (int32_t)taosArrayGetSize(pIndex->pIndex)); + + if (suid) { + CTG_ERR_RET(ctgUpdateRentStbVersion(pCtg, dbFName, tbName, dbCache->dbId, pIndex->suid, &cache)); + } + + return TSDB_CODE_SUCCESS; + } + + if (pCache->pIndex) { + if (0 == suid) { + suid = pCache->pIndex->suid; + } + taosArrayDestroyEx(pCache->pIndex->pIndex, tFreeSTableIndexInfo); + taosMemoryFreeClear(pCache->pIndex); + } + + pCache->pIndex = pIndex; + *index = NULL; + + ctgDebug("table %s index updated to cache, ver:%d, num:%d", tbName, pIndex->version, (int32_t)taosArrayGetSize(pIndex->pIndex)); + + if (suid) { + CTG_ERR_RET(ctgUpdateRentStbVersion(pCtg, dbFName, tbName, dbCache->dbId, suid, pCache)); + } return TSDB_CODE_SUCCESS; } @@ -1305,12 +1487,67 @@ _return: CTG_RET(code); } - int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) { int32_t code = 0; SCtgUpdateVgMsg *msg = operation->data; + SDBVgInfo* dbInfo = msg->dbInfo; + char* dbFName = msg->dbFName; + SCatalog* pCtg = msg->pCtg; - CTG_ERR_JRET(ctgWriteDBVgInfoToCache(msg->pCtg, msg->dbFName, msg->dbId, &msg->dbInfo)); + if (NULL == dbInfo->vgHash) { + return TSDB_CODE_SUCCESS; + } + + if (dbInfo->vgVersion < 0 || taosHashGetSize(dbInfo->vgHash) <= 0) { + ctgError("invalid db vgInfo, dbFName:%s, vgHash:%p, vgVersion:%d, vgHashSize:%d", + dbFName, dbInfo->vgHash, dbInfo->vgVersion, taosHashGetSize(dbInfo->vgHash)); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + bool newAdded = false; + SDbVgVersion vgVersion = {.dbId = msg->dbId, .vgVersion = dbInfo->vgVersion, .numOfTable = dbInfo->numOfTable}; + + SCtgDBCache *dbCache = NULL; + CTG_ERR_RET(ctgGetAddDBCache(msg->pCtg, dbFName, msg->dbId, &dbCache)); + if (NULL == dbCache) { + ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:%"PRIx64, dbFName, msg->dbId); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + SCtgVgCache *vgCache = &dbCache->vgCache; + CTG_ERR_RET(ctgWLockVgInfo(msg->pCtg, dbCache)); + + if (vgCache->vgInfo) { + SDBVgInfo *vgInfo = vgCache->vgInfo; + + if (dbInfo->vgVersion < vgInfo->vgVersion) { + ctgDebug("db vgVer is old, dbFName:%s, vgVer:%d, curVer:%d", dbFName, dbInfo->vgVersion, vgInfo->vgVersion); + ctgWUnlockVgInfo(dbCache); + + return TSDB_CODE_SUCCESS; + } + + if (dbInfo->vgVersion == vgInfo->vgVersion && dbInfo->numOfTable == vgInfo->numOfTable) { + ctgDebug("no new db vgVer or numOfTable, dbFName:%s, vgVer:%d, numOfTable:%d", dbFName, dbInfo->vgVersion, dbInfo->numOfTable); + ctgWUnlockVgInfo(dbCache); + + return TSDB_CODE_SUCCESS; + } + + ctgFreeVgInfo(vgInfo); + } + + vgCache->vgInfo = dbInfo; + msg->dbInfo = NULL; + + ctgDebug("db vgInfo updated, dbFName:%s, vgVer:%d, dbId:%"PRIx64, dbFName, vgVersion.vgVersion, vgVersion.dbId); + + ctgWUnlockVgInfo(dbCache); + + dbCache = NULL; + + strncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); + CTG_ERR_RET(ctgMetaRentUpdate(&msg->pCtg->dbRent, &vgVersion, vgVersion.dbId, sizeof(SDbVgVersion), ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare)); _return: @@ -1356,14 +1593,14 @@ int32_t ctgOpDropDbVgroup(SCtgCacheOperation *operation) { goto _return; } - CTG_ERR_RET(ctgWAcquireVgInfo(pCtg, dbCache)); + CTG_ERR_RET(ctgWLockVgInfo(pCtg, dbCache)); - ctgFreeVgInfo(dbCache->vgInfo); - dbCache->vgInfo = NULL; + ctgFreeVgInfo(dbCache->vgCache.vgInfo); + dbCache->vgCache.vgInfo = NULL; ctgDebug("db vgInfo removed, dbFName:%s", msg->dbFName); - ctgWReleaseVgInfo(dbCache); + ctgWUnlockVgInfo(dbCache); _return: @@ -1375,42 +1612,47 @@ _return: int32_t ctgOpUpdateTbMeta(SCtgCacheOperation *operation) { int32_t code = 0; - SCtgUpdateTblMsg *msg = operation->data; + SCtgUpdateTbMetaMsg *msg = operation->data; SCatalog* pCtg = msg->pCtg; - STableMetaOutput* output = msg->output; + STableMetaOutput* pMeta = msg->pMeta; SCtgDBCache *dbCache = NULL; - if ((!CTG_IS_META_CTABLE(output->metaType)) && NULL == output->tbMeta) { - ctgError("no valid tbmeta got from meta rsp, dbFName:%s, tbName:%s", output->dbFName, output->tbName); + if ((!CTG_IS_META_CTABLE(pMeta->metaType)) && NULL == pMeta->tbMeta) { + ctgError("no valid tbmeta got from meta rsp, dbFName:%s, tbName:%s", pMeta->dbFName, pMeta->tbName); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } - if (CTG_IS_META_BOTH(output->metaType) && TSDB_SUPER_TABLE != output->tbMeta->tableType) { - ctgError("table type error, expected:%d, actual:%d", TSDB_SUPER_TABLE, output->tbMeta->tableType); + if (CTG_IS_META_BOTH(pMeta->metaType) && TSDB_SUPER_TABLE != pMeta->tbMeta->tableType) { + ctgError("table type error, expected:%d, actual:%d", TSDB_SUPER_TABLE, pMeta->tbMeta->tableType); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } - CTG_ERR_JRET(ctgGetAddDBCache(pCtg, output->dbFName, output->dbId, &dbCache)); + CTG_ERR_JRET(ctgGetAddDBCache(pCtg, pMeta->dbFName, pMeta->dbId, &dbCache)); if (NULL == dbCache) { - ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:%"PRIx64, output->dbFName, output->dbId); + ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:%" PRIx64, pMeta->dbFName, pMeta->dbId); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } - if (CTG_IS_META_TABLE(output->metaType) || CTG_IS_META_BOTH(output->metaType)) { - int32_t metaSize = CTG_META_SIZE(output->tbMeta); - - CTG_ERR_JRET(ctgWriteTbMetaToCache(pCtg, dbCache, output->dbFName, output->dbId, output->tbName, output->tbMeta, metaSize)); + if (CTG_IS_META_TABLE(pMeta->metaType) || CTG_IS_META_BOTH(pMeta->metaType)) { + int32_t metaSize = CTG_META_SIZE(pMeta->tbMeta); + CTG_ERR_JRET(ctgWriteTbMetaToCache(pCtg, dbCache, pMeta->dbFName, pMeta->dbId, pMeta->tbName, pMeta->tbMeta, metaSize)); + pMeta->tbMeta = NULL; } - if (CTG_IS_META_CTABLE(output->metaType) || CTG_IS_META_BOTH(output->metaType)) { - CTG_ERR_JRET(ctgWriteTbMetaToCache(pCtg, dbCache, output->dbFName, output->dbId, output->ctbName, (STableMeta *)&output->ctbMeta, sizeof(output->ctbMeta))); + if (CTG_IS_META_CTABLE(pMeta->metaType) || CTG_IS_META_BOTH(pMeta->metaType)) { + SCTableMeta* ctbMeta = taosMemoryMalloc(sizeof(SCTableMeta)); + if (NULL == ctbMeta) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + memcpy(ctbMeta, &pMeta->ctbMeta, sizeof(SCTableMeta)); + CTG_ERR_JRET(ctgWriteTbMetaToCache(pCtg, dbCache, pMeta->dbFName, pMeta->dbId, pMeta->ctbName, (STableMeta *)ctbMeta, sizeof(SCTableMeta))); } _return: - if (output) { - taosMemoryFreeClear(output->tbMeta); - taosMemoryFreeClear(output); + if (pMeta) { + taosMemoryFreeClear(pMeta->tbMeta); + taosMemoryFreeClear(pMeta); } taosMemoryFreeClear(msg); @@ -1435,22 +1677,17 @@ int32_t ctgOpDropStbMeta(SCtgCacheOperation *operation) { return TSDB_CODE_SUCCESS; } - CTG_LOCK(CTG_WRITE, &dbCache->tbCache.stbLock); - if (taosHashRemove(dbCache->tbCache.stbCache, &msg->suid, sizeof(msg->suid))) { + if (taosHashRemove(dbCache->stbCache, &msg->suid, sizeof(msg->suid))) { ctgDebug("stb not exist in stbCache, may be removed, dbFName:%s, stb:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); } else { CTG_CACHE_STAT_DEC(stblNum, 1); } - CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); - if (taosHashRemove(dbCache->tbCache.metaCache, msg->stbName, strlen(msg->stbName))) { + if (taosHashRemove(dbCache->tbCache, msg->stbName, strlen(msg->stbName))) { ctgError("stb not exist in cache, dbFName:%s, stb:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); } else { CTG_CACHE_STAT_DEC(tblNum, 1); } - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - - CTG_UNLOCK(CTG_WRITE, &dbCache->tbCache.stbLock); ctgInfo("stb removed from cache, dbFName:%s, stbName:%s, suid:%"PRIx64, msg->dbFName, msg->stbName, msg->suid); @@ -1477,21 +1714,18 @@ int32_t ctgOpDropTbMeta(SCtgCacheOperation *operation) { } if (dbCache->dbId != msg->dbId) { - ctgDebug("dbId already modified, dbFName:%s, current:%"PRIx64", dbId:%"PRIx64", tbName:%s", msg->dbFName, dbCache->dbId, msg->dbId, msg->tbName); + ctgDebug("dbId %" PRIx64 " not match with curId %"PRIx64", dbFName:%s, tbName:%s", msg->dbId, dbCache->dbId, msg->dbFName, msg->tbName); return TSDB_CODE_SUCCESS; } - CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock); - if (taosHashRemove(dbCache->tbCache.metaCache, msg->tbName, strlen(msg->tbName))) { - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - ctgError("stb not exist in cache, dbFName:%s, tbName:%s", msg->dbFName, msg->tbName); - CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + if (taosHashRemove(dbCache->tbCache, msg->tbName, strlen(msg->tbName))) { + ctgError("tb %s not exist in cache, dbFName:%s", msg->tbName, msg->dbFName); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } else { CTG_CACHE_STAT_DEC(tblNum, 1); } - CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock); - ctgInfo("table removed from cache, dbFName:%s, tbName:%s", msg->dbFName, msg->tbName); + ctgDebug("table %s removed from cache, dbFName:%s", msg->tbName, msg->dbFName); _return: @@ -1553,7 +1787,6 @@ int32_t ctgOpUpdateUser(SCtgCacheOperation *operation) { _return: - taosHashCleanup(msg->userAuth.createdDbs); taosHashCleanup(msg->userAuth.readDbs); taosHashCleanup(msg->userAuth.writeDbs); @@ -1569,24 +1802,22 @@ int32_t ctgOpUpdateEpset(SCtgCacheOperation *operation) { SCatalog* pCtg = msg->pCtg; SCtgDBCache *dbCache = NULL; - CTG_ERR_RET(ctgAcquireDBCache(pCtg, msg->dbFName, &dbCache)); + CTG_ERR_JRET(ctgGetDBCache(pCtg, msg->dbFName, &dbCache)); if (NULL == dbCache) { ctgDebug("db %s not exist, ignore epset update", msg->dbFName); goto _return; } - SDBVgInfo *vgInfo = NULL; - CTG_ERR_RET(ctgWAcquireVgInfo(pCtg, dbCache)); - - if (NULL == dbCache->vgInfo) { - ctgWReleaseVgInfo(dbCache); + CTG_ERR_JRET(ctgWLockVgInfo(pCtg, dbCache)); + + SDBVgInfo *vgInfo = dbCache->vgCache.vgInfo; + if (NULL == vgInfo) { ctgDebug("vgroup in db %s not cached, ignore epset update", msg->dbFName); goto _return; } - SVgroupInfo* pInfo = taosHashGet(dbCache->vgInfo->vgHash, &msg->vgId, sizeof(msg->vgId)); + SVgroupInfo* pInfo = taosHashGet(vgInfo->vgHash, &msg->vgId, sizeof(msg->vgId)); if (NULL == pInfo) { - ctgWReleaseVgInfo(dbCache); ctgDebug("no vgroup %d in db %s, ignore epset update", msg->vgId, msg->dbFName); goto _return; } @@ -1599,12 +1830,10 @@ int32_t ctgOpUpdateEpset(SCtgCacheOperation *operation) { pInfo->epSet = msg->epSet; - ctgWReleaseVgInfo(dbCache); - _return: if (dbCache) { - ctgReleaseDBCache(msg->pCtg, dbCache); + ctgWUnlockVgInfo(dbCache); } taosMemoryFreeClear(msg); @@ -1612,6 +1841,62 @@ _return: CTG_RET(code); } +int32_t ctgOpUpdateTbIndex(SCtgCacheOperation *operation) { + int32_t code = 0; + SCtgUpdateTbIndexMsg *msg = operation->data; + SCatalog* pCtg = msg->pCtg; + STableIndex* pIndex = msg->pIndex; + SCtgDBCache *dbCache = NULL; + + CTG_ERR_JRET(ctgGetAddDBCache(pCtg, pIndex->dbFName, 0, &dbCache)); + + CTG_ERR_JRET(ctgWriteTbIndexToCache(pCtg, dbCache, pIndex->dbFName, pIndex->tbName, &pIndex)); + +_return: + + if (pIndex) { + taosArrayDestroyEx(pIndex->pIndex, tFreeSTableIndexInfo); + taosMemoryFreeClear(pIndex); + } + + taosMemoryFreeClear(msg); + + CTG_RET(code); +} + +int32_t ctgOpDropTbIndex(SCtgCacheOperation *operation) { + int32_t code = 0; + SCtgDropTbIndexMsg *msg = operation->data; + SCatalog* pCtg = msg->pCtg; + SCtgDBCache *dbCache = NULL; + + CTG_ERR_JRET(ctgGetDBCache(pCtg, msg->dbFName, &dbCache)); + if (NULL == dbCache) { + return TSDB_CODE_SUCCESS; + } + + STableIndex* pIndex = taosMemoryCalloc(1, sizeof(STableIndex)); + if (NULL == pIndex) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + strcpy(pIndex->tbName, msg->tbName); + strcpy(pIndex->dbFName, msg->dbFName); + pIndex->version = -1; + + CTG_ERR_JRET(ctgWriteTbIndexToCache(pCtg, dbCache, pIndex->dbFName, pIndex->tbName, &pIndex)); + +_return: + + if (pIndex) { + taosArrayDestroyEx(pIndex->pIndex, tFreeSTableIndexInfo); + taosMemoryFreeClear(pIndex); + } + + taosMemoryFreeClear(msg); + + CTG_RET(code); +} + void ctgUpdateThreadUnexpectedStopped(void) { if (CTG_IS_LOCKED(&gCtgMgmt.lock) > 0) CTG_UNLOCK(CTG_READ, &gCtgMgmt.lock); diff --git a/source/libs/catalog/src/ctgDbg.c b/source/libs/catalog/src/ctgDbg.c index d70a176631..9de1ea22be 100644 --- a/source/libs/catalog/src/ctgDbg.c +++ b/source/libs/catalog/src/ctgDbg.c @@ -19,7 +19,7 @@ #include "catalogInt.h" extern SCatalogMgmt gCtgMgmt; -SCtgDebug gCTGDebug = {.apiEnable = true}; +SCtgDebug gCTGDebug = {0}; void ctgdUserCallback(SMetaData* pResult, void* param, int32_t code) { ASSERT(*(int32_t*)param == 1); @@ -266,11 +266,11 @@ int32_t ctgdGetStatNum(char *option, void *res) { } int32_t ctgdGetTbMetaNum(SCtgDBCache *dbCache) { - return dbCache->tbCache.metaCache ? (int32_t)taosHashGetSize(dbCache->tbCache.metaCache) : 0; + return dbCache->tbCache ? (int32_t)taosHashGetSize(dbCache->tbCache) : 0; } int32_t ctgdGetStbNum(SCtgDBCache *dbCache) { - return dbCache->tbCache.stbCache ? (int32_t)taosHashGetSize(dbCache->tbCache.stbCache) : 0; + return dbCache->stbCache ? (int32_t)taosHashGetSize(dbCache->stbCache) : 0; } int32_t ctgdGetRentNum(SCtgRentMgmt *rent) { @@ -363,17 +363,17 @@ void ctgdShowDBCache(SCatalog* pCtg, SHashObj *dbHash) { dbFName = taosHashGetKey(pIter, &len); - int32_t metaNum = dbCache->tbCache.metaCache ? taosHashGetSize(dbCache->tbCache.metaCache) : 0; - int32_t stbNum = dbCache->tbCache.stbCache ? taosHashGetSize(dbCache->tbCache.stbCache) : 0; + int32_t metaNum = dbCache->tbCache ? taosHashGetSize(dbCache->tbCache) : 0; + int32_t stbNum = dbCache->stbCache ? taosHashGetSize(dbCache->stbCache) : 0; int32_t vgVersion = CTG_DEFAULT_INVALID_VERSION; int32_t hashMethod = -1; int32_t vgNum = 0; - if (dbCache->vgInfo) { - vgVersion = dbCache->vgInfo->vgVersion; - hashMethod = dbCache->vgInfo->hashMethod; - if (dbCache->vgInfo->vgHash) { - vgNum = taosHashGetSize(dbCache->vgInfo->vgHash); + if (dbCache->vgCache.vgInfo) { + vgVersion = dbCache->vgCache.vgInfo->vgVersion; + hashMethod = dbCache->vgCache.vgInfo->hashMethod; + if (dbCache->vgCache.vgInfo->vgHash) { + vgNum = taosHashGetSize(dbCache->vgCache.vgInfo->vgHash); } } diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c index bfd5057c69..178025704f 100644 --- a/source/libs/catalog/src/ctgRemote.c +++ b/source/libs/catalog/src/ctgRemote.c @@ -431,7 +431,7 @@ int32_t ctgGetIndexInfoFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const return TSDB_CODE_SUCCESS; } -int32_t ctgGetTbIndexFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SName *name, SArray** out, SCtgTask* pTask) { +int32_t ctgGetTbIndexFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SName *name, STableIndex* out, SCtgTask* pTask) { char *msg = NULL; int32_t msgLen = 0; int32_t reqType = TDMT_MND_GET_TABLE_INDEX; @@ -448,10 +448,11 @@ int32_t ctgGetTbIndexFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SName *n } if (pTask) { - void* pOut = taosMemoryCalloc(1, POINTER_BYTES); + void* pOut = taosMemoryCalloc(1, sizeof(STableIndex)); if (NULL == pOut) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } + CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, (char*)tbFName)); CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask, reqType, msg, msgLen)); diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index a792c555ee..1cb2b99a67 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -44,6 +44,16 @@ char *ctgTaskTypeStr(CTG_TASK_TYPE type) { } } +void ctgFreeSTableIndex(void *info) { + if (NULL == info) { + return; + } + + STableIndex *pInfo = (STableIndex *)info; + + taosArrayDestroyEx(pInfo->pIndex, tFreeSTableIndexInfo); +} + void ctgFreeSMetaData(SMetaData* pData) { taosArrayDestroy(pData->pTableMeta); pData->pTableMeta = NULL; @@ -110,25 +120,39 @@ void ctgFreeMetaRent(SCtgRentMgmt *mgmt) { taosMemoryFreeClear(mgmt->slots); } - -void ctgFreeTbMetaCache(SCtgTbMetaCache *cache) { - CTG_LOCK(CTG_WRITE, &cache->stbLock); - if (cache->stbCache) { - int32_t stblNum = taosHashGetSize(cache->stbCache); - taosHashCleanup(cache->stbCache); - cache->stbCache = NULL; - CTG_CACHE_STAT_DEC(stblNum, stblNum); +void ctgFreeStbMetaCache(SCtgDBCache *dbCache) { + if (NULL == dbCache->stbCache) { + return; } - CTG_UNLOCK(CTG_WRITE, &cache->stbLock); - CTG_LOCK(CTG_WRITE, &cache->metaLock); - if (cache->metaCache) { - int32_t tblNum = taosHashGetSize(cache->metaCache); - taosHashCleanup(cache->metaCache); - cache->metaCache = NULL; - CTG_CACHE_STAT_DEC(tblNum, tblNum); + int32_t stblNum = taosHashGetSize(dbCache->stbCache); + taosHashCleanup(dbCache->stbCache); + dbCache->stbCache = NULL; + CTG_CACHE_STAT_DEC(stblNum, stblNum); +} + +void ctgFreeTbCacheImpl(SCtgTbCache *pCache) { + taosMemoryFreeClear(pCache->pMeta); + if (pCache->pIndex) { + taosArrayDestroyEx(pCache->pIndex->pIndex, tFreeSTableIndexInfo); + taosMemoryFreeClear(pCache->pIndex); } - CTG_UNLOCK(CTG_WRITE, &cache->metaLock); +} + +void ctgFreeTbCache(SCtgDBCache *dbCache) { + if (NULL == dbCache->tbCache) { + return; + } + + int32_t tblNum = taosHashGetSize(dbCache->tbCache); + SCtgTbCache *pCache = taosHashIterate(dbCache->tbCache, NULL); + while (NULL != pCache) { + ctgFreeTbCacheImpl(pCache); + pCache = taosHashIterate(dbCache->tbCache, pCache); + } + taosHashCleanup(dbCache->tbCache); + dbCache->tbCache = NULL; + CTG_CACHE_STAT_DEC(tblNum, tblNum); } void ctgFreeVgInfo(SDBVgInfo *vgInfo) { @@ -144,16 +168,18 @@ void ctgFreeVgInfo(SDBVgInfo *vgInfo) { taosMemoryFreeClear(vgInfo); } +void ctgFreeVgInfoCache(SCtgDBCache *dbCache) { + ctgFreeVgInfo(dbCache->vgCache.vgInfo); +} + void ctgFreeDbCache(SCtgDBCache *dbCache) { if (NULL == dbCache) { return; } - CTG_LOCK(CTG_WRITE, &dbCache->vgLock); - ctgFreeVgInfo (dbCache->vgInfo); - CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); - - ctgFreeTbMetaCache(&dbCache->tbCache); + ctgFreeVgInfoCache(dbCache); + ctgFreeStbMetaCache(dbCache); + ctgFreeTbCache(dbCache); } @@ -167,16 +193,13 @@ void ctgFreeHandle(SCatalog* pCtg) { void *pIter = taosHashIterate(pCtg->dbCache, NULL); while (pIter) { SCtgDBCache *dbCache = pIter; - atomic_store_8(&dbCache->deleted, 1); - ctgFreeDbCache(dbCache); pIter = taosHashIterate(pCtg->dbCache, pIter); } taosHashCleanup(pCtg->dbCache); - CTG_CACHE_STAT_DEC(dbNum, dbNum); } @@ -186,14 +209,12 @@ void ctgFreeHandle(SCatalog* pCtg) { void *pIter = taosHashIterate(pCtg->userCache, NULL); while (pIter) { SCtgUserAuth *userCache = pIter; - ctgFreeSCtgUserAuth(userCache); pIter = taosHashIterate(pCtg->userCache, pIter); } taosHashCleanup(pCtg->userCache); - CTG_CACHE_STAT_DEC(userNum, userNum); } @@ -252,9 +273,9 @@ void ctgFreeMsgCtx(SCtgMsgCtx* pCtx) { break; } case TDMT_MND_GET_TABLE_INDEX: { - SArray** pOut = (SArray**)pCtx->out; + STableIndex* pOut = (STableIndex*)pCtx->out; if (pOut) { - taosArrayDestroyEx(*pOut, tFreeSTableIndexInfo); + taosArrayDestroyEx(pOut->pIndex, tFreeSTableIndexInfo); taosMemoryFreeClear(pCtx->out); } break; @@ -535,9 +556,9 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog *pCtg, SDBVgInfo *dbInfo, const SName } int32_t ctgStbVersionSearchCompare(const void* key1, const void* key2) { - if (*(uint64_t *)key1 < ((SSTableMetaVersion*)key2)->suid) { + if (*(uint64_t *)key1 < ((SSTableVersion*)key2)->suid) { return -1; - } else if (*(uint64_t *)key1 > ((SSTableMetaVersion*)key2)->suid) { + } else if (*(uint64_t *)key1 > ((SSTableVersion*)key2)->suid) { return 1; } else { return 0; @@ -555,9 +576,9 @@ int32_t ctgDbVgVersionSearchCompare(const void* key1, const void* key2) { } int32_t ctgStbVersionSortCompare(const void* key1, const void* key2) { - if (((SSTableMetaVersion*)key1)->suid < ((SSTableMetaVersion*)key2)->suid) { + if (((SSTableVersion*)key1)->suid < ((SSTableVersion*)key2)->suid) { return -1; - } else if (((SSTableMetaVersion*)key1)->suid > ((SSTableMetaVersion*)key2)->suid) { + } else if (((SSTableVersion*)key1)->suid > ((SSTableVersion*)key2)->suid) { return 1; } else { return 0; @@ -640,6 +661,27 @@ int32_t ctgCloneMetaOutput(STableMetaOutput *output, STableMetaOutput **pOutput) return TSDB_CODE_SUCCESS; } +int32_t ctgCloneTableIndex(SArray* pIndex, SArray** pRes) { + if (NULL == pIndex) { + *pRes = NULL; + return TSDB_CODE_SUCCESS; + } + + int32_t num = taosArrayGetSize(pIndex); + *pRes = taosArrayInit(num, sizeof(STableIndexInfo)); + if (NULL == *pRes) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + for (int32_t i = 0; i < num; ++i) { + STableIndexInfo *pInfo = taosArrayGet(pIndex, i); + taosArrayPush(*pRes, pInfo); + } + + return TSDB_CODE_SUCCESS; +} + + int32_t ctgUpdateSendTargetInfo(SMsgSendInfo *pMsgSendInfo, int32_t msgType, SCtgTask* pTask) { if (msgType == TDMT_VND_TABLE_META) { SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx; diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index 04dbc26774..aab341544f 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -895,9 +895,9 @@ void *ctgTestSetCtableMetaThread(void *param) { output = (STableMetaOutput *)taosMemoryMalloc(sizeof(STableMetaOutput)); ctgTestBuildCTableMetaOutput(output); - SCtgUpdateTblMsg *msg = (SCtgUpdateTblMsg *)taosMemoryMalloc(sizeof(SCtgUpdateTblMsg)); + SCtgUpdateTbMetaMsg *msg = (SCtgUpdateTbMetaMsg *)taosMemoryMalloc(sizeof(SCtgUpdateTbMetaMsg)); msg->pCtg = pCtg; - msg->output = output; + msg->pMeta = output; operation.data = msg; code = ctgOpUpdateTbMeta(&operation); @@ -989,7 +989,7 @@ TEST(tableMeta, normalTable) { ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); SDbVgVersion *dbs = NULL; - SSTableMetaVersion *stb = NULL; + SSTableVersion *stb = NULL; uint32_t dbNum = 0, stbNum = 0, allDbNum = 0, allStbNum = 0; int32_t i = 0; while (i < 5) { @@ -1098,7 +1098,7 @@ TEST(tableMeta, childTableCase) { ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); SDbVgVersion *dbs = NULL; - SSTableMetaVersion *stb = NULL; + SSTableVersion *stb = NULL; uint32_t dbNum = 0, stbNum = 0, allDbNum = 0, allStbNum = 0; int32_t i = 0; while (i < 5) { @@ -1220,7 +1220,7 @@ TEST(tableMeta, superTableCase) { ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); SDbVgVersion *dbs = NULL; - SSTableMetaVersion *stb = NULL; + SSTableVersion *stb = NULL; uint32_t dbNum = 0, stbNum = 0, allDbNum = 0, allStbNum = 0; int32_t i = 0; while (i < 5) { @@ -2299,7 +2299,7 @@ TEST(rentTest, allRent) { SArray *vgList = NULL; ctgTestStop = false; SDbVgVersion *dbs = NULL; - SSTableMetaVersion *stable = NULL; + SSTableVersion *stable = NULL; uint32_t num = 0; ctgTestInitLogFile(); diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index 6197edd474..90aafb1097 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -97,7 +97,14 @@ static int32_t sifGetOperParamNum(EOperatorType ty) { } return 2; } -static int32_t sifValidateColumn(SColumnNode *cn) { +static int32_t sifValidOp(EOperatorType ty) { + if ((ty >= OP_TYPE_ADD && ty <= OP_TYPE_BIT_OR) || (ty == OP_TYPE_IN || ty == OP_TYPE_NOT_IN) || + (ty == OP_TYPE_LIKE || ty == OP_TYPE_NOT_LIKE || ty == OP_TYPE_MATCH || ty == OP_TYPE_NMATCH)) { + return -1; + } + return 0; +} +static int32_t sifValidColumn(SColumnNode *cn) { // add more check if (cn == NULL) { return TSDB_CODE_QRY_INVALID_INPUT; @@ -176,6 +183,7 @@ static int32_t sifInitJsonParam(SNode *node, SIFParam *param, SIFCtx *ctx) { memcpy(param->colName, r->literal, strlen(r->literal)); // sprintf(param->colName, "%s_%s", l->colName, r->literal); param->colValType = r->typeData; + param->status = SFLT_COARSE_INDEX; return 0; // memcpy(param->colName, l->colName, sizeof(l->colName)); } @@ -197,7 +205,7 @@ static int32_t sifInitParam(SNode *node, SIFParam *param, SIFCtx *ctx) { case QUERY_NODE_COLUMN: { SColumnNode *cn = (SColumnNode *)node; /*only support tag column*/ - SIF_ERR_RET(sifValidateColumn(cn)); + SIF_ERR_RET(sifValidColumn(cn)); param->colId = cn->colId; param->colValType = cn->node.resType.type; @@ -247,11 +255,13 @@ static int32_t sifInitOperParams(SIFParam **params, SOperatorNode *node, SIFCtx return code; } SIFParam *paramList = taosMemoryCalloc(nParam, sizeof(SIFParam)); + if (NULL == paramList) { SIF_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - if (nodeType(node->pLeft) == QUERY_NODE_OPERATOR) { + if (nodeType(node->pLeft) == QUERY_NODE_OPERATOR && + (((SOperatorNode *)(node->pLeft))->opType == OP_TYPE_JSON_GET_VALUE)) { SNode *interNode = (node->pLeft); SIF_ERR_JRET(sifInitJsonParam(interNode, ¶mList[0], ctx)); if (nParam > 1) { @@ -505,6 +515,11 @@ static int32_t sifGetOperFn(int32_t funcId, sif_func_t *func, SIdxFltStatus *sta static int32_t sifExecOper(SOperatorNode *node, SIFCtx *ctx, SIFParam *output) { int32_t code = 0; + if (sifValidOp(node->opType) < 0) { + output->status = SFLT_NOT_INDEX; + return code; + } + int32_t nParam = sifGetOperParamNum(node->opType); if (nParam <= 1) { output->status = SFLT_NOT_INDEX; @@ -513,9 +528,15 @@ static int32_t sifExecOper(SOperatorNode *node, SIFCtx *ctx, SIFParam *output) { if (node->opType == OP_TYPE_JSON_GET_VALUE) { return code; } - SIFParam *params = NULL; + SIFParam *params = NULL; SIF_ERR_RET(sifInitOperParams(¶ms, node, ctx)); + + if (params[0].status == SFLT_NOT_INDEX || (nParam > 1 && params[1].status == SFLT_NOT_INDEX)) { + output->status = SFLT_NOT_INDEX; + return code; + } + // ugly code, refactor later output->arg = ctx->arg; sif_func_t operFn = sifNullFunc; diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 10d0f4a44a..b89c220519 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -484,13 +484,11 @@ int32_t queryProcessGetTbIndexRsp(void *output, char *msg, int32_t msgSize) { return TSDB_CODE_TSC_INVALID_INPUT; } - STableIndexRsp out = {0}; - if (tDeserializeSTableIndexRsp(msg, msgSize, &out) != 0) { + STableIndexRsp *out = (STableIndexRsp*)output; + if (tDeserializeSTableIndexRsp(msg, msgSize, out) != 0) { qError("tDeserializeSTableIndexRsp failed, msgSize:%d", msgSize); return TSDB_CODE_INVALID_MSG; } - - *(void **)output = out.pIndex; return TSDB_CODE_SUCCESS; } diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index ae0eda5449..451607e7d0 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -837,7 +837,6 @@ void qwProcessHbTimerEvent(void *param, void *tmrId) { SQWorker *mgmt = qwAcquire(refId); if (NULL == mgmt) { QW_DLOG("qwAcquire %" PRIx64 "failed", refId); - taosMemoryFree(param); return; } diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 6704bfdde3..a446aa7bdb 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -1476,6 +1476,11 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) for (uint32_t i = 0; i < info->fields[FLD_TYPE_VALUE].num; ++i) { SFilterField *field = &info->fields[FLD_TYPE_VALUE].fields[i]; if (field->desc) { + if (QUERY_NODE_VALUE != nodeType(field->desc)) { + qDebug("VAL%d => [type:not value node][val:NIL]", i); //TODO + continue; + } + SValueNode *var = (SValueNode *)field->desc; SDataType *dType = &var->node.resType; if (dType->type == TSDB_DATA_TYPE_VALUE_ARRAY) { diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index f3faa60d34..e7f11b9a7d 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1658,14 +1658,24 @@ static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) { syncTimeout2RpcMsg(pSyncMsg, &rpcMsg); syncRpcMsgLog2((char*)"==syncNodeEqHeartbeatTimer==", &rpcMsg); if (pSyncNode->FpEqMsg != NULL) { - pSyncNode->FpEqMsg(pSyncNode->msgcb, &rpcMsg); + int32_t code = pSyncNode->FpEqMsg(pSyncNode->msgcb, &rpcMsg); + if (code != 0) { + sError("vgId:%d sync enqueue timer msg error, code:%d", pSyncNode->vgId, code); + rpcFreeCont(rpcMsg.pCont); + return; + } + } else { sTrace("syncNodeEqHeartbeatTimer pSyncNode->FpEqMsg is NULL"); } syncTimeoutDestroy(pSyncMsg); + if (gSyncEnv != NULL) { taosTmrReset(syncNodeEqHeartbeatTimer, pSyncNode->heartbeatTimerMS, pSyncNode, gSyncEnv->pTimerManager, &pSyncNode->pHeartbeatTimer); + } else { + sError("sync env is already stop"); + } } else { sTrace("==syncNodeEqHeartbeatTimer== heartbeatTimerLogicClock:%" PRIu64 ", heartbeatTimerLogicClockUser:%" PRIu64 "", diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index a34b332b9d..2357f760d1 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -458,6 +458,9 @@ static void taosArrayInsertSort(SArray* pArray, __ext_compar_fn_t fn, const void } SArray* taosArrayDeepCopy(const SArray* pSrc, FCopy deepCopy) { + if (NULL == pSrc) { + return NULL; + } ASSERT(pSrc->elemSize == sizeof(void*)); SArray* pArray = taosArrayInit(pSrc->size, sizeof(void*)); for (int32_t i = 0; i < pSrc->size; i++) { diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 1c694bf7fb..db8a055362 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -58,7 +58,7 @@ # ---- mnode ./test.sh -f tsim/mnode/basic1.sim ./test.sh -f tsim/mnode/basic2.sim -#./test.sh -f tsim/mnode/basic3.sim +./test.sh -f tsim/mnode/basic3.sim ./test.sh -f tsim/mnode/basic4.sim ./test.sh -f tsim/mnode/basic5.sim diff --git a/tests/script/tsim/stable/tag_filter.sim b/tests/script/tsim/stable/tag_filter.sim index 04b8f21de9..1f400eb803 100644 --- a/tests/script/tsim/stable/tag_filter.sim +++ b/tests/script/tsim/stable/tag_filter.sim @@ -73,5 +73,80 @@ if $rows != 6 then endi +print ========== prepare stbBin and ctbBin +sql create table db.stbBin (ts timestamp, c1 int, c2 binary(4)) tags(t1 binary(16)) + +sql create table db.ctbBin using db.stbBin tags("a") +sql insert into db.ctbBin values(now, 1, "2") + +sql create table db.ctbBin1 using db.stbBin tags("b") +sql insert into db.ctbBin1 values(now, 2, "2") + +sql create table db.ctbBin2 using db.stbBin tags("c") +sql insert into db.ctbBin2 values(now, 3, "2") + +sql create table db.ctbBin3 using db.stbBin tags("d") +sql insert into db.ctbBin3 values(now, 4, "2") + +sql select * from db.stbBin where t1 = "a" +if $rows != 1 then + return -1 +endi + + +sql select * from db.stbBin where t1 < "a" +if $rows != 0 then + return -=1 +endi + +sql select * from db.stbBin where t1 < "b" +if $rows != 1 then + return -1 +endi + + +sql select * from db.stbBin where t1 between "a" and "e" +if $rows != 4 then + return -1 +endi + + +print ========== prepare stbNc and ctbNc +sql create table db.stbNc (ts timestamp, c1 int, c2 binary(4)) tags(t1 nchar(16)) + + +sql create table db.ctbNc using db.stbNc tags("a") +sql insert into db.ctbNc values(now, 1, "2") + +sql create table db.ctbNc1 using db.stbNc tags("b") +sql insert into db.ctbNc1 values(now, 2, "2") + +sql create table db.ctbNc2 using db.stbNc tags("c") +sql insert into db.ctbNc2 values(now, 3, "2") + +sql create table db.ctbNc3 using db.stbNc tags("d") +sql insert into db.ctbNc3 values(now, 4, "2") + +sql select * from db.stbNc where t1 = "a" +if $rows != 1 then + return -1 +endi + + +sql select * from db.stbNc where t1 < "a" +if $rows != 0 then + return -=1 +endi + +sql select * from db.stbNc where t1 < "b" +if $rows != 1 then + return -1 +endi + + +sql select * from db.stbNc where t1 between "a" and "e" +if $rows != 4 then + return -1 +endi system sh/exec.sh -n dnode1 -s stop -x SIGINT