From 6be54c112e2de4e38386cca041a33b6a612fab11 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 28 Jan 2022 17:50:16 +0800 Subject: [PATCH 1/7] catalog refactor --- include/common/tmsg.h | 14 +- include/libs/catalog/catalog.h | 4 +- include/libs/qcom/query.h | 12 +- source/client/src/clientHb.c | 17 +- source/client/src/clientMsgHandler.c | 7 +- source/dnode/mgmt/impl/test/sut/src/sut.cpp | 4 +- source/dnode/mnode/impl/src/mndBnode.c | 2 +- source/dnode/mnode/impl/src/mndCluster.c | 4 +- source/dnode/mnode/impl/src/mndDb.c | 2 +- source/dnode/mnode/impl/src/mndDnode.c | 4 +- source/dnode/mnode/impl/src/mndFunc.c | 2 +- source/dnode/mnode/impl/src/mndMnode.c | 2 +- source/dnode/mnode/impl/src/mndProfile.c | 4 +- source/dnode/mnode/impl/src/mndQnode.c | 2 +- source/dnode/mnode/impl/src/mndSnode.c | 2 +- source/dnode/mnode/impl/src/mndStb.c | 22 +- source/dnode/mnode/impl/src/mndSubscribe.c | 4 +- source/dnode/mnode/impl/src/mndTopic.c | 4 +- source/dnode/mnode/impl/src/mndUser.c | 2 +- source/dnode/mnode/impl/src/mndVgroup.c | 4 +- source/dnode/mnode/impl/test/show/show.cpp | 2 +- source/dnode/mnode/impl/test/stb/stb.cpp | 7 +- source/dnode/vnode/src/vnd/vnodeQuery.c | 14 +- source/libs/catalog/inc/catalogInt.h | 53 +- source/libs/catalog/src/catalog.c | 702 ++++++++++++-------- source/libs/catalog/test/catalogTests.cpp | 34 +- source/libs/qcom/src/querymsg.c | 52 +- 27 files changed, 548 insertions(+), 434 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index bbe3975cfb..eace9bd90d 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -149,8 +149,8 @@ typedef enum _mgmt_table { #define TSDB_COL_REQ_NULL(f) (((f)&TSDB_COL_NULL) != 0) typedef struct { int32_t vgId; - char* dbName; - char* tableFullName; + char* dbFName; + char* tbName; } SBuildTableMetaInput; typedef struct { @@ -691,8 +691,8 @@ typedef struct { typedef struct { SMsgHead header; - char dbFname[TSDB_DB_FNAME_LEN]; - char tableFname[TSDB_TABLE_FNAME_LEN]; + char dbFName[TSDB_DB_FNAME_LEN]; + char tbName[TSDB_TABLE_NAME_LEN]; } STableInfoReq; typedef struct { @@ -717,9 +717,9 @@ typedef struct { } SVgroupsInfo; typedef struct { - char tbFname[TSDB_TABLE_FNAME_LEN]; // table full name - char stbFname[TSDB_TABLE_FNAME_LEN]; - char dbFname[TSDB_DB_FNAME_LEN]; + char tbName[TSDB_TABLE_NAME_LEN]; + char stbName[TSDB_TABLE_NAME_LEN]; + char dbFName[TSDB_DB_FNAME_LEN]; int32_t numOfTags; int32_t numOfColumns; int8_t precision; diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 0b465b7b4e..6f4be556e4 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -49,7 +49,7 @@ typedef struct SCatalogCfg { uint32_t maxTblCacheNum; uint32_t maxDBCacheNum; uint32_t dbRentSec; - uint32_t stableRentSec; + uint32_t stbRentSec; } SCatalogCfg; typedef struct SSTableMetaVersion { @@ -99,7 +99,7 @@ int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pTransporter, const int32_t catalogUpdateDBVgroup(struct SCatalog* pCatalog, const char* dbName, SDBVgroupInfo* dbInfo); -int32_t catalogRemoveDBVgroup(struct SCatalog* pCatalog, SDbVgVersion* dbInfo); +int32_t catalogRemoveDB(struct SCatalog* pCatalog, const char* dbName, uint64_t dbId); /** * Get a table's meta data. diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index c7ca2d9e1d..2e4093590d 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -81,16 +81,15 @@ typedef struct STableMeta { } STableMeta; typedef struct SDBVgroupInfo { - SRWLatch lock; uint64_t dbId; int32_t vgVersion; int8_t hashMethod; - SHashObj *vgInfo; //key:vgId, value:SVgroupInfo + SHashObj *vgHash; //key:vgId, value:SVgroupInfo } SDBVgroupInfo; typedef struct SUseDbOutput { - char db[TSDB_DB_FNAME_LEN]; - SDBVgroupInfo dbVgroup; + char db[TSDB_DB_FNAME_LEN]; + SDBVgroupInfo *dbVgroup; } SUseDbOutput; enum { @@ -103,8 +102,9 @@ enum { typedef struct STableMetaOutput { int32_t metaType; - char ctbFname[TSDB_TABLE_FNAME_LEN]; - char tbFname[TSDB_TABLE_FNAME_LEN]; + char dbFName[TSDB_DB_FNAME_LEN]; + char ctbName[TSDB_TABLE_NAME_LEN]; + char tbName[TSDB_TABLE_NAME_LEN]; SCTableMeta ctbMeta; STableMeta *tbMeta; } STableMetaOutput; diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index f6c473333c..354df00fa7 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -41,19 +41,14 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog tscDebug("hb db rsp, db:%s, vgVersion:%d, uid:%"PRIx64, rsp->db, rsp->vgVersion, rsp->uid); if (rsp->vgVersion < 0) { - SDbVgVersion dbInfo; - strcpy(dbInfo.dbName, rsp->db); - dbInfo.dbId = rsp->uid; - dbInfo.vgVersion = rsp->vgVersion; - - code = catalogRemoveDBVgroup(pCatalog, &dbInfo); + code = catalogRemoveDB(pCatalog, rsp->db, rsp->uid); } else { SDBVgroupInfo vgInfo = {0}; vgInfo.dbId = rsp->uid; vgInfo.vgVersion = rsp->vgVersion; vgInfo.hashMethod = rsp->hashMethod; - vgInfo.vgInfo = taosHashInit(rsp->vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); - if (NULL == vgInfo.vgInfo) { + vgInfo.vgHash = taosHashInit(rsp->vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); + if (NULL == vgInfo.vgHash) { tscError("hash init[%d] failed", rsp->vgNum); return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -67,16 +62,16 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog rsp->vgroupInfo[i].epset.eps[n].port = ntohs(rsp->vgroupInfo[i].epset.eps[n].port); } - if (0 != taosHashPut(vgInfo.vgInfo, &rsp->vgroupInfo[i].vgId, sizeof(rsp->vgroupInfo[i].vgId), &rsp->vgroupInfo[i], sizeof(rsp->vgroupInfo[i]))) { + if (0 != taosHashPut(vgInfo.vgHash, &rsp->vgroupInfo[i].vgId, sizeof(rsp->vgroupInfo[i].vgId), &rsp->vgroupInfo[i], sizeof(rsp->vgroupInfo[i]))) { tscError("hash push failed, errno:%d", errno); - taosHashCleanup(vgInfo.vgInfo); + taosHashCleanup(vgInfo.vgHash); return TSDB_CODE_TSC_OUT_OF_MEMORY; } } code = catalogUpdateDBVgroup(pCatalog, rsp->db, &vgInfo); if (code) { - taosHashCleanup(vgInfo.vgInfo); + taosHashCleanup(vgInfo.vgHash); } } diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 24d1c0f03c..35d8a2a2ba 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -301,15 +301,12 @@ int32_t processDropDbRsp(void* param, const SDataBuf* pMsg, int32_t code) { SDropDbRsp *rsp = (SDropDbRsp *)pMsg->pData; - SDbVgVersion dbVer = {0}; struct SCatalog *pCatalog = NULL; - - strncpy(dbVer.dbName, rsp->db, sizeof(dbVer.dbName)); - dbVer.dbId = be64toh(rsp->uid); + rsp->uid = be64toh(rsp->uid); catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); - catalogRemoveDBVgroup(pCatalog, &dbVer); + catalogRemoveDB(pCatalog, rsp->db, rsp->uid); tsem_post(&pRequest->body.rspSem); return code; diff --git a/source/dnode/mgmt/impl/test/sut/src/sut.cpp b/source/dnode/mgmt/impl/test/sut/src/sut.cpp index c6e6c0006d..dc8390fba3 100644 --- a/source/dnode/mgmt/impl/test/sut/src/sut.cpp +++ b/source/dnode/mgmt/impl/test/sut/src/sut.cpp @@ -125,7 +125,7 @@ const char* Testbase::GetMetaName(int32_t index) { int32_t Testbase::GetMetaNum() { return pMeta->numOfColumns; } -const char* Testbase::GetMetaTbName() { return pMeta->tbFname; } +const char* Testbase::GetMetaTbName() { return pMeta->tbName; } void Testbase::SendShowRetrieveReq() { int32_t contLen = sizeof(SRetrieveTableReq); @@ -144,7 +144,7 @@ void Testbase::SendShowRetrieveReq() { pos = 0; } -const char* Testbase::GetShowName() { return pMeta->tbFname; } +const char* Testbase::GetShowName() { return pMeta->tbName; } int8_t Testbase::GetShowInt8() { int8_t data = *((int8_t*)(pData + pos)); diff --git a/source/dnode/mnode/impl/src/mndBnode.c b/source/dnode/mnode/impl/src/mndBnode.c index 189cbfea6d..c754494c24 100644 --- a/source/dnode/mnode/impl/src/mndBnode.c +++ b/source/dnode/mnode/impl/src/mndBnode.c @@ -426,7 +426,7 @@ static int32_t mndGetBnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * pShow->numOfRows = sdbGetSize(pSdb, SDB_BNODE); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbName, mndShowStr(pShow->type)); return 0; } diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index 5f80672369..a53298efb3 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -186,7 +186,7 @@ static int32_t mndGetClusterMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp cols++; pMeta->numOfColumns = htonl(cols); - strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbName, mndShowStr(pShow->type)); pShow->numOfColumns = cols; pShow->offset[0] = 0; @@ -196,7 +196,7 @@ static int32_t mndGetClusterMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp pShow->numOfRows = 1; pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbName, mndShowStr(pShow->type)); return 0; } diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 5e6f457a67..f71f3ac478 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -1113,7 +1113,7 @@ static int32_t mndGetDbMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMe pShow->numOfRows = sdbGetSize(pSdb, SDB_DB); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbName, mndShowStr(pShow->type)); return 0; } diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 9a96bdb0df..3b43cd8081 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -608,7 +608,7 @@ static int32_t mndGetConfigMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp pShow->numOfRows = TSDB_CONFIG_NUMBER; pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbName, mndShowStr(pShow->type)); return 0; } @@ -715,7 +715,7 @@ static int32_t mndGetDnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * pShow->numOfRows = sdbGetSize(pSdb, SDB_DNODE); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbName, mndShowStr(pShow->type)); return 0; } diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index 0e086fdb22..b3fc0e1da0 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -482,7 +482,7 @@ static int32_t mndGetFuncMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *p pShow->numOfRows = sdbGetSize(pSdb, SDB_FUNC); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbName, mndShowStr(pShow->type)); return 0; } diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index a969c0162f..dde48853ad 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -624,7 +624,7 @@ static int32_t mndGetMnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * pShow->numOfRows = sdbGetSize(pSdb, SDB_MNODE); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbName, mndShowStr(pShow->type)); mndUpdateMnodeRole(pMnode); return 0; diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index d379d294ab..3d9e3ff3de 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -623,7 +623,7 @@ static int32_t mndGetConnsMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * pShow->numOfRows = taosHashGetSize(pMgmt->cache->pHashTable); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbName, mndShowStr(pShow->type)); return 0; } @@ -792,7 +792,7 @@ static int32_t mndGetQueryMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * pShow->numOfRows = 1000000; pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbName, mndShowStr(pShow->type)); return 0; } diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c index 6077a95a7b..e29ab5a5bd 100644 --- a/source/dnode/mnode/impl/src/mndQnode.c +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -426,7 +426,7 @@ static int32_t mndGetQnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * pShow->numOfRows = sdbGetSize(pSdb, SDB_QNODE); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbName, mndShowStr(pShow->type)); return 0; } diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c index 7688ea76ab..cbba659719 100644 --- a/source/dnode/mnode/impl/src/mndSnode.c +++ b/source/dnode/mnode/impl/src/mndSnode.c @@ -428,7 +428,7 @@ static int32_t mndGetSnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * pShow->numOfRows = sdbGetSize(pSdb, SDB_SNODE); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbName, mndShowStr(pShow->type)); return 0; } diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 45b8428663..fcefa4838a 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -684,20 +684,23 @@ static int32_t mndProcessStbMetaReq(SMnodeMsg *pReq) { SMnode *pMnode = pReq->pMnode; STableInfoReq *pInfo = pReq->rpcMsg.pCont; - mDebug("stb:%s, start to retrieve meta", pInfo->tableFname); + char tbFName[TSDB_TABLE_FNAME_LEN] = {0}; + snprintf(tbFName, sizeof(tbFName), "%s.%s", pInfo->dbFName, pInfo->tbName); - SDbObj *pDb = mndAcquireDbByStb(pMnode, pInfo->tableFname); + mDebug("stb:%s, start to retrieve meta", tbFName); + + SDbObj *pDb = mndAcquireDbByStb(pMnode, tbFName); if (pDb == NULL) { terrno = TSDB_CODE_MND_DB_NOT_SELECTED; - mError("stb:%s, failed to retrieve meta since %s", pInfo->tableFname, terrstr()); + mError("stb:%s, failed to retrieve meta since %s", tbFName, terrstr()); return -1; } - SStbObj *pStb = mndAcquireStb(pMnode, pInfo->tableFname); + SStbObj *pStb = mndAcquireStb(pMnode, tbFName); if (pStb == NULL) { mndReleaseDb(pMnode, pDb); terrno = TSDB_CODE_MND_INVALID_STB; - mError("stb:%s, failed to get meta since %s", pInfo->tableFname, terrstr()); + mError("stb:%s, failed to get meta since %s", tbFName, terrstr()); return -1; } @@ -711,11 +714,12 @@ static int32_t mndProcessStbMetaReq(SMnodeMsg *pReq) { mndReleaseDb(pMnode, pDb); mndReleaseStb(pMnode, pStb); terrno = TSDB_CODE_OUT_OF_MEMORY; - mError("stb:%s, failed to get meta since %s", pInfo->tableFname, terrstr()); + mError("stb:%s, failed to get meta since %s", tbFName, terrstr()); return -1; } - memcpy(pMeta->tbFname, pStb->name, TSDB_TABLE_FNAME_LEN); + strcpy(pMeta->dbFName, pStb->db); + strcpy(pMeta->tbName, pInfo->tbName); pMeta->numOfTags = htonl(pStb->numOfTags); pMeta->numOfColumns = htonl(pStb->numOfColumns); pMeta->precision = pDb->cfg.precision; @@ -740,7 +744,7 @@ static int32_t mndProcessStbMetaReq(SMnodeMsg *pReq) { pReq->pCont = pMeta; pReq->contLen = contLen; - mDebug("stb:%s, meta is retrieved, cols:%d tags:%d", pInfo->tableFname, pStb->numOfColumns, pStb->numOfTags); + mDebug("stb:%s, meta is retrieved, cols:%d tags:%d", tbFName, pStb->numOfColumns, pStb->numOfTags); return 0; } @@ -816,7 +820,7 @@ static int32_t mndGetStbMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pM pShow->numOfRows = sdbGetSize(pSdb, SDB_STB); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbName, mndShowStr(pShow->type)); return 0; } diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 4269196c2e..f93cad0a4d 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -646,7 +646,7 @@ static int32_t mndProcessConsumerMetaMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; STableInfoReq *pInfo = pMsg->rpcMsg.pCont; - mDebug("subscribe:%s, start to retrieve meta", pInfo->tableFname); + mDebug("subscribe:%s, start to retrieve meta", pInfo->tbName); #if 0 SDbObj *pDb = mndAcquireDbByConsumer(pMnode, pInfo->tableFname); @@ -777,7 +777,7 @@ static int32_t mndGetConsumerMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRs pShow->numOfRows = sdbGetSize(pSdb, SDB_CONSUMER); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbName, mndShowStr(pShow->type)); return 0; } diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 6b4cb4ba59..9b821fe615 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -339,7 +339,7 @@ static int32_t mndProcessTopicMetaMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; STableInfoReq *pInfo = pMsg->rpcMsg.pCont; - mDebug("topic:%s, start to retrieve meta", pInfo->tableFname); + mDebug("topic:%s, start to retrieve meta", pInfo->tbName); #if 0 SDbObj *pDb = mndAcquireDbByTopic(pMnode, pInfo->tableFname); @@ -470,7 +470,7 @@ static int32_t mndGetTopicMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp * pShow->numOfRows = sdbGetSize(pSdb, SDB_TOPIC); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbName, mndShowStr(pShow->type)); return 0; } diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index c36e128893..3040cd8a28 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -474,7 +474,7 @@ static int32_t mndGetUserMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *p pShow->numOfRows = sdbGetSize(pSdb, SDB_USER); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbName, mndShowStr(pShow->type)); return 0; } diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 01d39ebdc2..33aa4ce07c 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -525,7 +525,7 @@ static int32_t mndGetVgroupMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp } pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbName, mndShowStr(pShow->type)); return 0; } @@ -638,7 +638,7 @@ static int32_t mndGetVnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * pShow->replica = dnodeId; pShow->numOfRows = mndGetVnodesNum(pMnode, dnodeId); pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - strcpy(pMeta->tbFname, mndShowStr(pShow->type)); + strcpy(pMeta->tbName, mndShowStr(pShow->type)); return 0; } diff --git a/source/dnode/mnode/impl/test/show/show.cpp b/source/dnode/mnode/impl/test/show/show.cpp index bfdc0f42b6..bc31630ffe 100644 --- a/source/dnode/mnode/impl/test/show/show.cpp +++ b/source/dnode/mnode/impl/test/show/show.cpp @@ -64,7 +64,7 @@ TEST_F(MndTestShow, 03_ShowMsg_Conn) { test.SendShowMetaReq(TSDB_MGMT_TABLE_CONNS, ""); STableMetaRsp* pMeta = test.GetShowMeta(); - EXPECT_STREQ(pMeta->tbFname, "show connections"); + EXPECT_STREQ(pMeta->tbName, "show connections"); EXPECT_EQ(pMeta->numOfTags, 0); EXPECT_EQ(pMeta->numOfColumns, 7); EXPECT_EQ(pMeta->precision, 0); diff --git a/source/dnode/mnode/impl/test/stb/stb.cpp b/source/dnode/mnode/impl/test/stb/stb.cpp index beb52d68c1..887fde312d 100644 --- a/source/dnode/mnode/impl/test/stb/stb.cpp +++ b/source/dnode/mnode/impl/test/stb/stb.cpp @@ -126,7 +126,8 @@ TEST_F(MndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { int32_t contLen = sizeof(STableInfoReq); STableInfoReq* pReq = (STableInfoReq*)rpcMallocCont(contLen); - strcpy(pReq->tableFname, "1.d1.stb"); + strcpy(pReq->dbFName, "1.d1"); + strcpy(pReq->tbName, "stb"); SRpcMsg* pMsg = test.SendReq(TDMT_MND_STB_META, pReq, contLen); ASSERT_NE(pMsg, nullptr); @@ -146,8 +147,8 @@ TEST_F(MndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { pSchema->bytes = htonl(pSchema->bytes); } - EXPECT_STREQ(pRsp->tbFname, "1.d1.stb"); - EXPECT_STREQ(pRsp->stbFname, ""); + EXPECT_STREQ(pRsp->tbName, "1.d1"); + EXPECT_STREQ(pRsp->stbName, "stb"); EXPECT_EQ(pRsp->numOfColumns, 2); EXPECT_EQ(pRsp->numOfTags, 3); EXPECT_EQ(pRsp->precision, TSDB_TIME_PRECISION_MILLI); diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 6692264c54..5575deef03 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -82,7 +82,11 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg) { int msgLen = 0; int32_t code = TSDB_CODE_VND_APP_ERROR; - pTbCfg = metaGetTbInfoByName(pVnode->pMeta, pReq->tableFname, &uid); + char tbFName[TSDB_TABLE_FNAME_LEN] = {0}; + snprintf(tbFName, sizeof(tbFName), "%s.%s", pReq->dbFName, pReq->tbName); + + + pTbCfg = metaGetTbInfoByName(pVnode->pMeta, tbFName, &uid); if (pTbCfg == NULL) { code = TSDB_CODE_VND_TB_NOT_EXIST; goto _exit; @@ -117,13 +121,13 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg) { goto _exit; } - memcpy(pTbMetaMsg->dbFname, pReq->dbFname, sizeof(pTbMetaMsg->dbFname)); - strcpy(pTbMetaMsg->tbFname, pTbCfg->name); + memcpy(pTbMetaMsg->dbFName, pReq->dbFName, sizeof(pTbMetaMsg->dbFName)); + strcpy(pTbMetaMsg->tbName, pReq->tbName); if (pTbCfg->type == META_CHILD_TABLE) { - strcpy(pTbMetaMsg->stbFname, pStbCfg->name); + strcpy(pTbMetaMsg->stbName, pStbCfg->name + strlen(pReq->dbFName) + 1); pTbMetaMsg->suid = htobe64(pTbCfg->ctbCfg.suid); } else if (pTbCfg->type == META_SUPER_TABLE) { - strcpy(pTbMetaMsg->stbFname, pTbCfg->name); + strcpy(pTbMetaMsg->stbName, pTbCfg->name + strlen(pReq->dbFName) + 1); pTbMetaMsg->suid = htobe64(uid); } pTbMetaMsg->numOfTags = htonl(nTagCols); diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 7128e143bf..22997feb3c 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -47,55 +47,52 @@ enum { CTG_RENT_STABLE, }; -typedef struct SCTGDebug { +typedef struct SCtgDebug { int32_t lockDebug; -} SCTGDebug; +} SCtgDebug; -typedef struct SVgroupListCache { - int32_t vgroupVersion; - SHashObj *cache; // key:vgId, value:SVgroupInfo -} SVgroupListCache; +typedef struct SCtgTbMetaCache { + SRWLatch stbLock; + SHashObj *cache; //key:tbname, value:STableMeta + SHashObj *stbCache; //key:suid, value:STableMeta* +} SCtgTbMetaCache; -typedef struct SDBVgroupCache { - SHashObj *cache; //key:dbname, value:SDBVgroupInfo -} SDBVgroupCache; +typedef struct SCtgDBCache { + SRWLatch vgLock; + int8_t deleted; + SDBVgroupInfo *vgInfo; + SCtgTbMetaCache tbCache; +} SCtgDBCache; -typedef struct STableMetaCache { - SRWLatch stableLock; - SHashObj *cache; //key:fulltablename, value:STableMeta - SHashObj *stableCache; //key:suid, value:STableMeta* -} STableMetaCache; - -typedef struct SRentSlotInfo { +typedef struct SCtgRentSlot { SRWLatch lock; bool needSort; SArray *meta; // element is SDbVgVersion or SSTableMetaVersion -} SRentSlotInfo; +} SCtgRentSlot; -typedef struct SMetaRentMgmt { +typedef struct SCtgRentMgmt { int8_t type; uint16_t slotNum; uint16_t slotRIdx; int64_t lastReadMsec; - SRentSlotInfo *slots; -} SMetaRentMgmt; + SCtgRentSlot *slots; +} SCtgRentMgmt; typedef struct SCatalog { - uint64_t clusterId; - SDBVgroupCache dbCache; - STableMetaCache tableCache; - SMetaRentMgmt dbRent; - SMetaRentMgmt stableRent; + uint64_t clusterId; + SHashObj *dbCache; //key:dbname, value:SCtgDBCache + SCtgRentMgmt dbRent; + SCtgRentMgmt stbRent; } SCatalog; typedef struct SCtgApiStat { } SCtgApiStat; -typedef struct SCtgResourceStat { +typedef struct SCtgRuntimeStat { -} SCtgResourceStat; +} SCtgRuntimeStat; typedef struct SCtgCacheStat { @@ -103,7 +100,7 @@ typedef struct SCtgCacheStat { typedef struct SCatalogStat { SCtgApiStat api; - SCtgResourceStat resource; + SCtgRuntimeStat runtime; SCtgCacheStat cache; } SCatalogStat; diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 1b4d711aad..996597de20 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -20,31 +20,106 @@ SCatalogMgmt ctgMgmt = {0}; -SCTGDebug gCTGDebug = {0}; +SCtgDebug gCTGDebug = {0}; -int32_t ctgGetDBVgroupFromCache(struct SCatalog* pCatalog, const char *dbName, SDBVgroupInfo **dbInfo, bool *inCache) { - if (NULL == pCatalog->dbCache.cache) { +void ctgFreeMetaRent(SCtgRentMgmt *mgmt) { + if (NULL == mgmt->slots) { + return; + } + + for (int32_t i = 0; i < mgmt->slotNum; ++i) { + SCtgRentSlot *slot = &mgmt->slots[i]; + if (slot->meta) { + taosArrayDestroy(slot->meta); + slot->meta = NULL; + } + } + + tfree(mgmt->slots); +} + + +void ctgFreeTableMetaCache(SCtgTbMetaCache *table) { + CTG_LOCK(CTG_WRITE, &table->stbLock); + if (table->stbCache) { + taosHashCleanup(table->stbCache); + table->stbCache = NULL; + } + CTG_UNLOCK(CTG_WRITE, &table->stbLock); + + if (table->cache) { + taosHashCleanup(table->cache); + table->cache = NULL; + } +} + +void ctgFreeDbCache(SCtgDBCache *dbCache) { + if (NULL == dbCache) { + return; + } + + atomic_store_8(&dbCache->deleted, 1); + + SDBVgroupInfo *dbInfo = NULL; + if (dbCache->vgInfo) { + CTG_LOCK(CTG_WRITE, &dbCache->vgLock); + + if (dbCache->vgInfo->vgHash) { + taosHashCleanup(dbCache->vgInfo->vgHash); + dbCache->vgInfo->vgHash = NULL; + } + + tfree(dbCache->vgInfo); + CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); + } + + ctgFreeTableMetaCache(&dbCache->tbCache); +} + + +void ctgFreeHandle(struct SCatalog* pCatalog) { + ctgFreeMetaRent(&pCatalog->dbRent); + ctgFreeMetaRent(&pCatalog->stbRent); + if (pCatalog->dbCache) { + void *pIter = taosHashIterate(pCatalog->dbCache, NULL); + while (pIter) { + SCtgDBCache *dbCache = pIter; + + ctgFreeDbCache(dbCache); + + pIter = taosHashIterate(pCatalog->dbCache, pIter); + } + + taosHashCleanup(pCatalog->dbCache); + } + + free(pCatalog); +} + + +int32_t ctgGetDBVgroupFromCache(struct SCatalog* pCatalog, const char *dbName, SCtgDBCache **dbCache, bool *inCache) { + if (NULL == pCatalog->dbCache) { *inCache = false; ctgWarn("empty db cache, dbName:%s", dbName); return TSDB_CODE_SUCCESS; } - SDBVgroupInfo *info = NULL; + SCtgDBCache *cache = NULL; while (true) { - info = taosHashAcquire(pCatalog->dbCache.cache, dbName, strlen(dbName)); + cache = taosHashAcquire(pCatalog->dbCache, dbName, strlen(dbName)); - if (NULL == info) { + if (NULL == cache) { *inCache = false; ctgWarn("not in db vgroup cache, dbName:%s", dbName); return TSDB_CODE_SUCCESS; } - CTG_LOCK(CTG_READ, &info->lock); - if (NULL == info->vgInfo) { - CTG_UNLOCK(CTG_READ, &info->lock); - taosHashRelease(pCatalog->dbCache.cache, info); + CTG_LOCK(CTG_READ, &cache->vgLock); + if (NULL == cache->vgInfo) { + CTG_UNLOCK(CTG_READ, &cache->vgLock); + taosHashRelease(pCatalog->dbCache, cache); ctgWarn("db cache vgInfo is NULL, dbName:%s", dbName); continue; @@ -53,7 +128,7 @@ int32_t ctgGetDBVgroupFromCache(struct SCatalog* pCatalog, const char *dbName, S break; } - *dbInfo = info; + *dbCache = cache; *inCache = true; ctgDebug("Got db vgroup from cache, dbName:%s", dbName); @@ -98,48 +173,73 @@ int32_t ctgGetDBVgroupFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEp return TSDB_CODE_SUCCESS; } -int32_t ctgIsTableMetaExistInCache(struct SCatalog* pCatalog, const char* tbFullName, int32_t *exist) { - if (NULL == pCatalog->tableCache.cache) { +int32_t ctgIsTableMetaExistInCache(struct SCatalog* pCatalog, char *dbFName, char* tbName, int32_t *exist) { + if (NULL == pCatalog->dbCache) { *exist = 0; - ctgWarn("empty tablemeta cache, tbName:%s", tbFullName); + ctgWarn("empty db cache, dbFName:%s, tbName:%s", dbFName, tbName); return TSDB_CODE_SUCCESS; } - size_t sz = 0; - STableMeta *tbMeta = taosHashGet(pCatalog->tableCache.cache, tbFullName, strlen(tbFullName)); - - if (NULL == tbMeta) { + SCtgDBCache *dbCache = taosHashAcquire(pCatalog->dbCache, dbFName, strlen(dbFName)); + if (NULL == dbCache) { *exist = 0; - ctgDebug("tablemeta not in cache, tbName:%s", tbFullName); + ctgWarn("db not exist in cache, dbFName:%s", dbFName); + return TSDB_CODE_SUCCESS; + } + + + size_t sz = 0; + STableMeta *tbMeta = taosHashGet(dbCache->tbCache.cache, tbName, strlen(tbName)); + if (NULL == tbMeta) { + taosHashRelease(pCatalog->dbCache, dbCache); + + *exist = 0; + ctgDebug("tbmeta not in cache, dbFName:%s, tbName:%s", dbFName, tbName); return TSDB_CODE_SUCCESS; } *exist = 1; + + taosHashRelease(pCatalog->dbCache, dbCache); - ctgDebug("tablemeta is in cache, tbName:%s", tbFullName); + ctgDebug("tbmeta is in cache, dbFName:%s, tbName:%s", dbFName, tbName); return TSDB_CODE_SUCCESS; } int32_t ctgGetTableMetaFromCache(struct SCatalog* pCatalog, const SName* pTableName, STableMeta** pTableMeta, int32_t *exist) { - if (NULL == pCatalog->tableCache.cache) { + if (NULL == pCatalog->dbCache) { *exist = 0; - ctgWarn("empty tablemeta cache, tbName:%s", pTableName->tname); + ctgWarn("empty tbmeta cache, tbName:%s", pTableName->tname); return TSDB_CODE_SUCCESS; } - char tbFullName[TSDB_TABLE_FNAME_LEN]; - tNameExtractFullName(pTableName, tbFullName); + char db[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(pTableName, db); *pTableMeta = NULL; - size_t sz = 0; - STableMeta *tbMeta = taosHashGetCloneExt(pCatalog->tableCache.cache, tbFullName, strlen(tbFullName), NULL, (void **)pTableMeta, &sz); + SCtgDBCache *dbCache = taosHashAcquire(pCatalog->dbCache, db, strlen(db)); + if (NULL == dbCache) { + *exist = 0; + ctgWarn("no db cache, dbFName:%s, tbName:%s", db, pTableName->tname); + return TSDB_CODE_SUCCESS; + } + if (NULL == dbCache->tbCache.cache) { + *exist = 0; + taosHashRelease(pCatalog->dbCache, dbCache); + ctgWarn("empty tbmeta cache, dbFName:%s, tbName:%s", db, pTableName->tname); + return TSDB_CODE_SUCCESS; + } + + size_t sz = 0; + STableMeta *tbMeta = taosHashGetCloneExt(dbCache->tbCache.cache, pTableName->tname, strlen(pTableName->tname), NULL, (void **)pTableMeta, &sz); if (NULL == *pTableMeta) { *exist = 0; - ctgDebug("tablemeta not in cache, tbName:%s", tbFullName); + taosHashRelease(pCatalog->dbCache, dbCache); + ctgDebug("tbmeta not in cache, dbFName:%s, tbName:%s", db, pTableName->tname); return TSDB_CODE_SUCCESS; } @@ -148,80 +248,85 @@ int32_t ctgGetTableMetaFromCache(struct SCatalog* pCatalog, const SName* pTableN tbMeta = *pTableMeta; if (tbMeta->tableType != TSDB_CHILD_TABLE) { - ctgDebug("Got tablemeta from cache, tbName:%s", tbFullName); - + ctgDebug("Got tbmeta from cache, type:%d, dbFName:%s, tbName:%s", tbMeta->tableType, db, pTableName->tname); return TSDB_CODE_SUCCESS; } - CTG_LOCK(CTG_READ, &pCatalog->tableCache.stableLock); + CTG_LOCK(CTG_READ, &dbCache->tbCache.stbLock); - STableMeta **stbMeta = taosHashGet(pCatalog->tableCache.stableCache, &tbMeta->suid, sizeof(tbMeta->suid)); + STableMeta **stbMeta = taosHashGet(dbCache->tbCache.stbCache, &tbMeta->suid, sizeof(tbMeta->suid)); if (NULL == stbMeta || NULL == *stbMeta) { - CTG_UNLOCK(CTG_READ, &pCatalog->tableCache.stableLock); - ctgError("stable not in stableCache, suid:%"PRIx64, tbMeta->suid); + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); + ctgError("stable not in stbCache, suid:%"PRIx64, tbMeta->suid); tfree(*pTableMeta); *exist = 0; return TSDB_CODE_SUCCESS; } if ((*stbMeta)->suid != tbMeta->suid) { - CTG_UNLOCK(CTG_READ, &pCatalog->tableCache.stableLock); + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); tfree(*pTableMeta); - ctgError("stable suid in stableCache mis-match, expected suid:%"PRIx64 ",actual suid:%"PRIx64, tbMeta->suid, (*stbMeta)->suid); + ctgError("stable suid in stbCache mis-match, expected suid:%"PRIx64 ",actual suid:%"PRIx64, tbMeta->suid, (*stbMeta)->suid); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } int32_t metaSize = sizeof(STableMeta) + ((*stbMeta)->tableInfo.numOfTags + (*stbMeta)->tableInfo.numOfColumns) * sizeof(SSchema); *pTableMeta = realloc(*pTableMeta, metaSize); if (NULL == *pTableMeta) { - CTG_UNLOCK(CTG_READ, &pCatalog->tableCache.stableLock); + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); ctgError("realloc size[%d] failed", metaSize); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } memcpy(&(*pTableMeta)->sversion, &(*stbMeta)->sversion, metaSize - sizeof(SCTableMeta)); - CTG_UNLOCK(CTG_READ, &pCatalog->tableCache.stableLock); + CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); - ctgDebug("Got tablemeta from cache, tbName:%s", tbFullName); + ctgDebug("Got tbmeta from cache, dbFName:%s, tbName:%s", db, pTableName->tname); return TSDB_CODE_SUCCESS; } int32_t ctgGetTableTypeFromCache(struct SCatalog* pCatalog, const SName* pTableName, int32_t *tbType) { - if (NULL == pCatalog->tableCache.cache) { - ctgWarn("empty tablemeta cache, tbName:%s", pTableName->tname); + if (NULL == pCatalog->dbCache) { + ctgWarn("empty db cache, tbName:%s", pTableName->tname); return TSDB_CODE_SUCCESS; } - char tbFullName[TSDB_TABLE_FNAME_LEN]; - tNameExtractFullName(pTableName, tbFullName); + char dbName[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(pTableName, dbName); - size_t sz = 0; - STableMeta *pTableMeta = NULL; - - taosHashGetCloneExt(pCatalog->tableCache.cache, tbFullName, strlen(tbFullName), NULL, (void **)&pTableMeta, &sz); + SCtgDBCache *dbCache = taosHashAcquire(pCatalog->dbCache, dbName, strlen(dbName)); + if (NULL == dbCache) { + ctgInfo("db not in cache, dbFName:%s", dbName); + return TSDB_CODE_SUCCESS; + } + STableMeta *pTableMeta = (STableMeta *)taosHashAcquire(dbCache->tbCache.cache, pTableName->tname, strlen(pTableName->tname)); if (NULL == pTableMeta) { - ctgWarn("tablemeta not in cache, tbName:%s", tbFullName); - + ctgWarn("tbmeta not in cache, dbFName:%s, tbName:%s", dbName, pTableName->tname); + taosHashRelease(pCatalog->dbCache, dbCache); + return TSDB_CODE_SUCCESS; } - *tbType = pTableMeta->tableType; + *tbType = atomic_load_8(&pTableMeta->tableType); - ctgDebug("Got tabletype from cache, tbName:%s, type:%d", tbFullName, *tbType); + taosHashRelease(dbCache->tbCache.cache, dbCache); + taosHashRelease(pCatalog->dbCache, dbCache); + + ctgDebug("Got tbtype from cache, dbFName:%s, tbName:%s, type:%d", dbName, pTableName->tname, *tbType); return TSDB_CODE_SUCCESS; } -int32_t ctgGetTableMetaFromMnodeImpl(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, char* tbFullName, STableMetaOutput* output) { - SBuildTableMetaInput bInput = {.vgId = 0, .dbName = NULL, .tableFullName = tbFullName}; +int32_t ctgGetTableMetaFromMnodeImpl(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, char *dbFName, char* tbName, STableMetaOutput* output) { + SBuildTableMetaInput bInput = {.vgId = 0, .dbFName = dbFName, .tbName = tbName}; char *msg = NULL; SEpSet *pVnodeEpSet = NULL; int32_t msgLen = 0; - ctgDebug("try to get table meta from mnode, tbName:%s", tbFullName); + ctgDebug("try to get table meta from mnode, dbFName:%s, tbName:%s", dbFName, tbName); int32_t code = queryBuildMsg[TMSG_INDEX(TDMT_MND_STB_META)](&bInput, &msg, 0, &msgLen); if (code) { @@ -242,30 +347,30 @@ int32_t ctgGetTableMetaFromMnodeImpl(struct SCatalog* pCatalog, void *pTransport if (TSDB_CODE_SUCCESS != rpcRsp.code) { if (CTG_TABLE_NOT_EXIST(rpcRsp.code)) { SET_META_TYPE_NULL(output->metaType); - ctgDebug("stablemeta not exist in mnode, tbName:%s", tbFullName); + ctgDebug("stablemeta not exist in mnode, dbFName:%s, tbName:%s", dbFName, tbName); return TSDB_CODE_SUCCESS; } - ctgError("error rsp for stablemeta from mnode, code:%s, tbName:%s", tstrerror(rpcRsp.code), tbFullName); + ctgError("error rsp for stablemeta from mnode, code:%s, dbFName:%s, tbName:%s", tstrerror(rpcRsp.code), dbFName, tbName); CTG_ERR_RET(rpcRsp.code); } code = queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_STB_META)](output, rpcRsp.pCont, rpcRsp.contLen); if (code) { - ctgError("Process mnode stablemeta rsp failed, code:%x, tbName:%s", code, tbFullName); + ctgError("Process mnode stablemeta rsp failed, code:%x, dbFName:%s, tbName:%s", code, dbFName, tbName); CTG_ERR_RET(code); } - ctgDebug("Got table meta from mnode, tbName:%s", tbFullName); + ctgDebug("Got table meta from mnode, dbFName:%s, tbName:%s", dbFName, tbName); return TSDB_CODE_SUCCESS; } int32_t ctgGetTableMetaFromMnode(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMetaOutput* output) { - char tbFullName[TSDB_TABLE_FNAME_LEN]; - tNameExtractFullName(pTableName, tbFullName); + char dbFName[TSDB_DB_FNAME_LEN]; + tNameGetFullDbName(pTableName, dbFName); - return ctgGetTableMetaFromMnodeImpl(pCatalog, pTransporter, pMgmtEps, tbFullName, output); + return ctgGetTableMetaFromMnodeImpl(pCatalog, pTransporter, pMgmtEps, dbFName, (char *)pTableName->tname, output); } int32_t ctgGetTableMetaFromVnode(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* output) { @@ -273,18 +378,18 @@ int32_t ctgGetTableMetaFromVnode(struct SCatalog* pCatalog, void *pTransporter, CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } - char dbFullName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(pTableName, dbFullName); + char dbFName[TSDB_DB_FNAME_LEN]; + tNameGetFullDbName(pTableName, dbFName); - ctgDebug("try to get table meta from vnode, db:%s, tbName:%s", dbFullName, tNameGetTableName(pTableName)); + ctgDebug("try to get table meta from vnode, dbFName:%s, tbName:%s", dbFName, tNameGetTableName(pTableName)); - SBuildTableMetaInput bInput = {.vgId = vgroupInfo->vgId, .dbName = dbFullName, .tableFullName = (char *)tNameGetTableName(pTableName)}; + SBuildTableMetaInput bInput = {.vgId = vgroupInfo->vgId, .dbFName = dbFName, .tbName = (char *)tNameGetTableName(pTableName)}; char *msg = NULL; int32_t msgLen = 0; int32_t code = queryBuildMsg[TMSG_INDEX(TDMT_VND_TABLE_META)](&bInput, &msg, 0, &msgLen); if (code) { - ctgError("Build vnode tablemeta msg failed, code:%x, tbName:%s", code, tNameGetTableName(pTableName)); + ctgError("Build vnode tablemeta msg failed, code:%x, dbFName:%s, tbName:%s", code, dbFName, tNameGetTableName(pTableName)); CTG_ERR_RET(code); } @@ -300,21 +405,21 @@ int32_t ctgGetTableMetaFromVnode(struct SCatalog* pCatalog, void *pTransporter, if (TSDB_CODE_SUCCESS != rpcRsp.code) { if (CTG_TABLE_NOT_EXIST(rpcRsp.code)) { SET_META_TYPE_NULL(output->metaType); - ctgDebug("tablemeta not exist in vnode, tbName:%s", tNameGetTableName(pTableName)); + ctgDebug("tablemeta not exist in vnode, dbFName:%s, tbName:%s", dbFName, tNameGetTableName(pTableName)); return TSDB_CODE_SUCCESS; } - ctgError("error rsp for table meta from vnode, code:%s, tbName:%s", tstrerror(rpcRsp.code), tNameGetTableName(pTableName)); + ctgError("error rsp for table meta from vnode, code:%s, dbFName:%s, tbName:%s", tstrerror(rpcRsp.code), dbFName, tNameGetTableName(pTableName)); CTG_ERR_RET(rpcRsp.code); } code = queryProcessMsgRsp[TMSG_INDEX(TDMT_VND_TABLE_META)](output, rpcRsp.pCont, rpcRsp.contLen); if (code) { - ctgError("Process vnode tablemeta rsp failed, code:%s, tbName:%s", tstrerror(code), tNameGetTableName(pTableName)); + ctgError("Process vnode tablemeta rsp failed, code:%s, dbFName:%s, tbName:%s", tstrerror(code), dbFName, tNameGetTableName(pTableName)); CTG_ERR_RET(code); } - ctgDebug("Got table meta from vnode, db:%s, tbName:%s", dbFullName, tNameGetTableName(pTableName)); + ctgDebug("Got table meta from vnode, dbFName:%s, tbName:%s", dbFName, tNameGetTableName(pTableName)); return TSDB_CODE_SUCCESS; } @@ -334,7 +439,7 @@ int32_t ctgGetVgInfoFromDB(struct SCatalog *pCatalog, void *pRpc, const SEpSet * SVgroupInfo *vgInfo = NULL; SArray *vgList = NULL; int32_t code = 0; - int32_t vgNum = taosHashGetSize(dbInfo->vgInfo); + int32_t vgNum = taosHashGetSize(dbInfo->vgHash); vgList = taosArrayInit(vgNum, sizeof(SVgroupInfo)); if (NULL == vgList) { @@ -342,7 +447,7 @@ int32_t ctgGetVgInfoFromDB(struct SCatalog *pCatalog, void *pRpc, const SEpSet * CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } - void *pIter = taosHashIterate(dbInfo->vgInfo, NULL); + void *pIter = taosHashIterate(dbInfo->vgHash, NULL); while (pIter) { vgInfo = pIter; @@ -351,7 +456,7 @@ int32_t ctgGetVgInfoFromDB(struct SCatalog *pCatalog, void *pRpc, const SEpSet * CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } - pIter = taosHashIterate(dbInfo->vgInfo, pIter); + pIter = taosHashIterate(dbInfo->vgHash, pIter); vgInfo = NULL; } @@ -374,7 +479,7 @@ _return: int32_t ctgGetVgInfoFromHashValue(struct SCatalog *pCatalog, SDBVgroupInfo *dbInfo, const SName *pTableName, SVgroupInfo *pVgroup) { int32_t code = 0; - int32_t vgNum = taosHashGetSize(dbInfo->vgInfo); + int32_t vgNum = taosHashGetSize(dbInfo->vgHash); char db[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(pTableName, db); @@ -386,33 +491,32 @@ int32_t ctgGetVgInfoFromHashValue(struct SCatalog *pCatalog, SDBVgroupInfo *dbIn tableNameHashFp fp = NULL; SVgroupInfo *vgInfo = NULL; - CTG_ERR_JRET(ctgGetHashFunction(dbInfo->hashMethod, &fp)); + CTG_ERR_RET(ctgGetHashFunction(dbInfo->hashMethod, &fp)); char tbFullName[TSDB_TABLE_FNAME_LEN]; tNameExtractFullName(pTableName, tbFullName); uint32_t hashValue = (*fp)(tbFullName, (uint32_t)strlen(tbFullName)); - void *pIter = taosHashIterate(dbInfo->vgInfo, NULL); + void *pIter = taosHashIterate(dbInfo->vgHash, NULL); while (pIter) { vgInfo = pIter; if (hashValue >= vgInfo->hashBegin && hashValue <= vgInfo->hashEnd) { - taosHashCancelIterate(dbInfo->vgInfo, pIter); + taosHashCancelIterate(dbInfo->vgHash, pIter); break; } - pIter = taosHashIterate(dbInfo->vgInfo, pIter); + pIter = taosHashIterate(dbInfo->vgHash, pIter); vgInfo = NULL; } if (NULL == vgInfo) { - ctgError("no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, db, taosHashGetSize(dbInfo->vgInfo)); - CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + ctgError("no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, db, taosHashGetSize(dbInfo->vgHash)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } *pVgroup = *vgInfo; -_return: CTG_RET(code); } @@ -437,12 +541,12 @@ int32_t ctgDbVgVersionCompare(const void* key1, const void* key2) { } -int32_t ctgMetaRentInit(SMetaRentMgmt *mgmt, uint32_t rentSec, int8_t type) { +int32_t ctgMetaRentInit(SCtgRentMgmt *mgmt, uint32_t rentSec, int8_t type) { mgmt->slotRIdx = 0; mgmt->slotNum = rentSec / CTG_RENT_SLOT_SECOND; mgmt->type = type; - size_t msgSize = sizeof(SRentSlotInfo) * mgmt->slotNum; + size_t msgSize = sizeof(SCtgRentSlot) * mgmt->slotNum; mgmt->slots = calloc(1, msgSize); if (NULL == mgmt->slots) { @@ -456,10 +560,10 @@ int32_t ctgMetaRentInit(SMetaRentMgmt *mgmt, uint32_t rentSec, int8_t type) { } -int32_t ctgMetaRentAdd(SMetaRentMgmt *mgmt, void *meta, int64_t id, int32_t size) { +int32_t ctgMetaRentAdd(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t size) { int16_t widx = abs(id % mgmt->slotNum); - SRentSlotInfo *slot = &mgmt->slots[widx]; + SCtgRentSlot *slot = &mgmt->slots[widx]; int32_t code = 0; CTG_LOCK(CTG_WRITE, &slot->lock); @@ -486,10 +590,10 @@ _return: CTG_RET(code); } -int32_t ctgMetaRentUpdate(SMetaRentMgmt *mgmt, void *meta, int64_t id, int32_t size, __compar_fn_t compare) { +int32_t ctgMetaRentUpdate(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t size, __compar_fn_t compare) { int16_t widx = abs(id % mgmt->slotNum); - SRentSlotInfo *slot = &mgmt->slots[widx]; + SCtgRentSlot *slot = &mgmt->slots[widx]; int32_t code = 0; CTG_LOCK(CTG_WRITE, &slot->lock); @@ -526,10 +630,10 @@ _return: CTG_RET(code); } -int32_t ctgMetaRentRemove(SMetaRentMgmt *mgmt, int64_t id, __compar_fn_t compare) { +int32_t ctgMetaRentRemove(SCtgRentMgmt *mgmt, int64_t id, __compar_fn_t compare) { int16_t widx = abs(id % mgmt->slotNum); - SRentSlotInfo *slot = &mgmt->slots[widx]; + SCtgRentSlot *slot = &mgmt->slots[widx]; int32_t code = 0; CTG_LOCK(CTG_WRITE, &slot->lock); @@ -562,14 +666,14 @@ _return: } -int32_t ctgMetaRentGetImpl(SMetaRentMgmt *mgmt, void **res, uint32_t *num, int32_t size) { +int32_t ctgMetaRentGetImpl(SCtgRentMgmt *mgmt, void **res, uint32_t *num, int32_t size) { int16_t ridx = atomic_add_fetch_16(&mgmt->slotRIdx, 1); if (ridx >= mgmt->slotNum) { ridx %= mgmt->slotNum; atomic_store_16(&mgmt->slotRIdx, ridx); } - SRentSlotInfo *slot = &mgmt->slots[ridx]; + SCtgRentSlot *slot = &mgmt->slots[ridx]; int32_t code = 0; CTG_LOCK(CTG_READ, &slot->lock); @@ -608,7 +712,7 @@ _return: CTG_RET(code); } -int32_t ctgMetaRentGet(SMetaRentMgmt *mgmt, void **res, uint32_t *num, int32_t size) { +int32_t ctgMetaRentGet(SCtgRentMgmt *mgmt, void **res, uint32_t *num, int32_t size) { while (true) { int64_t msec = taosGetTimestampMs(); int64_t lsec = atomic_load_64(&mgmt->lastReadMsec); @@ -635,52 +739,79 @@ int32_t ctgMetaRentGet(SMetaRentMgmt *mgmt, void **res, uint32_t *num, int32_t s int32_t ctgUpdateTableMetaCache(struct SCatalog *pCatalog, STableMetaOutput *output) { int32_t code = 0; + SCtgDBCache *dbCache = NULL; if ((!CTG_IS_META_CTABLE(output->metaType)) && NULL == output->tbMeta) { - ctgError("no valid table meta got from meta rsp, tbName:%s", output->tbFname); + ctgError("no valid tbmeta got from meta rsp, dbFName:%s, tbName:%s", output->dbFName, output->tbName); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - if (NULL == pCatalog->tableCache.cache) { - SHashObj *cache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + if (NULL == pCatalog->dbCache) { + SHashObj *cache = taosHashInit(ctgMgmt.cfg.maxDBCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); if (NULL == cache) { - ctgError("taosHashInit failed, num:%d", ctgMgmt.cfg.maxTblCacheNum); + ctgError("taosHashInit %d failed", CTG_DEFAULT_CACHE_DB_NUMBER); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } - if (NULL != atomic_val_compare_exchange_ptr(&pCatalog->tableCache.cache, NULL, cache)) { + if (NULL != atomic_val_compare_exchange_ptr(&pCatalog->dbCache, NULL, cache)) { taosHashCleanup(cache); } } - if (NULL == pCatalog->tableCache.stableCache) { + while (true) { + dbCache = (SCtgDBCache *)taosHashAcquire(pCatalog->dbCache, output->dbFName, strlen(output->dbFName)); + if (dbCache) { + break; + } + + SCtgDBCache newDbCache = {0}; + + if (taosHashPut(pCatalog->dbCache, output->dbFName, strlen(output->dbFName), &newDbCache, sizeof(newDbCache))) { + ctgError("taosHashPut db to cache failed, db:%s", output->dbFName); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + } + + if (NULL == dbCache->tbCache.cache) { + SHashObj *cache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + if (NULL == cache) { + ctgError("taosHashInit failed, num:%d", ctgMgmt.cfg.maxTblCacheNum); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); + } + + if (NULL != atomic_val_compare_exchange_ptr(&dbCache->tbCache.cache, NULL, cache)) { + taosHashCleanup(cache); + } + } + + if (NULL == dbCache->tbCache.stbCache) { SHashObj *cache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_ENTRY_LOCK); if (NULL == cache) { ctgError("taosHashInit failed, num:%d", ctgMgmt.cfg.maxTblCacheNum); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } - if (NULL != atomic_val_compare_exchange_ptr(&pCatalog->tableCache.stableCache, NULL, cache)) { + if (NULL != atomic_val_compare_exchange_ptr(&dbCache->tbCache.stbCache, NULL, cache)) { taosHashCleanup(cache); } } if (CTG_IS_META_CTABLE(output->metaType) || CTG_IS_META_BOTH(output->metaType)) { - if (taosHashPut(pCatalog->tableCache.cache, output->ctbFname, strlen(output->ctbFname), &output->ctbMeta, sizeof(output->ctbMeta)) != 0) { - ctgError("taosHashPut ctablemeta to cache failed, ctbName:%s", output->ctbFname); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + if (taosHashPut(dbCache->tbCache.cache, output->ctbName, strlen(output->ctbName), &output->ctbMeta, sizeof(output->ctbMeta)) != 0) { + ctgError("taosHashPut ctbmeta to cache failed, ctbName:%s", output->ctbName); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } - ctgDebug("update child tablemeta to cache, tbName:%s", output->ctbFname); + ctgDebug("ctbmeta updated to cache, ctbName:%s", output->ctbName); } if (CTG_IS_META_CTABLE(output->metaType)) { - return TSDB_CODE_SUCCESS; + goto _return; } if (CTG_IS_META_BOTH(output->metaType) && TSDB_SUPER_TABLE != output->tbMeta->tableType) { ctgError("table type error, expected:%d, actual:%d", TSDB_SUPER_TABLE, output->tbMeta->tableType); - CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } int32_t tbSize = sizeof(*output->tbMeta) + sizeof(SSchema) * (output->tbMeta->tableInfo.numOfColumns + output->tbMeta->tableInfo.numOfTags); @@ -689,44 +820,50 @@ int32_t ctgUpdateTableMetaCache(struct SCatalog *pCatalog, STableMetaOutput *out bool newAdded = false; SSTableMetaVersion metaRent = {.suid = output->tbMeta->suid, .sversion = output->tbMeta->sversion, .tversion = output->tbMeta->tversion}; - CTG_LOCK(CTG_WRITE, &pCatalog->tableCache.stableLock); - if (taosHashPut(pCatalog->tableCache.cache, output->tbFname, strlen(output->tbFname), output->tbMeta, tbSize) != 0) { - CTG_UNLOCK(CTG_WRITE, &pCatalog->tableCache.stableLock); - ctgError("taosHashPut tablemeta to cache failed, tbName:%s", output->tbFname); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + CTG_LOCK(CTG_WRITE, &dbCache->tbCache.stbLock); + if (taosHashPut(dbCache->tbCache.cache, output->tbName, strlen(output->tbName), output->tbMeta, tbSize) != 0) { + CTG_UNLOCK(CTG_WRITE, &dbCache->tbCache.stbLock); + ctgError("taosHashPut tablemeta to cache failed, dbFName:%s, tbName:%s", output->dbFName, output->tbName); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } - STableMeta *tbMeta = taosHashGet(pCatalog->tableCache.cache, output->tbFname, strlen(output->tbFname)); - if (taosHashPutExt(pCatalog->tableCache.stableCache, &output->tbMeta->suid, sizeof(output->tbMeta->suid), &tbMeta, POINTER_BYTES, &newAdded) != 0) { - CTG_UNLOCK(CTG_WRITE, &pCatalog->tableCache.stableLock); + STableMeta *tbMeta = taosHashGet(dbCache->tbCache.cache, output->tbName, strlen(output->tbName)); + if (taosHashPutExt(dbCache->tbCache.stbCache, &output->tbMeta->suid, sizeof(output->tbMeta->suid), &tbMeta, POINTER_BYTES, &newAdded) != 0) { + CTG_UNLOCK(CTG_WRITE, &dbCache->tbCache.stbLock); ctgError("taosHashPutExt stable to stable cache failed, suid:%"PRIx64, output->tbMeta->suid); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } - CTG_UNLOCK(CTG_WRITE, &pCatalog->tableCache.stableLock); + CTG_UNLOCK(CTG_WRITE, &dbCache->tbCache.stbLock); ctgDebug("update stable to cache, suid:%"PRIx64, output->tbMeta->suid); if (newAdded) { - CTG_ERR_RET(ctgMetaRentAdd(&pCatalog->stableRent, &metaRent, metaRent.suid, sizeof(SSTableMetaVersion))); + CTG_ERR_JRET(ctgMetaRentAdd(&pCatalog->stbRent, &metaRent, metaRent.suid, sizeof(SSTableMetaVersion))); } else { - CTG_ERR_RET(ctgMetaRentUpdate(&pCatalog->stableRent, &metaRent, metaRent.suid, sizeof(SSTableMetaVersion), ctgSTableVersionCompare)); + CTG_ERR_JRET(ctgMetaRentUpdate(&pCatalog->stbRent, &metaRent, metaRent.suid, sizeof(SSTableMetaVersion), ctgSTableVersionCompare)); } } else { - if (taosHashPut(pCatalog->tableCache.cache, output->tbFname, strlen(output->tbFname), output->tbMeta, tbSize) != 0) { - ctgError("taosHashPut tablemeta to cache failed, tbName:%s", output->tbFname); - CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + if (taosHashPut(dbCache->tbCache.cache, output->tbName, strlen(output->tbName), output->tbMeta, tbSize) != 0) { + ctgError("taosHashPut tablemeta to cache failed, dbFName:%s, tbName:%s", output->dbFName, output->tbName); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } } - ctgDebug("update tablemeta to cache, tbName:%s", output->tbFname); + ctgDebug("update tablemeta to cache, dbFName:%s, tbName:%s", output->dbFName, output->tbName); + +_return: + + if (dbCache) { + taosHashRelease(pCatalog->dbCache, dbCache); + } CTG_RET(code); } -int32_t ctgGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* dbName, bool forceUpdate, SDBVgroupInfo** dbInfo) { +int32_t ctgGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* dbName, bool forceUpdate, SCtgDBCache** dbCache) { bool inCache = false; if (!forceUpdate) { - CTG_ERR_RET(ctgGetDBVgroupFromCache(pCatalog, dbName, dbInfo, &inCache)); + CTG_ERR_RET(ctgGetDBVgroupFromCache(pCatalog, dbName, dbCache, &inCache)); if (inCache) { return TSDB_CODE_SUCCESS; } @@ -742,8 +879,8 @@ int32_t ctgGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgm while (true) { CTG_ERR_RET(ctgGetDBVgroupFromMnode(pCatalog, pRpc, pMgmtEps, &input, &DbOut)); - CTG_ERR_RET(catalogUpdateDBVgroup(pCatalog, dbName, &DbOut.dbVgroup)); - CTG_ERR_RET(ctgGetDBVgroupFromCache(pCatalog, dbName, dbInfo, &inCache)); + CTG_ERR_RET(catalogUpdateDBVgroup(pCatalog, dbName, DbOut.dbVgroup)); + CTG_ERR_RET(ctgGetDBVgroupFromCache(pCatalog, dbName, dbCache, &inCache)); if (!inCache) { ctgWarn("can't get db vgroup from cache, will retry, db:%s", dbName); @@ -757,66 +894,51 @@ int32_t ctgGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgm } -int32_t ctgValidateAndFreeDbInfo(struct SCatalog* pCatalog, const char* dbName, SDBVgroupInfo* dbInfo) { - SDBVgroupInfo *oldInfo = (SDBVgroupInfo *)taosHashAcquire(pCatalog->dbCache.cache, dbName, strlen(dbName)); - if (oldInfo) { - CTG_LOCK(CTG_WRITE, &oldInfo->lock); - if (dbInfo->vgVersion <= oldInfo->vgVersion) { - ctgInfo("db vgVersion is not new, db:%s, vgVersion:%d, current:%d", dbName, dbInfo->vgVersion, oldInfo->vgVersion); - CTG_UNLOCK(CTG_WRITE, &oldInfo->lock); - taosHashRelease(pCatalog->dbCache.cache, oldInfo); - - return TSDB_CODE_SUCCESS; - } - - if (oldInfo->vgInfo) { - ctgInfo("cleanup db vgInfo, db:%s", dbName); - taosHashCleanup(oldInfo->vgInfo); - oldInfo->vgInfo = NULL; - } - - CTG_UNLOCK(CTG_WRITE, &oldInfo->lock); - - taosHashRelease(pCatalog->dbCache.cache, oldInfo); - } - - return TSDB_CODE_SUCCESS; -} - -int32_t ctgValidateAndRemoveDbInfo(struct SCatalog* pCatalog, SDbVgVersion* target, bool *removed) { +int32_t ctgValidateAndRemoveDb(struct SCatalog* pCatalog, const char* dbName, uint64_t dbId, bool *removed) { *removed = false; - SDBVgroupInfo *info = (SDBVgroupInfo *)taosHashAcquire(pCatalog->dbCache.cache, target->dbName, strlen(target->dbName)); - if (NULL == info) { - ctgInfo("db not exist in dbCache, may be removed, db:%s", target->dbName); + SCtgDBCache *dbCache = (SCtgDBCache *)taosHashAcquire(pCatalog->dbCache, dbName, strlen(dbName)); + if (NULL == dbCache) { + ctgInfo("db not exist in dbCache, may be removed, db:%s", dbName); return TSDB_CODE_SUCCESS; } - CTG_LOCK(CTG_WRITE, &info->lock); + CTG_LOCK(CTG_WRITE, &dbCache->vgLock); - if (info->dbId != target->dbId) { - ctgInfo("db id already updated, db:%s, dbId:%"PRIx64 ", targetId:%"PRIx64, target->dbName, info->dbId, target->dbId); - CTG_UNLOCK(CTG_WRITE, &info->lock); - taosHashRelease(pCatalog->dbCache.cache, info); + if (NULL == dbCache->vgInfo) { + ctgInfo("db vgInfo not in dbCache, may be removed, db:%s, dbId:%"PRIx64, dbName, dbId); + CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); + taosHashRelease(pCatalog->dbCache, dbCache); + return TSDB_CODE_SUCCESS; + } + + if (dbCache->vgInfo->dbId != dbId) { + ctgInfo("db id already updated, db:%s, dbId:%"PRIx64 ", targetId:%"PRIx64, dbName, dbCache->vgInfo->dbId, dbId); + CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); + taosHashRelease(pCatalog->dbCache, dbCache); return TSDB_CODE_SUCCESS; } - if (info->vgInfo) { - ctgInfo("cleanup db vgInfo, db:%s", target->dbName); - taosHashCleanup(info->vgInfo); - info->vgInfo = NULL; + if (dbCache->vgInfo->vgHash) { + ctgInfo("cleanup db vgInfo, db:%s, dbId:%"PRIx64, dbName, dbId); + taosHashCleanup(dbCache->vgInfo->vgHash); + tfree(dbCache->vgInfo); } - if (taosHashRemove(pCatalog->dbCache.cache, target->dbName, strlen(target->dbName))) { - ctgError("taosHashRemove from dbCache failed, db:%s", target->dbName); - CTG_UNLOCK(CTG_WRITE, &info->lock); - taosHashRelease(pCatalog->dbCache.cache, info); + if (taosHashRemove(pCatalog->dbCache, dbName, strlen(dbName))) { + ctgError("taosHashRemove from dbCache failed, db:%s", dbName); + CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); + taosHashRelease(pCatalog->dbCache, dbCache); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - - CTG_UNLOCK(CTG_WRITE, &info->lock); - taosHashRelease(pCatalog->dbCache.cache, info); + dbCache->deleted = true; + + CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); + + ctgFreeTableMetaCache(&dbCache->tbCache); + + taosHashRelease(pCatalog->dbCache, dbCache); *removed = true; @@ -839,7 +961,7 @@ int32_t ctgRenewTableMetaImpl(struct SCatalog* pCatalog, void *pTransporter, con STableMetaOutput *output = &voutput; if (CTG_IS_STABLE(isSTable)) { - ctgDebug("will renew table meta, supposed to be stable, tbName:%s", tNameGetTableName(pTableName)); + ctgDebug("will renew tbmeta, supposed to be stb, tbName:%s", tNameGetTableName(pTableName)); // if get from mnode failed, will not try vnode CTG_ERR_JRET(ctgGetTableMetaFromMnode(pCatalog, pTransporter, pMgmtEps, pTableName, &moutput)); @@ -850,15 +972,15 @@ int32_t ctgRenewTableMetaImpl(struct SCatalog* pCatalog, void *pTransporter, con output = &moutput; } } else { - ctgDebug("will renew table meta, not supposed to be stable, tbName:%s, isStable:%d", tNameGetTableName(pTableName), isSTable); + ctgDebug("will renew tbmeta, not supposed to be stb, tbName:%s, isStable:%d", tNameGetTableName(pTableName), isSTable); // if get from vnode failed or no table meta, will not try mnode CTG_ERR_JRET(ctgGetTableMetaFromVnode(pCatalog, pTransporter, pMgmtEps, pTableName, &vgroupInfo, &voutput)); if (CTG_IS_META_TABLE(voutput.metaType) && TSDB_SUPER_TABLE == voutput.tbMeta->tableType) { - ctgDebug("will continue to renew table meta since got stable, tbName:%s, metaType:%d", tNameGetTableName(pTableName), voutput.metaType); + ctgDebug("will continue to renew tbmeta since got stb, tbName:%s, metaType:%d", tNameGetTableName(pTableName), voutput.metaType); - CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCatalog, pTransporter, pMgmtEps, voutput.tbFname, &moutput)); + CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCatalog, pTransporter, pMgmtEps, voutput.dbFName, voutput.tbName, &moutput)); voutput.metaType = moutput.metaType; @@ -867,9 +989,9 @@ int32_t ctgRenewTableMetaImpl(struct SCatalog* pCatalog, void *pTransporter, con moutput.tbMeta = NULL; } else if (CTG_IS_META_BOTH(voutput.metaType)) { int32_t exist = 0; - CTG_ERR_JRET(ctgIsTableMetaExistInCache(pCatalog, voutput.tbFname, &exist)); + CTG_ERR_JRET(ctgIsTableMetaExistInCache(pCatalog, voutput.dbFName, voutput.tbName, &exist)); if (0 == exist) { - CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCatalog, pTransporter, pMgmtEps, voutput.tbFname, &moutput)); + CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCatalog, pTransporter, pMgmtEps, voutput.dbFName, voutput.tbName, &moutput)); if (CTG_IS_META_NULL(moutput.metaType)) { SET_META_TYPE_NULL(voutput.metaType); @@ -934,64 +1056,6 @@ int32_t ctgGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMg return TSDB_CODE_SUCCESS; } -void ctgFreeMetaRent(SMetaRentMgmt *mgmt) { - if (NULL == mgmt->slots) { - return; - } - - for (int32_t i = 0; i < mgmt->slotNum; ++i) { - SRentSlotInfo *slot = &mgmt->slots[i]; - if (slot->meta) { - taosArrayDestroy(slot->meta); - slot->meta = NULL; - } - } - - tfree(mgmt->slots); -} - -void ctgFreeDbCache(SDBVgroupCache *db) { - if (NULL == db->cache) { - return; - } - - SDBVgroupInfo *dbInfo = NULL; - void *pIter = taosHashIterate(db->cache, NULL); - while (pIter) { - dbInfo = pIter; - - if (dbInfo->vgInfo) { - taosHashCleanup(dbInfo->vgInfo); - dbInfo->vgInfo = NULL; - } - - pIter = taosHashIterate(db->cache, pIter); - } - - taosHashCleanup(db->cache); - db->cache = NULL; -} - -void ctgFreeTableMetaCache(STableMetaCache *table) { - if (table->stableCache) { - taosHashCleanup(table->stableCache); - table->stableCache = NULL; - } - - if (table->cache) { - taosHashCleanup(table->cache); - table->cache = NULL; - } -} - -void ctgFreeHandle(struct SCatalog* pCatalog) { - ctgFreeMetaRent(&pCatalog->dbRent); - ctgFreeMetaRent(&pCatalog->stableRent); - ctgFreeDbCache(&pCatalog->dbCache); - ctgFreeTableMetaCache(&pCatalog->tableCache); - - free(pCatalog); -} int32_t catalogInit(SCatalogCfg *cfg) { if (ctgMgmt.pCluster) { @@ -1014,14 +1078,14 @@ int32_t catalogInit(SCatalogCfg *cfg) { ctgMgmt.cfg.dbRentSec = CTG_DEFAULT_RENT_SECOND; } - if (ctgMgmt.cfg.stableRentSec == 0) { - ctgMgmt.cfg.stableRentSec = CTG_DEFAULT_RENT_SECOND; + if (ctgMgmt.cfg.stbRentSec == 0) { + ctgMgmt.cfg.stbRentSec = CTG_DEFAULT_RENT_SECOND; } } else { ctgMgmt.cfg.maxDBCacheNum = CTG_DEFAULT_CACHE_DB_NUMBER; ctgMgmt.cfg.maxTblCacheNum = CTG_DEFAULT_CACHE_TABLEMETA_NUMBER; ctgMgmt.cfg.dbRentSec = CTG_DEFAULT_RENT_SECOND; - ctgMgmt.cfg.stableRentSec = CTG_DEFAULT_RENT_SECOND; + ctgMgmt.cfg.stbRentSec = CTG_DEFAULT_RENT_SECOND; } ctgMgmt.pCluster = taosHashInit(CTG_DEFAULT_CACHE_CLUSTER_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); @@ -1030,7 +1094,7 @@ int32_t catalogInit(SCatalogCfg *cfg) { CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - qDebug("catalog initialized, maxDb:%u, maxTbl:%u, dbRentSec:%u, stableRentSec:%u", ctgMgmt.cfg.maxDBCacheNum, ctgMgmt.cfg.maxTblCacheNum, ctgMgmt.cfg.dbRentSec, ctgMgmt.cfg.stableRentSec); + qDebug("catalog initialized, maxDb:%u, maxTbl:%u, dbRentSec:%u, stbRentSec:%u", ctgMgmt.cfg.maxDBCacheNum, ctgMgmt.cfg.maxTblCacheNum, ctgMgmt.cfg.dbRentSec, ctgMgmt.cfg.stbRentSec); return TSDB_CODE_SUCCESS; } @@ -1066,7 +1130,7 @@ int32_t catalogGetHandle(uint64_t clusterId, struct SCatalog** catalogHandle) { clusterCtg->clusterId = clusterId; CTG_ERR_JRET(ctgMetaRentInit(&clusterCtg->dbRent, ctgMgmt.cfg.dbRentSec, CTG_RENT_DB)); - CTG_ERR_JRET(ctgMetaRentInit(&clusterCtg->stableRent, ctgMgmt.cfg.stableRentSec, CTG_RENT_STABLE)); + CTG_ERR_JRET(ctgMetaRentInit(&clusterCtg->stbRent, ctgMgmt.cfg.stbRentSec, CTG_RENT_STABLE)); code = taosHashPut(ctgMgmt.pCluster, &clusterId, sizeof(clusterId), &clusterCtg, POINTER_BYTES); if (code) { @@ -1117,21 +1181,33 @@ int32_t catalogGetDBVgroupVersion(struct SCatalog* pCatalog, const char* dbName, CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } - if (NULL == pCatalog->dbCache.cache) { + if (NULL == pCatalog->dbCache) { *version = CTG_DEFAULT_INVALID_VERSION; ctgInfo("empty db cache, dbName:%s", dbName); return TSDB_CODE_SUCCESS; } - SDBVgroupInfo * dbInfo = taosHashAcquire(pCatalog->dbCache.cache, dbName, strlen(dbName)); - if (NULL == dbInfo) { + SCtgDBCache *db = taosHashAcquire(pCatalog->dbCache, dbName, strlen(dbName)); + if (NULL == db) { *version = CTG_DEFAULT_INVALID_VERSION; ctgInfo("db not in cache, dbName:%s", dbName); return TSDB_CODE_SUCCESS; } - *version = dbInfo->vgVersion; - taosHashRelease(pCatalog->dbCache.cache, dbInfo); + CTG_LOCK(CTG_READ, &db->vgLock); + + if (NULL == db->vgInfo) { + CTG_UNLOCK(CTG_READ, &db->vgLock); + + *version = CTG_DEFAULT_INVALID_VERSION; + ctgInfo("db not in cache, dbName:%s", dbName); + return TSDB_CODE_SUCCESS; + } + + *version = db->vgInfo->vgVersion; + CTG_UNLOCK(CTG_READ, &db->vgLock); + + taosHashRelease(pCatalog->dbCache, db); ctgDebug("Got db vgVersion from cache, dbName:%s, vgVersion:%d", dbName, *version); @@ -1143,29 +1219,31 @@ int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } - SDBVgroupInfo* db = NULL; + SCtgDBCache* dbCache = NULL; SVgroupInfo *vgInfo = NULL; int32_t code = 0; SArray *vgList = NULL; - CTG_ERR_JRET(ctgGetDBVgroup(pCatalog, pRpc, pMgmtEps, dbName, forceUpdate, &db)); + CTG_ERR_JRET(ctgGetDBVgroup(pCatalog, pRpc, pMgmtEps, dbName, forceUpdate, &dbCache)); - vgList = taosArrayInit(taosHashGetSize(db->vgInfo), sizeof(SVgroupInfo)); + int32_t vgNum = (int32_t)taosHashGetSize(dbCache->vgInfo->vgHash); + vgList = taosArrayInit(vgNum, sizeof(SVgroupInfo)); if (NULL == vgList) { - ctgError("taosArrayInit %d failed", taosHashGetSize(db->vgInfo)); + ctgError("taosArrayInit %d failed", vgNum); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } - void *pIter = taosHashIterate(db->vgInfo, NULL); + void *pIter = taosHashIterate(dbCache->vgInfo->vgHash, NULL); while (pIter) { vgInfo = pIter; if (NULL == taosArrayPush(vgList, vgInfo)) { ctgError("taosArrayPush failed, vgId:%d", vgInfo->vgId); + taosHashCancelIterate(dbCache->vgInfo->vgHash, pIter); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } - pIter = taosHashIterate(db->vgInfo, pIter); + pIter = taosHashIterate(dbCache->vgInfo->vgHash, pIter); vgInfo = NULL; } @@ -1173,9 +1251,10 @@ int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* vgList = NULL; _return: - if (db) { - CTG_UNLOCK(CTG_READ, &db->lock); - taosHashRelease(pCatalog->dbCache.cache, db); + + if (dbCache) { + CTG_UNLOCK(CTG_READ, &dbCache->vgLock); + taosHashRelease(pCatalog->dbCache, dbCache); } if (vgList) { @@ -1194,80 +1273,114 @@ int32_t catalogUpdateDBVgroup(struct SCatalog* pCatalog, const char* dbName, SDB CTG_ERR_JRET(TSDB_CODE_CTG_INVALID_INPUT); } - if (NULL == dbInfo->vgInfo || dbInfo->vgVersion < 0 || taosHashGetSize(dbInfo->vgInfo) <= 0) { - ctgError("invalid db vgInfo, dbName:%s, vgInfo:%p, vgVersion:%d", dbName, dbInfo->vgInfo, dbInfo->vgVersion); + if (NULL == dbInfo->vgHash || dbInfo->vgVersion < 0 || taosHashGetSize(dbInfo->vgHash) <= 0) { + ctgError("invalid db vgInfo, dbName:%s, vgHash:%p, vgVersion:%d", dbName, dbInfo->vgHash, dbInfo->vgVersion); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } - if (NULL == pCatalog->dbCache.cache) { + if (NULL == pCatalog->dbCache) { SHashObj *cache = taosHashInit(ctgMgmt.cfg.maxDBCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); if (NULL == cache) { ctgError("taosHashInit %d failed", CTG_DEFAULT_CACHE_DB_NUMBER); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); } - if (NULL != atomic_val_compare_exchange_ptr(&pCatalog->dbCache.cache, NULL, cache)) { + if (NULL != atomic_val_compare_exchange_ptr(&pCatalog->dbCache, NULL, cache)) { taosHashCleanup(cache); } - } else { - CTG_ERR_JRET(ctgValidateAndFreeDbInfo(pCatalog, dbName, dbInfo)); } bool newAdded = false; - - dbInfo->lock = 0; - if (taosHashPutExt(pCatalog->dbCache.cache, dbName, strlen(dbName), dbInfo, sizeof(*dbInfo), &newAdded) != 0) { - ctgError("taosHashPutExt db vgroup to cache failed, db:%s", dbName); - CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); + SDbVgVersion vgVersion = {.dbId = dbInfo->dbId, .vgVersion = dbInfo->vgVersion}; + + SCtgDBCache *dbCache = (SCtgDBCache *)taosHashAcquire(pCatalog->dbCache, dbName, strlen(dbName)); + if (dbCache) { + CTG_LOCK(CTG_WRITE, &dbCache->vgLock); + + if (NULL == dbCache->vgInfo) { + newAdded = true; + + dbCache->vgInfo = dbInfo; + } else { + if (dbCache->vgInfo->dbId != dbInfo->dbId) { + ctgMetaRentRemove(&pCatalog->dbRent, dbCache->vgInfo->dbId, ctgDbVgVersionCompare); + newAdded = true; + } else if (dbInfo->vgVersion <= dbCache->vgInfo->vgVersion) { + ctgInfo("db vgVersion is not new, db:%s, vgVersion:%d, current:%d", dbName, dbInfo->vgVersion, dbCache->vgInfo->vgVersion); + CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); + taosHashRelease(pCatalog->dbCache, dbCache); + + goto _return; + } + + if (dbCache->vgInfo->vgHash) { + ctgInfo("cleanup db vgHash, db:%s", dbName); + taosHashCleanup(dbCache->vgInfo->vgHash); + dbCache->vgInfo->vgHash = NULL; + } + } + + CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); + taosHashRelease(pCatalog->dbCache, dbCache); + } else { + SCtgDBCache newDBCache = {0}; + newDBCache.vgInfo = dbInfo; + + if (taosHashPut(pCatalog->dbCache, dbName, strlen(dbName), &newDBCache, sizeof(newDBCache)) != 0) { + ctgError("taosHashPut db & db vgroup to cache failed, db:%s", dbName); + CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); + } + + newAdded = true; } - dbInfo->vgInfo = NULL; + dbInfo = NULL; - SDbVgVersion vgVersion = {.dbId = dbInfo->dbId, .vgVersion = dbInfo->vgVersion}; strncpy(vgVersion.dbName, dbName, sizeof(vgVersion.dbName)); if (newAdded) { - CTG_ERR_JRET(ctgMetaRentAdd(&pCatalog->dbRent, &vgVersion, dbInfo->dbId, sizeof(SDbVgVersion))); + CTG_ERR_JRET(ctgMetaRentAdd(&pCatalog->dbRent, &vgVersion, vgVersion.dbId, sizeof(SDbVgVersion))); } else { - CTG_ERR_JRET(ctgMetaRentUpdate(&pCatalog->dbRent, &vgVersion, dbInfo->dbId, sizeof(SDbVgVersion), ctgDbVgVersionCompare)); + CTG_ERR_JRET(ctgMetaRentUpdate(&pCatalog->dbRent, &vgVersion, vgVersion.dbId, sizeof(SDbVgVersion), ctgDbVgVersionCompare)); } - ctgDebug("dbName:%s vgroup updated, vgVersion:%d", dbName, dbInfo->vgVersion); + ctgDebug("dbName:%s vgroup updated, vgVersion:%d", dbName, vgVersion.vgVersion); _return: - if (dbInfo && dbInfo->vgInfo) { - taosHashCleanup(dbInfo->vgInfo); - dbInfo->vgInfo = NULL; + if (dbInfo) { + taosHashCleanup(dbInfo->vgHash); + dbInfo->vgHash = NULL; + tfree(dbInfo); } CTG_RET(code); } -int32_t catalogRemoveDBVgroup(struct SCatalog* pCatalog, SDbVgVersion* dbInfo) { +int32_t catalogRemoveDB(struct SCatalog* pCatalog, const char* dbName, uint64_t dbId) { int32_t code = 0; bool removed = false; - if (NULL == pCatalog || NULL == dbInfo) { + if (NULL == pCatalog || NULL == dbName) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } - if (NULL == pCatalog->dbCache.cache) { + if (NULL == pCatalog->dbCache) { return TSDB_CODE_SUCCESS; } - CTG_ERR_RET(ctgValidateAndRemoveDbInfo(pCatalog, dbInfo, &removed)); + CTG_ERR_RET(ctgValidateAndRemoveDb(pCatalog, dbName, dbId, &removed)); if (!removed) { return TSDB_CODE_SUCCESS; } - ctgInfo("db removed from cache, db:%s", dbInfo->dbName); + ctgInfo("db removed from cache, db:%s, uid:%"PRIx64, dbName, dbId); - CTG_ERR_RET(ctgMetaRentRemove(&pCatalog->dbRent, dbInfo->dbId, ctgDbVgVersionCompare)); + CTG_ERR_RET(ctgMetaRentRemove(&pCatalog->dbRent, dbId, ctgDbVgVersionCompare)); - ctgDebug("db removed from rent, db:%s", dbInfo->dbName); + ctgDebug("db removed from rent, db:%s, uid:%"PRIx64, dbName, dbId); CTG_RET(code); } @@ -1301,7 +1414,7 @@ int32_t catalogGetTableDistVgroup(struct SCatalog* pCatalog, void *pRpc, const S STableMeta *tbMeta = NULL; int32_t code = 0; SVgroupInfo vgroupInfo = {0}; - SDBVgroupInfo* dbVgroup = NULL; + SCtgDBCache* dbCache = NULL; SArray *vgList = NULL; *pVgroupList = NULL; @@ -1310,7 +1423,7 @@ int32_t catalogGetTableDistVgroup(struct SCatalog* pCatalog, void *pRpc, const S char db[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(pTableName, db); - CTG_ERR_JRET(ctgGetDBVgroup(pCatalog, pRpc, pMgmtEps, db, false, &dbVgroup)); + CTG_ERR_JRET(ctgGetDBVgroup(pCatalog, pRpc, pMgmtEps, db, false, &dbCache)); // REMOEV THIS .... if (0 == tbMeta->vgId) { @@ -1323,10 +1436,10 @@ int32_t catalogGetTableDistVgroup(struct SCatalog* pCatalog, void *pRpc, const S // REMOVE THIS .... if (tbMeta->tableType == TSDB_SUPER_TABLE) { - CTG_ERR_JRET(ctgGetVgInfoFromDB(pCatalog, pRpc, pMgmtEps, dbVgroup, pVgroupList)); + CTG_ERR_JRET(ctgGetVgInfoFromDB(pCatalog, pRpc, pMgmtEps, dbCache->vgInfo, pVgroupList)); } else { int32_t vgId = tbMeta->vgId; - if (NULL == taosHashGetClone(dbVgroup->vgInfo, &vgId, sizeof(vgId), &vgroupInfo)) { + if (NULL == taosHashGetClone(dbCache->vgInfo->vgHash, &vgId, sizeof(vgId), &vgroupInfo)) { ctgError("table's vgId not found in vgroup list, vgId:%d, tbName:%s", vgId, tNameGetTableName(pTableName)); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } @@ -1349,9 +1462,9 @@ int32_t catalogGetTableDistVgroup(struct SCatalog* pCatalog, void *pRpc, const S _return: tfree(tbMeta); - if (dbVgroup) { - CTG_UNLOCK(CTG_READ, &dbVgroup->lock); - taosHashRelease(pCatalog->dbCache.cache, dbVgroup); + if (dbCache) { + CTG_UNLOCK(CTG_READ, &dbCache->vgLock); + taosHashRelease(pCatalog->dbCache, dbCache); } if (vgList) { @@ -1364,21 +1477,21 @@ _return: int32_t catalogGetTableHashVgroup(struct SCatalog *pCatalog, void *pTransporter, const SEpSet *pMgmtEps, const SName *pTableName, SVgroupInfo *pVgroup) { - SDBVgroupInfo* dbInfo = NULL; + SCtgDBCache* dbCache = NULL; int32_t code = 0; char db[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(pTableName, db); - CTG_ERR_RET(ctgGetDBVgroup(pCatalog, pTransporter, pMgmtEps, db, false, &dbInfo)); + CTG_ERR_RET(ctgGetDBVgroup(pCatalog, pTransporter, pMgmtEps, db, false, &dbCache)); - CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCatalog, dbInfo, pTableName, pVgroup)); + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCatalog, dbCache->vgInfo, pTableName, pVgroup)); _return: - if (dbInfo) { - CTG_UNLOCK(CTG_READ, &dbInfo->lock); - taosHashRelease(pCatalog->dbCache.cache, dbInfo); + if (dbCache) { + CTG_UNLOCK(CTG_READ, &dbCache->vgLock); + taosHashRelease(pCatalog->dbCache, dbCache); } CTG_RET(code); @@ -1422,6 +1535,7 @@ int32_t catalogGetAllMeta(struct SCatalog* pCatalog, void *pTransporter, const S return TSDB_CODE_SUCCESS; _return: + if (pRsp->pTableMeta) { int32_t aSize = taosArrayGetSize(pRsp->pTableMeta); for (int32_t i = 0; i < aSize; ++i) { @@ -1451,7 +1565,7 @@ int32_t catalogGetExpiredSTables(struct SCatalog* pCatalog, SSTableMetaVersion * CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } - CTG_RET(ctgMetaRentGet(&pCatalog->stableRent, (void **)stables, num, sizeof(SSTableMetaVersion))); + CTG_RET(ctgMetaRentGet(&pCatalog->stbRent, (void **)stables, num, sizeof(SSTableMetaVersion))); } int32_t catalogGetExpiredDBs(struct SCatalog* pCatalog, SDbVgVersion **dbs, uint32_t *num) { diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index c6e5e359bf..b8bea04b2d 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -128,15 +128,11 @@ void ctgTestBuildCTableMetaOutput(STableMetaOutput *output) { strcpy(sn.dbname, "db1"); strcpy(sn.tname, ctgTestSTablename); - char tbFullName[TSDB_TABLE_FNAME_LEN]; - tNameExtractFullName(&cn, tbFullName); - + strcpy(output->dbFName, cn.dbname); SET_META_TYPE_BOTH_TABLE(output->metaType); - strcpy(output->ctbFname, tbFullName); - - tNameExtractFullName(&cn, tbFullName); - strcpy(output->tbFname, tbFullName); + strcpy(output->ctbName, cn.tname); + strcpy(output->tbName, sn.tname); output->ctbMeta.vgId = 9; output->ctbMeta.tableType = TSDB_CHILD_TABLE; @@ -186,7 +182,7 @@ void ctgTestBuildDBVgroup(SDBVgroupInfo *dbVgroup) { dbVgroup->hashMethod = 0; dbVgroup->dbId = ctgTestDbId; - dbVgroup->vgInfo = taosHashInit(ctgTestVgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); + dbVgroup->vgHash = taosHashInit(ctgTestVgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); vgNum = ctgTestGetVgNumFromVgVersion(dbVgroup->vgVersion); uint32_t hashUnit = UINT32_MAX / vgNum; @@ -203,7 +199,7 @@ void ctgTestBuildDBVgroup(SDBVgroupInfo *dbVgroup) { addr->port = htons(n + 22); } - taosHashPut(dbVgroup->vgInfo, &vgInfo.vgId, sizeof(vgInfo.vgId), &vgInfo, sizeof(vgInfo)); + taosHashPut(dbVgroup->vgHash, &vgInfo.vgId, sizeof(vgInfo.vgId), &vgInfo, sizeof(vgInfo)); } } @@ -250,7 +246,8 @@ void ctgTestPrepareTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcM pRsp->contLen = sizeof(STableMetaRsp) + (ctgTestColNum + ctgTestTagNum) * sizeof(SSchema); pRsp->pCont = calloc(1, pRsp->contLen); rspMsg = (STableMetaRsp *)pRsp->pCont; - sprintf(rspMsg->tbFname, "%s.%s", ctgTestDbname, ctgTestTablename); + strcpy(rspMsg->dbFName, ctgTestDbname); + strcpy(rspMsg->tbName, ctgTestTablename); rspMsg->numOfTags = 0; rspMsg->numOfColumns = htonl(ctgTestColNum); rspMsg->precision = 1; @@ -285,8 +282,9 @@ void ctgTestPrepareCTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpc pRsp->contLen = sizeof(STableMetaRsp) + (ctgTestColNum + ctgTestTagNum) * sizeof(SSchema); pRsp->pCont = calloc(1, pRsp->contLen); rspMsg = (STableMetaRsp *)pRsp->pCont; - sprintf(rspMsg->tbFname, "%s.%s", ctgTestDbname, ctgTestCTablename); - sprintf(rspMsg->stbFname, "%s.%s", ctgTestDbname, ctgTestSTablename); + strcpy(rspMsg->dbFName, ctgTestDbname); + strcpy(rspMsg->tbName, ctgTestCTablename); + strcpy(rspMsg->stbName, ctgTestSTablename); rspMsg->numOfTags = htonl(ctgTestTagNum); rspMsg->numOfColumns = htonl(ctgTestColNum); rspMsg->precision = 1; @@ -327,8 +325,9 @@ void ctgTestPrepareSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpc pRsp->contLen = sizeof(STableMetaRsp) + (ctgTestColNum + ctgTestTagNum) * sizeof(SSchema); pRsp->pCont = calloc(1, pRsp->contLen); rspMsg = (STableMetaRsp *)pRsp->pCont; - sprintf(rspMsg->tbFname, "%s.%s", ctgTestDbname, ctgTestSTablename); - sprintf(rspMsg->stbFname, "%s.%s", ctgTestDbname, ctgTestSTablename); + strcpy(rspMsg->dbFName, ctgTestDbname); + strcpy(rspMsg->tbName, ctgTestSTablename); + strcpy(rspMsg->stbName, ctgTestSTablename); rspMsg->numOfTags = htonl(ctgTestTagNum); rspMsg->numOfColumns = htonl(ctgTestColNum); rspMsg->precision = 1; @@ -370,8 +369,9 @@ void ctgTestPrepareMultiSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, pRsp->contLen = sizeof(STableMetaRsp) + (ctgTestColNum + ctgTestTagNum) * sizeof(SSchema); pRsp->pCont = calloc(1, pRsp->contLen); rspMsg = (STableMetaRsp *)pRsp->pCont; - sprintf(rspMsg->tbFname, "%s.%s_%d", ctgTestDbname, ctgTestSTablename, idx); - sprintf(rspMsg->stbFname, "%s.%s_%d", ctgTestDbname, ctgTestSTablename, idx); + strcpy(rspMsg->dbFName, ctgTestDbname); + strcpy(rspMsg->tbName, ctgTestSTablename); + sprintf(rspMsg->stbName, "%s_%d", ctgTestSTablename, idx); rspMsg->numOfTags = htonl(ctgTestTagNum); rspMsg->numOfColumns = htonl(ctgTestColNum); rspMsg->precision = 1; @@ -1291,4 +1291,4 @@ int main(int argc, char **argv) { return RUN_ALL_TESTS(); } -#pragma GCC diagnostic pop \ No newline at end of file +#pragma GCC diagnostic pop diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 38ce005094..e7b3d08bc5 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -45,11 +45,11 @@ int32_t queryBuildTableMetaReqMsg(void* input, char **msg, int32_t msgSize, int3 bMsg->header.vgId = htonl(bInput->vgId); - if (bInput->dbName) { - tstrncpy(bMsg->dbFname, bInput->dbName, tListLen(bMsg->dbFname)); + if (bInput->dbFName) { + tstrncpy(bMsg->dbFName, bInput->dbFName, tListLen(bMsg->dbFName)); } - tstrncpy(bMsg->tableFname, bInput->tableFullName, tListLen(bMsg->tableFname)); + tstrncpy(bMsg->tbName, bInput->tbName, tListLen(bMsg->tbName)); *msgLen = (int32_t)sizeof(*bMsg); return TSDB_CODE_SUCCESS; @@ -113,12 +113,19 @@ int32_t queryProcessUseDBRsp(void* output, char *msg, int32_t msgSize) { return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; } - pOut->dbVgroup.vgVersion = pRsp->vgVersion; - pOut->dbVgroup.hashMethod = pRsp->hashMethod; - pOut->dbVgroup.dbId = pRsp->uid; - pOut->dbVgroup.vgInfo = taosHashInit(pRsp->vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); - if (NULL == pOut->dbVgroup.vgInfo) { - qError("hash init[%d] failed", pRsp->vgNum); + pOut->dbVgroup = calloc(1, sizeof(SDBVgroupInfo)); + if (NULL == pOut->dbVgroup) { + qError("calloc %d failed", (int32_t)sizeof(SDBVgroupInfo)); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + pOut->dbVgroup->vgVersion = pRsp->vgVersion; + pOut->dbVgroup->hashMethod = pRsp->hashMethod; + pOut->dbVgroup->dbId = pRsp->uid; + pOut->dbVgroup->vgHash = taosHashInit(pRsp->vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); + if (NULL == pOut->dbVgroup->vgHash) { + qError("taosHashInit %d failed", pRsp->vgNum); + tfree(pOut->dbVgroup); return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -131,8 +138,8 @@ int32_t queryProcessUseDBRsp(void* output, char *msg, int32_t msgSize) { pRsp->vgroupInfo[i].epset.eps[n].port = ntohs(pRsp->vgroupInfo[i].epset.eps[n].port); } - if (0 != taosHashPut(pOut->dbVgroup.vgInfo, &pRsp->vgroupInfo[i].vgId, sizeof(pRsp->vgroupInfo[i].vgId), &pRsp->vgroupInfo[i], sizeof(pRsp->vgroupInfo[i]))) { - qError("hash push failed"); + if (0 != taosHashPut(pOut->dbVgroup->vgHash, &pRsp->vgroupInfo[i].vgId, sizeof(pRsp->vgroupInfo[i].vgId), &pRsp->vgroupInfo[i], sizeof(pRsp->vgroupInfo[i]))) { + qError("taosHashPut failed"); goto _return; } } @@ -142,8 +149,10 @@ int32_t queryProcessUseDBRsp(void* output, char *msg, int32_t msgSize) { return code; _return: + if (pOut) { - tfree(pOut->dbVgroup.vgInfo); + taosHashCleanup(pOut->dbVgroup->vgHash); + tfree(pOut->dbVgroup); } return code; @@ -248,16 +257,13 @@ int32_t queryProcessTableMetaRsp(void* output, char *msg, int32_t msgSize) { return TSDB_CODE_TSC_INVALID_VALUE; } + strcpy(pOut->dbFName, pMetaMsg->dbFName); + if (pMetaMsg->tableType == TSDB_CHILD_TABLE) { SET_META_TYPE_BOTH_TABLE(pOut->metaType); - if (pMetaMsg->dbFname[0]) { - snprintf(pOut->ctbFname, sizeof(pOut->ctbFname), "%s.%s", pMetaMsg->dbFname, pMetaMsg->tbFname); - snprintf(pOut->tbFname, sizeof(pOut->tbFname), "%s.%s", pMetaMsg->dbFname, pMetaMsg->stbFname); - } else { - memcpy(pOut->ctbFname, pMetaMsg->tbFname, sizeof(pOut->ctbFname)); - memcpy(pOut->tbFname, pMetaMsg->stbFname, sizeof(pOut->tbFname)); - } + strcpy(pOut->ctbName, pMetaMsg->tbName); + strcpy(pOut->tbName, pMetaMsg->stbName); pOut->ctbMeta.vgId = pMetaMsg->vgId; pOut->ctbMeta.tableType = pMetaMsg->tableType; @@ -268,11 +274,7 @@ int32_t queryProcessTableMetaRsp(void* output, char *msg, int32_t msgSize) { } else { SET_META_TYPE_TABLE(pOut->metaType); - if (pMetaMsg->dbFname[0]) { - snprintf(pOut->tbFname, sizeof(pOut->tbFname), "%s.%s", pMetaMsg->dbFname, pMetaMsg->tbFname); - } else { - memcpy(pOut->tbFname, pMetaMsg->tbFname, sizeof(pOut->tbFname)); - } + strcpy(pOut->tbName, pMetaMsg->tbName); code = queryCreateTableMetaFromMsg(pMetaMsg, (pMetaMsg->tableType == TSDB_SUPER_TABLE), &pOut->tbMeta); } @@ -291,4 +293,4 @@ void initQueryModuleMsgHandle() { queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_USE_DB)] = queryProcessUseDBRsp; } -#pragma GCC diagnostic pop \ No newline at end of file +#pragma GCC diagnostic pop From 573aad5cc14aa6d1f7f3d18f96ff38ec04863bc7 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 3 Feb 2022 19:47:27 +0800 Subject: [PATCH 2/7] fix uninitialized --- .gitignore | 2 + example/src/tmq.c | 21 +++++--- include/client/taos.h | 2 +- include/os/osEndian.h | 2 +- include/os/osMath.h | 30 +++++------ source/client/src/tmq.c | 8 +-- source/common/src/ttypes.c | 8 --- source/dnode/mnode/impl/src/mndConsumer.c | 7 ++- source/dnode/mnode/impl/src/mndSubscribe.c | 7 ++- source/dnode/vnode/src/inc/tqInt.h | 61 ++++++++++++++-------- source/dnode/vnode/src/tq/tq.c | 9 +--- source/libs/wal/src/walRead.c | 4 +- source/os/src/osDir.c | 2 + source/util/src/tbuffer.c | 2 + source/util/src/tlog.c | 1 - source/util/src/tpagedfile.c | 17 ++++++ 16 files changed, 112 insertions(+), 71 deletions(-) diff --git a/.gitignore b/.gitignore index f0067d26ba..b9b5341b06 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,8 @@ build/ compile_commands.json .cache .ycm_extra_conf.py +.tasks +.vimspector.json .vscode/ .idea/ cmake-build-debug/ diff --git a/example/src/tmq.c b/example/src/tmq.c index c1c6677b02..31bfe4197f 100644 --- a/example/src/tmq.c +++ b/example/src/tmq.c @@ -16,6 +16,7 @@ #include #include #include +#include #include "taos.h" static int running = 1; @@ -65,7 +66,7 @@ int32_t init_env() { taos_free_result(pRes); - char* sql = "select * from st1"; + const char* sql = "select * from st1"; pRes = tmq_create_topic(pConn, "test_stb_topic_1", sql, strlen(sql)); /*if (taos_errno(pRes) != 0) {*/ /*printf("failed to create topic test_stb_topic_1, reason:%s\n", taos_errstr(pRes));*/ @@ -112,14 +113,20 @@ void basic_consume_loop(tmq_t *tmq, printf("subscribe err\n"); return; } - + int32_t cnt = 0; + clock_t startTime = clock(); while (running) { - tmq_message_t *tmqmessage = tmq_consumer_poll(tmq, 500); - if (tmq) { - msg_process(tmqmessage); + tmq_message_t *tmqmessage = tmq_consumer_poll(tmq, 0); + if (tmqmessage) { + cnt++; + /*msg_process(tmqmessage);*/ tmq_message_destroy(tmqmessage); + } else { + break; } } + clock_t endTime = clock(); + printf("log cnt: %d %f s\n", cnt, (double)(endTime - startTime) / CLOCKS_PER_SEC); err = tmq_consumer_close(tmq); if (err) @@ -163,6 +170,6 @@ int main() { code = init_env(); tmq_t* tmq = build_consumer(); tmq_list_t* topic_list = build_topic_list(); - /*basic_consume_loop(tmq, topic_list);*/ - sync_consume_loop(tmq, topic_list); + basic_consume_loop(tmq, topic_list); + /*sync_consume_loop(tmq, topic_list);*/ } diff --git a/include/client/taos.h b/include/client/taos.h index 7c5cc56b20..a8627a43da 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -213,7 +213,7 @@ typedef struct tmq_message_t tmq_message_t; typedef void(tmq_commit_cb(tmq_t *, tmq_resp_err_t, tmq_topic_vgroup_list_t *, void *param)); DLL_EXPORT tmq_list_t *tmq_list_new(); -DLL_EXPORT int32_t tmq_list_append(tmq_list_t *, char *); +DLL_EXPORT int32_t tmq_list_append(tmq_list_t *, const char *); DLL_EXPORT TAOS_RES *tmq_create_topic(TAOS *taos, const char *name, const char *sql, int sqlLen); DLL_EXPORT tmq_t *tmq_consumer_new(void *conn, tmq_conf_t *conf, char *errstr, int32_t errstrLen); diff --git a/include/os/osEndian.h b/include/os/osEndian.h index e573ba0a75..1f52ece535 100644 --- a/include/os/osEndian.h +++ b/include/os/osEndian.h @@ -30,4 +30,4 @@ static const int32_t endian_test_var = 1; } #endif -#endif /*_TD_OS_ENDIAN_H_*/ \ No newline at end of file +#endif /*_TD_OS_ENDIAN_H_*/ diff --git a/include/os/osMath.h b/include/os/osMath.h index 948fbbf665..3fe46d557e 100644 --- a/include/os/osMath.h +++ b/include/os/osMath.h @@ -36,25 +36,25 @@ extern "C" { #else - #define TSWAP(a, b, c) \ - do { \ - typeof(a) __tmp = (a); \ - (a) = (b); \ - (b) = __tmp; \ + #define TSWAP(a, b, c) \ + do { \ + __typeof(a) __tmp = (a); \ + (a) = (b); \ + (b) = __tmp; \ } while (0) - #define TMAX(a, b) \ - ({ \ - typeof(a) __a = (a); \ - typeof(b) __b = (b); \ - (__a > __b) ? __a : __b; \ + #define TMAX(a, b) \ + ({ \ + __typeof(a) __a = (a); \ + __typeof(b) __b = (b); \ + (__a > __b) ? __a : __b; \ }) - #define TMIN(a, b) \ - ({ \ - typeof(a) __a = (a); \ - typeof(b) __b = (b); \ - (__a < __b) ? __a : __b; \ + #define TMIN(a, b) \ + ({ \ + __typeof(a) __a = (a); \ + __typeof(b) __b = (b); \ + (__a < __b) ? __a : __b; \ }) #endif diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index facc8c7216..e57901ed2e 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -13,6 +13,8 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE + #include "clientInt.h" #include "clientLog.h" #include "parser.h" @@ -146,7 +148,7 @@ tmq_list_t* tmq_list_new() { return ptr; } -int32_t tmq_list_append(tmq_list_t* ptr, char* src) { +int32_t tmq_list_append(tmq_list_t* ptr, const char* src) { if (ptr->cnt >= ptr->tot - 1) return -1; ptr->elems[ptr->cnt] = strdup(src); ptr->cnt++; @@ -366,7 +368,7 @@ TAOS_RES* tmq_create_topic(TAOS* taos, const char* topicName, const char* sql, i .igExists = 1, .physicalPlan = (char*)pStr, .sql = (char*)sql, - .logicalPlan = "no logic plan", + .logicalPlan = (char*)"no logic plan", }; int tlen = tSerializeSCMCreateTopicReq(NULL, &req); @@ -512,7 +514,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { return -1; } tDecodeSMqConsumeRsp(pMsg->pData, pRsp); - /*printf("rsp %ld %ld\n", pRsp->committedOffset, pRsp->rspOffset);*/ + /*printf("rsp %ld %ld %d\n", pRsp->committedOffset, pRsp->rspOffset, pRsp->numOfTopics);*/ if (pRsp->numOfTopics == 0) { /*printf("no data\n");*/ free(pRsp); diff --git a/source/common/src/ttypes.c b/source/common/src/ttypes.c index 73f755bf1a..1cab413c3f 100644 --- a/source/common/src/ttypes.c +++ b/source/common/src/ttypes.c @@ -630,14 +630,6 @@ void operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type) { } } -#define TSWAP(a, b, c) \ - do { \ - typeof(a) __tmp = (a); \ - (a) = (b); \ - (b) = __tmp; \ - } while (0) - - void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf) { switch (type) { case TSDB_DATA_TYPE_INT: diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 6fd61baee3..fd960eb63e 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -14,6 +14,7 @@ */ #define _DEFAULT_SOURCE + #include "mndConsumer.h" #include "mndDb.h" #include "mndDnode.h" @@ -54,13 +55,14 @@ void mndCleanupConsumer(SMnode *pMnode) {} SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer) { terrno = TSDB_CODE_OUT_OF_MEMORY; + void* buf = NULL; int32_t tlen = tEncodeSMqConsumerObj(NULL, pConsumer); int32_t size = sizeof(int32_t) + tlen + MND_CONSUMER_RESERVE_SIZE; SSdbRaw *pRaw = sdbAllocRaw(SDB_CONSUMER, MND_CONSUMER_VER_NUMBER, size); if (pRaw == NULL) goto CM_ENCODE_OVER; - void *buf = malloc(tlen); + buf = malloc(tlen); if (buf == NULL) goto CM_ENCODE_OVER; void *abuf = buf; @@ -88,6 +90,7 @@ CM_ENCODE_OVER: SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) { terrno = TSDB_CODE_OUT_OF_MEMORY; + void* buf = NULL; int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto CM_DECODE_OVER; @@ -106,7 +109,7 @@ SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) { int32_t dataPos = 0; int32_t len; SDB_GET_INT32(pRaw, dataPos, &len, CM_DECODE_OVER); - void *buf = malloc(len); + buf = malloc(len); if (buf == NULL) goto CM_DECODE_OVER; SDB_GET_BINARY(pRaw, dataPos, buf, len, CM_DECODE_OVER); SDB_GET_RESERVE(pRaw, dataPos, MND_CONSUMER_RESERVE_SIZE, CM_DECODE_OVER); diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index bf138d335b..938a3eb89e 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -12,6 +12,7 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "mndSubscribe.h" #include "mndConsumer.h" @@ -372,13 +373,14 @@ void mndCleanupSubscribe(SMnode *pMnode) {} static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *pSub) { terrno = TSDB_CODE_OUT_OF_MEMORY; + void* buf = NULL; int32_t tlen = tEncodeSubscribeObj(NULL, pSub); int32_t size = sizeof(int32_t) + tlen + MND_SUBSCRIBE_RESERVE_SIZE; SSdbRaw *pRaw = sdbAllocRaw(SDB_SUBSCRIBE, MND_SUBSCRIBE_VER_NUMBER, size); if (pRaw == NULL) goto SUB_ENCODE_OVER; - void *buf = malloc(tlen); + buf = malloc(tlen); if (buf == NULL) goto SUB_ENCODE_OVER; void *abuf = buf; @@ -406,6 +408,7 @@ SUB_ENCODE_OVER: static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw) { terrno = TSDB_CODE_OUT_OF_MEMORY; + void* buf = NULL; int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto SUB_DECODE_OVER; @@ -425,7 +428,7 @@ static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw) { int32_t dataPos = 0; int32_t tlen; SDB_GET_INT32(pRaw, dataPos, &tlen, SUB_DECODE_OVER); - void *buf = malloc(tlen + 1); + buf = malloc(tlen + 1); if (buf == NULL) goto SUB_DECODE_OVER; SDB_GET_BINARY(pRaw, dataPos, buf, tlen, SUB_DECODE_OVER); SDB_GET_RESERVE(pRaw, dataPos, MND_SUBSCRIBE_RESERVE_SIZE, SUB_DECODE_OVER); diff --git a/source/dnode/vnode/src/inc/tqInt.h b/source/dnode/vnode/src/inc/tqInt.h index a9ba825a29..cefe9eff72 100644 --- a/source/dnode/vnode/src/inc/tqInt.h +++ b/source/dnode/vnode/src/inc/tqInt.h @@ -16,9 +16,9 @@ #ifndef _TD_TQ_INT_H_ #define _TD_TQ_INT_H_ -#include "tq.h" #include "meta.h" #include "tlog.h" +#include "tq.h" #include "trpc.h" #ifdef __cplusplus extern "C" { @@ -26,29 +26,48 @@ extern "C" { extern int32_t tqDebugFlag; -#define tqFatal(...) { if (tqDebugFlag & DEBUG_FATAL) { taosPrintLog("TQ FATAL ", 255, __VA_ARGS__); }} -#define tqError(...) { if (tqDebugFlag & DEBUG_ERROR) { taosPrintLog("TQ ERROR ", 255, __VA_ARGS__); }} -#define tqWarn(...) { if (tqDebugFlag & DEBUG_WARN) { taosPrintLog("TQ WARN ", 255, __VA_ARGS__); }} -#define tqInfo(...) { if (tqDebugFlag & DEBUG_INFO) { taosPrintLog("TQ ", 255, __VA_ARGS__); }} -#define tqDebug(...) { if (tqDebugFlag & DEBUG_DEBUG) { taosPrintLog("TQ ", tqDebugFlag, __VA_ARGS__); }} -#define tqTrace(...) { if (tqDebugFlag & DEBUG_TRACE) { taosPrintLog("TQ ", tqDebugFlag, __VA_ARGS__); }} +#define tqFatal(...) \ + { \ + if (tqDebugFlag & DEBUG_FATAL) { \ + taosPrintLog("TQ FATAL ", 255, __VA_ARGS__); \ + } \ + } +#define tqError(...) \ + { \ + if (tqDebugFlag & DEBUG_ERROR) { \ + taosPrintLog("TQ ERROR ", 255, __VA_ARGS__); \ + } \ + } +#define tqWarn(...) \ + { \ + if (tqDebugFlag & DEBUG_WARN) { \ + taosPrintLog("TQ WARN ", 255, __VA_ARGS__); \ + } \ + } +#define tqInfo(...) \ + { \ + if (tqDebugFlag & DEBUG_INFO) { \ + taosPrintLog("TQ ", 255, __VA_ARGS__); \ + } \ + } +#define tqDebug(...) \ + { \ + if (tqDebugFlag & DEBUG_DEBUG) { \ + taosPrintLog("TQ ", tqDebugFlag, __VA_ARGS__); \ + } \ + } +#define tqTrace(...) \ + { \ + if (tqDebugFlag & DEBUG_TRACE) { \ + taosPrintLog("TQ ", tqDebugFlag, __VA_ARGS__); \ + } \ + } -// create persistent storage for meta info such as consuming offset -// return value > 0: cgId -// return value <= 0: error code -// int tqCreateTCGroup(STQ*, const char* topic, int cgId, tqBufferHandle** handle); -// create ring buffer in memory and load consuming offset -// int tqOpenTCGroup(STQ*, const char* topic, int cgId); -// destroy ring buffer and persist consuming offset -// int tqCloseTCGroup(STQ*, const char* topic, int cgId); -// delete persistent storage for meta info -// int tqDropTCGroup(STQ*, const char* topic, int cgId); +int tqSerializeConsumer(const STqConsumerHandle*, STqSerializedHead**); +const void* tqDeserializeConsumer(const STqSerializedHead* pHead, STqConsumerHandle**); -//int tqSerializeGroup(const STqGroup*, STqSerializedHead**); -//const void* tqDeserializeGroup(const STqSerializedHead* pHead, STqGroup** ppGroup); -int tqSerializeConsumer(const STqConsumerHandle*, STqSerializedHead**); -const void* tqDeserializeConsumer(const STqSerializedHead* pHead, STqConsumerHandle**); static int FORCE_INLINE tqQueryExecuting(int32_t status) { return status; } + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 4e52ecd858..a625980505 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -12,19 +12,12 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "tcompare.h" #include "tqInt.h" #include "tqMetaStore.h" -// static -// read next version data -// -// send to fetch queue -// -// handle management message -// - int tqInit() { int8_t old = atomic_val_compare_exchange_8(&tqMgmt.inited, 0, 1); if (old == 1) return 0; diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 41fff4d228..1d9201f69d 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -170,7 +170,7 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { } if (pRead->pHead->head.version != ver) { - /*wError("unexpected wal log version: %ld, read request version:%ld", pRead->pHead->head.version, ver);*/ + wError("unexpected wal log version: %ld, read request version:%ld", pRead->pHead->head.version, ver); pRead->curVersion = -1; terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; @@ -178,7 +178,7 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { code = walValidBodyCksum(pRead->pHead); if (code != 0) { - /*wError("unexpected wal log version: checksum not passed");*/ + wError("unexpected wal log version: checksum not passed"); pRead->curVersion = -1; terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index 04efbef5b3..8999646f6a 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -13,6 +13,8 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE + #include "os.h" #include "osString.h" diff --git a/source/util/src/tbuffer.c b/source/util/src/tbuffer.c index ddd283ae0f..0456d6a2ee 100644 --- a/source/util/src/tbuffer.c +++ b/source/util/src/tbuffer.c @@ -13,6 +13,8 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE + #include "tbuffer.h" #include "exception.h" #include "os.h" diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 5ce69f1c2e..12aa77214f 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -19,7 +19,6 @@ #include "tnote.h" #include "tutil.h" #include "ulog.h" -//#include "zlib.h" #define MAX_LOGLINE_SIZE (1000) #define MAX_LOGLINE_BUFFER_SIZE (MAX_LOGLINE_SIZE + 10) diff --git a/source/util/src/tpagedfile.c b/source/util/src/tpagedfile.c index 3373d09876..3cdba580d4 100644 --- a/source/util/src/tpagedfile.c +++ b/source/util/src/tpagedfile.c @@ -1,3 +1,20 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE + #include "tpagedfile.h" #include "thash.h" #include "stddef.h" From 70644256323e69ae7c5819c83c57d2233e6f19ac Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 4 Feb 2022 21:10:36 +0800 Subject: [PATCH 3/7] fix compile --- source/util/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/source/util/CMakeLists.txt b/source/util/CMakeLists.txt index 6d8dfa51e7..760a66a4fb 100644 --- a/source/util/CMakeLists.txt +++ b/source/util/CMakeLists.txt @@ -10,6 +10,7 @@ target_link_libraries( util PRIVATE os PUBLIC lz4_static + PUBLIC api ) if(${BUILD_TEST}) From 259635777c9b1857ebc44fa7dcfb825ae7f9e296 Mon Sep 17 00:00:00 2001 From: dapan Date: Sat, 5 Feb 2022 12:21:21 +0800 Subject: [PATCH 4/7] feature/qnode3 --- source/dnode/vnode/src/vnd/vnodeQuery.c | 10 +++---- source/libs/catalog/src/catalog.c | 9 +++++++ source/libs/catalog/test/catalogTests.cpp | 32 +++++++++++++++++------ source/libs/scheduler/inc/schedulerInt.h | 2 +- source/libs/scheduler/src/scheduler.c | 4 +-- 5 files changed, 39 insertions(+), 18 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 5575deef03..f4776fa3c5 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -82,11 +82,7 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg) { int msgLen = 0; int32_t code = TSDB_CODE_VND_APP_ERROR; - char tbFName[TSDB_TABLE_FNAME_LEN] = {0}; - snprintf(tbFName, sizeof(tbFName), "%s.%s", pReq->dbFName, pReq->tbName); - - - pTbCfg = metaGetTbInfoByName(pVnode->pMeta, tbFName, &uid); + pTbCfg = metaGetTbInfoByName(pVnode->pMeta, pReq->tbName, &uid); if (pTbCfg == NULL) { code = TSDB_CODE_VND_TB_NOT_EXIST; goto _exit; @@ -124,10 +120,10 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg) { memcpy(pTbMetaMsg->dbFName, pReq->dbFName, sizeof(pTbMetaMsg->dbFName)); strcpy(pTbMetaMsg->tbName, pReq->tbName); if (pTbCfg->type == META_CHILD_TABLE) { - strcpy(pTbMetaMsg->stbName, pStbCfg->name + strlen(pReq->dbFName) + 1); + strcpy(pTbMetaMsg->stbName, pStbCfg->name); pTbMetaMsg->suid = htobe64(pTbCfg->ctbCfg.suid); } else if (pTbCfg->type == META_SUPER_TABLE) { - strcpy(pTbMetaMsg->stbName, pTbCfg->name + strlen(pReq->dbFName) + 1); + strcpy(pTbMetaMsg->stbName, pTbCfg->name); pTbMetaMsg->suid = htobe64(uid); } pTbMetaMsg->numOfTags = htonl(nTagCols); diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 996597de20..95e60415c7 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -248,6 +248,7 @@ int32_t ctgGetTableMetaFromCache(struct SCatalog* pCatalog, const SName* pTableN tbMeta = *pTableMeta; if (tbMeta->tableType != TSDB_CHILD_TABLE) { + taosHashRelease(pCatalog->dbCache, dbCache); ctgDebug("Got tbmeta from cache, type:%d, dbFName:%s, tbName:%s", tbMeta->tableType, db, pTableName->tname); return TSDB_CODE_SUCCESS; } @@ -257,6 +258,7 @@ int32_t ctgGetTableMetaFromCache(struct SCatalog* pCatalog, const SName* pTableN STableMeta **stbMeta = taosHashGet(dbCache->tbCache.stbCache, &tbMeta->suid, sizeof(tbMeta->suid)); if (NULL == stbMeta || NULL == *stbMeta) { CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); + taosHashRelease(pCatalog->dbCache, dbCache); ctgError("stable not in stbCache, suid:%"PRIx64, tbMeta->suid); tfree(*pTableMeta); *exist = 0; @@ -265,6 +267,7 @@ int32_t ctgGetTableMetaFromCache(struct SCatalog* pCatalog, const SName* pTableN if ((*stbMeta)->suid != tbMeta->suid) { CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); + taosHashRelease(pCatalog->dbCache, dbCache); tfree(*pTableMeta); ctgError("stable suid in stbCache mis-match, expected suid:%"PRIx64 ",actual suid:%"PRIx64, tbMeta->suid, (*stbMeta)->suid); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); @@ -274,6 +277,7 @@ int32_t ctgGetTableMetaFromCache(struct SCatalog* pCatalog, const SName* pTableN *pTableMeta = realloc(*pTableMeta, metaSize); if (NULL == *pTableMeta) { CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); + taosHashRelease(pCatalog->dbCache, dbCache); ctgError("realloc size[%d] failed", metaSize); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } @@ -282,6 +286,8 @@ int32_t ctgGetTableMetaFromCache(struct SCatalog* pCatalog, const SName* pTableN CTG_UNLOCK(CTG_READ, &dbCache->tbCache.stbLock); + taosHashRelease(pCatalog->dbCache, dbCache); + ctgDebug("Got tbmeta from cache, dbFName:%s, tbName:%s", db, pTableName->tname); return TSDB_CODE_SUCCESS; @@ -1318,6 +1324,9 @@ int32_t catalogUpdateDBVgroup(struct SCatalog* pCatalog, const char* dbName, SDB taosHashCleanup(dbCache->vgInfo->vgHash); dbCache->vgInfo->vgHash = NULL; } + + tfree(dbCache->vgInfo); + dbCache->vgInfo = dbInfo; } CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index b8bea04b2d..bfb950e5db 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -128,7 +128,10 @@ void ctgTestBuildCTableMetaOutput(STableMetaOutput *output) { strcpy(sn.dbname, "db1"); strcpy(sn.tname, ctgTestSTablename); - strcpy(output->dbFName, cn.dbname); + char db[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(&cn, db); + + strcpy(output->dbFName, db); SET_META_TYPE_BOTH_TABLE(output->metaType); strcpy(output->ctbName, cn.tname); @@ -171,10 +174,11 @@ void ctgTestBuildCTableMetaOutput(STableMetaOutput *output) { strcpy(s->name, "tag1s"); } -void ctgTestBuildDBVgroup(SDBVgroupInfo *dbVgroup) { +void ctgTestBuildDBVgroup(SDBVgroupInfo **pdbVgroup) { static int32_t vgVersion = ctgTestVgVersion + 1; int32_t vgNum = 0; SVgroupInfo vgInfo = {0}; + SDBVgroupInfo *dbVgroup = (SDBVgroupInfo *)calloc(1, sizeof(SDBVgroupInfo)); dbVgroup->vgVersion = vgVersion++; @@ -201,6 +205,8 @@ void ctgTestBuildDBVgroup(SDBVgroupInfo *dbVgroup) { taosHashPut(dbVgroup->vgHash, &vgInfo.vgId, sizeof(vgInfo.vgId), &vgInfo, sizeof(vgInfo)); } + + *pdbVgroup = dbVgroup; } void ctgTestPrepareDbVgroups(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { @@ -370,7 +376,7 @@ void ctgTestPrepareMultiSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, pRsp->pCont = calloc(1, pRsp->contLen); rspMsg = (STableMetaRsp *)pRsp->pCont; strcpy(rspMsg->dbFName, ctgTestDbname); - strcpy(rspMsg->tbName, ctgTestSTablename); + sprintf(rspMsg->tbName, "%s_%d", ctgTestSTablename, idx); sprintf(rspMsg->stbName, "%s_%d", ctgTestSTablename, idx); rspMsg->numOfTags = htonl(ctgTestTagNum); rspMsg->numOfColumns = htonl(ctgTestColNum); @@ -589,12 +595,12 @@ void *ctgTestGetDbVgroupThread(void *param) { void *ctgTestSetDbVgroupThread(void *param) { struct SCatalog *pCtg = (struct SCatalog *)param; int32_t code = 0; - SDBVgroupInfo dbVgroup = {0}; + SDBVgroupInfo *dbVgroup = NULL; int32_t n = 0; while (!ctgTestStop) { ctgTestBuildDBVgroup(&dbVgroup); - code = catalogUpdateDBVgroup(pCtg, ctgTestDbname, &dbVgroup); + code = catalogUpdateDBVgroup(pCtg, ctgTestDbname, dbVgroup); if (code) { assert(0); } @@ -669,6 +675,7 @@ void *ctgTestSetCtableMetaThread(void *param) { return NULL; } + TEST(tableMeta, normalTable) { struct SCatalog *pCtg = NULL; void *mockPointer = (void *)0x1; @@ -1062,9 +1069,11 @@ TEST(dbVgroup, getSetDbVgroupCase) { void *mockPointer = (void *)0x1; SVgroupInfo vgInfo = {0}; SVgroupInfo *pvgInfo = NULL; - SDBVgroupInfo dbVgroup = {0}; + SDBVgroupInfo *dbVgroup = NULL; SArray *vgList = NULL; + ctgTestInitLogFile(); + ctgTestSetPrepareDbVgroupsAndNormalMeta(); initQueryModuleMsgHandle(); @@ -1099,7 +1108,7 @@ TEST(dbVgroup, getSetDbVgroupCase) { taosArrayDestroy(vgList); ctgTestBuildDBVgroup(&dbVgroup); - code = catalogUpdateDBVgroup(pCtg, ctgTestDbname, &dbVgroup); + code = catalogUpdateDBVgroup(pCtg, ctgTestDbname, dbVgroup); ASSERT_EQ(code, 0); code = catalogGetTableHashVgroup(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &vgInfo); @@ -1169,6 +1178,7 @@ TEST(multiThread, getSetDbVgroupCase) { catalogDestroy(); } + TEST(multiThread, ctableMeta) { struct SCatalog *pCtg = NULL; void *mockPointer = (void *)0x1; @@ -1178,6 +1188,8 @@ TEST(multiThread, ctableMeta) { SArray *vgList = NULL; ctgTestStop = false; + ctgTestInitLogFile(); + ctgTestSetPrepareDbVgroupsAndChildMeta(); initQueryModuleMsgHandle(); @@ -1212,11 +1224,13 @@ TEST(multiThread, ctableMeta) { } ctgTestStop = true; - sleep(1); + sleep(2); catalogDestroy(); } + + TEST(rentTest, allRent) { struct SCatalog *pCtg = NULL; void *mockPointer = (void *)0x1; @@ -1229,6 +1243,8 @@ TEST(rentTest, allRent) { SSTableMetaVersion *stable = NULL; uint32_t num = 0; + ctgTestInitLogFile(); + ctgTestSetPrepareDbVgroupsAndMultiSuperMeta(); initQueryModuleMsgHandle(); diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index 2ace1c66f5..df62e72770 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -131,7 +131,7 @@ typedef struct SSchJob { SQueryProfileSummary summary; } SSchJob; -#define SCH_TASK_READY_TO_LUNCH(task) (atomic_load_32(&(task)->childReady) >= taosArrayGetSize((task)->children)) +#define SCH_TASK_READY_TO_LUNCH(readyNum, task) ((readyNum) >= taosArrayGetSize((task)->children)) #define SCH_IS_DATA_SRC_TASK(task) ((task)->plan->type == QUERY_TYPE_SCAN) #define SCH_TASK_NEED_WAIT_ALL(task) ((task)->plan->type == QUERY_TYPE_MODIFY) diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 621ed68bae..ab35993966 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -779,14 +779,14 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { SSchTask *par = *(SSchTask **)taosArrayGet(pTask->parents, i); pErrTask = par; - atomic_add_fetch_32(&par->childReady, 1); + int32_t readyNum = atomic_add_fetch_32(&par->childReady, 1); SCH_LOCK(SCH_WRITE, &par->lock); SDownstreamSource source = {.taskId = pTask->taskId, .schedId = schMgmt.sId, .addr = pTask->succeedAddr}; qSetSubplanExecutionNode(par->plan, pTask->plan->id.templateId, &source); SCH_UNLOCK(SCH_WRITE, &par->lock); - if (SCH_TASK_READY_TO_LUNCH(par)) { + if (SCH_TASK_READY_TO_LUNCH(readyNum, par)) { SCH_ERR_RET(schLaunchTask(pJob, par)); } } From dd0a08c920a8663f2eac5e3b8b904424e8de79c4 Mon Sep 17 00:00:00 2001 From: dapan Date: Sat, 5 Feb 2022 12:52:40 +0800 Subject: [PATCH 5/7] feature/qnode --- source/dnode/mnode/impl/src/mndStb.c | 1 + source/dnode/mnode/impl/test/stb/stb.cpp | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index f855252d1d..9944c0c7fc 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -759,6 +759,7 @@ static int32_t mndProcessStbMetaReq(SMnodeMsg *pReq) { strcpy(pMeta->dbFName, pStb->db); strcpy(pMeta->tbName, pInfo->tbName); + strcpy(pMeta->stbName, pInfo->tbName); pMeta->numOfTags = htonl(pStb->numOfTags); pMeta->numOfColumns = htonl(pStb->numOfColumns); pMeta->precision = pDb->cfg.precision; diff --git a/source/dnode/mnode/impl/test/stb/stb.cpp b/source/dnode/mnode/impl/test/stb/stb.cpp index 887fde312d..a0e2460334 100644 --- a/source/dnode/mnode/impl/test/stb/stb.cpp +++ b/source/dnode/mnode/impl/test/stb/stb.cpp @@ -147,7 +147,8 @@ TEST_F(MndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { pSchema->bytes = htonl(pSchema->bytes); } - EXPECT_STREQ(pRsp->tbName, "1.d1"); + EXPECT_STREQ(pRsp->dbFName, "1.d1"); + EXPECT_STREQ(pRsp->tbName, "stb"); EXPECT_STREQ(pRsp->stbName, "stb"); EXPECT_EQ(pRsp->numOfColumns, 2); EXPECT_EQ(pRsp->numOfTags, 3); From de439d47875e4e0c873ee2492181a916f32a0db1 Mon Sep 17 00:00:00 2001 From: dapan Date: Sun, 6 Feb 2022 15:37:16 +0800 Subject: [PATCH 6/7] feature/qnode --- include/libs/catalog/catalog.h | 6 +- source/client/src/clientHb.c | 127 +++++++++++++++++++++- source/client/src/clientImpl.c | 2 +- source/client/src/clientMsgHandler.c | 2 +- source/dnode/mnode/impl/inc/mndStb.h | 3 + source/dnode/mnode/impl/src/mndDb.c | 6 +- source/dnode/mnode/impl/src/mndProfile.c | 12 +- source/dnode/mnode/impl/src/mndStb.c | 105 +++++++++++++++++- source/libs/catalog/inc/catalogInt.h | 2 +- source/libs/catalog/src/catalog.c | 75 +++++++++++-- source/libs/catalog/test/catalogTests.cpp | 9 +- 11 files changed, 323 insertions(+), 26 deletions(-) diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 6f4be556e4..c291ebd8fd 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -53,13 +53,15 @@ typedef struct SCatalogCfg { } SCatalogCfg; typedef struct SSTableMetaVersion { + char dbFName[TSDB_DB_FNAME_LEN]; + char stbName[TSDB_TABLE_NAME_LEN]; uint64_t suid; int16_t sversion; int16_t tversion; } SSTableMetaVersion; typedef struct SDbVgVersion { - char dbName[TSDB_DB_FNAME_LEN]; + char dbFName[TSDB_DB_FNAME_LEN]; int64_t dbId; int32_t vgVersion; } SDbVgVersion; @@ -101,6 +103,8 @@ int32_t catalogUpdateDBVgroup(struct SCatalog* pCatalog, const char* dbName, SDB int32_t catalogRemoveDB(struct SCatalog* pCatalog, const char* dbName, uint64_t dbId); +int32_t catalogRemoveSTableMeta(struct SCatalog* pCatalog, const char* dbName, const char* stbName, uint64_t suid); + /** * Get a table's meta data. * @param pCatalog (input, got with catalogGetHandle) diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index b09c3ee9d8..d265ffaa94 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -85,6 +85,75 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog return TSDB_CODE_SUCCESS; } +static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) { + int32_t msgLen = 0; + int32_t code = 0; + int32_t schemaNum = 0; + + while (msgLen < valueLen) { + STableMetaRsp *rsp = (STableMetaRsp *)((char *)value + msgLen); + + rsp->numOfColumns = ntohl(rsp->numOfColumns); + rsp->suid = be64toh(rsp->suid); + + if (rsp->numOfColumns < 0) { + schemaNum = 0; + + tscDebug("hb remove stb, db:%s, stb:%s", rsp->dbFName, rsp->stbName); + + code = catalogRemoveSTableMeta(pCatalog, rsp->dbFName, rsp->stbName, rsp->suid); + } else { + rsp->numOfTags = ntohl(rsp->numOfTags); + + schemaNum = rsp->numOfColumns + rsp->numOfTags; +/* + rsp->vgNum = ntohl(rsp->vgNum); + rsp->uid = be64toh(rsp->uid); + + SDBVgroupInfo vgInfo = {0}; + vgInfo.dbId = rsp->uid; + vgInfo.vgVersion = rsp->vgVersion; + vgInfo.hashMethod = rsp->hashMethod; + vgInfo.vgHash = taosHashInit(rsp->vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); + if (NULL == vgInfo.vgHash) { + tscError("hash init[%d] failed", rsp->vgNum); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + for (int32_t i = 0; i < rsp->vgNum; ++i) { + rsp->vgroupInfo[i].vgId = ntohl(rsp->vgroupInfo[i].vgId); + rsp->vgroupInfo[i].hashBegin = ntohl(rsp->vgroupInfo[i].hashBegin); + rsp->vgroupInfo[i].hashEnd = ntohl(rsp->vgroupInfo[i].hashEnd); + + for (int32_t n = 0; n < rsp->vgroupInfo[i].epset.numOfEps; ++n) { + rsp->vgroupInfo[i].epset.eps[n].port = ntohs(rsp->vgroupInfo[i].epset.eps[n].port); + } + + if (0 != taosHashPut(vgInfo.vgHash, &rsp->vgroupInfo[i].vgId, sizeof(rsp->vgroupInfo[i].vgId), &rsp->vgroupInfo[i], sizeof(rsp->vgroupInfo[i]))) { + tscError("hash push failed, errno:%d", errno); + taosHashCleanup(vgInfo.vgHash); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + } + + code = catalogUpdateDBVgroup(pCatalog, rsp->db, &vgInfo); + if (code) { + taosHashCleanup(vgInfo.vgHash); + } +*/ + } + + if (code) { + return code; + } + + msgLen += sizeof(STableMetaRsp) + schemaNum * sizeof(SSchema); + } + + return TSDB_CODE_SUCCESS; +} + + static int32_t hbQueryHbRspHandle(struct SAppHbMgr *pAppHbMgr, SClientHbRsp* pRsp) { SHbConnInfo * info = taosHashGet(pAppHbMgr->connInfo, &pRsp->connKey, sizeof(SClientHbKey)); if (NULL == info) { @@ -117,9 +186,24 @@ static int32_t hbQueryHbRspHandle(struct SAppHbMgr *pAppHbMgr, SClientHbRsp* pRs hbProcessDBInfoRsp(kv->value, kv->valueLen, pCatalog); break; } - case HEARTBEAT_KEY_STBINFO: + case HEARTBEAT_KEY_STBINFO:{ + if (kv->valueLen <= 0 || NULL == kv->value) { + tscError("invalid hb stb info, len:%d, value:%p", kv->valueLen, kv->value); + break; + } + int64_t *clusterId = (int64_t *)info->param; + struct SCatalog *pCatalog = NULL; + + int32_t code = catalogGetHandle(*clusterId, &pCatalog); + if (code != TSDB_CODE_SUCCESS) { + tscWarn("catalogGetHandle failed, clusterId:%"PRIx64", error:%s", *clusterId, tstrerror(code)); + break; + } + + hbProcessStbInfoRsp(kv->value, kv->valueLen, pCatalog); break; + } default: tscError("invalid hb key type:%d", kv->key); break; @@ -152,7 +236,7 @@ static int32_t hbMqAsyncCallBack(void* param, const SDataBuf* pMsg, int32_t code tfree(param); if (rspNum) { - tscDebug("hb got %d rsp, %d empty rsp prior", rspNum, atomic_val_compare_exchange_32(&emptyRspNum, emptyRspNum, 0)); + tscDebug("hb got %d rsp, %d empty rsp received before", rspNum, atomic_val_compare_exchange_32(&emptyRspNum, emptyRspNum, 0)); } else { atomic_add_fetch_32(&emptyRspNum, 1); } @@ -199,6 +283,37 @@ int32_t hbGetExpiredDBInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SCl return TSDB_CODE_SUCCESS; } +int32_t hbGetExpiredStbInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SClientHbReq *req) { + SSTableMetaVersion *stbs = NULL; + uint32_t stbNum = 0; + int32_t code = 0; + + code = catalogGetExpiredSTables(pCatalog, &stbs, &stbNum); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + + if (stbNum <= 0) { + return TSDB_CODE_SUCCESS; + } + + for (int32_t i = 0; i < stbNum; ++i) { + SSTableMetaVersion *stb = &stbs[i]; + stb->suid = htobe64(stb->suid); + stb->sversion = htons(stb->sversion); + stb->tversion = htons(stb->tversion); + } + + SKv kv = {.key = HEARTBEAT_KEY_STBINFO, .valueLen = sizeof(SSTableMetaVersion) * stbNum, .value = stbs}; + + tscDebug("hb got %d expired stb, valueLen:%d", stbNum, kv.valueLen); + + taosHashPut(req->info, &kv.key, sizeof(kv.key), &kv, sizeof(kv)); + + return TSDB_CODE_SUCCESS; +} + + int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void* param, SClientHbReq *req) { int64_t *clusterId = (int64_t *)param; struct SCatalog *pCatalog = NULL; @@ -214,6 +329,11 @@ int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void* param, SClientHbReq *req return code; } + code = hbGetExpiredStbInfo(connKey, pCatalog, req); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + return TSDB_CODE_SUCCESS; } @@ -384,7 +504,6 @@ static void hbStopThread() { } SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo, char *key) { - return NULL; hbMgrInit(); SAppHbMgr* pAppHbMgr = malloc(sizeof(SAppHbMgr)); if (pAppHbMgr == NULL) { @@ -442,7 +561,6 @@ void appHbMgrCleanup(void) { } int hbMgrInit() { - return 0; // init once int8_t old = atomic_val_compare_exchange_8(&clientHbMgr.inited, 0, 1); if (old == 1) return 0; @@ -499,7 +617,6 @@ int hbRegisterConnImpl(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, SHbConnInfo * } int hbRegisterConn(SAppHbMgr* pAppHbMgr, int32_t connId, int64_t clusterId, int32_t hbType) { - return 0; SClientHbKey connKey = {.connId = connId, .hbType = HEARTBEAT_TYPE_QUERY}; SHbConnInfo info = {0}; diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index c60b89f78e..ff498e68d1 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -109,7 +109,7 @@ TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, SAppInstInfo* p = calloc(1, sizeof(struct SAppInstInfo)); p->mgmtEp = epSet; p->pTransporter = openTransporter(user, secretEncrypt, tsNumOfCores); - /*p->pAppHbMgr = appHbMgrInit(p, key);*/ + p->pAppHbMgr = appHbMgrInit(p, key); taosHashPut(appInfo.pInstMap, key, strlen(key), &p, POINTER_BYTES); pInst = &p; diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 0652618e6c..8ab1880069 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -76,7 +76,7 @@ int processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) { pTscObj->connType = HEARTBEAT_TYPE_QUERY; - /*hbRegisterConn(pTscObj->pAppInfo->pAppHbMgr, pConnect->connId, pConnect->clusterId, HEARTBEAT_TYPE_QUERY);*/ + hbRegisterConn(pTscObj->pAppInfo->pAppHbMgr, pConnect->connId, pConnect->clusterId, HEARTBEAT_TYPE_QUERY); // pRequest->body.resInfo.pRspMsg = pMsg->pData; tscDebug("0x%" PRIx64 " clusterId:%" PRId64 ", totalConn:%" PRId64, pRequest->requestId, pConnect->clusterId, diff --git a/source/dnode/mnode/impl/inc/mndStb.h b/source/dnode/mnode/impl/inc/mndStb.h index 48847dc6a3..0855b5bd4d 100644 --- a/source/dnode/mnode/impl/inc/mndStb.h +++ b/source/dnode/mnode/impl/inc/mndStb.h @@ -28,6 +28,9 @@ void mndCleanupStb(SMnode *pMnode); SStbObj *mndAcquireStb(SMnode *pMnode, char *stbName); void mndReleaseStb(SMnode *pMnode, SStbObj *pStb); +int32_t mndValidateStbInfo(SMnode *pMnode, SSTableMetaVersion *stbs, int32_t num, void **rsp, int32_t *rspLen); + + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index f71f3ac478..1bfe8c9d9a 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -907,9 +907,9 @@ int32_t mndValidateDBInfo(SMnode *pMnode, SDbVgVersion *dbs, int32_t num, void * len = 0; - SDbObj *pDb = mndAcquireDb(pMnode, db->dbName); + SDbObj *pDb = mndAcquireDb(pMnode, db->dbFName); if (pDb == NULL) { - mInfo("db %s not exist", db->dbName); + mInfo("db %s not exist", db->dbFName); len = sizeof(SUseDbRsp); } else if (pDb->uid != db->dbId || db->vgVersion < pDb->vgVersion) { @@ -929,7 +929,7 @@ int32_t mndValidateDBInfo(SMnode *pMnode, SDbVgVersion *dbs, int32_t num, void * } pRsp = (SUseDbRsp *)((char *)buf + bufOffset); - memcpy(pRsp->db, db->dbName, TSDB_DB_FNAME_LEN); + memcpy(pRsp->db, db->dbFName, TSDB_DB_FNAME_LEN); if (pDb) { int32_t vgNum = 0; mndBuildDBVgroupInfo(pDb, pMnode, pRsp->vgroupInfo, &vgNum); diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 3d9e3ff3de..d63ade4320 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -18,6 +18,7 @@ #include "mndProfile.h" //#include "mndConsumer.h" #include "mndDb.h" +#include "mndStb.h" #include "mndMnode.h" #include "mndShow.h" //#include "mndTopic.h" @@ -376,9 +377,16 @@ static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) { } break; } - case HEARTBEAT_KEY_STBINFO: - + case HEARTBEAT_KEY_STBINFO: { + void *rspMsg = NULL; + int32_t rspLen = 0; + mndValidateStbInfo(pMnode, (SSTableMetaVersion *)kv->value, kv->valueLen/sizeof(SSTableMetaVersion), &rspMsg, &rspLen); + if (rspMsg && rspLen > 0) { + SKv kv = {.key = HEARTBEAT_KEY_STBINFO, .valueLen = rspLen, .value = rspMsg}; + taosArrayPush(hbRsp.info, &kv); + } break; + } default: mError("invalid kv key:%d", kv->key); hbRsp.status = TSDB_CODE_MND_APP_ERROR; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 9944c0c7fc..4ccd4b63c4 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -728,7 +728,7 @@ static int32_t mndProcessStbMetaReq(SMnodeMsg *pReq) { mDebug("stb:%s, start to retrieve meta", tbFName); - SDbObj *pDb = mndAcquireDbByStb(pMnode, tbFName); + SDbObj *pDb = mndAcquireDb(pMnode, pInfo->dbFName); if (pDb == NULL) { terrno = TSDB_CODE_MND_DB_NOT_SELECTED; mError("stb:%s, failed to retrieve meta since %s", tbFName, terrstr()); @@ -788,6 +788,109 @@ static int32_t mndProcessStbMetaReq(SMnodeMsg *pReq) { return 0; } +int32_t mndValidateStbInfo(SMnode *pMnode, SSTableMetaVersion *stbs, int32_t num, void **rsp, int32_t *rspLen) { + SSdb *pSdb = pMnode->pSdb; + int32_t bufSize = num * (sizeof(STableMetaRsp) + 4 * sizeof(SSchema)); + void *buf = malloc(bufSize); + int32_t len = 0; + int32_t contLen = 0; + STableMetaRsp *pRsp = NULL; + + for (int32_t i = 0; i < num; ++i) { + SSTableMetaVersion *stb = &stbs[i]; + stb->suid = be64toh(stb->suid); + stb->sversion = ntohs(stb->sversion); + stb->tversion = ntohs(stb->tversion); + + if ((contLen + sizeof(STableMetaRsp)) > bufSize) { + bufSize = contLen + (num -i) * (sizeof(STableMetaRsp) + 4 * sizeof(SSchema)); + buf = realloc(buf, bufSize); + } + + pRsp = (STableMetaRsp *)((char *)buf + contLen); + + strcpy(pRsp->dbFName, stb->dbFName); + strcpy(pRsp->tbName, stb->stbName); + strcpy(pRsp->stbName, stb->stbName); + + mDebug("start to retrieve meta, db:%s, stb:%s", stb->dbFName, stb->stbName); + + SDbObj *pDb = mndAcquireDb(pMnode, stb->dbFName); + if (pDb == NULL) { + pRsp->numOfColumns = -1; + pRsp->suid = htobe64(stb->suid); + contLen += sizeof(STableMetaRsp); + mWarn("db:%s, failed to require db since %s", stb->dbFName, terrstr()); + continue; + } + + char tbFName[TSDB_TABLE_FNAME_LEN] = {0}; + snprintf(tbFName, sizeof(tbFName), "%s.%s", stb->dbFName, stb->stbName); + + SStbObj *pStb = mndAcquireStb(pMnode, tbFName); + if (pStb == NULL) { + mndReleaseDb(pMnode, pDb); + pRsp->numOfColumns = -1; + pRsp->suid = htobe64(stb->suid); + contLen += sizeof(STableMetaRsp); + mWarn("stb:%s, failed to get meta since %s", tbFName, terrstr()); + continue; + } + + taosRLockLatch(&pStb->lock); + + if (stb->suid == pStb->uid && stb->sversion == pStb->version) { + taosRUnLockLatch(&pStb->lock); + mndReleaseDb(pMnode, pDb); + mndReleaseStb(pMnode, pStb); + continue; + } + + int32_t totalCols = pStb->numOfColumns + pStb->numOfTags; + int32_t len = totalCols * sizeof(SSchema); + + contLen += sizeof(STableMetaRsp) + len; + + if (contLen > bufSize) { + bufSize = contLen + (num -i - 1) * (sizeof(STableMetaRsp) + 4 * sizeof(SSchema)); + buf = realloc(buf, bufSize); + } + + pRsp->numOfTags = htonl(pStb->numOfTags); + pRsp->numOfColumns = htonl(pStb->numOfColumns); + pRsp->precision = pDb->cfg.precision; + pRsp->tableType = TSDB_SUPER_TABLE; + pRsp->update = pDb->cfg.update; + pRsp->sversion = htonl(pStb->version); + pRsp->suid = htobe64(pStb->uid); + pRsp->tuid = htobe64(pStb->uid); + + for (int32_t i = 0; i < totalCols; ++i) { + SSchema *pSchema = &pRsp->pSchema[i]; + SSchema *pSrcSchema = &pStb->pSchema[i]; + memcpy(pSchema->name, pSrcSchema->name, TSDB_COL_NAME_LEN); + pSchema->type = pSrcSchema->type; + pSchema->colId = htonl(pSrcSchema->colId); + pSchema->bytes = htonl(pSrcSchema->bytes); + } + taosRUnLockLatch(&pStb->lock); + mndReleaseDb(pMnode, pDb); + mndReleaseStb(pMnode, pStb); + } + + if (contLen > 0) { + *rsp = buf; + *rspLen = contLen; + } else { + *rsp = NULL; + tfree(buf); + *rspLen = 0; + } + + return 0; +} + + static int32_t mndGetNumOfStbs(SMnode *pMnode, char *dbName, int32_t *pNumOfStbs) { SSdb *pSdb = pMnode->pSdb; diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 22997feb3c..9c041d76c7 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -27,7 +27,7 @@ extern "C" { #define CTG_DEFAULT_CACHE_CLUSTER_NUMBER 6 #define CTG_DEFAULT_CACHE_VGROUP_NUMBER 100 #define CTG_DEFAULT_CACHE_DB_NUMBER 20 -#define CTG_DEFAULT_CACHE_TABLEMETA_NUMBER 100000 +#define CTG_DEFAULT_CACHE_TABLEMETA_NUMBER 10000 #define CTG_DEFAULT_RENT_SECOND 10 #define CTG_DEFAULT_RENT_SLOT_SIZE 10 diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 95e60415c7..02773fe533 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -527,9 +527,9 @@ int32_t ctgGetVgInfoFromHashValue(struct SCatalog *pCatalog, SDBVgroupInfo *dbIn } int32_t ctgSTableVersionCompare(const void* key1, const void* key2) { - if (((SSTableMetaVersion*)key1)->suid < ((SSTableMetaVersion*)key2)->suid) { + if (*(uint64_t *)key1 < ((SSTableMetaVersion*)key2)->suid) { return -1; - } else if (((SSTableMetaVersion*)key1)->suid > ((SSTableMetaVersion*)key2)->suid) { + } else if (*(uint64_t *)key1 > ((SSTableMetaVersion*)key2)->suid) { return 1; } else { return 0; @@ -557,7 +557,7 @@ int32_t ctgMetaRentInit(SCtgRentMgmt *mgmt, uint32_t rentSec, int8_t type) { mgmt->slots = calloc(1, msgSize); if (NULL == mgmt->slots) { qError("calloc %d failed", (int32_t)msgSize); - return TSDB_CODE_CTG_MEM_ERROR; + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); } qDebug("meta rent initialized, type:%d, slotNum:%d", type, mgmt->slotNum); @@ -825,6 +825,8 @@ int32_t ctgUpdateTableMetaCache(struct SCatalog *pCatalog, STableMetaOutput *out if (TSDB_SUPER_TABLE == output->tbMeta->tableType) { bool newAdded = false; SSTableMetaVersion metaRent = {.suid = output->tbMeta->suid, .sversion = output->tbMeta->sversion, .tversion = output->tbMeta->tversion}; + strcpy(metaRent.dbFName, output->dbFName); + strcpy(metaRent.stbName, output->tbName); CTG_LOCK(CTG_WRITE, &dbCache->tbCache.stbLock); if (taosHashPut(dbCache->tbCache.cache, output->tbName, strlen(output->tbName), output->tbMeta, tbSize) != 0) { @@ -951,6 +953,39 @@ int32_t ctgValidateAndRemoveDb(struct SCatalog* pCatalog, const char* dbName, ui return TSDB_CODE_SUCCESS; } +int32_t ctgValidateAndRemoveStbMeta(struct SCatalog* pCatalog, const char* dbName, const char* stbName, uint64_t suid, bool *removed) { + *removed = false; + + SCtgDBCache *dbCache = (SCtgDBCache *)taosHashAcquire(pCatalog->dbCache, dbName, strlen(dbName)); + if (NULL == dbCache) { + ctgInfo("db not exist in dbCache, may be removed, db:%s", dbName); + return TSDB_CODE_SUCCESS; + } + + CTG_LOCK(CTG_WRITE, &dbCache->tbCache.stbLock); + if (taosHashRemove(dbCache->tbCache.stbCache, &suid, sizeof(suid))) { + CTG_UNLOCK(CTG_WRITE, &dbCache->tbCache.stbLock); + taosHashRelease(pCatalog->dbCache, dbCache); + ctgInfo("stb not exist in stbCache, may be removed, db:%s, stb:%s, suid:%"PRIx64, dbName, stbName, suid); + return TSDB_CODE_SUCCESS; + } + + if (taosHashRemove(dbCache->tbCache.cache, stbName, strlen(stbName))) { + CTG_UNLOCK(CTG_WRITE, &dbCache->tbCache.stbLock); + taosHashRelease(pCatalog->dbCache, dbCache); + ctgError("stb not exist in cache, db:%s, stb:%s, suid:%"PRIx64, dbName, stbName, suid); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + CTG_UNLOCK(CTG_WRITE, &dbCache->tbCache.stbLock); + + taosHashRelease(pCatalog->dbCache, dbCache); + + *removed = true; + + return TSDB_CODE_SUCCESS; +} + + int32_t ctgRenewTableMetaImpl(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, int32_t isSTable) { if (NULL == pCatalog || NULL == pTransporter || NULL == pMgmtEps || NULL == pTableName) { @@ -1065,7 +1100,7 @@ int32_t ctgGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMg int32_t catalogInit(SCatalogCfg *cfg) { if (ctgMgmt.pCluster) { - qError("catalog already init"); + qError("catalog already initialized"); CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } @@ -1111,7 +1146,7 @@ int32_t catalogGetHandle(uint64_t clusterId, struct SCatalog** catalogHandle) { } if (NULL == ctgMgmt.pCluster) { - qError("cluster cache are not ready, clusterId:%"PRIx64, clusterId); + qError("catalog cluster cache are not ready, clusterId:%"PRIx64, clusterId); CTG_ERR_RET(TSDB_CODE_CTG_NOT_READY); } @@ -1312,7 +1347,7 @@ int32_t catalogUpdateDBVgroup(struct SCatalog* pCatalog, const char* dbName, SDB ctgMetaRentRemove(&pCatalog->dbRent, dbCache->vgInfo->dbId, ctgDbVgVersionCompare); newAdded = true; } else if (dbInfo->vgVersion <= dbCache->vgInfo->vgVersion) { - ctgInfo("db vgVersion is not new, db:%s, vgVersion:%d, current:%d", dbName, dbInfo->vgVersion, dbCache->vgInfo->vgVersion); + ctgInfo("db vgVersion is old, db:%s, vgVersion:%d, current:%d", dbName, dbInfo->vgVersion, dbCache->vgInfo->vgVersion); CTG_UNLOCK(CTG_WRITE, &dbCache->vgLock); taosHashRelease(pCatalog->dbCache, dbCache); @@ -1345,7 +1380,7 @@ int32_t catalogUpdateDBVgroup(struct SCatalog* pCatalog, const char* dbName, SDB dbInfo = NULL; - strncpy(vgVersion.dbName, dbName, sizeof(vgVersion.dbName)); + strncpy(vgVersion.dbFName, dbName, sizeof(vgVersion.dbFName)); if (newAdded) { CTG_ERR_JRET(ctgMetaRentAdd(&pCatalog->dbRent, &vgVersion, vgVersion.dbId, sizeof(SDbVgVersion))); @@ -1394,6 +1429,32 @@ int32_t catalogRemoveDB(struct SCatalog* pCatalog, const char* dbName, uint64_t CTG_RET(code); } +int32_t catalogRemoveSTableMeta(struct SCatalog* pCatalog, const char* dbName, const char* stbName, uint64_t suid) { + int32_t code = 0; + bool removed = false; + + if (NULL == pCatalog || NULL == dbName || NULL == stbName) { + CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); + } + + if (NULL == pCatalog->dbCache) { + return TSDB_CODE_SUCCESS; + } + + CTG_ERR_RET(ctgValidateAndRemoveStbMeta(pCatalog, dbName, stbName, suid, &removed)); + if (!removed) { + return TSDB_CODE_SUCCESS; + } + + ctgInfo("stb removed from cache, db:%s, stbName:%s, suid:%"PRIx64, dbName, stbName, suid); + + CTG_ERR_RET(ctgMetaRentRemove(&pCatalog->stbRent, suid, ctgSTableVersionCompare)); + + ctgDebug("stb removed from rent, db:%s, stbName:%s, suid:%"PRIx64, dbName, stbName, suid); + + CTG_RET(code); +} + int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) { return ctgGetTableMeta(pCatalog, pTransporter, pMgmtEps, pTableName, false, pTableMeta, -1); diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index bfb950e5db..751fa72347 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -748,7 +748,7 @@ TEST(tableMeta, normalTable) { } if (stbNum) { - printf("got expired stb,suid:%" PRId64 "\n", stb->suid); + printf("got expired stb,suid:%" PRId64 ",dbFName:%s, stbName:%s\n", stb->suid, stb->dbFName, stb->stbName); free(stb); stb = NULL; } else { @@ -844,7 +844,7 @@ TEST(tableMeta, childTableCase) { } if (stbNum) { - printf("got expired stb,suid:%" PRId64 "\n", stb->suid); + printf("got expired stb,suid:%" PRId64 ",dbFName:%s, stbName:%s\n", stb->suid, stb->dbFName, stb->stbName); free(stb); stb = NULL; } else { @@ -945,7 +945,8 @@ TEST(tableMeta, superTableCase) { } if (stbNum) { - printf("got expired stb,suid:%" PRId64 "\n", stb->suid); + printf("got expired stb,suid:%" PRId64 ",dbFName:%s, stbName:%s\n", stb->suid, stb->dbFName, stb->stbName); + free(stb); stb = NULL; } else { @@ -1289,7 +1290,7 @@ TEST(rentTest, allRent) { printf("%d - expired stableNum:%d\n", i, num); if (stable) { for (int32_t n = 0; n < num; ++n) { - printf("suid:%" PRId64 ", sversion:%d, tversion:%d\n", stable[n].suid, stable[n].sversion, stable[n].tversion); + printf("suid:%" PRId64 ", dbFName:%s, stbName:%s, sversion:%d, tversion:%d\n", stable[n].suid, stable[n].dbFName, stable[n].stbName, stable[n].sversion, stable[n].tversion); } free(stable); stable = NULL; From 57c2bdc57a3c2d2d1399c519811f2b5c6409f675 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sun, 6 Feb 2022 05:27:42 -0500 Subject: [PATCH 7/7] TD-13338 SELECT statement translate code --- include/nodes/nodes.h | 24 +- include/util/taoserror.h | 5 +- source/libs/parser/src/astCreateFuncs.c | 5 +- source/libs/parser/src/astTranslate.c | 18 -- source/libs/parser/src/parserImpl.c | 293 +++++++++++++++++++++- source/libs/parser/test/newParserTest.cpp | 4 +- source/nodes/src/nodesTraverseFuncs.c | 133 ++++++---- 7 files changed, 399 insertions(+), 83 deletions(-) delete mode 100644 source/libs/parser/src/astTranslate.c diff --git a/include/nodes/nodes.h b/include/nodes/nodes.h index 39638b3757..ccb135aa0d 100644 --- a/include/nodes/nodes.h +++ b/include/nodes/nodes.h @@ -78,6 +78,11 @@ typedef struct SNodeList { SListCell* pTail; } SNodeList; +typedef struct SNameStr { + int32_t len; + char* pName; +} SNameStr; + typedef struct SDataType { uint8_t type; uint8_t precision; @@ -89,6 +94,7 @@ typedef struct SExprNode { ENodeType nodeType; SDataType resType; char aliasName[TSDB_COL_NAME_LEN]; + SNodeList* pAssociationList; } SExprNode; typedef enum EColumnType { @@ -102,7 +108,9 @@ typedef struct SColumnNode { EColumnType colType; // column or tag char dbName[TSDB_DB_NAME_LEN]; char tableName[TSDB_TABLE_NAME_LEN]; + char tableAlias[TSDB_TABLE_NAME_LEN]; char colName[TSDB_COL_NAME_LEN]; + SNode* pProjectRef; } SColumnNode; typedef struct SValueNode { @@ -176,13 +184,16 @@ typedef struct SFunctionNode { typedef struct STableNode { ENodeType type; + char dbName[TSDB_DB_NAME_LEN]; char tableName[TSDB_TABLE_NAME_LEN]; - char tableAliasName[TSDB_COL_NAME_LEN]; + char tableAlias[TSDB_TABLE_NAME_LEN]; } STableNode; +struct STableMeta; + typedef struct SRealTableNode { STableNode table; // QUERY_NODE_REAL_TABLE - char dbName[TSDB_DB_NAME_LEN]; + struct STableMeta* pMeta; } SRealTableNode; typedef struct STempTableNode { @@ -273,7 +284,6 @@ typedef struct SFillNode { typedef struct SSelectStmt { ENodeType type; // QUERY_NODE_SELECT_STMT bool isDistinct; - bool isStar; SNodeList* pProjectionList; // SNode SNode* pFromTable; SNode* pWhere; @@ -295,6 +305,8 @@ typedef struct SSetOperator { ESetOperatorType opType; SNode* pLeft; SNode* pRight; + SNodeList* pOrderByList; // SOrderByExprNode + SNode* pLimit; } SSetOperator; SNode* nodesMakeNode(ENodeType type); @@ -306,8 +318,10 @@ void nodesDestroyList(SNodeList* pList); typedef bool (*FQueryNodeWalker)(SNode* pNode, void* pContext); -bool nodesWalkNode(SNode* pNode, FQueryNodeWalker walker, void* pContext); -bool nodesWalkList(SNodeList* pList, FQueryNodeWalker walker, void* pContext); +void nodesWalkNode(SNode* pNode, FQueryNodeWalker walker, void* pContext); +void nodesWalkList(SNodeList* pList, FQueryNodeWalker walker, void* pContext); +void nodesWalkNodePostOrder(SNode* pNode, FQueryNodeWalker walker, void* pContext); +void nodesWalkListPostOrder(SNodeList* pList, FQueryNodeWalker walker, void* pContext); bool nodesWalkStmt(SNode* pNode, FQueryNodeWalker walker, void* pContext); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 50af44a94c..b5740b0118 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -440,7 +440,10 @@ int32_t* taosGetErrno(); #define TSDB_CODE_SCH_STATUS_ERROR TAOS_DEF_ERROR_CODE(0, 0x2501) //scheduler status error #define TSDB_CODE_SCH_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x2502) //scheduler internal error - +//parser +#define TSDB_CODE_PARSER_INVALID_COLUMN TAOS_DEF_ERROR_CODE(0, 0x2601) //invalid column name +#define TSDB_CODE_PARSER_TABLE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x2602) //table not exist +#define TSDB_CODE_PARSER_AMBIGUOUS_COLUMN TAOS_DEF_ERROR_CODE(0, 0x2603) //ambiguous column #ifdef __cplusplus } diff --git a/source/libs/parser/src/astCreateFuncs.c b/source/libs/parser/src/astCreateFuncs.c index f054649e1a..75b2fb2e65 100644 --- a/source/libs/parser/src/astCreateFuncs.c +++ b/source/libs/parser/src/astCreateFuncs.c @@ -150,7 +150,7 @@ SNode* createRealTableNode(SAstCreateContext* pCxt, const SToken* pDbName, const SRealTableNode* realTable = (SRealTableNode*)nodesMakeNode(QUERY_NODE_REAL_TABLE); CHECK_OUT_OF_MEM(realTable); if (NULL != pDbName) { - strncpy(realTable->dbName, pDbName->z, pDbName->n); + strncpy(realTable->table.dbName, pDbName->z, pDbName->n); } strncpy(realTable->table.tableName, pTableName->z, pTableName->n); return (SNode*)realTable; @@ -288,9 +288,6 @@ SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pPr SSelectStmt* select = (SSelectStmt*)nodesMakeNode(QUERY_NODE_SELECT_STMT); CHECK_OUT_OF_MEM(select); select->isDistinct = isDistinct; - if (NULL == pProjectionList) { - select->isStar = true; - } select->pProjectionList = pProjectionList; select->pFromTable = pTable; return (SNode*)select; diff --git a/source/libs/parser/src/astTranslate.c b/source/libs/parser/src/astTranslate.c deleted file mode 100644 index ff68c4b22c..0000000000 --- a/source/libs/parser/src/astTranslate.c +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -// int32_t doTranslate() { - -// } diff --git a/source/libs/parser/src/parserImpl.c b/source/libs/parser/src/parserImpl.c index fdedafcbb4..971b6fe8ac 100644 --- a/source/libs/parser/src/parserImpl.c +++ b/source/libs/parser/src/parserImpl.c @@ -15,8 +15,9 @@ #include "parserImpl.h" -#include "ttoken.h" #include "astCreateContext.h" +#include "parserInt.h" +#include "ttoken.h" typedef void* (*FMalloc)(size_t); typedef void (*FFree)(void*); @@ -25,7 +26,7 @@ extern void* NewParseAlloc(FMalloc); extern void NewParse(void*, int, SToken, void*); extern void NewParseFree(void*, FFree); -uint32_t toNewTokenId(uint32_t tokenId) { +static uint32_t toNewTokenId(uint32_t tokenId) { switch (tokenId) { case TK_UNION: return NEW_TK_UNION; @@ -73,7 +74,7 @@ uint32_t toNewTokenId(uint32_t tokenId) { return tokenId; } -uint32_t getToken(const char* z, uint32_t* tokenId) { +static uint32_t getToken(const char* z, uint32_t* tokenId) { uint32_t n = tGetToken(z, tokenId); *tokenId = toNewTokenId(*tokenId); return n; @@ -137,3 +138,289 @@ abort_parse: pQuery->pRoot = cxt.pRootNode; return cxt.valid ? TSDB_CODE_SUCCESS : TSDB_CODE_FAILED; } + +// typedef struct SNamespace { +// int16_t level; // todo for correlated subquery +// char dbName[TSDB_DB_NAME_LEN]; +// char tableAlias[TSDB_TABLE_NAME_LEN]; +// SHashObj* pColHash; // key is colname, value is index of STableMeta.schema +// STableMeta* pMeta; +// } SNamespace; + +typedef struct STranslateContext { + SParseContext* pParseCxt; + int32_t errCode; + SMsgBuf msgBuf; + SArray* pNsLevel; // element is SArray*, the element of this subarray is STableNode* + int32_t currLevel; +} STranslateContext; + +static int32_t translateSubquery(STranslateContext* pCxt, SNode* pNode); + +static char* getSyntaxErrFormat(int32_t errCode) { + switch (errCode) { + case TSDB_CODE_PARSER_INVALID_COLUMN: + return "Invalid column name : %s"; + case TSDB_CODE_PARSER_TABLE_NOT_EXIST: + return "Table does not exist : %s"; + case TSDB_CODE_PARSER_AMBIGUOUS_COLUMN: + return "Column ambiguously defined : %s"; + default: + return "Unknown error"; + } +} + +static int32_t generateSyntaxErrMsg(STranslateContext* pCxt, int32_t errCode, const char* additionalInfo) { + snprintf(pCxt->msgBuf.buf, pCxt->msgBuf.len, getSyntaxErrFormat(errCode), additionalInfo); + pCxt->errCode = errCode; + return errCode; +} + +static int32_t addNamespace(STranslateContext* pCxt, void* pTable) { + SArray* pTables = NULL; + if (taosArrayGetSize(pCxt->pNsLevel) > pCxt->currLevel) { + pTables = taosArrayGetP(pCxt->pNsLevel, pCxt->currLevel); + } else { + pTables = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); + } + taosArrayPush(pTables, &pTable); + return TSDB_CODE_SUCCESS; +} + +static SName* toName(const SRealTableNode* pRealTable, SName* pName) { + strncpy(pName->dbname, pRealTable->table.dbName, strlen(pRealTable->table.dbName)); + strncpy(pName->dbname, pRealTable->table.tableName, strlen(pRealTable->table.tableName)); + return pName; +} + +static bool belongTable(const char* currentDb, const SColumnNode* pCol, const STableNode* pTable) { + int cmp = 0; + if ('\0' != pCol->dbName[0]) { + cmp = strcmp(pCol->dbName, pTable->dbName); + } else { + cmp = strcmp(currentDb, pTable->dbName); + } + if (0 == cmp) { + cmp = strcmp(pCol->tableAlias, pTable->tableAlias); + } + return (0 == cmp); +} + +static SNodeList* getProjectList(SNode* pNode) { + if (QUERY_NODE_SELECT_STMT == nodeType(pNode)) { + return ((SSelectStmt*)pNode)->pProjectionList; + } + return NULL; +} + +static int32_t createColumnNodeByTable(const STableNode* pTable, SNodeList* pList) { + if (QUERY_NODE_REAL_TABLE == nodeType(pTable)) { + const STableMeta* pMeta = ((SRealTableNode*)pTable)->pMeta; + int32_t nums = pMeta->tableInfo.numOfTags + pMeta->tableInfo.numOfColumns; + for (int32_t i = 0; i < nums; ++i) { + SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); + pCol->colId = pMeta->schema[i].colId; + pCol->colType = pMeta->schema[i].type; + pCol->node.resType.bytes = pMeta->schema[i].bytes; + nodesListAppend(pList, (SNode*)pCol); + } + } else { + SNodeList* pProjectList = getProjectList(((STempTableNode*)pTable)->pSubquery); + SNode* pNode; + FOREACH(pNode, pProjectList) { + SExprNode* pExpr = (SExprNode*)pNode; + SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); + pCol->pProjectRef = (SNode*)pExpr; + pExpr->pAssociationList = nodesListAppend(pExpr->pAssociationList, (SNode*)pCol); + pCol->node.resType = pExpr->resType; + nodesListAppend(pList, (SNode*)pCol); + } + } +} + +static bool findAndSetColumn(SColumnNode* pCol, const STableNode* pTable) { + bool found = false; + if (QUERY_NODE_REAL_TABLE == nodeType(pTable)) { + const STableMeta* pMeta = ((SRealTableNode*)pTable)->pMeta; + int32_t nums = pMeta->tableInfo.numOfTags + pMeta->tableInfo.numOfColumns; + for (int32_t i = 0; i < nums; ++i) { + if (0 == strcmp(pCol->colName, pMeta->schema[i].name)) { + pCol->colId = pMeta->schema[i].colId; + pCol->colType = pMeta->schema[i].type; + pCol->node.resType.bytes = pMeta->schema[i].bytes; + found = true; + break; + } + } + } else { + SNodeList* pProjectList = getProjectList(((STempTableNode*)pTable)->pSubquery); + SNode* pNode; + FOREACH(pNode, pProjectList) { + SExprNode* pExpr = (SExprNode*)pNode; + if (0 == strcmp(pCol->colName, pExpr->aliasName)) { + pCol->pProjectRef = (SNode*)pExpr; + pExpr->pAssociationList = nodesListAppend(pExpr->pAssociationList, (SNode*)pCol); + pCol->node.resType = pExpr->resType; + found = true; + break; + } + } + } + return found; +} + +static bool doTranslateExpr(SNode* pNode, void* pContext) { + STranslateContext* pCxt = (STranslateContext*)pContext; + switch (nodeType(pNode)) { + case QUERY_NODE_COLUMN: { + SColumnNode* pCol = (SColumnNode*)pNode; + SArray* pTables = taosArrayGetP(pCxt->pNsLevel, pCxt->currLevel); + size_t nums = taosArrayGetSize(pTables); + bool hasTableAlias = ('\0' != pCol->tableAlias[0]); + bool found = false; + for (size_t i = 0; i < nums; ++i) { + STableNode* pTable = taosArrayGetP(pTables, i); + if (hasTableAlias) { + if (belongTable(pCxt->pParseCxt->db, pCol, pTable)) { + if (findAndSetColumn(pCol, pTable)) { + break; + } + generateSyntaxErrMsg(pCxt, TSDB_CODE_PARSER_INVALID_COLUMN, pCol->colName); + return false; + } + } else { + if (findAndSetColumn(pCol, pTable)) { + if (found) { + generateSyntaxErrMsg(pCxt, TSDB_CODE_PARSER_AMBIGUOUS_COLUMN, pCol->colName); + return false; + } + found = true; + } + } + } + break; + } + case QUERY_NODE_VALUE: + break; // todo check literal format + case QUERY_NODE_OPERATOR: { + + break; + } + case QUERY_NODE_FUNCTION: + break; // todo + case QUERY_NODE_TEMP_TABLE: + return translateSubquery(pCxt, ((STempTableNode*)pNode)->pSubquery); + default: + break; + } + return true; +} + +static int32_t translateExpr(STranslateContext* pCxt, SNode* pNode) { + nodesWalkNodePostOrder(pNode, doTranslateExpr, pCxt); + return pCxt->errCode; +} + +static int32_t translateExprList(STranslateContext* pCxt, SNodeList* pList) { + nodesWalkListPostOrder(pList, doTranslateExpr, pCxt); + return pCxt->errCode; +} + +static int32_t translateTable(STranslateContext* pCxt, SNode* pTable) { + int32_t code = TSDB_CODE_SUCCESS; + switch (nodeType(pTable)) { + case QUERY_NODE_REAL_TABLE: { + SRealTableNode* pRealTable = (SRealTableNode*)pTable; + if ('\0' == pRealTable->table.dbName[0]) { + strcpy(pRealTable->table.dbName, pCxt->pParseCxt->db); + } + if ('\0' == pRealTable->table.tableAlias[0]) { + strcpy(pRealTable->table.tableAlias, pRealTable->table.tableName); + } + SName name; + code = catalogGetTableMeta( + pCxt->pParseCxt->pCatalog, pCxt->pParseCxt->pTransporter, &(pCxt->pParseCxt->mgmtEpSet), toName(pRealTable, &name), &(pRealTable->pMeta)); + if (TSDB_CODE_SUCCESS != code) { + return generateSyntaxErrMsg(pCxt, TSDB_CODE_PARSER_TABLE_NOT_EXIST, pRealTable->table.tableName); + } + code = addNamespace(pCxt, pRealTable); + break; + } + case QUERY_NODE_TEMP_TABLE: { + STempTableNode* pTempTable = (STempTableNode*)pTable; + code = translateSubquery(pCxt, pTempTable->pSubquery); + if (TSDB_CODE_SUCCESS == code) { + code = addNamespace(pCxt, pTempTable); + } + break; + } + case QUERY_NODE_JOIN_TABLE: { + SJoinTableNode* pJoinTable = (SJoinTableNode*)pTable; + code = translateTable(pCxt, pJoinTable->pLeft); + if (TSDB_CODE_SUCCESS == code) { + code = translateTable(pCxt, pJoinTable->pRight); + } + if (TSDB_CODE_SUCCESS == code) { + code = translateExpr(pCxt, pJoinTable->pOnCond); + } + break; + } + default: + break; + } + return code; +} + +static int32_t translateStar(STranslateContext* pCxt, SSelectStmt* pSelect, bool* pIsSelectStar) { + if (NULL == pSelect->pProjectionList) { // select * ... + SArray* pTables = taosArrayGetP(pCxt->pNsLevel, pCxt->currLevel); + size_t nums = taosArrayGetSize(pTables); + for (size_t i = 0; i < nums; ++i) { + STableNode* pTable = taosArrayGetP(pTables, i); + createColumnNodeByTable(pTable, pSelect->pProjectionList); + } + *pIsSelectStar = true; + } else { + + } +} + +static int32_t translateSelect(STranslateContext* pCxt, SSelectStmt* pSelect) { + int32_t code = TSDB_CODE_SUCCESS; + code = translateTable(pCxt, pSelect->pFromTable); + if (TSDB_CODE_SUCCESS == code) { + code = translateExpr(pCxt, pSelect->pWhere); + } + bool isSelectStar = false; + if (TSDB_CODE_SUCCESS == code) { + code = translateStar(pCxt, pSelect, &isSelectStar); + } + if (TSDB_CODE_SUCCESS == code && !isSelectStar) { + code = translateExprList(pCxt, pSelect->pProjectionList); + } + return code; +} + +static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) { + int32_t code = TSDB_CODE_SUCCESS; + switch (nodeType(pNode)) { + case QUERY_NODE_SELECT_STMT: + code = translateSelect(pCxt, (SSelectStmt*)pNode); + break; + default: + break; + } + return code; +} + +static int32_t translateSubquery(STranslateContext* pCxt, SNode* pNode) { + ++(pCxt->currLevel); + int32_t code = translateQuery(pCxt, pNode); + --(pCxt->currLevel); + return code; +} + +int32_t doTranslate(SParseContext* pParseCxt, SQuery* pQuery) { + STranslateContext cxt = { .pParseCxt = pParseCxt, .pNsLevel = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES), .currLevel = 0 }; + return translateQuery(&cxt, pQuery->pRoot); +} diff --git a/source/libs/parser/test/newParserTest.cpp b/source/libs/parser/test/newParserTest.cpp index 706deb443c..0f0fe6ac5d 100644 --- a/source/libs/parser/test/newParserTest.cpp +++ b/source/libs/parser/test/newParserTest.cpp @@ -71,8 +71,8 @@ private: switch (nodeType(node)) { case QUERY_NODE_REAL_TABLE: { SRealTableNode* realTable = (SRealTableNode*)table; - if ('\0' != realTable->dbName[0]) { - sql.append(realTable->dbName); + if ('\0' != realTable->table.dbName[0]) { + sql.append(realTable->table.dbName); sql.append("."); } sql.append(realTable->table.tableName); diff --git a/source/nodes/src/nodesTraverseFuncs.c b/source/nodes/src/nodesTraverseFuncs.c index ffe40073e6..444ff7cbcf 100644 --- a/source/nodes/src/nodesTraverseFuncs.c +++ b/source/nodes/src/nodesTraverseFuncs.c @@ -15,68 +15,101 @@ #include "nodes.h" -typedef bool (*FQueryNodeWalker)(SNode* pNode, void* pContext); +typedef enum ETraversalOrder { + TRAVERSAL_PREORDER = 1, + TRAVERSAL_POSTORDER +} ETraversalOrder; -bool nodesWalkNodeList(SNodeList* pNodeList, FQueryNodeWalker walker, void* pContext) { +static bool walkList(SNodeList* pNodeList, ETraversalOrder order, FQueryNodeWalker walker, void* pContext); + +static bool walkNode(SNode* pNode, ETraversalOrder order, FQueryNodeWalker walker, void* pContext) { + if (NULL == pNode) { + return true; + } + + if (TRAVERSAL_PREORDER == order && !walker(pNode, pContext)) { + return false; + } + + bool res = true; + switch (nodeType(pNode)) { + case QUERY_NODE_COLUMN: + case QUERY_NODE_VALUE: + case QUERY_NODE_LIMIT: + // these node types with no subnodes + break; + case QUERY_NODE_OPERATOR: { + SOperatorNode* pOpNode = (SOperatorNode*)pNode; + res = walkNode(pOpNode->pLeft, order, walker, pContext); + if (res) { + res = walkNode(pOpNode->pRight, order, walker, pContext); + } + break; + } + case QUERY_NODE_LOGIC_CONDITION: + res = walkList(((SLogicConditionNode*)pNode)->pParameterList, order, walker, pContext); + break; + case QUERY_NODE_IS_NULL_CONDITION: + res = walkNode(((SIsNullCondNode*)pNode)->pExpr, order, walker, pContext); + break; + case QUERY_NODE_FUNCTION: + res = walkList(((SFunctionNode*)pNode)->pParameterList, order, walker, pContext); + break; + case QUERY_NODE_REAL_TABLE: + case QUERY_NODE_TEMP_TABLE: + break; // todo + case QUERY_NODE_JOIN_TABLE: { + SJoinTableNode* pJoinTableNode = (SJoinTableNode*)pNode; + res = walkNode(pJoinTableNode->pLeft, order, walker, pContext); + if (res) { + res = walkNode(pJoinTableNode->pRight, order, walker, pContext); + } + if (res) { + res = walkNode(pJoinTableNode->pOnCond, order, walker, pContext); + } + break; + } + case QUERY_NODE_GROUPING_SET: + res = walkList(((SGroupingSetNode*)pNode)->pParameterList, order, walker, pContext); + break; + case QUERY_NODE_ORDER_BY_EXPR: + res = walkNode(((SOrderByExprNode*)pNode)->pExpr, order, walker, pContext); + break; + default: + break; + } + + if (res && TRAVERSAL_POSTORDER == order) { + res = walker(pNode, pContext); + } + + return res; +} + +static bool walkList(SNodeList* pNodeList, ETraversalOrder order, FQueryNodeWalker walker, void* pContext) { SNode* node; FOREACH(node, pNodeList) { - if (!nodesWalkNode(node, walker, pContext)) { + if (!walkNode(node, order, walker, pContext)) { return false; } } return true; } -bool nodesWalkNode(SNode* pNode, FQueryNodeWalker walker, void* pContext) { - if (NULL == pNode) { - return true; - } +void nodesWalkNode(SNode* pNode, FQueryNodeWalker walker, void* pContext) { + (void)walkNode(pNode, TRAVERSAL_PREORDER, walker, pContext); +} - if (!walker(pNode, pContext)) { - return false; - } +void nodesWalkList(SNodeList* pNodeList, FQueryNodeWalker walker, void* pContext) { + (void)walkList(pNodeList, TRAVERSAL_PREORDER, walker, pContext); +} - switch (nodeType(pNode)) { - case QUERY_NODE_COLUMN: - case QUERY_NODE_VALUE: - case QUERY_NODE_LIMIT: - // these node types with no subnodes - return true; - case QUERY_NODE_OPERATOR: { - SOperatorNode* pOpNode = (SOperatorNode*)pNode; - if (!nodesWalkNode(pOpNode->pLeft, walker, pContext)) { - return false; - } - return nodesWalkNode(pOpNode->pRight, walker, pContext); - } - case QUERY_NODE_LOGIC_CONDITION: - return nodesWalkNodeList(((SLogicConditionNode*)pNode)->pParameterList, walker, pContext); - case QUERY_NODE_IS_NULL_CONDITION: - return nodesWalkNode(((SIsNullCondNode*)pNode)->pExpr, walker, pContext); - case QUERY_NODE_FUNCTION: - return nodesWalkNodeList(((SFunctionNode*)pNode)->pParameterList, walker, pContext); - case QUERY_NODE_REAL_TABLE: - case QUERY_NODE_TEMP_TABLE: - return true; // todo - case QUERY_NODE_JOIN_TABLE: { - SJoinTableNode* pJoinTableNode = (SJoinTableNode*)pNode; - if (!nodesWalkNode(pJoinTableNode->pLeft, walker, pContext)) { - return false; - } - if (!nodesWalkNode(pJoinTableNode->pRight, walker, pContext)) { - return false; - } - return nodesWalkNode(pJoinTableNode->pOnCond, walker, pContext); - } - case QUERY_NODE_GROUPING_SET: - return nodesWalkNodeList(((SGroupingSetNode*)pNode)->pParameterList, walker, pContext); - case QUERY_NODE_ORDER_BY_EXPR: - return nodesWalkNode(((SOrderByExprNode*)pNode)->pExpr, walker, pContext); - default: - break; - } +void nodesWalkNodePostOrder(SNode* pNode, FQueryNodeWalker walker, void* pContext) { + (void)walkNode(pNode, TRAVERSAL_POSTORDER, walker, pContext); +} - return false; +void nodesWalkListPostOrder(SNodeList* pList, FQueryNodeWalker walker, void* pContext) { + (void)walkList(pList, TRAVERSAL_PREORDER, walker, pContext); } bool nodesWalkStmt(SNode* pNode, FQueryNodeWalker walker, void* pContext) {