From 4d8ba0f60f324d82b84746d4f974d28429b5e0f2 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 1 Feb 2024 17:02:08 +0800 Subject: [PATCH 01/19] feat: support uniq grant --- include/common/tgrant.h | 2 +- source/util/src/tbase64.c | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/include/common/tgrant.h b/include/common/tgrant.h index dbca2ac90c..688fac858e 100644 --- a/include/common/tgrant.h +++ b/include/common/tgrant.h @@ -69,7 +69,7 @@ int32_t grantAlterActiveCode(int32_t did, const char* old, const char* newer, ch {.name = "expire_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ {.name = "service_time", .bytes = 19 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ {.name = "expired", .bytes = 5 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ - {.name = "state", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ + {.name = "state", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ {.name = "timeseries", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ {.name = "dnodes", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ {.name = "cpu_cores", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, \ diff --git a/source/util/src/tbase64.c b/source/util/src/tbase64.c index f6f12fef97..a2f4ddbc51 100644 --- a/source/util/src/tbase64.c +++ b/source/util/src/tbase64.c @@ -15,8 +15,6 @@ #define _DEFAULT_SOURCE #include "tbase64.h" -#include -#include static char basis_64[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; From adee720128159c1f3236f28959d81c4e432bfb27 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 1 Feb 2024 17:48:29 +0800 Subject: [PATCH 02/19] feat: support uniq grant --- include/common/tgrant.h | 3 +- include/common/tmsg.h | 8 +++++ include/libs/catalog/catalog.h | 4 +++ source/client/src/clientHb.c | 40 +++++++++++++++++++++++ source/common/src/tgrant.c | 1 + source/common/src/tmsg.c | 29 ++++++++++++++++ source/dnode/mnode/impl/inc/mndGrant.h | 6 ++-- source/dnode/mnode/impl/src/mndConsumer.c | 7 ++-- source/dnode/mnode/impl/src/mndProfile.c | 11 +++++++ source/dnode/mnode/impl/src/mndStream.c | 2 +- source/dnode/mnode/impl/src/mndStreamHb.c | 2 +- source/libs/parser/src/parInsertSql.c | 12 +++++-- 12 files changed, 114 insertions(+), 11 deletions(-) diff --git a/include/common/tgrant.h b/include/common/tgrant.h index 688fac858e..9f7572ed63 100644 --- a/include/common/tgrant.h +++ b/include/common/tgrant.h @@ -54,7 +54,8 @@ typedef enum { TSDB_GRANT_BACKUP_RESTORE, } EGrantType; -int32_t grantCheck(EGrantType grant); +int32_t grantCheck(EGrantType grant); // less +int32_t grantCheckLE(EGrantType grant); // less or equal char* tGetMachineId(); #ifndef TD_UNIQ_GRANT int32_t grantAlterActiveCode(int32_t did, const char* old, const char* newer, char* out, int8_t type); diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 04502eb64a..9b493c4a40 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -110,6 +110,7 @@ enum { HEARTBEAT_KEY_TMQ, HEARTBEAT_KEY_DYN_VIEW, HEARTBEAT_KEY_VIEWINFO, + HEARTBEAT_KEY_GRANT, }; typedef enum _mgmt_table { @@ -1885,6 +1886,13 @@ int32_t tSerializeSViewHbRsp(void* buf, int32_t bufLen, SViewHbRsp* pRsp); int32_t tDeserializeSViewHbRsp(void* buf, int32_t bufLen, SViewHbRsp* pRsp); void tFreeSViewHbRsp(SViewHbRsp* pRsp); +typedef struct { + uint32_t flags; +} SGrantHbRsp; + +int32_t tSerializeSGrantHbRsp(void* buf, int32_t bufLen, SGrantHbRsp* pRsp); +int32_t tDeserializeSGrantHbRsp(void* buf, int32_t bufLen, SGrantHbRsp* pRsp); + typedef struct { int32_t numOfTables; int32_t numOfVgroup; diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index b34b998d76..3649f369b7 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -145,6 +145,10 @@ typedef struct SSTableVersion { int32_t smaVer; } SSTableVersion; +typedef struct SGrantVersion { + int32_t version; +} SGrantVersion; + typedef struct SDbCacheInfo { char dbFName[TSDB_DB_FNAME_LEN]; int64_t dbId; diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 63a65d7c95..f4c2fbec5a 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -327,6 +327,37 @@ static int32_t hbProcessViewInfoRsp(void *value, int32_t valueLen, struct SCatal return TSDB_CODE_SUCCESS; } +#if 0 +static int32_t hbProcessGrantInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) { + int32_t code = 0; + + SGrantHbRsp hbRsp = {0}; + if (tDeserializeSGrantHbRsp(value, valueLen, &hbRsp) != 0) { + taosArrayDestroyEx(hbRsp.pViewRsp, hbFreeSViewMetaInRsp); + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + int32_t numOfMeta = taosArrayGetSize(hbRsp.pViewRsp); + for (int32_t i = 0; i < numOfMeta; ++i) { + SViewMetaRsp *rsp = taosArrayGetP(hbRsp.pViewRsp, i); + + if (rsp->numOfCols < 0) { + tscDebug("hb to remove view, db:%s, view:%s", rsp->dbFName, rsp->name); + catalogRemoveViewMeta(pCatalog, rsp->dbFName, rsp->dbId, rsp->name, rsp->viewId); + tFreeSViewMetaRsp(rsp); + taosMemoryFreeClear(rsp); + } else { + tscDebug("hb to update view, db:%s, view:%s", rsp->dbFName, rsp->name); + catalogUpdateViewMeta(pCatalog, rsp); + } + } + + taosArrayDestroy(hbRsp.pViewRsp); + return TSDB_CODE_SUCCESS; +} +#endif + static void hbProcessQueryRspKvs(int32_t kvNum, SArray* pKvs, struct SCatalog *pCatalog, SAppHbMgr *pAppHbMgr) { for (int32_t i = 0; i < kvNum; ++i) { @@ -378,6 +409,15 @@ static void hbProcessQueryRspKvs(int32_t kvNum, SArray* pKvs, struct SCatalog *p hbProcessViewInfoRsp(kv->value, kv->valueLen, pCatalog); break; } + case HEARTBEAT_KEY_GRANT: { + if (kv->valueLen <= 0 || NULL == kv->value) { + tscError("invalid grant info, len:%d, value:%p", kv->valueLen, kv->value); + break; + } + + // hbProcessGrantInfoRsp(kv->value, kv->valueLen, pCatalog); + break; + } #endif default: tscError("invalid hb key type:%d", kv->key); diff --git a/source/common/src/tgrant.c b/source/common/src/tgrant.c index f212d71362..8e4fe9febb 100644 --- a/source/common/src/tgrant.c +++ b/source/common/src/tgrant.c @@ -19,5 +19,6 @@ #ifndef _GRANT int32_t grantCheck(EGrantType grant) {return TSDB_CODE_SUCCESS;} +int32_t grantCheckLE(EGrantType grant) {return TSDB_CODE_SUCCESS;} #endif \ No newline at end of file diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index de505ab3d2..c866aae209 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -9303,3 +9303,32 @@ void tFreeSViewHbRsp(SViewHbRsp *pRsp) { taosArrayDestroy(pRsp->pViewRsp); } + +int32_t tSerializeSGrantHbRsp(void *buf, int32_t bufLen, SGrantHbRsp *pRsp) { + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + + if (tStartEncode(&encoder) < 0) return -1; + + if (tEncodeU32v(&encoder, pRsp->flags) < 0) return -1; + + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSGrantHbRsp(void *buf, int32_t bufLen, SGrantHbRsp *pRsp) { + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, bufLen); + + if (tStartDecode(&decoder) < 0) return -1; + + if (tDecodeU32v(&decoder, &pRsp->flags) < 0) return -1; + + tEndDecode(&decoder); + + tDecoderClear(&decoder); + return 0; +} \ No newline at end of file diff --git a/source/dnode/mnode/impl/inc/mndGrant.h b/source/dnode/mnode/impl/inc/mndGrant.h index 82b3260860..a012d7a8f5 100644 --- a/source/dnode/mnode/impl/inc/mndGrant.h +++ b/source/dnode/mnode/impl/inc/mndGrant.h @@ -41,9 +41,9 @@ char **mergeActive); #endif - int32_t mndProcessConfigGrantReq(SMnode * pMnode, SRpcMsg * pReq, SMCfgClusterReq * pCfg); - int32_t mndProcessUpdGrantLog(SMnode * pMnode, SRpcMsg * pReq, SArray * pMachines, SGrantState * pState); - + int32_t mndProcessConfigGrantReq(SMnode * pMnode, SRpcMsg * pReq, SMCfgClusterReq * pCfg); + int32_t mndProcessUpdGrantLog(SMnode * pMnode, SRpcMsg * pReq, SArray * pMachines, SGrantState * pState); + int32_t mndValidateGrant(SMnode * pMnode, SGrantVersion * pGrantVersion, void **ppRsp, int32_t *pRspLen); int32_t mndGrantGetLastState(SMnode * pMnode, SGrantState * pState); SGrantLogObj *mndAcquireGrant(SMnode * pMnode, void **ppIter); void mndReleaseGrant(SMnode * pMnode, SGrantLogObj * pGrant, void *pIter); diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 3ce548a4f6..753076f1f3 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -107,7 +107,7 @@ static int32_t validateTopics(STrans *pTrans, const SArray *pTopicList, SMnode * goto FAILED; } - if ((terrno = grantCheck(TSDB_GRANT_SUBSCRIPTION)) < 0) { + if ((terrno = grantCheckLE(TSDB_GRANT_SUBSCRIPTION)) < 0) { code = terrno; goto FAILED; } @@ -240,9 +240,10 @@ static int32_t checkPrivilege(SMnode *pMnode, SMqConsumerObj *pConsumer, SMqHbR } STopicPrivilege *data = taosArrayReserve(rsp->topicPrivileges, 1); strcpy(data->topic, topic); - if (mndCheckTopicPrivilege(pMnode, user, MND_OPER_SUBSCRIBE, pTopic) != 0 || grantCheck(TSDB_GRANT_SUBSCRIPTION) < 0) { + if (mndCheckTopicPrivilege(pMnode, user, MND_OPER_SUBSCRIBE, pTopic) != 0 || + grantCheckLE(TSDB_GRANT_SUBSCRIPTION) < 0) { data->noPrivilege = 1; - } else{ + } else { data->noPrivilege = 0; } mndReleaseTopic(pMnode, pTopic); diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index e0559b4c48..d7e6d9609c 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -18,6 +18,7 @@ #include "audit.h" #include "mndDb.h" #include "mndDnode.h" +#include "mndGrant.h" #include "mndMnode.h" #include "mndPrivilege.h" #include "mndQnode.h" @@ -605,6 +606,16 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb } break; } + case HEARTBEAT_KEY_GRANT: { + void *rspMsg = NULL; + int32_t rspLen = 0; + mndValidateGrant(pMnode, kv->value, &rspMsg, &rspLen); + if (rspMsg && rspLen > 0) { + SKv kv1 = {.key = HEARTBEAT_KEY_GRANT, .valueLen = rspLen, .value = rspMsg}; + taosArrayPush(hbRsp.info, &kv1); + } + break; + } #endif default: mError("invalid kv key:%d", kv->key); diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 5f5fbdba13..530b3d5bcd 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -1628,7 +1628,7 @@ static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SStreamObj *pStream = NULL; - if(grantCheck(TSDB_GRANT_STREAMS) < 0){ + if(grantCheckLE(TSDB_GRANT_STREAMS) < 0){ terrno = TSDB_CODE_GRANT_EXPIRED; return -1; } diff --git a/source/dnode/mnode/impl/src/mndStreamHb.c b/source/dnode/mnode/impl/src/mndStreamHb.c index 4426ab0672..005caea31b 100644 --- a/source/dnode/mnode/impl/src/mndStreamHb.c +++ b/source/dnode/mnode/impl/src/mndStreamHb.c @@ -225,7 +225,7 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { SArray *pFailedTasks = taosArrayInit(4, sizeof(SFailedCheckpointInfo)); SArray *pOrphanTasks = taosArrayInit(3, sizeof(SOrphanTask)); - if(grantCheck(TSDB_GRANT_STREAMS) < 0){ + if(grantCheckLE(TSDB_GRANT_STREAMS) < 0){ if(suspendAllStreams(pMnode, &pReq->info) < 0){ return -1; } diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 512dfdaef2..8e32eca28e 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -2193,6 +2193,10 @@ static int32_t parseFileClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS return buildInvalidOperationMsg(&pCxt->msg, "proxy mode does not support csv loading"); } + // if ((terrno = grantCheck(TSDB_GRANT_CSV)) < 0) { + // return buildInvalidOperationMsg(&pCxt->msg, terrstr()); + // } + NEXT_TOKEN(pStmt->pSql, *pToken); if (0 == pToken->n || (TK_NK_STRING != pToken->type && TK_NK_ID != pToken->type)) { return buildSyntaxErrMsg(&pCxt->msg, "file path is required following keyword FILE", pToken->z); @@ -2754,11 +2758,15 @@ static int32_t parseInsertSqlFromStart(SInsertParseContext* pCxt, SVnodeModifyOp } static int32_t parseInsertSqlFromCsv(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; SRowsDataContext rowsDataCxt; + // if ((code = grantCheck(TSDB_GRANT_CSV)) < 0) { + // return code; + // } + if (!pStmt->stbSyntax) { - STableDataCxt* pTableCxt = NULL; + STableDataCxt* pTableCxt = NULL; code = getTableDataCxt(pCxt, pStmt, &pTableCxt); rowsDataCxt.pTableDataCxt = pTableCxt; } else { From 44d264456d951d1ce28770569bd5196cf44f59f3 Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 2 Feb 2024 08:41:45 +0800 Subject: [PATCH 03/19] feat: support uniq grant --- include/libs/catalog/catalog.h | 1 + source/client/src/clientHb.c | 33 +++---- source/libs/catalog/inc/catalogInt.h | 12 +++ source/libs/catalog/src/catalog.c | 15 +++ source/libs/catalog/src/ctgCache.c | 131 +++++++++++++++++++++++++++ 5 files changed, 170 insertions(+), 22 deletions(-) diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 3649f369b7..2e013413a3 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -383,6 +383,7 @@ int32_t catalogRemoveViewMeta(SCatalog* pCtg, const char* dbFName, uint64_t dbId int32_t catalogUpdateDynViewVer(SCatalog* pCtg, SDynViewVersion* pVer); int32_t catalogUpdateViewMeta(SCatalog* pCtg, SViewMetaRsp* pMsg); +int32_t catalogUpdateGrantInfo(SCatalog* pCtg, SGrantHbRsp* pMsg); int32_t catalogAsyncUpdateViewMeta(SCatalog* pCtg, SViewMetaRsp* pMsg); diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index f4c2fbec5a..755ab0eb91 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -327,37 +327,26 @@ static int32_t hbProcessViewInfoRsp(void *value, int32_t valueLen, struct SCatal return TSDB_CODE_SUCCESS; } -#if 0 static int32_t hbProcessGrantInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) { int32_t code = 0; - SGrantHbRsp hbRsp = {0}; - if (tDeserializeSGrantHbRsp(value, valueLen, &hbRsp) != 0) { - taosArrayDestroyEx(hbRsp.pViewRsp, hbFreeSViewMetaInRsp); + SGrantHbRsp *hbRsp = taosMemoryCalloc(1, sizeof(SGrantHbRsp)); + if (!hbRsp) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + if (tDeserializeSGrantHbRsp(value, valueLen, hbRsp) != 0) { + taosMemoryFree(hbRsp); terrno = TSDB_CODE_INVALID_MSG; return -1; } - int32_t numOfMeta = taosArrayGetSize(hbRsp.pViewRsp); - for (int32_t i = 0; i < numOfMeta; ++i) { - SViewMetaRsp *rsp = taosArrayGetP(hbRsp.pViewRsp, i); + tscInfo("hb to update grant info:%u", hbRsp->flags); + catalogUpdateGrantInfo(pCatalog, hbRsp); - if (rsp->numOfCols < 0) { - tscDebug("hb to remove view, db:%s, view:%s", rsp->dbFName, rsp->name); - catalogRemoveViewMeta(pCatalog, rsp->dbFName, rsp->dbId, rsp->name, rsp->viewId); - tFreeSViewMetaRsp(rsp); - taosMemoryFreeClear(rsp); - } else { - tscDebug("hb to update view, db:%s, view:%s", rsp->dbFName, rsp->name); - catalogUpdateViewMeta(pCatalog, rsp); - } - } - - taosArrayDestroy(hbRsp.pViewRsp); return TSDB_CODE_SUCCESS; } -#endif - static void hbProcessQueryRspKvs(int32_t kvNum, SArray* pKvs, struct SCatalog *pCatalog, SAppHbMgr *pAppHbMgr) { for (int32_t i = 0; i < kvNum; ++i) { @@ -415,7 +404,7 @@ static void hbProcessQueryRspKvs(int32_t kvNum, SArray* pKvs, struct SCatalog *p break; } - // hbProcessGrantInfoRsp(kv->value, kv->valueLen, pCatalog); + hbProcessGrantInfoRsp(kv->value, kv->valueLen, pCatalog); break; } #endif diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index f9f4ee7dfc..b9a0752178 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -69,6 +69,7 @@ typedef enum { CTG_CI_UDF, CTG_CI_SVR_VER, CTG_CI_VIEW, + CTG_CI_GRANT_INFO, CTG_CI_MAX_VALUE, } CTG_CACHE_ITEM; @@ -101,6 +102,8 @@ enum { CTG_OP_DROP_TB_INDEX, CTG_OP_UPDATE_VIEW_META, CTG_OP_DROP_VIEW_META, + CTG_OP_UPDATE_GRANT_INFO, + CTG_OP_DROP_GRANT_INFO, CTG_OP_CLEAR_CACHE, CTG_OP_MAX }; @@ -123,6 +126,7 @@ typedef enum { CTG_TASK_GET_TB_HASH_BATCH, CTG_TASK_GET_TB_TAG, CTG_TASK_GET_VIEW, + CTG_TASK_GET_GRANT_INFO, } CTG_TASK_TYPE; typedef enum { @@ -322,9 +326,12 @@ typedef struct SCatalog { SDynViewVersion dynViewVer; SHashObj* userCache; // key:user, value:SCtgUserAuth SHashObj* dbCache; // key:dbname, value:SCtgDBCache + SHashObj* grantCache; + SGrantHbRsp _grantCache; SCtgRentMgmt dbRent; SCtgRentMgmt stbRent; SCtgRentMgmt viewRent; + SCtgRentMgmt grantRent; SCtgCacheStat cacheStat; } SCatalog; @@ -548,6 +555,10 @@ typedef struct SCtgDropViewMetaMsg { uint64_t viewId; } SCtgDropViewMetaMsg; +typedef struct SCtgUpdateGrantInfoMsg { + SCatalog* pCtg; + SGrantHbRsp* pRsp; +} SCtgUpdateGrantInfoMsg; typedef struct SCtgCacheOperation { int32_t opId; @@ -948,6 +959,7 @@ int32_t ctgUpdateRentViewVersion(SCatalog *pCtg, char *dbFName, char *viewName, SCtgViewCache *pCache); int32_t ctgUpdateTbMetaToCache(SCatalog* pCtg, STableMetaOutput* pOut, bool syncReq); int32_t ctgUpdateViewMetaToCache(SCatalog *pCtg, SViewMetaRsp *pRsp, bool syncReq); +int32_t ctgUpdateGrantInfoToCache(SCatalog *pCtg, SGrantHbRsp *pRsp, bool syncReq); int32_t ctgStartUpdateThread(); int32_t ctgRelaunchGetTbMetaTask(SCtgTask* pTask); void ctgReleaseVgInfoToCache(SCatalog* pCtg, SCtgDBCache* dbCache); diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index c2d88e5ce3..f52787a61e 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -1747,6 +1747,21 @@ _return: CTG_API_LEAVE(code); } +int32_t catalogUpdateGrantInfo(SCatalog* pCtg, SGrantHbRsp* pMsg) { + CTG_API_ENTER(); + + if (NULL == pCtg || NULL == pMsg) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + + int32_t code = 0; + CTG_ERR_JRET(ctgUpdateGrantInfoToCache(pCtg, pMsg, true)); + +_return: + + CTG_API_LEAVE(code); +} + int32_t catalogAsyncUpdateViewMeta(SCatalog* pCtg, SViewMetaRsp* pMsg) { CTG_API_ENTER(); diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 1b693b4e07..05078fabb4 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -32,6 +32,8 @@ SCtgOperation gCtgCacheOperation[CTG_OP_MAX] = {{CTG_OP_UPDATE_VGROUP, "update v {CTG_OP_DROP_TB_INDEX, "drop tbIndex", ctgOpDropTbIndex}, {CTG_OP_UPDATE_VIEW_META, "update viewMeta", ctgOpUpdateViewMeta}, {CTG_OP_DROP_VIEW_META, "drop viewMeta", ctgOpDropViewMeta}, + {CTG_OP_UPDATE_GRANT_INFO, "update grantInfo", ctgOpUpdateGrantInfo}, + {CTG_OP_DROP_GRANT_INFO, "drop grantInfo", ctgOpDropGrantInfo}, {CTG_OP_CLEAR_CACHE, "clear cache", ctgOpClearCache}}; SCtgCacheItemInfo gCtgStatItem[CTG_CI_MAX_VALUE] = { @@ -1323,6 +1325,33 @@ _return: CTG_RET(code); } +int32_t ctgUpdateGrantInfoEnqueue(SCatalog *pCtg, SGrantHbRsp *pRsp, bool syncOp) { + int32_t code = 0; + SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); + op->opId = CTG_OP_UPDATE_GRANT_INFO; + op->syncOp = syncOp; + + SCtgUpdateGrantInfoMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateGrantInfoMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateGrantInfoMsg)); + taosMemoryFree(op); + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + + msg->pCtg = pCtg; + msg->pRsp = pRsp; + + op->data = msg; + + CTG_ERR_RET(ctgEnqueue(pCtg, op)); + + return TSDB_CODE_SUCCESS; +_return: + + taosMemoryFree(pRsp); + CTG_RET(code); +} + int32_t ctgDropViewMetaEnqueue(SCatalog *pCtg, const char *dbFName, uint64_t dbId, const char *viewName, uint64_t viewId, bool syncOp) { int32_t code = 0; SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); @@ -1715,6 +1744,10 @@ int32_t ctgUpdateViewMetaToCache(SCatalog *pCtg, SViewMetaRsp *pRsp, bool syncRe CTG_RET(ctgUpdateViewMetaEnqueue(pCtg, pRsp, syncReq)); } +int32_t ctgUpdateGrantInfoToCache(SCatalog *pCtg, SGrantHbRsp *pRsp, bool syncReq) { + CTG_RET(ctgUpdateGrantInfoEnqueue(pCtg, pRsp, syncReq)); +} + void ctgClearAllHandles(void) { SCatalog *pCtg = NULL; @@ -2450,6 +2483,104 @@ _return: CTG_RET(code); } +int32_t ctgOpUpdateGrantInfo(SCtgCacheOperation *operation) { + int32_t code = 0; + SCtgUpdateGrantInfoMsg *msg = operation->data; + SCatalog *pCtg = msg->pCtg; + SGrantHbRsp *pRsp = msg->pRsp; + SCtgDBCache *dbCache = NULL; + SViewMeta *pMeta = NULL; + + taosMemoryFreeClear(msg); + + if (pCtg->stopUpdate) { + goto _return; + } + + CTG_ERR_JRET(ctgGetAddDBCache(pCtg, pRsp->dbFName, pRsp->dbId, &dbCache)); + if (NULL == dbCache) { + ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:0x%" PRIx64, pRsp->dbFName, pRsp->dbId); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + pMeta = taosMemoryCalloc(1, sizeof(SViewMeta)); + if (NULL == pMeta) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + + CTG_ERR_JRET(dupViewMetaFromRsp(pRsp, pMeta)); + + code = ctgWriteViewMetaToCache(pCtg, dbCache, pRsp->dbFName, pRsp->name, pMeta); + pMeta = NULL; + +_return: + + tFreeSViewMetaRsp(pRsp); + taosMemoryFree(pRsp); + ctgFreeSViewMeta(pMeta); + taosMemoryFree(pMeta); + + CTG_RET(code); +} + +int32_t ctgOpDropViewMeta(SCtgCacheOperation *operation) { + int32_t code = 0; + SCtgDropViewMetaMsg *msg = operation->data; + SCatalog *pCtg = msg->pCtg; + int32_t tblType = 0; + + if (pCtg->stopUpdate) { + goto _return; + } + + SCtgDBCache *dbCache = NULL; + ctgGetDBCache(pCtg, msg->dbFName, &dbCache); + if (NULL == dbCache) { + goto _return; + } + + if ((0 != msg->dbId) && (dbCache->dbId != msg->dbId)) { + ctgDebug("dbId 0x%" PRIx64 " not match with curId 0x%" PRIx64 ", dbFName:%s, viewName:%s", msg->dbId, dbCache->dbId, + msg->dbFName, msg->viewName); + goto _return; + } + + SCtgViewCache *pViewCache = taosHashGet(dbCache->viewCache, msg->viewName, strlen(msg->viewName)); + if (NULL == pViewCache) { + ctgDebug("view %s already not in cache", msg->viewName); + goto _return; + } + + int64_t viewId = pViewCache->pMeta->viewId; + if (0 != msg->viewId && viewId != msg->viewId) { + ctgDebug("viewId 0x%" PRIx64 " not match with curId 0x%" PRIx64 ", viewName:%s", msg->viewId, viewId, msg->viewName); + goto _return; + } + + atomic_sub_fetch_64(&dbCache->dbCacheSize, ctgGetViewMetaCacheSize(pViewCache->pMeta)); + ctgFreeViewCacheImpl(pViewCache, true); + + if (taosHashRemove(dbCache->viewCache, msg->viewName, strlen(msg->viewName))) { + ctgError("view %s not exist in cache, dbFName:%s", msg->viewName, msg->dbFName); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); + } else { + atomic_sub_fetch_64(&dbCache->dbCacheSize, sizeof(SCtgViewCache) + strlen(msg->viewName)); + CTG_DB_NUM_DEC(CTG_CI_VIEW); + } + + ctgDebug("view %s removed from cache, dbFName:%s", msg->viewName, msg->dbFName); + + CTG_ERR_JRET(ctgMetaRentRemove(&msg->pCtg->viewRent, viewId, ctgViewVersionSortCompare, ctgViewVersionSearchCompare)); + + ctgDebug("view %s removed from rent, dbFName:%s, viewId:0x%" PRIx64, msg->viewName, msg->dbFName, viewId); + +_return: + + taosMemoryFreeClear(msg); + + CTG_RET(code); +} + void ctgClearFreeCache(SCtgCacheOperation *operation) { SCtgClearCacheMsg *msg = operation->data; From 015fb4d679d9f825d2f99ab7ff28bf57d105ad2d Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 2 Feb 2024 11:15:39 +0800 Subject: [PATCH 04/19] feat: support uniq grant --- source/libs/catalog/inc/catalogInt.h | 16 ++++- source/libs/catalog/src/ctgCache.c | 99 +++++++++++++++------------- 2 files changed, 68 insertions(+), 47 deletions(-) diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index b9a0752178..d86825c216 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -320,14 +320,19 @@ typedef struct SCtgUserAuth { uint64_t userCacheSize; } SCtgUserAuth; +typedef struct SCtgGrantCache { + SRWLatch lock; + SGrantHbRsp grantInfo; + uint64_t grantCacheSize; +} SCtgGrantCache; + typedef struct SCatalog { uint64_t clusterId; bool stopUpdate; SDynViewVersion dynViewVer; SHashObj* userCache; // key:user, value:SCtgUserAuth SHashObj* dbCache; // key:dbname, value:SCtgDBCache - SHashObj* grantCache; - SGrantHbRsp _grantCache; + SCtgGrantCache grantCache; SCtgRentMgmt dbRent; SCtgRentMgmt stbRent; SCtgRentMgmt viewRent; @@ -377,6 +382,7 @@ typedef struct SCtgJob { int32_t tbCfgNum; int32_t svrVerNum; int32_t viewNum; + int32_t grantNum; } SCtgJob; typedef struct SCtgMsgCtx { @@ -560,6 +566,10 @@ typedef struct SCtgUpdateGrantInfoMsg { SGrantHbRsp* pRsp; } SCtgUpdateGrantInfoMsg; +typedef struct SCtgDropGrantInfoMsg { + SCatalog* pCtg; +} SCtgDropGrantInfoMsg; + typedef struct SCtgCacheOperation { int32_t opId; void* data; @@ -922,6 +932,7 @@ int32_t ctgOpDropDbVgroup(SCtgCacheOperation* action); int32_t ctgOpDropStbMeta(SCtgCacheOperation* action); int32_t ctgOpDropTbMeta(SCtgCacheOperation* action); int32_t ctgOpDropViewMeta(SCtgCacheOperation* action); +int32_t ctgOpDropGrantInfo(SCtgCacheOperation* action); int32_t ctgOpUpdateUser(SCtgCacheOperation* action); int32_t ctgOpUpdateEpset(SCtgCacheOperation* operation); int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char* dbFName, SCtgDBCache** pCache); @@ -969,6 +980,7 @@ int32_t ctgOpDropTbIndex(SCtgCacheOperation* operation); int32_t ctgOpUpdateTbIndex(SCtgCacheOperation* operation); int32_t ctgOpClearCache(SCtgCacheOperation* operation); int32_t ctgOpUpdateViewMeta(SCtgCacheOperation *operation); +int32_t ctgOpUpdateGrantInfo(SCtgCacheOperation *operation); int32_t ctgReadTbTypeFromCache(SCatalog* pCtg, char* dbFName, char* tableName, int32_t* tbType); int32_t ctgGetTbHashVgroupFromCache(SCatalog* pCtg, const SName* pTableName, SVgroupInfo** pVgroup); int32_t ctgGetViewsFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgViewsCtx *ctx, int32_t dbIdx, diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 05078fabb4..92bda82bd3 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -160,6 +160,16 @@ int32_t ctgGetDBCache(SCatalog *pCtg, const char *dbFName, SCtgDBCache **pCache) CTG_RET(ctgAcquireDBCacheImpl(pCtg, dbFName, pCache, false)); } +int32_t ctgAcquireGrantCache(SCatalog *pCtg, SCtgGrantCache **ppCache) { + CTG_RET(ctgAcquireDBCacheImpl(pCtg, dbFName, pCache, true)); +} + +void ctgReleaseGrantCache(SCatalog *pCtg, SCtgGrantCache *pCache) { + CTG_UNLOCK(CTG_READ, &dbCache->dbLock); + taosHashRelease(pCtg->dbCache, dbCache); +} + + void ctgReleaseVgInfoToCache(SCatalog *pCtg, SCtgDBCache *dbCache) { ctgRUnlockVgInfo(dbCache); ctgReleaseDBCache(pCtg, dbCache); @@ -1512,6 +1522,47 @@ int32_t ctgGetAddDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId, SCt return TSDB_CODE_SUCCESS; } +int32_t ctgGetAddGrantCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId, SCtgDBCache **pCache) { + int32_t code = 0; + SCtgDBCache *dbCache = NULL; + ctgGetDBCache(pCtg, dbFName, &dbCache); + + if (dbCache) { + // TODO OPEN IT +#if 0 + if (dbCache->dbId == dbId) { + *pCache = dbCache; + return TSDB_CODE_SUCCESS; + } +#else + if (0 == dbId) { + *pCache = dbCache; + return TSDB_CODE_SUCCESS; + } + + if (dbId && (dbCache->dbId == 0)) { + dbCache->dbId = dbId; + *pCache = dbCache; + return TSDB_CODE_SUCCESS; + } + + if (dbCache->dbId == dbId) { + *pCache = dbCache; + return TSDB_CODE_SUCCESS; + } +#endif + CTG_ERR_RET(ctgRemoveDBFromCache(pCtg, dbCache, dbFName)); + } + + CTG_ERR_RET(ctgAddNewDBCache(pCtg, dbFName, dbId)); + + ctgGetDBCache(pCtg, dbFName, &dbCache); + + *pCache = dbCache; + + return TSDB_CODE_SUCCESS; +} + int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, uint64_t dbId, char *tbName, STableMeta *meta, int32_t metaSize) { if (NULL == dbCache->tbCache || NULL == dbCache->stbCache) { @@ -2488,8 +2539,6 @@ int32_t ctgOpUpdateGrantInfo(SCtgCacheOperation *operation) { SCtgUpdateGrantInfoMsg *msg = operation->data; SCatalog *pCtg = msg->pCtg; SGrantHbRsp *pRsp = msg->pRsp; - SCtgDBCache *dbCache = NULL; - SViewMeta *pMeta = NULL; taosMemoryFreeClear(msg); @@ -2523,56 +2572,16 @@ _return: CTG_RET(code); } -int32_t ctgOpDropViewMeta(SCtgCacheOperation *operation) { +int32_t ctgOpDropGrantInfo(SCtgCacheOperation *operation) { int32_t code = 0; - SCtgDropViewMetaMsg *msg = operation->data; + SCtgDropGrantInfoMsg *msg = operation->data; SCatalog *pCtg = msg->pCtg; - int32_t tblType = 0; if (pCtg->stopUpdate) { goto _return; } - SCtgDBCache *dbCache = NULL; - ctgGetDBCache(pCtg, msg->dbFName, &dbCache); - if (NULL == dbCache) { - goto _return; - } - - if ((0 != msg->dbId) && (dbCache->dbId != msg->dbId)) { - ctgDebug("dbId 0x%" PRIx64 " not match with curId 0x%" PRIx64 ", dbFName:%s, viewName:%s", msg->dbId, dbCache->dbId, - msg->dbFName, msg->viewName); - goto _return; - } - - SCtgViewCache *pViewCache = taosHashGet(dbCache->viewCache, msg->viewName, strlen(msg->viewName)); - if (NULL == pViewCache) { - ctgDebug("view %s already not in cache", msg->viewName); - goto _return; - } - - int64_t viewId = pViewCache->pMeta->viewId; - if (0 != msg->viewId && viewId != msg->viewId) { - ctgDebug("viewId 0x%" PRIx64 " not match with curId 0x%" PRIx64 ", viewName:%s", msg->viewId, viewId, msg->viewName); - goto _return; - } - - atomic_sub_fetch_64(&dbCache->dbCacheSize, ctgGetViewMetaCacheSize(pViewCache->pMeta)); - ctgFreeViewCacheImpl(pViewCache, true); - - if (taosHashRemove(dbCache->viewCache, msg->viewName, strlen(msg->viewName))) { - ctgError("view %s not exist in cache, dbFName:%s", msg->viewName, msg->dbFName); - CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); - } else { - atomic_sub_fetch_64(&dbCache->dbCacheSize, sizeof(SCtgViewCache) + strlen(msg->viewName)); - CTG_DB_NUM_DEC(CTG_CI_VIEW); - } - - ctgDebug("view %s removed from cache, dbFName:%s", msg->viewName, msg->dbFName); - - CTG_ERR_JRET(ctgMetaRentRemove(&msg->pCtg->viewRent, viewId, ctgViewVersionSortCompare, ctgViewVersionSearchCompare)); - - ctgDebug("view %s removed from rent, dbFName:%s, viewId:0x%" PRIx64, msg->viewName, msg->dbFName, viewId); + printf("prop:%s:%d grant %s removed from rent\n"); _return: From a4524802497c61c170495bc102f8d9baeb09ecbf Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 2 Feb 2024 12:28:51 +0800 Subject: [PATCH 05/19] feat: support uniq grant --- source/libs/catalog/src/ctgCache.c | 39 +++--------------------------- 1 file changed, 4 insertions(+), 35 deletions(-) diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 92bda82bd3..fc471d2513 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -161,14 +161,12 @@ int32_t ctgGetDBCache(SCatalog *pCtg, const char *dbFName, SCtgDBCache **pCache) } int32_t ctgAcquireGrantCache(SCatalog *pCtg, SCtgGrantCache **ppCache) { - CTG_RET(ctgAcquireDBCacheImpl(pCtg, dbFName, pCache, true)); -} - -void ctgReleaseGrantCache(SCatalog *pCtg, SCtgGrantCache *pCache) { - CTG_UNLOCK(CTG_READ, &dbCache->dbLock); - taosHashRelease(pCtg->dbCache, dbCache); + CTG_LOCK(CTG_READ, &pCtg->grantCache.lock); + *ppCache = &pCtg->grantCache; + CTG_CACHE_HIT_INC(CTG_CI_GRANT_INFO, 1); } +void ctgReleaseGrantCache(SCatalog *pCtg, SCtgGrantCache *pCache) { CTG_UNLOCK(CTG_READ, &pCache->lock); } void ctgReleaseVgInfoToCache(SCatalog *pCtg, SCtgDBCache *dbCache) { ctgRUnlockVgInfo(dbCache); @@ -1527,36 +1525,7 @@ int32_t ctgGetAddGrantCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId, SCtgDBCache *dbCache = NULL; ctgGetDBCache(pCtg, dbFName, &dbCache); - if (dbCache) { - // TODO OPEN IT -#if 0 - if (dbCache->dbId == dbId) { - *pCache = dbCache; - return TSDB_CODE_SUCCESS; - } -#else - if (0 == dbId) { - *pCache = dbCache; - return TSDB_CODE_SUCCESS; - } - if (dbId && (dbCache->dbId == 0)) { - dbCache->dbId = dbId; - *pCache = dbCache; - return TSDB_CODE_SUCCESS; - } - - if (dbCache->dbId == dbId) { - *pCache = dbCache; - return TSDB_CODE_SUCCESS; - } -#endif - CTG_ERR_RET(ctgRemoveDBFromCache(pCtg, dbCache, dbFName)); - } - - CTG_ERR_RET(ctgAddNewDBCache(pCtg, dbFName, dbId)); - - ctgGetDBCache(pCtg, dbFName, &dbCache); *pCache = dbCache; From 3f061a8460c9c1e64b9d90a16deece217a440ac4 Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 2 Feb 2024 13:44:33 +0800 Subject: [PATCH 06/19] feat: support uniq grant --- include/common/tmsg.h | 1 + include/libs/catalog/catalog.h | 1 + source/libs/catalog/src/ctgCache.c | 54 ++++++++++++++---------------- source/libs/catalog/src/ctgRent.c | 14 ++++++++ source/libs/catalog/src/ctgUtil.c | 19 +++++++++++ 5 files changed, 61 insertions(+), 28 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index dfbdc5a9f4..524f8864f3 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1887,6 +1887,7 @@ int32_t tDeserializeSViewHbRsp(void* buf, int32_t bufLen, SViewHbRsp* pRsp); void tFreeSViewHbRsp(SViewHbRsp* pRsp); typedef struct { + int32_t version; uint32_t flags; } SGrantHbRsp; diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 2e013413a3..ba59ab85e9 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -146,6 +146,7 @@ typedef struct SSTableVersion { } SSTableVersion; typedef struct SGrantVersion { + int32_t grantId; int32_t version; } SGrantVersion; diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index fc471d2513..800e1edb80 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -1520,18 +1520,6 @@ int32_t ctgGetAddDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId, SCt return TSDB_CODE_SUCCESS; } -int32_t ctgGetAddGrantCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId, SCtgDBCache **pCache) { - int32_t code = 0; - SCtgDBCache *dbCache = NULL; - ctgGetDBCache(pCtg, dbFName, &dbCache); - - - - *pCache = dbCache; - - return TSDB_CODE_SUCCESS; -} - int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, uint64_t dbId, char *tbName, STableMeta *meta, int32_t metaSize) { if (NULL == dbCache->tbCache || NULL == dbCache->stbCache) { @@ -1743,6 +1731,30 @@ _return: CTG_RET(code); } +int32_t ctgWriteGrantInfoToCache(SCatalog *pCtg, SGrantHbRsp *pRsp) { + int32_t code = TSDB_CODE_SUCCESS; + SCtgGrantCache *pCache = &pCtg->grantCache; + + CTG_LOCK(CTG_WRITE, &pCache->lock); + pCache->grantInfo = *pRsp; + CTG_UNLOCK(CTG_WRITE, &pCache->lock); + + ctgDebug("grant info updated to cache, version"); + + CTG_ERR_RET(ctgUpdateRentViewVersion(pCtg, dbFName, viewName, dbCache->dbId, pMeta->viewId, pCache)); + + pMeta = NULL; + +_return: + + if (pMeta) { + ctgFreeSViewMeta(pMeta); + taosMemoryFree(pMeta); + } + + CTG_RET(code); +} + int32_t ctgUpdateTbMetaToCache(SCatalog *pCtg, STableMetaOutput *pOut, bool syncReq) { STableMetaOutput *pOutput = NULL; int32_t code = 0; @@ -2508,6 +2520,7 @@ int32_t ctgOpUpdateGrantInfo(SCtgCacheOperation *operation) { SCtgUpdateGrantInfoMsg *msg = operation->data; SCatalog *pCtg = msg->pCtg; SGrantHbRsp *pRsp = msg->pRsp; + SCtgGrantCache *pGrantCache = NULL; taosMemoryFreeClear(msg); @@ -2515,29 +2528,14 @@ int32_t ctgOpUpdateGrantInfo(SCtgCacheOperation *operation) { goto _return; } - CTG_ERR_JRET(ctgGetAddDBCache(pCtg, pRsp->dbFName, pRsp->dbId, &dbCache)); - if (NULL == dbCache) { - ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:0x%" PRIx64, pRsp->dbFName, pRsp->dbId); - CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); - } - - pMeta = taosMemoryCalloc(1, sizeof(SViewMeta)); - if (NULL == pMeta) { - CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); - } - - CTG_ERR_JRET(dupViewMetaFromRsp(pRsp, pMeta)); + CTG_ERR_JRET(ctgAcquireGrantCache(pCtg, &pGrantCache)); code = ctgWriteViewMetaToCache(pCtg, dbCache, pRsp->dbFName, pRsp->name, pMeta); pMeta = NULL; _return: - tFreeSViewMetaRsp(pRsp); taosMemoryFree(pRsp); - ctgFreeSViewMeta(pMeta); - taosMemoryFree(pMeta); - CTG_RET(code); } diff --git a/source/libs/catalog/src/ctgRent.c b/source/libs/catalog/src/ctgRent.c index 457285b147..799c76d5e5 100755 --- a/source/libs/catalog/src/ctgRent.c +++ b/source/libs/catalog/src/ctgRent.c @@ -301,6 +301,20 @@ int32_t ctgUpdateRentViewVersion(SCatalog *pCtg, char *dbFName, char *viewName, return TSDB_CODE_SUCCESS; } +int32_t ctgUpdateRentGrantVersion(SCatalog *pCtg, int32_t grantId, + SCtgGrantCache *pCache) { + SGrantVersion metaRent = {.grantId = grantId, .version = pCache->grantInfo.version}; + + + CTG_ERR_RET(ctgMetaRentUpdate(&pCtg->grantRent, &metaRent, metaRent.grantId, sizeof(SGrantVersion), + ctgGrantVersionSortCompare, ctgGrantVersionSearchCompare)); + + ctgDebug("grant %d version %d updated to grantRent", grantId, metaRent.version); + + return TSDB_CODE_SUCCESS; +} + + diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index cacdfcb8d6..e157028bee 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -1328,6 +1328,16 @@ int32_t ctgViewVersionSearchCompare(const void* key1, const void* key2) { } } +int32_t ctgGrantVersionSearchCompare(const void* key1, const void* key2) { + if (*(int32_t*)key1 == ((SGrantVersion*)key2)->grantId) { + return 0; + } else if (*(int32_t*)key1 < ((SGrantVersion*)key2)->grantId) { + return -1; + } else { + return 1; + } +} + int32_t ctgStbVersionSortCompare(const void* key1, const void* key2) { if (((SSTableVersion*)key1)->suid < ((SSTableVersion*)key2)->suid) { @@ -1359,6 +1369,15 @@ int32_t ctgViewVersionSortCompare(const void* key1, const void* key2) { } } +int32_t ctgGrantVersionSortCompare(const void* key1, const void* key2) { + if (((SGrantVersion*)key1)->grantId == ((SGrantVersion*)key2)->grantId) { + return 0; + } else if (((SGrantVersion*)key1)->grantId < ((SGrantVersion*)key2)->grantId) { + return -1; + } else { + return 1; + } +} int32_t ctgMakeVgArray(SDBVgInfo* dbInfo) { if (NULL == dbInfo) { From 3e7884a4f789efe75b121c9a30afc68dd90045cd Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 2 Feb 2024 13:59:45 +0800 Subject: [PATCH 07/19] feat: support uniq grant --- include/libs/catalog/catalog.h | 2 +- source/libs/catalog/inc/catalogInt.h | 3 +++ source/libs/catalog/src/ctgCache.c | 12 +++++++----- source/libs/catalog/src/ctgRent.c | 9 +-------- 4 files changed, 12 insertions(+), 14 deletions(-) diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index ba59ab85e9..914ecb64ff 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -146,7 +146,7 @@ typedef struct SSTableVersion { } SSTableVersion; typedef struct SGrantVersion { - int32_t grantId; + int64_t grantId; int32_t version; } SGrantVersion; diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index d86825c216..4d759d6bd7 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -568,6 +568,7 @@ typedef struct SCtgUpdateGrantInfoMsg { typedef struct SCtgDropGrantInfoMsg { SCatalog* pCtg; + int64_t grantId; } SCtgDropGrantInfoMsg; typedef struct SCtgCacheOperation { @@ -1040,10 +1041,12 @@ void ctgResetTbMetaTask(SCtgTask* pTask); void ctgFreeDbCache(SCtgDBCache* dbCache); int32_t ctgStbVersionSortCompare(const void* key1, const void* key2); int32_t ctgViewVersionSortCompare(const void* key1, const void* key2); +int32_t ctgGrantVersionSortCompare(const void* key1, const void* key2); int32_t ctgDbCacheInfoSortCompare(const void* key1, const void* key2); int32_t ctgStbVersionSearchCompare(const void* key1, const void* key2); int32_t ctgDbCacheInfoSearchCompare(const void* key1, const void* key2); int32_t ctgViewVersionSearchCompare(const void* key1, const void* key2); +int32_t ctgGrantVersionSearchCompare(const void* key1, const void* key2); void ctgFreeSTableMetaOutput(STableMetaOutput* pOutput); int32_t ctgUpdateMsgCtx(SCtgMsgCtx* pCtx, int32_t reqType, void* out, char* target); int32_t ctgAddMsgCtx(SArray* pCtxs, int32_t reqType, void* out, char* target); diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 800e1edb80..0f93ba317a 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -1739,7 +1739,7 @@ int32_t ctgWriteGrantInfoToCache(SCatalog *pCtg, SGrantHbRsp *pRsp) { pCache->grantInfo = *pRsp; CTG_UNLOCK(CTG_WRITE, &pCache->lock); - ctgDebug("grant info updated to cache, version"); + ctgDebug("grant info updated to cache, flags:%u, version:%d", pRsp->flags, pRsp->version); CTG_ERR_RET(ctgUpdateRentViewVersion(pCtg, dbFName, viewName, dbCache->dbId, pMeta->viewId, pCache)); @@ -2540,15 +2540,18 @@ _return: } int32_t ctgOpDropGrantInfo(SCtgCacheOperation *operation) { - int32_t code = 0; + int32_t code = 0; SCtgDropGrantInfoMsg *msg = operation->data; - SCatalog *pCtg = msg->pCtg; + SCatalog *pCtg = msg->pCtg; if (pCtg->stopUpdate) { goto _return; } - printf("prop:%s:%d grant %s removed from rent\n"); + CTG_ERR_JRET( + ctgMetaRentRemove(&pCtg->grantRent, msg->grantId, ctgGrantVersionSortCompare, ctgGrantVersionSearchCompare)); + + printf("prop:grant:0x%" PRIx64 "removed from rent", msg->grantId); _return: @@ -2557,7 +2560,6 @@ _return: CTG_RET(code); } - void ctgClearFreeCache(SCtgCacheOperation *operation) { SCtgClearCacheMsg *msg = operation->data; SCatalog *pCtg = msg->pCtg; diff --git a/source/libs/catalog/src/ctgRent.c b/source/libs/catalog/src/ctgRent.c index 799c76d5e5..9ef1cf5928 100755 --- a/source/libs/catalog/src/ctgRent.c +++ b/source/libs/catalog/src/ctgRent.c @@ -301,11 +301,9 @@ int32_t ctgUpdateRentViewVersion(SCatalog *pCtg, char *dbFName, char *viewName, return TSDB_CODE_SUCCESS; } -int32_t ctgUpdateRentGrantVersion(SCatalog *pCtg, int32_t grantId, - SCtgGrantCache *pCache) { +int32_t ctgUpdateRentGrantVersion(SCatalog *pCtg, int32_t grantId, SCtgGrantCache *pCache) { SGrantVersion metaRent = {.grantId = grantId, .version = pCache->grantInfo.version}; - CTG_ERR_RET(ctgMetaRentUpdate(&pCtg->grantRent, &metaRent, metaRent.grantId, sizeof(SGrantVersion), ctgGrantVersionSortCompare, ctgGrantVersionSearchCompare)); @@ -313,8 +311,3 @@ int32_t ctgUpdateRentGrantVersion(SCatalog *pCtg, int32_t grantId, return TSDB_CODE_SUCCESS; } - - - - - From 092fc8d0110bb834968bb56ad703d2d50d8b27cf Mon Sep 17 00:00:00 2001 From: kailixu Date: Sat, 3 Feb 2024 12:10:37 +0800 Subject: [PATCH 08/19] feat: support uniq grant --- include/util/tdef.h | 1 + source/common/src/systable.c | 16 ++++++++-------- source/common/src/tmsg.c | 2 ++ source/libs/catalog/inc/catalogInt.h | 4 +++- source/libs/catalog/src/ctgCache.c | 15 ++++----------- source/libs/catalog/src/ctgRent.c | 4 ++-- 6 files changed, 20 insertions(+), 22 deletions(-) diff --git a/include/util/tdef.h b/include/util/tdef.h index 4698d50e67..f136005026 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -287,6 +287,7 @@ typedef enum ELogicConditionType { #define TSDB_DNODE_VALUE_LEN 256 #define TSDB_CLUSTER_VALUE_LEN 1000 +#define TSDB_GRANT_LOG_COL_LEN 15072 #define TSDB_ACTIVE_KEY_LEN 109 #define TSDB_CONN_ACTIVE_KEY_LEN 255 diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 77083d0425..47eac317ec 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -349,21 +349,21 @@ static const SSysDbTableSchema userCompactsDetailSchema[] = { }; static const SSysDbTableSchema useGrantsFullSchema[] = { - {.name = "grant_name", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, - {.name = "display_name", .bytes = 256 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, - {.name = "expire", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, - {.name = "limits", .bytes = 512 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "grant_name", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "display_name", .bytes = 256 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "expire", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "limits", .bytes = 512 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, }; static const SSysDbTableSchema useGrantsLogsSchema[] = { - {.name = "state", .bytes = 1536 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, - {.name = "active", .bytes = 512 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, - {.name = "machine", .bytes = 9088 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "state", .bytes = 1536 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "active", .bytes = 512 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "machine", .bytes = TSDB_GRANT_LOG_COL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, }; static const SSysDbTableSchema useMachinesSchema[] = { {.name = "id", .bytes = TSDB_CLUSTER_ID_LEN + 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, - {.name = "machine", .bytes = 6016 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "machine", .bytes = 7552 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, }; static const SSysTableMeta infosMeta[] = { diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index df62e8d8fc..72c17756b5 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -9341,6 +9341,7 @@ int32_t tSerializeSGrantHbRsp(void *buf, int32_t bufLen, SGrantHbRsp *pRsp) { if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI32v(&encoder, pRsp->version) < 0) return -1; if (tEncodeU32v(&encoder, pRsp->flags) < 0) return -1; tEndEncode(&encoder); @@ -9356,6 +9357,7 @@ int32_t tDeserializeSGrantHbRsp(void *buf, int32_t bufLen, SGrantHbRsp *pRsp) { if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI32v(&decoder, &pRsp->version) < 0) return -1; if (tDecodeU32v(&decoder, &pRsp->flags) < 0) return -1; tEndDecode(&decoder); diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 4d759d6bd7..b880c0a155 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -39,6 +39,7 @@ extern "C" { #define CTG_MAX_COMMAND_LEN 512 #define CTG_DEFAULT_CACHE_MON_MSEC 5000 #define CTG_CLEAR_CACHE_ROUND_TB_NUM 3000 +#define CGT_GRANT_ID 0 #define CTG_RENT_SLOT_SECOND 1.5 @@ -968,7 +969,8 @@ void ctgRemoveViewRent(SCatalog *pCtg, SCtgDBCache *dbCache); int32_t ctgUpdateRentStbVersion(SCatalog *pCtg, char *dbFName, char *tbName, uint64_t dbId, uint64_t suid, SCtgTbCache *pCache); int32_t ctgUpdateRentViewVersion(SCatalog *pCtg, char *dbFName, char *viewName, uint64_t dbId, uint64_t viewId, - SCtgViewCache *pCache); + SCtgViewCache *pCache); +int32_t ctgUpdateRentGrantVersion(SCatalog* pCtg, int32_t grantId, SGrantHbRsp* pGrant); int32_t ctgUpdateTbMetaToCache(SCatalog* pCtg, STableMetaOutput* pOut, bool syncReq); int32_t ctgUpdateViewMetaToCache(SCatalog *pCtg, SViewMetaRsp *pRsp, bool syncReq); int32_t ctgUpdateGrantInfoToCache(SCatalog *pCtg, SGrantHbRsp *pRsp, bool syncReq); diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 0f93ba317a..3b8a18e908 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -164,6 +164,7 @@ int32_t ctgAcquireGrantCache(SCatalog *pCtg, SCtgGrantCache **ppCache) { CTG_LOCK(CTG_READ, &pCtg->grantCache.lock); *ppCache = &pCtg->grantCache; CTG_CACHE_HIT_INC(CTG_CI_GRANT_INFO, 1); + return TSDB_CODE_SUCCESS; } void ctgReleaseGrantCache(SCatalog *pCtg, SCtgGrantCache *pCache) { CTG_UNLOCK(CTG_READ, &pCache->lock); } @@ -1741,17 +1742,10 @@ int32_t ctgWriteGrantInfoToCache(SCatalog *pCtg, SGrantHbRsp *pRsp) { ctgDebug("grant info updated to cache, flags:%u, version:%d", pRsp->flags, pRsp->version); - CTG_ERR_RET(ctgUpdateRentViewVersion(pCtg, dbFName, viewName, dbCache->dbId, pMeta->viewId, pCache)); - - pMeta = NULL; + CTG_ERR_RET(ctgUpdateRentGrantVersion(pCtg, CGT_GRANT_ID, pRsp)); _return: - if (pMeta) { - ctgFreeSViewMeta(pMeta); - taosMemoryFree(pMeta); - } - CTG_RET(code); } @@ -2528,10 +2522,9 @@ int32_t ctgOpUpdateGrantInfo(SCtgCacheOperation *operation) { goto _return; } - CTG_ERR_JRET(ctgAcquireGrantCache(pCtg, &pGrantCache)); + // CTG_ERR_JRET(ctgAcquireGrantCache(pCtg, &pGrantCache)); // TODO: inc/dec or cacheSize ? - code = ctgWriteViewMetaToCache(pCtg, dbCache, pRsp->dbFName, pRsp->name, pMeta); - pMeta = NULL; + code = ctgWriteGrantInfoToCache(pCtg, pRsp); _return: diff --git a/source/libs/catalog/src/ctgRent.c b/source/libs/catalog/src/ctgRent.c index 9ef1cf5928..f757bd856f 100755 --- a/source/libs/catalog/src/ctgRent.c +++ b/source/libs/catalog/src/ctgRent.c @@ -301,8 +301,8 @@ int32_t ctgUpdateRentViewVersion(SCatalog *pCtg, char *dbFName, char *viewName, return TSDB_CODE_SUCCESS; } -int32_t ctgUpdateRentGrantVersion(SCatalog *pCtg, int32_t grantId, SCtgGrantCache *pCache) { - SGrantVersion metaRent = {.grantId = grantId, .version = pCache->grantInfo.version}; +int32_t ctgUpdateRentGrantVersion(SCatalog *pCtg, int32_t grantId, SGrantHbRsp *pGrant) { + SGrantVersion metaRent = {.grantId = grantId, .version = pGrant->version}; CTG_ERR_RET(ctgMetaRentUpdate(&pCtg->grantRent, &metaRent, metaRent.grantId, sizeof(SGrantVersion), ctgGrantVersionSortCompare, ctgGrantVersionSearchCompare)); From 3a0892631978a8126cf6b2531342503393b43511 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sat, 3 Feb 2024 17:32:02 +0800 Subject: [PATCH 09/19] feat: support uniq grant --- include/common/tmsg.h | 2 +- include/libs/catalog/catalog.h | 3 +- source/client/src/clientHb.c | 54 ++++++++++++++++++++++-- source/dnode/mnode/impl/src/mndProfile.c | 4 +- source/libs/catalog/inc/catalogInt.h | 8 ++-- source/libs/catalog/src/catalog.c | 17 ++++++++ source/libs/catalog/src/ctgCache.c | 7 --- source/libs/catalog/src/ctgRent.c | 10 ----- source/libs/catalog/src/ctgUtil.c | 19 --------- 9 files changed, 76 insertions(+), 48 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 524f8864f3..96bf9677b7 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -110,7 +110,7 @@ enum { HEARTBEAT_KEY_TMQ, HEARTBEAT_KEY_DYN_VIEW, HEARTBEAT_KEY_VIEWINFO, - HEARTBEAT_KEY_GRANT, + HEARTBEAT_KEY_GRANTINFO, }; typedef enum _mgmt_table { diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 914ecb64ff..16deda205a 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -146,7 +146,6 @@ typedef struct SSTableVersion { } SSTableVersion; typedef struct SGrantVersion { - int64_t grantId; int32_t version; } SGrantVersion; @@ -343,6 +342,8 @@ int32_t catalogGetExpiredSTables(SCatalog* pCatalog, SSTableVersion** stables, u int32_t catalogGetExpiredViews(SCatalog* pCtg, SViewVersion** views, uint32_t* num, SDynViewVersion** dynViewVersion); +int32_t catalogGetExpiredGrants(SCatalog* pCtg, SGrantVersion** grants, uint32_t* num); + int32_t catalogGetExpiredDBs(SCatalog* pCatalog, SDbCacheInfo** dbs, uint32_t* num); int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion** users, uint32_t* num); diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 755ab0eb91..6972ca3c38 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -342,7 +342,8 @@ static int32_t hbProcessGrantInfoRsp(void *value, int32_t valueLen, struct SCata return -1; } - tscInfo("hb to update grant info:%u", hbRsp->flags); + tscDebug("hb to update grant info, version:%d, flags:%u", hbRsp->version, hbRsp->flags); + catalogUpdateGrantInfo(pCatalog, hbRsp); return TSDB_CODE_SUCCESS; @@ -398,11 +399,12 @@ static void hbProcessQueryRspKvs(int32_t kvNum, SArray* pKvs, struct SCatalog *p hbProcessViewInfoRsp(kv->value, kv->valueLen, pCatalog); break; } - case HEARTBEAT_KEY_GRANT: { + case HEARTBEAT_KEY_GRANTINFO: { if (kv->valueLen <= 0 || NULL == kv->value) { tscError("invalid grant info, len:%d, value:%p", kv->valueLen, kv->value); break; } + assert(0); hbProcessGrantInfoRsp(kv->value, kv->valueLen, pCatalog); break; @@ -872,7 +874,7 @@ int32_t hbGetExpiredViewInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, S view->version = htonl(view->version); } - tscDebug("hb got %d expired view, valueLen:%lu", viewNum, sizeof(SViewVersion) * viewNum); + tscDebug("hb got %u expired view, valueLen:%lu", viewNum, sizeof(SViewVersion) * viewNum); if (NULL == req->info) { req->info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); @@ -895,6 +897,47 @@ int32_t hbGetExpiredViewInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, S return TSDB_CODE_SUCCESS; } +int32_t hbGetExpiredGrantInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SClientHbReq *req) { + SGrantVersion *grants = NULL; + uint32_t grantNum = 0; + int32_t code = 0; + + code = catalogGetExpiredGrants(pCatalog, &grants, &grantNum); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + + if (grantNum <= 0) { + taosMemoryFree(grants); + return code; + } + + for (int32_t i = 0; i < grantNum; ++i) { + SGrantVersion *gv = &grants[i]; + gv->version = htonl(gv->version); + } + + tscDebug("hb got %d expired grant, valueLen:%u", grantNum, (int32_t)sizeof(SGrantVersion) * grantNum); + + if (!req->info) { + req->info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); + if (!req->info) { + code = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(grants); + return code; + taosMemoryFree(grants); + } + } + + SKv kv = { + .key = HEARTBEAT_KEY_GRANTINFO, + .valueLen = sizeof(SGrantVersion) * grantNum, + .value = grants, + }; + taosHashPut(req->info, &kv.key, sizeof(kv.key), &kv, sizeof(kv)); + + return TSDB_CODE_SUCCESS; +} int32_t hbGetAppInfo(int64_t clusterId, SClientHbReq *req) { SAppHbReq *pApp = taosHashGet(clientHbMgr.appSummary, &clusterId, sizeof(clusterId)); @@ -958,6 +1001,11 @@ int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req if (TSDB_CODE_SUCCESS != code) { return code; } + + code = hbGetExpiredGrantInfo(connKey, pCatalog, req); + if (TSDB_CODE_SUCCESS != code) { + return code; + } #endif } else { req->app.appId = 0; diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index d7e6d9609c..2d2d12a668 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -606,12 +606,12 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb } break; } - case HEARTBEAT_KEY_GRANT: { + case HEARTBEAT_KEY_GRANTINFO: { void *rspMsg = NULL; int32_t rspLen = 0; mndValidateGrant(pMnode, kv->value, &rspMsg, &rspLen); if (rspMsg && rspLen > 0) { - SKv kv1 = {.key = HEARTBEAT_KEY_GRANT, .valueLen = rspLen, .value = rspMsg}; + SKv kv1 = {.key = HEARTBEAT_KEY_GRANTINFO, .valueLen = rspLen, .value = rspMsg}; taosArrayPush(hbRsp.info, &kv1); } break; diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index b880c0a155..89fd50657d 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -337,7 +337,6 @@ typedef struct SCatalog { SCtgRentMgmt dbRent; SCtgRentMgmt stbRent; SCtgRentMgmt viewRent; - SCtgRentMgmt grantRent; SCtgCacheStat cacheStat; } SCatalog; @@ -569,7 +568,6 @@ typedef struct SCtgUpdateGrantInfoMsg { typedef struct SCtgDropGrantInfoMsg { SCatalog* pCtg; - int64_t grantId; } SCtgDropGrantInfoMsg; typedef struct SCtgCacheOperation { @@ -970,7 +968,7 @@ int32_t ctgUpdateRentStbVersion(SCatalog *pCtg, char *dbFName, char *tbName, uin SCtgTbCache *pCache); int32_t ctgUpdateRentViewVersion(SCatalog *pCtg, char *dbFName, char *viewName, uint64_t dbId, uint64_t viewId, SCtgViewCache *pCache); -int32_t ctgUpdateRentGrantVersion(SCatalog* pCtg, int32_t grantId, SGrantHbRsp* pGrant); +// int32_t ctgUpdateRentGrantVersion(SCatalog* pCtg, int32_t grantId, SGrantHbRsp* pGrant); int32_t ctgUpdateTbMetaToCache(SCatalog* pCtg, STableMetaOutput* pOut, bool syncReq); int32_t ctgUpdateViewMetaToCache(SCatalog *pCtg, SViewMetaRsp *pRsp, bool syncReq); int32_t ctgUpdateGrantInfoToCache(SCatalog *pCtg, SGrantHbRsp *pRsp, bool syncReq); @@ -1043,12 +1041,12 @@ void ctgResetTbMetaTask(SCtgTask* pTask); void ctgFreeDbCache(SCtgDBCache* dbCache); int32_t ctgStbVersionSortCompare(const void* key1, const void* key2); int32_t ctgViewVersionSortCompare(const void* key1, const void* key2); -int32_t ctgGrantVersionSortCompare(const void* key1, const void* key2); +// int32_t ctgGrantVersionSortCompare(const void* key1, const void* key2); int32_t ctgDbCacheInfoSortCompare(const void* key1, const void* key2); int32_t ctgStbVersionSearchCompare(const void* key1, const void* key2); int32_t ctgDbCacheInfoSearchCompare(const void* key1, const void* key2); int32_t ctgViewVersionSearchCompare(const void* key1, const void* key2); -int32_t ctgGrantVersionSearchCompare(const void* key1, const void* key2); +// int32_t ctgGrantVersionSearchCompare(const void* key1, const void* key2); void ctgFreeSTableMetaOutput(STableMetaOutput* pOutput); int32_t ctgUpdateMsgCtx(SCtgMsgCtx* pCtx, int32_t reqType, void* out, char* target); int32_t ctgAddMsgCtx(SArray* pCtxs, int32_t reqType, void* out, char* target); diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index f52787a61e..c1bf641303 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -1513,6 +1513,23 @@ int32_t catalogGetExpiredViews(SCatalog* pCtg, SViewVersion** views, uint32_t* n CTG_API_LEAVE(ctgMetaRentGet(&pCtg->viewRent, (void**)views, num, sizeof(SViewVersion))); } +int32_t catalogGetExpiredGrants(SCatalog* pCtg, SGrantVersion** grants, uint32_t* num) { + CTG_API_ENTER(); + + if (NULL == pCtg || NULL == grants || NULL == num) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + + *num = 1; + *grants = taosMemoryCalloc(*num, sizeof(SGrantVersion)); + if (!(*grants)) { + ctgError("calloc %d grantVersion failed", *num); + CTG_API_LEAVE(TSDB_CODE_OUT_OF_MEMORY); + } + (*grants)[0].version = pCtg->grantCache.grantInfo.version; + + CTG_API_LEAVE(TSDB_CODE_SUCCESS); +} int32_t catalogGetExpiredDBs(SCatalog* pCtg, SDbCacheInfo** dbs, uint32_t* num) { CTG_API_ENTER(); diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 3b8a18e908..12c9907ffd 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -1742,8 +1742,6 @@ int32_t ctgWriteGrantInfoToCache(SCatalog *pCtg, SGrantHbRsp *pRsp) { ctgDebug("grant info updated to cache, flags:%u, version:%d", pRsp->flags, pRsp->version); - CTG_ERR_RET(ctgUpdateRentGrantVersion(pCtg, CGT_GRANT_ID, pRsp)); - _return: CTG_RET(code); @@ -2541,11 +2539,6 @@ int32_t ctgOpDropGrantInfo(SCtgCacheOperation *operation) { goto _return; } - CTG_ERR_JRET( - ctgMetaRentRemove(&pCtg->grantRent, msg->grantId, ctgGrantVersionSortCompare, ctgGrantVersionSearchCompare)); - - printf("prop:grant:0x%" PRIx64 "removed from rent", msg->grantId); - _return: taosMemoryFreeClear(msg); diff --git a/source/libs/catalog/src/ctgRent.c b/source/libs/catalog/src/ctgRent.c index f757bd856f..0ac133b8e7 100755 --- a/source/libs/catalog/src/ctgRent.c +++ b/source/libs/catalog/src/ctgRent.c @@ -301,13 +301,3 @@ int32_t ctgUpdateRentViewVersion(SCatalog *pCtg, char *dbFName, char *viewName, return TSDB_CODE_SUCCESS; } -int32_t ctgUpdateRentGrantVersion(SCatalog *pCtg, int32_t grantId, SGrantHbRsp *pGrant) { - SGrantVersion metaRent = {.grantId = grantId, .version = pGrant->version}; - - CTG_ERR_RET(ctgMetaRentUpdate(&pCtg->grantRent, &metaRent, metaRent.grantId, sizeof(SGrantVersion), - ctgGrantVersionSortCompare, ctgGrantVersionSearchCompare)); - - ctgDebug("grant %d version %d updated to grantRent", grantId, metaRent.version); - - return TSDB_CODE_SUCCESS; -} diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index e157028bee..cacdfcb8d6 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -1328,16 +1328,6 @@ int32_t ctgViewVersionSearchCompare(const void* key1, const void* key2) { } } -int32_t ctgGrantVersionSearchCompare(const void* key1, const void* key2) { - if (*(int32_t*)key1 == ((SGrantVersion*)key2)->grantId) { - return 0; - } else if (*(int32_t*)key1 < ((SGrantVersion*)key2)->grantId) { - return -1; - } else { - return 1; - } -} - int32_t ctgStbVersionSortCompare(const void* key1, const void* key2) { if (((SSTableVersion*)key1)->suid < ((SSTableVersion*)key2)->suid) { @@ -1369,15 +1359,6 @@ int32_t ctgViewVersionSortCompare(const void* key1, const void* key2) { } } -int32_t ctgGrantVersionSortCompare(const void* key1, const void* key2) { - if (((SGrantVersion*)key1)->grantId == ((SGrantVersion*)key2)->grantId) { - return 0; - } else if (((SGrantVersion*)key1)->grantId < ((SGrantVersion*)key2)->grantId) { - return -1; - } else { - return 1; - } -} int32_t ctgMakeVgArray(SDBVgInfo* dbInfo) { if (NULL == dbInfo) { From f53d0bce5d12669e21d75007b46a864a3ae84fca Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 4 Feb 2024 00:24:42 +0800 Subject: [PATCH 10/19] feat: support uniq grant --- include/common/tgrant.h | 5 +++ include/common/tmsg.h | 1 + include/libs/catalog/catalog.h | 5 ++- include/util/taoserror.h | 2 -- source/dnode/vnode/src/vnd/vnodeSvr.c | 5 +++ source/libs/catalog/inc/catalogInt.h | 1 - source/libs/catalog/src/catalog.c | 47 ++++++++++++++++++++++++++ source/libs/catalog/src/ctgCache.c | 2 -- source/libs/parser/inc/parInsertUtil.h | 3 +- source/libs/parser/src/parInsertSml.c | 6 ++-- source/libs/parser/src/parInsertSql.c | 24 ++++++------- source/libs/parser/src/parInsertUtil.c | 9 ++--- source/util/src/terror.c | 2 -- 13 files changed, 84 insertions(+), 28 deletions(-) diff --git a/include/common/tgrant.h b/include/common/tgrant.h index 9f7572ed63..66408e9fa4 100644 --- a/include/common/tgrant.h +++ b/include/common/tgrant.h @@ -30,6 +30,10 @@ extern "C" { #define GRANT_HEART_BEAT_MIN 2 #define GRANT_ACTIVE_CODE "activeCode" +#define GRANT_ALL_FLAG (0x01) +#define GRANT_AUDIT_FLAG (0x02) +#define GRANT_CSV_FLAG (0x04) +#define GRANT_VIEW_FLAG (0x08) typedef enum { TSDB_GRANT_ALL, @@ -50,6 +54,7 @@ typedef enum { TSDB_GRANT_SUBSCRIPTION, TSDB_GRANT_AUDIT, TSDB_GRANT_CSV, + TSDB_GRANT_VIEW, TSDB_GRANT_MULTI_TIER, TSDB_GRANT_BACKUP_RESTORE, } EGrantType; diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 96bf9677b7..899ce62a9d 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -3931,6 +3931,7 @@ int32_t tDeserializeSMqSeekReq(void* buf, int32_t bufLen, SMqSeekReq* pReq); #define SUBMIT_REQ_AUTO_CREATE_TABLE 0x1 #define SUBMIT_REQ_COLUMN_DATA_FORMAT 0x2 +#define SUBMIT_REQ_FROM_FILE 0x4 typedef struct { int32_t flags; diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 16deda205a..6ce1769f5f 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -25,6 +25,7 @@ extern "C" { #include "taosdef.h" #include "tarray.h" #include "tcommon.h" +#include "tgrant.h" #include "thash.h" #include "tmsg.h" #include "tname.h" @@ -364,7 +365,9 @@ int32_t catalogGetUdfInfo(SCatalog* pCtg, SRequestConnInfo* pConn, const char* f int32_t catalogChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, SUserAuthInfo *pAuth, SUserAuthRes* pRes); -int32_t catalogChkAuthFromCache(SCatalog* pCtg, SUserAuthInfo *pAuth, SUserAuthRes* pRes, bool* exists); +int32_t catalogChkAuthFromCache(SCatalog* pCtg, SUserAuthInfo* pAuth, SUserAuthRes* pRes, bool* exists); + +int32_t catalogChkGrant(SCatalog* pCtg, EGrantType grant); int32_t catalogUpdateUserAuthInfo(SCatalog* pCtg, SGetUserAuthRsp* pAuth); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 94fe80b901..c0c20a0fde 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -575,8 +575,6 @@ int32_t* taosGetErrno(); #define TSDB_CODE_GRANT_OPT_EXPIRE_TOO_LARGE TAOS_DEF_ERROR_CODE(0, 0x0821) #define TSDB_CODE_GRANT_DUPLICATED_ACTIVE TAOS_DEF_ERROR_CODE(0, 0x0822) #define TSDB_CODE_GRANT_VIEW_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0823) -#define TSDB_CODE_GRANT_CSV_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0824) -#define TSDB_CODE_GRANT_AUDIT_LIMITED TAOS_DEF_ERROR_CODE(0, 0x0825) // sync // #define TSDB_CODE_SYN_INVALID_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0900) // 2.x diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 3ec6adee41..e4d7b11176 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -238,6 +238,11 @@ static int32_t vnodePreProcessSubmitTbData(SVnode *pVnode, SDecoder *pCoder, int TSDB_CHECK_CODE(code, lino, _exit); } + if (submitTbData.flags & SUBMIT_REQ_FROM_FILE) { + code = grantCheck(TSDB_GRANT_CSV); + TSDB_CHECK_CODE(code, lino, _exit); + } + int64_t uid; if (submitTbData.flags & SUBMIT_REQ_AUTO_CREATE_TABLE) { code = vnodePreprocessCreateTableReq(pVnode, pCoder, btimeMs, &uid); diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 89fd50657d..0934aa0a27 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -324,7 +324,6 @@ typedef struct SCtgUserAuth { typedef struct SCtgGrantCache { SRWLatch lock; SGrantHbRsp grantInfo; - uint64_t grantCacheSize; } SCtgGrantCache; typedef struct SCatalog { diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index c1bf641303..3d256f07d3 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -353,6 +353,39 @@ _return: CTG_RET(code); } +int32_t ctgChkGrant(SCatalog* pCtg, EGrantType grant) { + int32_t code = 0; + int32_t flag = 0; + SCtgGrantCache* pCache = &pCtg->grantCache; + + CTG_LOCK(CTG_READ, &pCache->lock); + switch (grant) { + case TSDB_GRANT_ALL: { + flag = pCache->grantInfo.flags & GRANT_ALL_FLAG; + break; + } + case TSDB_GRANT_AUDIT: { + flag = pCache->grantInfo.flags & GRANT_AUDIT_FLAG; + break; + } + case TSDB_GRANT_CSV: { + flag = pCache->grantInfo.flags & GRANT_CSV_FLAG; + break; + } + case TSDB_GRANT_VIEW: { + flag = pCache->grantInfo.flags & GRANT_VIEW_FLAG; + break; + } + } + CTG_UNLOCK(CTG_READ, &pCache->lock); + + if (flag) code = TSDB_CODE_GRANT_EXPIRED; + +_return: + + CTG_RET(code); +} + int32_t ctgGetTbType(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTableName, int32_t* tbType) { char dbFName[TSDB_DB_FNAME_LEN]; tNameGetFullDbName(pTableName, dbFName); @@ -1693,6 +1726,20 @@ _return: CTG_API_LEAVE(code); } +int32_t catalogChkGrant(SCatalog* pCtg, EGrantType grant) { + CTG_API_ENTER(); + + if (NULL == pCtg) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + + int32_t code = 0; + CTG_ERR_JRET(ctgChkGrant(pCtg, grant)); + +_return: + + CTG_API_LEAVE(code); +} int32_t catalogGetServerVersion(SCatalog* pCtg, SRequestConnInfo* pConn, char** pVersion) { CTG_API_ENTER(); diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 12c9907ffd..ad48aa3fb1 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -2520,8 +2520,6 @@ int32_t ctgOpUpdateGrantInfo(SCtgCacheOperation *operation) { goto _return; } - // CTG_ERR_JRET(ctgAcquireGrantCache(pCtg, &pGrantCache)); // TODO: inc/dec or cacheSize ? - code = ctgWriteGrantInfoToCache(pCtg, pRsp); _return: diff --git a/source/libs/parser/inc/parInsertUtil.h b/source/libs/parser/inc/parInsertUtil.h index b20587dd43..f1ba58f6de 100644 --- a/source/libs/parser/inc/parInsertUtil.h +++ b/source/libs/parser/inc/parInsertUtil.h @@ -48,7 +48,8 @@ int32_t insInitBoundColsInfo(int32_t numOfBound, SBoundColInfo *pInfo); void insInitColValues(STableMeta* pTableMeta, SArray* aColValues); void insCheckTableDataOrder(STableDataCxt *pTableCxt, TSKEY tsKey); int32_t insGetTableDataCxt(SHashObj *pHash, void *id, int32_t idLen, STableMeta *pTableMeta, - SVCreateTbReq **pCreateTbReq, STableDataCxt **pTableCxt, bool colMode, bool ignoreColVals); + SVCreateTbReq **pCreateTbReq, STableDataCxt **pTableCxt, bool colMode, bool ignoreColVals, + uint32_t insertType); int32_t initTableColSubmitData(STableDataCxt *pTableCxt); int32_t insMergeTableDataCxt(SHashObj *pTableHash, SArray **pVgDataBlocks, bool isRebuild); int32_t insBuildVgDataBlocks(SHashObj *pVgroupsHashObj, SArray *pVgDataBlocks, SArray **pDataBlocks); diff --git a/source/libs/parser/src/parInsertSml.c b/source/libs/parser/src/parInsertSml.c index 13c4431b62..ffb4eacaea 100644 --- a/source/libs/parser/src/parInsertSml.c +++ b/source/libs/parser/src/parInsertSml.c @@ -169,7 +169,8 @@ STableDataCxt* smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta) { STableDataCxt* pTableCxt = NULL; SVCreateTbReq* pCreateTbReq = NULL; int ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, - sizeof(pTableMeta->uid), pTableMeta, &pCreateTbReq, &pTableCxt, false, false); + sizeof(pTableMeta->uid), pTableMeta, &pCreateTbReq, &pTableCxt, false, false, + ((SVnodeModifyOpStmt*)(query->pRoot))->insertType); if (ret != TSDB_CODE_SUCCESS) { return NULL; } @@ -313,7 +314,8 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc STableDataCxt* pTableCxt = NULL; ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, - sizeof(pTableMeta->uid), pTableMeta, &pCreateTblReq, &pTableCxt, false, false); + sizeof(pTableMeta->uid), pTableMeta, &pCreateTblReq, &pTableCxt, false, false, + ((SVnodeModifyOpStmt*)(query->pRoot))->insertType); if (ret != TSDB_CODE_SUCCESS) { buildInvalidOperationMsg(&pBuf, "insGetTableDataCxt error"); goto end; diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 8e32eca28e..b6df2e47ed 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -1333,7 +1333,7 @@ static int32_t preParseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModif static int32_t getTableDataCxt(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt** pTableCxt) { if (pCxt->pComCxt->async) { return insGetTableDataCxt(pStmt->pTableBlockHashObj, &pStmt->pTableMeta->uid, sizeof(pStmt->pTableMeta->uid), - pStmt->pTableMeta, &pStmt->pCreateTblReq, pTableCxt, false, false); + pStmt->pTableMeta, &pStmt->pCreateTblReq, pTableCxt, false, false, pStmt->insertType); } char tbFName[TSDB_TABLE_FNAME_LEN]; @@ -1342,7 +1342,7 @@ static int32_t getTableDataCxt(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS pStmt->pTableMeta->uid = 0; } return insGetTableDataCxt(pStmt->pTableBlockHashObj, tbFName, strlen(tbFName), pStmt->pTableMeta, - &pStmt->pCreateTblReq, pTableCxt, NULL != pCxt->pComCxt->pStmtCb, false); + &pStmt->pCreateTblReq, pTableCxt, NULL != pCxt->pComCxt->pStmtCb, false, pStmt->insertType); } static int32_t parseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt* pTableCxt) { @@ -1931,7 +1931,7 @@ static int32_t parseOneStbRow(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pSt } code = insGetTableDataCxt(pStmt->pTableBlockHashObj, &pStbRowsCxt->pCtbMeta->uid, sizeof(pStbRowsCxt->pCtbMeta->uid), - pStbRowsCxt->pCtbMeta, &pStbRowsCxt->pCreateCtbReq, ppTableDataCxt, false, true); + pStbRowsCxt->pCtbMeta, &pStbRowsCxt->pCreateCtbReq, ppTableDataCxt, false, true, pStmt->insertType); initTableColSubmitData(*ppTableDataCxt); if (code == TSDB_CODE_SUCCESS) { SRow** pRow = taosArrayReserve((*ppTableDataCxt)->pData->aRowP, 1); @@ -2139,14 +2139,20 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt } static int32_t parseDataFromFileImpl(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SRowsDataContext rowsDataCxt) { + int32_t code = 0; + int32_t numOfRows = 0; + + if ((code = catalogChkGrant(pCxt->pComCxt->pCatalog, TSDB_GRANT_CSV)) < 0) { + return code; + } + // init only for file if (NULL == pStmt->pTableCxtHashObj) { pStmt->pTableCxtHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); } - int32_t numOfRows = 0; - int32_t code = parseCsvFile(pCxt, pStmt, rowsDataCxt, &numOfRows); + code = parseCsvFile(pCxt, pStmt, rowsDataCxt, &numOfRows); if (TSDB_CODE_SUCCESS == code) { pStmt->totalRowsNum += numOfRows; pStmt->totalTbNum += 1; @@ -2193,10 +2199,6 @@ static int32_t parseFileClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS return buildInvalidOperationMsg(&pCxt->msg, "proxy mode does not support csv loading"); } - // if ((terrno = grantCheck(TSDB_GRANT_CSV)) < 0) { - // return buildInvalidOperationMsg(&pCxt->msg, terrstr()); - // } - NEXT_TOKEN(pStmt->pSql, *pToken); if (0 == pToken->n || (TK_NK_STRING != pToken->type && TK_NK_ID != pToken->type)) { return buildSyntaxErrMsg(&pCxt->msg, "file path is required following keyword FILE", pToken->z); @@ -2761,10 +2763,6 @@ static int32_t parseInsertSqlFromCsv(SInsertParseContext* pCxt, SVnodeModifyOpSt int32_t code = TSDB_CODE_SUCCESS; SRowsDataContext rowsDataCxt; - // if ((code = grantCheck(TSDB_GRANT_CSV)) < 0) { - // return code; - // } - if (!pStmt->stbSyntax) { STableDataCxt* pTableCxt = NULL; code = getTableDataCxt(pCxt, pStmt, &pTableCxt); diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 6b655bfae6..5c7de2519c 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -208,7 +208,7 @@ void insCheckTableDataOrder(STableDataCxt* pTableCxt, TSKEY tsKey) { void insDestroyBoundColInfo(SBoundColInfo* pInfo) { taosMemoryFreeClear(pInfo->pColIndex); } static int32_t createTableDataCxt(STableMeta* pTableMeta, SVCreateTbReq** pCreateTbReq, STableDataCxt** pOutput, - bool colMode, bool ignoreColVals) { + bool colMode, bool ignoreColVals, uint32_t insertType) { STableDataCxt* pTableCxt = taosMemoryCalloc(1, sizeof(STableDataCxt)); if (NULL == pTableCxt) { return TSDB_CODE_OUT_OF_MEMORY; @@ -249,6 +249,7 @@ static int32_t createTableDataCxt(STableMeta* pTableMeta, SVCreateTbReq** pCreat } else { pTableCxt->pData->flags = (pCreateTbReq != NULL && NULL != *pCreateTbReq) ? SUBMIT_REQ_AUTO_CREATE_TABLE : 0; pTableCxt->pData->flags |= colMode ? SUBMIT_REQ_COLUMN_DATA_FORMAT : 0; + pTableCxt->pData->flags |= insertType == TSDB_QUERY_TYPE_FILE_INSERT ? SUBMIT_REQ_FROM_FILE : 0; pTableCxt->pData->suid = pTableMeta->suid; pTableCxt->pData->uid = pTableMeta->uid; pTableCxt->pData->sver = pTableMeta->sversion; @@ -330,7 +331,7 @@ static void resetColValues(SArray* pValues) { } int32_t insGetTableDataCxt(SHashObj* pHash, void* id, int32_t idLen, STableMeta* pTableMeta, - SVCreateTbReq** pCreateTbReq, STableDataCxt** pTableCxt, bool colMode, bool ignoreColVals) { + SVCreateTbReq** pCreateTbReq, STableDataCxt** pTableCxt, bool colMode, bool ignoreColVals, uint32_t insertType) { STableDataCxt** tmp = (STableDataCxt**)taosHashGet(pHash, id, idLen); if (NULL != tmp) { *pTableCxt = *tmp; @@ -339,7 +340,7 @@ int32_t insGetTableDataCxt(SHashObj* pHash, void* id, int32_t idLen, STableMeta* } return TSDB_CODE_SUCCESS; } - int32_t code = createTableDataCxt(pTableMeta, pCreateTbReq, pTableCxt, colMode, ignoreColVals); + int32_t code = createTableDataCxt(pTableMeta, pCreateTbReq, pTableCxt, colMode, ignoreColVals, insertType); if (TSDB_CODE_SUCCESS == code) { void* pData = *pTableCxt; // deal scan coverity code = taosHashPut(pHash, id, idLen, &pData, POINTER_BYTES); @@ -645,7 +646,7 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate void* tmp = taosHashGet(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, sizeof(pTableMeta->uid)); STableDataCxt* pTableCxt = NULL; int ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, - sizeof(pTableMeta->uid), pTableMeta, pCreateTb, &pTableCxt, true, false); + sizeof(pTableMeta->uid), pTableMeta, pCreateTb, &pTableCxt, true, false, ((SVnodeModifyOpStmt*)(query->pRoot))->insertType); if (ret != TSDB_CODE_SUCCESS) { uError("insGetTableDataCxt error"); goto end; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 9fcca86744..7510b89736 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -462,8 +462,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_MACHINES_MISMATCH, "Cluster machines mism TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_OPT_EXPIRE_TOO_LARGE, "Expire time of optional grant item is too large") TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_DUPLICATED_ACTIVE, "The active code can't be activated repeatedly") TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_VIEW_LIMITED, "Number of view has reached the licensed upper limit") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_CSV_LIMITED, "Csv has reached the licensed upper limit") -TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_AUDIT_LIMITED, "Audit has reached the licensed upper limit") // sync TAOS_DEFINE_ERROR(TSDB_CODE_SYN_TIMEOUT, "Sync timeout") From a9316c37100a8d8aec51041d8a1ec7423eb34626 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 4 Feb 2024 12:28:00 +0800 Subject: [PATCH 11/19] feat: support uniq grant --- include/libs/nodes/plannodes.h | 6 +++++- include/libs/nodes/querynodes.h | 2 ++ include/libs/parser/parser.h | 4 +++- include/libs/planner/planner.h | 2 ++ include/libs/qcom/query.h | 5 +++++ source/client/src/clientHb.c | 4 +--- source/client/src/clientImpl.c | 4 ++++ source/libs/nodes/src/nodesCloneFuncs.c | 2 ++ source/libs/nodes/src/nodesCodeFuncs.c | 12 ++++++++++++ source/libs/nodes/src/nodesMsgFuncs.c | 13 ++++++++++++- source/libs/parser/inc/parInsertUtil.h | 3 +-- source/libs/parser/inc/parTranslater.h | 2 ++ source/libs/parser/src/parInsertSml.c | 6 ++---- source/libs/parser/src/parInsertSql.c | 9 ++++++--- source/libs/parser/src/parInsertUtil.c | 9 ++++----- source/libs/parser/src/parUtil.c | 2 ++ source/libs/planner/src/planLogicCreater.c | 2 ++ source/libs/scheduler/src/schRemote.c | 3 +++ 18 files changed, 70 insertions(+), 20 deletions(-) diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 8e375e9da8..3ff50ca2e8 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -92,6 +92,8 @@ typedef struct SScanLogicNode { STimeWindow scanRange; SName tableName; bool showRewrite; + bool isView; + bool isAudit; double ratio; SNodeList* pDynamicScanFuncs; int32_t dataRequired; @@ -713,8 +715,10 @@ typedef struct SSubplan { SNode* pTagCond; SNode* pTagIndexCond; bool showRewrite; - int32_t rowsThreshold; + bool isView; + bool isAudit; bool dynamicRowThreshold; + int32_t rowsThreshold; } SSubplan; typedef enum EExplainMode { EXPLAIN_MODE_DISABLE = 1, EXPLAIN_MODE_STATIC, EXPLAIN_MODE_ANALYZE } EExplainMode; diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 9647c0adac..97a20c2fc0 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -517,6 +517,8 @@ typedef struct SQuery { SArray* pTableList; SArray* pDbList; bool showRewrite; + // bool isView; + // bool isAudit; int32_t placeholderNum; SArray* pPlaceholderValues; SNode* pPrepareRoot; diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 6a41f4607b..2ac2c3ccbd 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -86,8 +86,10 @@ typedef struct SParseContext { bool enableSysInfo; bool async; bool hasInvisibleCol; - const char* svrVer; + bool isView; + bool isAudit; bool nodeOffline; + const char* svrVer; SArray* pTableMetaPos; // sql table pos => catalog data pos SArray* pTableVgroupPos; // sql table pos => catalog data pos int64_t allocatorId; diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index 1b523c0323..707d70b71b 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -32,6 +32,8 @@ typedef struct SPlanContext { bool streamQuery; bool rSmaQuery; bool showRewrite; + bool isView; + bool isAudit; int8_t triggerType; int64_t watermark; int64_t deleteMark; diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index f79a0a0718..ad8de4707d 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -66,7 +66,12 @@ typedef enum { #define QUERY_RSP_POLICY_QUICK 1 #define QUERY_MSG_MASK_SHOW_REWRITE() (1 << 0) +#define QUERY_MSG_MASK_VIEW() (1 << 1) +#define QUERY_MSG_MASK_AUDIT() (1 << 2) #define TEST_SHOW_REWRITE_MASK(m) (((m)&QUERY_MSG_MASK_SHOW_REWRITE()) != 0) +#define TEST_VIEW_MASK(m) (((m)&QUERY_MSG_MASK_VIEW()) != 0) +#define TEST_AUDIT_MASK(m) (((m)&QUERY_MSG_MASK_MASK()) != 0) + typedef struct STableComInfo { uint8_t numOfTags; // the number of tags in schema diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 6972ca3c38..e7529e7ed3 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -404,8 +404,6 @@ static void hbProcessQueryRspKvs(int32_t kvNum, SArray* pKvs, struct SCatalog *p tscError("invalid grant info, len:%d, value:%p", kv->valueLen, kv->value); break; } - assert(0); - hbProcessGrantInfoRsp(kv->value, kv->valueLen, pCatalog); break; } @@ -917,7 +915,7 @@ int32_t hbGetExpiredGrantInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, gv->version = htonl(gv->version); } - tscDebug("hb got %d expired grant, valueLen:%u", grantNum, (int32_t)sizeof(SGrantVersion) * grantNum); + tscDebug("hb got %d expired grant, valueLen:%d", grantNum, (int32_t)(sizeof(SGrantVersion) * grantNum)); if (!req->info) { req->info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index c1e1da617d..ddcf8ea105 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -458,6 +458,8 @@ int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArra .mgmtEpSet = getEpSet_s(&pAppInfo->mgmtEp), .pAstRoot = pQuery->pRoot, .showRewrite = pQuery->showRewrite, + .isView = pQuery->isView; + .isAudit = pQuery->isAudit; .pMsg = pRequest->msgBuf, .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE, .pUser = pRequest->pTscObj->user, @@ -1154,6 +1156,8 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat .mgmtEpSet = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp), .pAstRoot = pQuery->pRoot, .showRewrite = pQuery->showRewrite, + .isView = pWrapper->pParseCtx->isView, + .isAudit = pWrapper->pParseCtx->isAudit, .pMsg = pRequest->msgBuf, .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE, .pUser = pRequest->pTscObj->user, diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index bc9839792c..b882271175 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -423,6 +423,8 @@ static int32_t logicScanCopy(const SScanLogicNode* pSrc, SScanLogicNode* pDst) { COPY_OBJECT_FIELD(scanRange, sizeof(STimeWindow)); COPY_OBJECT_FIELD(tableName, sizeof(SName)); COPY_SCALAR_FIELD(showRewrite); + COPY_SCALAR_FIELD(isView); + COPY_SCALAR_FIELD(isAudit); COPY_SCALAR_FIELD(ratio); CLONE_NODE_LIST_FIELD(pDynamicScanFuncs); COPY_SCALAR_FIELD(dataRequired); diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index cae6ffa5fa..b05c9b019e 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -3332,6 +3332,12 @@ static int32_t subplanToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddBoolToObject(pJson, jkSubplanShowRewrite, pNode->showRewrite); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkSubplanShowRewrite, pNode->isView); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkSubplanShowRewrite, pNode->isAudit); + } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkSubplanRowsThreshold, pNode->rowsThreshold); } @@ -3379,6 +3385,12 @@ static int32_t jsonToSubplan(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonGetBoolValue(pJson, jkSubplanShowRewrite, &pNode->showRewrite); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkSubplanShowRewrite, &pNode->isView); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkSubplanShowRewrite, &pNode->isAudit); + } if (TSDB_CODE_SUCCESS == code) { code = tjsonGetIntValue(pJson, jkSubplanRowsThreshold, &pNode->rowsThreshold); } diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c index 3fd219b8d8..7d8cd72cdb 100644 --- a/source/libs/nodes/src/nodesMsgFuncs.c +++ b/source/libs/nodes/src/nodesMsgFuncs.c @@ -3930,6 +3930,12 @@ static int32_t subplanInlineToMsg(const void* pObj, STlvEncoder* pEncoder) { if (TSDB_CODE_SUCCESS == code) { code = tlvEncodeValueBool(pEncoder, pNode->dynamicRowThreshold); } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeValueBool(pEncoder, pNode->isView); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeValueBool(pEncoder, pNode->isAudit); + } return code; } @@ -3985,7 +3991,12 @@ static int32_t msgToSubplanInline(STlvDecoder* pDecoder, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tlvDecodeValueBool(pDecoder, &pNode->dynamicRowThreshold); } - + if (TSDB_CODE_SUCCESS == code) { + code = tlvDecodeValueBool(pDecoder, &pNode->isView); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvDecodeValueBool(pDecoder, &pNode->isAudit); + } return code; } diff --git a/source/libs/parser/inc/parInsertUtil.h b/source/libs/parser/inc/parInsertUtil.h index f1ba58f6de..b20587dd43 100644 --- a/source/libs/parser/inc/parInsertUtil.h +++ b/source/libs/parser/inc/parInsertUtil.h @@ -48,8 +48,7 @@ int32_t insInitBoundColsInfo(int32_t numOfBound, SBoundColInfo *pInfo); void insInitColValues(STableMeta* pTableMeta, SArray* aColValues); void insCheckTableDataOrder(STableDataCxt *pTableCxt, TSKEY tsKey); int32_t insGetTableDataCxt(SHashObj *pHash, void *id, int32_t idLen, STableMeta *pTableMeta, - SVCreateTbReq **pCreateTbReq, STableDataCxt **pTableCxt, bool colMode, bool ignoreColVals, - uint32_t insertType); + SVCreateTbReq **pCreateTbReq, STableDataCxt **pTableCxt, bool colMode, bool ignoreColVals); int32_t initTableColSubmitData(STableDataCxt *pTableCxt); int32_t insMergeTableDataCxt(SHashObj *pTableHash, SArray **pVgDataBlocks, bool isRebuild); int32_t insBuildVgDataBlocks(SHashObj *pVgroupsHashObj, SArray *pVgDataBlocks, SArray **pDataBlocks); diff --git a/source/libs/parser/inc/parTranslater.h b/source/libs/parser/inc/parTranslater.h index 55ea71a368..de885bb1be 100644 --- a/source/libs/parser/inc/parTranslater.h +++ b/source/libs/parser/inc/parTranslater.h @@ -43,6 +43,8 @@ typedef struct STranslateContext { bool createStream; bool stableQuery; bool showRewrite; + bool isView; + bool isAudit; SNode* pPrevRoot; SNode* pPostRoot; } STranslateContext; diff --git a/source/libs/parser/src/parInsertSml.c b/source/libs/parser/src/parInsertSml.c index ffb4eacaea..13c4431b62 100644 --- a/source/libs/parser/src/parInsertSml.c +++ b/source/libs/parser/src/parInsertSml.c @@ -169,8 +169,7 @@ STableDataCxt* smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta) { STableDataCxt* pTableCxt = NULL; SVCreateTbReq* pCreateTbReq = NULL; int ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, - sizeof(pTableMeta->uid), pTableMeta, &pCreateTbReq, &pTableCxt, false, false, - ((SVnodeModifyOpStmt*)(query->pRoot))->insertType); + sizeof(pTableMeta->uid), pTableMeta, &pCreateTbReq, &pTableCxt, false, false); if (ret != TSDB_CODE_SUCCESS) { return NULL; } @@ -314,8 +313,7 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc STableDataCxt* pTableCxt = NULL; ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, - sizeof(pTableMeta->uid), pTableMeta, &pCreateTblReq, &pTableCxt, false, false, - ((SVnodeModifyOpStmt*)(query->pRoot))->insertType); + sizeof(pTableMeta->uid), pTableMeta, &pCreateTblReq, &pTableCxt, false, false); if (ret != TSDB_CODE_SUCCESS) { buildInvalidOperationMsg(&pBuf, "insGetTableDataCxt error"); goto end; diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index b6df2e47ed..83a38c4b15 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -1333,7 +1333,7 @@ static int32_t preParseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModif static int32_t getTableDataCxt(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt** pTableCxt) { if (pCxt->pComCxt->async) { return insGetTableDataCxt(pStmt->pTableBlockHashObj, &pStmt->pTableMeta->uid, sizeof(pStmt->pTableMeta->uid), - pStmt->pTableMeta, &pStmt->pCreateTblReq, pTableCxt, false, false, pStmt->insertType); + pStmt->pTableMeta, &pStmt->pCreateTblReq, pTableCxt, false, false); } char tbFName[TSDB_TABLE_FNAME_LEN]; @@ -1342,7 +1342,7 @@ static int32_t getTableDataCxt(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS pStmt->pTableMeta->uid = 0; } return insGetTableDataCxt(pStmt->pTableBlockHashObj, tbFName, strlen(tbFName), pStmt->pTableMeta, - &pStmt->pCreateTblReq, pTableCxt, NULL != pCxt->pComCxt->pStmtCb, false, pStmt->insertType); + &pStmt->pCreateTblReq, pTableCxt, NULL != pCxt->pComCxt->pStmtCb, false); } static int32_t parseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, STableDataCxt* pTableCxt) { @@ -1931,7 +1931,7 @@ static int32_t parseOneStbRow(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pSt } code = insGetTableDataCxt(pStmt->pTableBlockHashObj, &pStbRowsCxt->pCtbMeta->uid, sizeof(pStbRowsCxt->pCtbMeta->uid), - pStbRowsCxt->pCtbMeta, &pStbRowsCxt->pCreateCtbReq, ppTableDataCxt, false, true, pStmt->insertType); + pStbRowsCxt->pCtbMeta, &pStbRowsCxt->pCreateCtbReq, ppTableDataCxt, false, true); initTableColSubmitData(*ppTableDataCxt); if (code == TSDB_CODE_SUCCESS) { SRow** pRow = taosArrayReserve((*ppTableDataCxt)->pData->aRowP, 1); @@ -2157,6 +2157,9 @@ static int32_t parseDataFromFileImpl(SInsertParseContext* pCxt, SVnodeModifyOpSt pStmt->totalRowsNum += numOfRows; pStmt->totalTbNum += 1; TSDB_QUERY_SET_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_FILE_INSERT); + if (rowsDataCxt.pTableDataCxt && rowsDataCxt.pTableDataCxt->pData) { + rowsDataCxt.pTableDataCxt->pData->flags |= SUBMIT_REQ_FROM_FILE; + } if (!pStmt->fileProcessing) { taosCloseFile(&pStmt->fp); } else { diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 5c7de2519c..6b655bfae6 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -208,7 +208,7 @@ void insCheckTableDataOrder(STableDataCxt* pTableCxt, TSKEY tsKey) { void insDestroyBoundColInfo(SBoundColInfo* pInfo) { taosMemoryFreeClear(pInfo->pColIndex); } static int32_t createTableDataCxt(STableMeta* pTableMeta, SVCreateTbReq** pCreateTbReq, STableDataCxt** pOutput, - bool colMode, bool ignoreColVals, uint32_t insertType) { + bool colMode, bool ignoreColVals) { STableDataCxt* pTableCxt = taosMemoryCalloc(1, sizeof(STableDataCxt)); if (NULL == pTableCxt) { return TSDB_CODE_OUT_OF_MEMORY; @@ -249,7 +249,6 @@ static int32_t createTableDataCxt(STableMeta* pTableMeta, SVCreateTbReq** pCreat } else { pTableCxt->pData->flags = (pCreateTbReq != NULL && NULL != *pCreateTbReq) ? SUBMIT_REQ_AUTO_CREATE_TABLE : 0; pTableCxt->pData->flags |= colMode ? SUBMIT_REQ_COLUMN_DATA_FORMAT : 0; - pTableCxt->pData->flags |= insertType == TSDB_QUERY_TYPE_FILE_INSERT ? SUBMIT_REQ_FROM_FILE : 0; pTableCxt->pData->suid = pTableMeta->suid; pTableCxt->pData->uid = pTableMeta->uid; pTableCxt->pData->sver = pTableMeta->sversion; @@ -331,7 +330,7 @@ static void resetColValues(SArray* pValues) { } int32_t insGetTableDataCxt(SHashObj* pHash, void* id, int32_t idLen, STableMeta* pTableMeta, - SVCreateTbReq** pCreateTbReq, STableDataCxt** pTableCxt, bool colMode, bool ignoreColVals, uint32_t insertType) { + SVCreateTbReq** pCreateTbReq, STableDataCxt** pTableCxt, bool colMode, bool ignoreColVals) { STableDataCxt** tmp = (STableDataCxt**)taosHashGet(pHash, id, idLen); if (NULL != tmp) { *pTableCxt = *tmp; @@ -340,7 +339,7 @@ int32_t insGetTableDataCxt(SHashObj* pHash, void* id, int32_t idLen, STableMeta* } return TSDB_CODE_SUCCESS; } - int32_t code = createTableDataCxt(pTableMeta, pCreateTbReq, pTableCxt, colMode, ignoreColVals, insertType); + int32_t code = createTableDataCxt(pTableMeta, pCreateTbReq, pTableCxt, colMode, ignoreColVals); if (TSDB_CODE_SUCCESS == code) { void* pData = *pTableCxt; // deal scan coverity code = taosHashPut(pHash, id, idLen, &pData, POINTER_BYTES); @@ -646,7 +645,7 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate void* tmp = taosHashGet(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, sizeof(pTableMeta->uid)); STableDataCxt* pTableCxt = NULL; int ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, - sizeof(pTableMeta->uid), pTableMeta, pCreateTb, &pTableCxt, true, false, ((SVnodeModifyOpStmt*)(query->pRoot))->insertType); + sizeof(pTableMeta->uid), pTableMeta, pCreateTb, &pTableCxt, true, false); if (ret != TSDB_CODE_SUCCESS) { uError("insGetTableDataCxt error"); goto end; diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index dfe33ce55e..8aa37f5784 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -192,6 +192,8 @@ static char* getSyntaxErrFormat(int32_t errCode) { return "Out of memory"; case TSDB_CODE_PAR_ORDERBY_AMBIGUOUS: return "ORDER BY \"%s\" is ambiguous"; + case TSDB_CODE_GRANT_EXPIRED: + return "License expired"; default: return "Unknown error"; } diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 12b7360165..a5828af969 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -340,6 +340,8 @@ static int32_t makeScanLogicNode(SLogicPlanContext* pCxt, SRealTableNode* pRealT strcpy(pScan->tableName.dbname, pRealTable->table.dbName); strcpy(pScan->tableName.tname, pRealTable->table.tableName); pScan->showRewrite = pCxt->pPlanCxt->showRewrite; + pScan->isView = pCxt->pPlanCxt->isView; + pScan->isAudit = pCxt->pPlanCxt->isAudit; pScan->ratio = pRealTable->ratio; pScan->dataRequired = FUNC_DATA_REQUIRED_DATA_LOAD; pScan->cacheLastMode = pRealTable->cacheLastMode; diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 5c67c7974f..1a9c37ffe9 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -1109,6 +1109,9 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, qMsg.refId = pJob->refId; qMsg.execId = pTask->execId; qMsg.msgMask = (pTask->plan->showRewrite) ? QUERY_MSG_MASK_SHOW_REWRITE() : 0; + qMsg.msgMask |= (pTask->plan->isView) ? QUERY_MSG_MASK_VIEW() : 0; + qMsg.msgMask |= (pTask->plan->isAudit) ? QUERY_MSG_MASK_AUDIT() : 0; + qMsg.taskType = TASK_TYPE_TEMP; qMsg.explain = SCH_IS_EXPLAIN_JOB(pJob); qMsg.needFetch = SCH_TASK_NEED_FETCH(pTask); From 6cf7809e8d793de9ebf76300528dee1057c4c027 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 4 Feb 2024 14:59:47 +0800 Subject: [PATCH 12/19] feat: support uniq grant --- include/common/tglobal.h | 2 +- include/libs/nodes/plannodes.h | 2 -- include/libs/nodes/querynodes.h | 2 -- include/libs/qcom/query.h | 2 +- source/client/src/clientImpl.c | 2 -- source/common/src/tglobal.c | 15 ++++++++++++++- source/libs/parser/inc/parTranslater.h | 4 ++-- source/libs/planner/src/planPhysiCreater.c | 2 ++ source/libs/qworker/src/qwMsg.c | 22 ++++++++++++++++++---- 9 files changed, 38 insertions(+), 15 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index f23bb4d51b..04e9b5a380 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -232,7 +232,7 @@ struct SConfig *taosGetCfg(); void taosSetAllDebugFlag(int32_t flag); void taosSetDebugFlag(int32_t *pFlagPtr, const char *flagName, int32_t flagVal); void taosLocalCfgForbiddenToChange(char *name, bool *forbidden); -int8_t taosGranted(); +int8_t taosGranted(int8_t type); #ifdef __cplusplus } diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 3ff50ca2e8..84934b4f4f 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -92,8 +92,6 @@ typedef struct SScanLogicNode { STimeWindow scanRange; SName tableName; bool showRewrite; - bool isView; - bool isAudit; double ratio; SNodeList* pDynamicScanFuncs; int32_t dataRequired; diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 97a20c2fc0..9647c0adac 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -517,8 +517,6 @@ typedef struct SQuery { SArray* pTableList; SArray* pDbList; bool showRewrite; - // bool isView; - // bool isAudit; int32_t placeholderNum; SArray* pPlaceholderValues; SNode* pPrepareRoot; diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index ad8de4707d..b63d3d5db4 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -70,7 +70,7 @@ typedef enum { #define QUERY_MSG_MASK_AUDIT() (1 << 2) #define TEST_SHOW_REWRITE_MASK(m) (((m)&QUERY_MSG_MASK_SHOW_REWRITE()) != 0) #define TEST_VIEW_MASK(m) (((m)&QUERY_MSG_MASK_VIEW()) != 0) -#define TEST_AUDIT_MASK(m) (((m)&QUERY_MSG_MASK_MASK()) != 0) +#define TEST_AUDIT_MASK(m) (((m)&QUERY_MSG_MASK_AUDIT()) != 0) typedef struct STableComInfo { diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index ddcf8ea105..d3d8ee1dc1 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -458,8 +458,6 @@ int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArra .mgmtEpSet = getEpSet_s(&pAppInfo->mgmtEp), .pAstRoot = pQuery->pRoot, .showRewrite = pQuery->showRewrite, - .isView = pQuery->isView; - .isAudit = pQuery->isAudit; .pMsg = pRequest->msgBuf, .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE, .pUser = pRequest->pTscObj->user, diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 46c4d613fb..d46b91276b 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -1801,4 +1801,17 @@ void taosSetAllDebugFlag(int32_t flag) { if (terrno == TSDB_CODE_CFG_NOT_FOUND) terrno = TSDB_CODE_SUCCESS; // ignore not exist } -int8_t taosGranted() { return atomic_load_8(&tsGrant); } +int8_t taosGranted(int8_t type) { + switch (type) { + case TSDB_GRANT_ALL: + return atomic_load_8(&tsGrant) & GRANT_ALL_FLAG; + case TSDB_GRANT_AUDIT: + return atomic_load_8(&tsGrant) & GRANT_AUDIT_FLAG; + case TSDB_GRANT_VIEW: + return atomic_load_8(&tsGrant) & GRANT_VIEW_FLAG; + default: + ASSERTS(0, "undefined grant type:%" PRIi8, type); + break; + } + return 0; +} \ No newline at end of file diff --git a/source/libs/parser/inc/parTranslater.h b/source/libs/parser/inc/parTranslater.h index de885bb1be..00f37cb002 100644 --- a/source/libs/parser/inc/parTranslater.h +++ b/source/libs/parser/inc/parTranslater.h @@ -43,8 +43,8 @@ typedef struct STranslateContext { bool createStream; bool stableQuery; bool showRewrite; - bool isView; - bool isAudit; + // bool isView; + // bool isAudit; SNode* pPrevRoot; SNode* pPostRoot; } STranslateContext; diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 21c637116f..4b0024d098 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -2164,6 +2164,8 @@ static SSubplan* makeSubplan(SPhysiPlanContext* pCxt, SLogicSubplan* pLogicSubpl pSubplan->level = pLogicSubplan->level; pSubplan->rowsThreshold = 4096; pSubplan->dynamicRowThreshold = false; + pSubplan->isView = pCxt->pPlanCxt->isView; + pSubplan->isAudit = pCxt->pPlanCxt->isAudit; if (NULL != pCxt->pPlanCxt->pUser) { snprintf(pSubplan->user, sizeof(pSubplan->user), "%s", pCxt->pPlanCxt->pUser); } diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c index 9a1c309ab0..c9d3cb51e3 100644 --- a/source/libs/qworker/src/qwMsg.c +++ b/source/libs/qworker/src/qwMsg.c @@ -360,10 +360,24 @@ int32_t qWorkerPreprocessQueryMsg(void *qWorkerMgmt, SRpcMsg *pMsg, bool chkGran QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - if (chkGrant && (!TEST_SHOW_REWRITE_MASK(msg.msgMask)) && !taosGranted()) { - QW_ELOG("query failed cause of grant expired, msgMask:%d", msg.msgMask); - tFreeSSubQueryMsg(&msg); - QW_ERR_RET(TSDB_CODE_GRANT_EXPIRED); + if (chkGrant) { + if ((!TEST_SHOW_REWRITE_MASK(msg.msgMask)) && !taosGranted(TSDB_GRANT_ALL)) { + QW_ELOG("query failed cause of grant expired, msgMask:%d", msg.msgMask); + tFreeSSubQueryMsg(&msg); + QW_ERR_RET(TSDB_CODE_GRANT_EXPIRED); + } + + if ((TEST_VIEW_MASK(msg.msgMask)) && !taosGranted(TSDB_GRANT_VIEW)) { + QW_ELOG("query failed cause of view grant expired, msgMask:%d", msg.msgMask); + tFreeSSubQueryMsg(&msg); + QW_ERR_RET(TSDB_CODE_GRANT_EXPIRED); + } + + if ((TEST_AUDIT_MASK(msg.msgMask)) && !taosGranted(TSDB_GRANT_AUDIT)) { + QW_ELOG("query failed cause of audit grant expired, msgMask:%d", msg.msgMask); + tFreeSSubQueryMsg(&msg); + QW_ERR_RET(TSDB_CODE_GRANT_EXPIRED); + } } uint64_t sId = msg.sId; From 3ebc26e1fb35dd0cb589ce859949905496b3b48a Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 4 Feb 2024 16:33:43 +0800 Subject: [PATCH 13/19] feat: support uniq grant --- include/common/systable.h | 5 +++++ include/libs/qcom/query.h | 5 +++++ source/dnode/mnode/impl/src/mndCluster.c | 2 +- source/libs/nodes/src/nodesCloneFuncs.c | 2 -- source/libs/parser/src/parInsertSql.c | 6 +++--- source/libs/parser/src/parTranslater.c | 16 +++++++++++++++- source/libs/planner/src/planLogicCreater.c | 2 -- 7 files changed, 29 insertions(+), 9 deletions(-) diff --git a/include/common/systable.h b/include/common/systable.h index 37593144d8..f0f8ac8cf6 100644 --- a/include/common/systable.h +++ b/include/common/systable.h @@ -65,6 +65,11 @@ extern "C" { #define TSDB_PERFS_TABLE_TRANS "perf_trans" #define TSDB_PERFS_TABLE_APPS "perf_apps" +#define TSDB_AUDIT_DB "audit" +#define TSDB_AUDIT_STB_OPERATION "operations" +#define TSDB_AUDIT_CTB_OPERATION "t_operations_" +#define TSDB_AUDIT_CTB_OPERATION_LEN 13 + typedef struct SSysDbTableSchema { const char* name; const int32_t type; diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index b63d3d5db4..4c33c36d1b 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -343,6 +343,11 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t #define IS_SYS_DBNAME(_dbname) (IS_INFORMATION_SCHEMA_DB(_dbname) || IS_PERFORMANCE_SCHEMA_DB(_dbname)) +#define IS_AUDIT_DBNAME(_dbname) ((*(_dbname) == 'a') && (0 == strcmp(_dbname, TSDB_AUDIT_DB))) +#define IS_AUDIT_STB_NAME(_stbname) ((*(_stbname) == 'o') && (0 == strcmp(_stbname, TSDB_AUDIT_STB_OPERATION))) +#define IS_AUDIT_CTB_NAME(_ctbname) \ + ((*(_ctbname) == 't') && (0 == strncmp(_ctbname, TSDB_AUDIT_CTB_OPERATION, TSDB_AUDIT_CTB_OPERATION_LEN))) + #define qFatal(...) \ do { \ if (qDebugFlag & DEBUG_FATAL) { \ diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index 1a55a161bf..36b079d347 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -409,7 +409,7 @@ int32_t mndProcessConfigClusterReq(SRpcMsg *pReq) { } { // audit - auditRecord(pReq, pMnode->clusterId, "alterCluster", "", "", cfgReq.sql, cfgReq.sqlLen); + auditRecord(pReq, pMnode->clusterId, "alterCluster", "", "", cfgReq.sql, MIN(cfgReq.sqlLen, GRANT_ACTIVE_HEAD_LEN << 1)); } _exit: tFreeSMCfgClusterReq(&cfgReq); diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index b882271175..bc9839792c 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -423,8 +423,6 @@ static int32_t logicScanCopy(const SScanLogicNode* pSrc, SScanLogicNode* pDst) { COPY_OBJECT_FIELD(scanRange, sizeof(STimeWindow)); COPY_OBJECT_FIELD(tableName, sizeof(SName)); COPY_SCALAR_FIELD(showRewrite); - COPY_SCALAR_FIELD(isView); - COPY_SCALAR_FIELD(isAudit); COPY_SCALAR_FIELD(ratio); CLONE_NODE_LIST_FIELD(pDynamicScanFuncs); COPY_SCALAR_FIELD(dataRequired); diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 83a38c4b15..a229d2658e 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -2142,9 +2142,9 @@ static int32_t parseDataFromFileImpl(SInsertParseContext* pCxt, SVnodeModifyOpSt int32_t code = 0; int32_t numOfRows = 0; - if ((code = catalogChkGrant(pCxt->pComCxt->pCatalog, TSDB_GRANT_CSV)) < 0) { - return code; - } + // if ((code = catalogChkGrant(pCxt->pComCxt->pCatalog, TSDB_GRANT_CSV)) < 0) { + // return code; + // } // init only for file if (NULL == pStmt->pTableCxtHashObj) { diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 7dabae0c3a..de667b2f20 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3165,6 +3165,19 @@ static int32_t checkJoinTable(STranslateContext* pCxt, SJoinTableNode* pJoinTabl return TSDB_CODE_SUCCESS; } +static int32_t translateAudit(STranslateContext* pCxt, SRealTableNode* pRealTable, SName* pName) { + if (pRealTable->pMeta->tableType == TSDB_SUPER_TABLE) { + if (IS_AUDIT_DBNAME(pName->dbname) && IS_AUDIT_STB_NAME(pName->tname)) { + pCxt->pParseCxt->isAudit = true; + } + } else if (pRealTable->pMeta->tableType == TSDB_CHILD_TABLE) { + if (IS_AUDIT_DBNAME(pName->dbname) && IS_AUDIT_CTB_NAME(pName->tname)) { + pCxt->pParseCxt->isAudit = true; + } + } + return 0; +} + int32_t translateTable(STranslateContext* pCxt, SNode** pTable) { int32_t code = TSDB_CODE_SUCCESS; switch (nodeType(*pTable)) { @@ -3184,7 +3197,8 @@ int32_t translateTable(STranslateContext* pCxt, SNode** pTable) { if (TSDB_VIEW_TABLE == pRealTable->pMeta->tableType) { return translateView(pCxt, pTable, &name); } -#endif + translateAudit(pCxt, pRealTable, &name); +#endif code = setTableVgroupList(pCxt, &name, pRealTable); if (TSDB_CODE_SUCCESS == code) { code = setTableIndex(pCxt, &name, pRealTable); diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index a5828af969..12b7360165 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -340,8 +340,6 @@ static int32_t makeScanLogicNode(SLogicPlanContext* pCxt, SRealTableNode* pRealT strcpy(pScan->tableName.dbname, pRealTable->table.dbName); strcpy(pScan->tableName.tname, pRealTable->table.tableName); pScan->showRewrite = pCxt->pPlanCxt->showRewrite; - pScan->isView = pCxt->pPlanCxt->isView; - pScan->isAudit = pCxt->pPlanCxt->isAudit; pScan->ratio = pRealTable->ratio; pScan->dataRequired = FUNC_DATA_REQUIRED_DATA_LOAD; pScan->cacheLastMode = pRealTable->cacheLastMode; From d281bc70b0f0ed2618e23cda284439f9ae122109 Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 4 Feb 2024 16:46:24 +0800 Subject: [PATCH 14/19] feat: remove hb of grantInfo --- include/common/tmsg.h | 9 --- include/libs/catalog/catalog.h | 5 -- source/client/src/clientHb.c | 77 ------------------- source/common/src/tmsg.c | 31 -------- source/dnode/mnode/impl/inc/mndGrant.h | 1 - source/dnode/mnode/impl/src/mndProfile.c | 10 --- source/libs/catalog/inc/catalogInt.h | 27 ------- source/libs/catalog/src/catalog.c | 82 --------------------- source/libs/catalog/src/ctgCache.c | 94 ------------------------ source/libs/parser/src/parInsertSql.c | 4 - 10 files changed, 340 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 899ce62a9d..9cd9acaee4 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -110,7 +110,6 @@ enum { HEARTBEAT_KEY_TMQ, HEARTBEAT_KEY_DYN_VIEW, HEARTBEAT_KEY_VIEWINFO, - HEARTBEAT_KEY_GRANTINFO, }; typedef enum _mgmt_table { @@ -1886,14 +1885,6 @@ int32_t tSerializeSViewHbRsp(void* buf, int32_t bufLen, SViewHbRsp* pRsp); int32_t tDeserializeSViewHbRsp(void* buf, int32_t bufLen, SViewHbRsp* pRsp); void tFreeSViewHbRsp(SViewHbRsp* pRsp); -typedef struct { - int32_t version; - uint32_t flags; -} SGrantHbRsp; - -int32_t tSerializeSGrantHbRsp(void* buf, int32_t bufLen, SGrantHbRsp* pRsp); -int32_t tDeserializeSGrantHbRsp(void* buf, int32_t bufLen, SGrantHbRsp* pRsp); - typedef struct { int32_t numOfTables; int32_t numOfVgroup; diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 6ce1769f5f..96a05ccbf9 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -343,8 +343,6 @@ int32_t catalogGetExpiredSTables(SCatalog* pCatalog, SSTableVersion** stables, u int32_t catalogGetExpiredViews(SCatalog* pCtg, SViewVersion** views, uint32_t* num, SDynViewVersion** dynViewVersion); -int32_t catalogGetExpiredGrants(SCatalog* pCtg, SGrantVersion** grants, uint32_t* num); - int32_t catalogGetExpiredDBs(SCatalog* pCatalog, SDbCacheInfo** dbs, uint32_t* num); int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion** users, uint32_t* num); @@ -367,8 +365,6 @@ int32_t catalogChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, SUserAuthInfo *p int32_t catalogChkAuthFromCache(SCatalog* pCtg, SUserAuthInfo* pAuth, SUserAuthRes* pRes, bool* exists); -int32_t catalogChkGrant(SCatalog* pCtg, EGrantType grant); - int32_t catalogUpdateUserAuthInfo(SCatalog* pCtg, SGetUserAuthRsp* pAuth); int32_t catalogUpdateVgEpSet(SCatalog* pCtg, const char* dbFName, int32_t vgId, SEpSet* epSet); @@ -388,7 +384,6 @@ int32_t catalogRemoveViewMeta(SCatalog* pCtg, const char* dbFName, uint64_t dbId int32_t catalogUpdateDynViewVer(SCatalog* pCtg, SDynViewVersion* pVer); int32_t catalogUpdateViewMeta(SCatalog* pCtg, SViewMetaRsp* pMsg); -int32_t catalogUpdateGrantInfo(SCatalog* pCtg, SGrantHbRsp* pMsg); int32_t catalogAsyncUpdateViewMeta(SCatalog* pCtg, SViewMetaRsp* pMsg); diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index e7529e7ed3..53a64cec86 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -327,28 +327,6 @@ static int32_t hbProcessViewInfoRsp(void *value, int32_t valueLen, struct SCatal return TSDB_CODE_SUCCESS; } -static int32_t hbProcessGrantInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) { - int32_t code = 0; - - SGrantHbRsp *hbRsp = taosMemoryCalloc(1, sizeof(SGrantHbRsp)); - if (!hbRsp) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - if (tDeserializeSGrantHbRsp(value, valueLen, hbRsp) != 0) { - taosMemoryFree(hbRsp); - terrno = TSDB_CODE_INVALID_MSG; - return -1; - } - - tscDebug("hb to update grant info, version:%d, flags:%u", hbRsp->version, hbRsp->flags); - - catalogUpdateGrantInfo(pCatalog, hbRsp); - - return TSDB_CODE_SUCCESS; -} - static void hbProcessQueryRspKvs(int32_t kvNum, SArray* pKvs, struct SCatalog *pCatalog, SAppHbMgr *pAppHbMgr) { for (int32_t i = 0; i < kvNum; ++i) { SKv *kv = taosArrayGet(pKvs, i); @@ -399,14 +377,6 @@ static void hbProcessQueryRspKvs(int32_t kvNum, SArray* pKvs, struct SCatalog *p hbProcessViewInfoRsp(kv->value, kv->valueLen, pCatalog); break; } - case HEARTBEAT_KEY_GRANTINFO: { - if (kv->valueLen <= 0 || NULL == kv->value) { - tscError("invalid grant info, len:%d, value:%p", kv->valueLen, kv->value); - break; - } - hbProcessGrantInfoRsp(kv->value, kv->valueLen, pCatalog); - break; - } #endif default: tscError("invalid hb key type:%d", kv->key); @@ -895,48 +865,6 @@ int32_t hbGetExpiredViewInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, S return TSDB_CODE_SUCCESS; } -int32_t hbGetExpiredGrantInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SClientHbReq *req) { - SGrantVersion *grants = NULL; - uint32_t grantNum = 0; - int32_t code = 0; - - code = catalogGetExpiredGrants(pCatalog, &grants, &grantNum); - if (TSDB_CODE_SUCCESS != code) { - return code; - } - - if (grantNum <= 0) { - taosMemoryFree(grants); - return code; - } - - for (int32_t i = 0; i < grantNum; ++i) { - SGrantVersion *gv = &grants[i]; - gv->version = htonl(gv->version); - } - - tscDebug("hb got %d expired grant, valueLen:%d", grantNum, (int32_t)(sizeof(SGrantVersion) * grantNum)); - - if (!req->info) { - req->info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); - if (!req->info) { - code = TSDB_CODE_OUT_OF_MEMORY; - taosMemoryFree(grants); - return code; - taosMemoryFree(grants); - } - } - - SKv kv = { - .key = HEARTBEAT_KEY_GRANTINFO, - .valueLen = sizeof(SGrantVersion) * grantNum, - .value = grants, - }; - taosHashPut(req->info, &kv.key, sizeof(kv.key), &kv, sizeof(kv)); - - return TSDB_CODE_SUCCESS; -} - int32_t hbGetAppInfo(int64_t clusterId, SClientHbReq *req) { SAppHbReq *pApp = taosHashGet(clientHbMgr.appSummary, &clusterId, sizeof(clusterId)); if (NULL != pApp) { @@ -999,11 +927,6 @@ int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req if (TSDB_CODE_SUCCESS != code) { return code; } - - code = hbGetExpiredGrantInfo(connKey, pCatalog, req); - if (TSDB_CODE_SUCCESS != code) { - return code; - } #endif } else { req->app.appId = 0; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 72c17756b5..785cb1878e 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -9333,35 +9333,4 @@ void tFreeSViewHbRsp(SViewHbRsp *pRsp) { } taosArrayDestroy(pRsp->pViewRsp); -} - -int32_t tSerializeSGrantHbRsp(void *buf, int32_t bufLen, SGrantHbRsp *pRsp) { - SEncoder encoder = {0}; - tEncoderInit(&encoder, buf, bufLen); - - if (tStartEncode(&encoder) < 0) return -1; - - if (tEncodeI32v(&encoder, pRsp->version) < 0) return -1; - if (tEncodeU32v(&encoder, pRsp->flags) < 0) return -1; - - tEndEncode(&encoder); - - int32_t tlen = encoder.pos; - tEncoderClear(&encoder); - return tlen; -} - -int32_t tDeserializeSGrantHbRsp(void *buf, int32_t bufLen, SGrantHbRsp *pRsp) { - SDecoder decoder = {0}; - tDecoderInit(&decoder, buf, bufLen); - - if (tStartDecode(&decoder) < 0) return -1; - - if (tDecodeI32v(&decoder, &pRsp->version) < 0) return -1; - if (tDecodeU32v(&decoder, &pRsp->flags) < 0) return -1; - - tEndDecode(&decoder); - - tDecoderClear(&decoder); - return 0; } \ No newline at end of file diff --git a/source/dnode/mnode/impl/inc/mndGrant.h b/source/dnode/mnode/impl/inc/mndGrant.h index a012d7a8f5..bda5cebf01 100644 --- a/source/dnode/mnode/impl/inc/mndGrant.h +++ b/source/dnode/mnode/impl/inc/mndGrant.h @@ -43,7 +43,6 @@ int32_t mndProcessConfigGrantReq(SMnode * pMnode, SRpcMsg * pReq, SMCfgClusterReq * pCfg); int32_t mndProcessUpdGrantLog(SMnode * pMnode, SRpcMsg * pReq, SArray * pMachines, SGrantState * pState); - int32_t mndValidateGrant(SMnode * pMnode, SGrantVersion * pGrantVersion, void **ppRsp, int32_t *pRspLen); int32_t mndGrantGetLastState(SMnode * pMnode, SGrantState * pState); SGrantLogObj *mndAcquireGrant(SMnode * pMnode, void **ppIter); void mndReleaseGrant(SMnode * pMnode, SGrantLogObj * pGrant, void *pIter); diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 2d2d12a668..f42cdfa4f3 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -606,16 +606,6 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb } break; } - case HEARTBEAT_KEY_GRANTINFO: { - void *rspMsg = NULL; - int32_t rspLen = 0; - mndValidateGrant(pMnode, kv->value, &rspMsg, &rspLen); - if (rspMsg && rspLen > 0) { - SKv kv1 = {.key = HEARTBEAT_KEY_GRANTINFO, .valueLen = rspLen, .value = rspMsg}; - taosArrayPush(hbRsp.info, &kv1); - } - break; - } #endif default: mError("invalid kv key:%d", kv->key); diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 0934aa0a27..a35eec0ec5 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -39,7 +39,6 @@ extern "C" { #define CTG_MAX_COMMAND_LEN 512 #define CTG_DEFAULT_CACHE_MON_MSEC 5000 #define CTG_CLEAR_CACHE_ROUND_TB_NUM 3000 -#define CGT_GRANT_ID 0 #define CTG_RENT_SLOT_SECOND 1.5 @@ -70,7 +69,6 @@ typedef enum { CTG_CI_UDF, CTG_CI_SVR_VER, CTG_CI_VIEW, - CTG_CI_GRANT_INFO, CTG_CI_MAX_VALUE, } CTG_CACHE_ITEM; @@ -103,8 +101,6 @@ enum { CTG_OP_DROP_TB_INDEX, CTG_OP_UPDATE_VIEW_META, CTG_OP_DROP_VIEW_META, - CTG_OP_UPDATE_GRANT_INFO, - CTG_OP_DROP_GRANT_INFO, CTG_OP_CLEAR_CACHE, CTG_OP_MAX }; @@ -127,7 +123,6 @@ typedef enum { CTG_TASK_GET_TB_HASH_BATCH, CTG_TASK_GET_TB_TAG, CTG_TASK_GET_VIEW, - CTG_TASK_GET_GRANT_INFO, } CTG_TASK_TYPE; typedef enum { @@ -321,18 +316,12 @@ typedef struct SCtgUserAuth { uint64_t userCacheSize; } SCtgUserAuth; -typedef struct SCtgGrantCache { - SRWLatch lock; - SGrantHbRsp grantInfo; -} SCtgGrantCache; - typedef struct SCatalog { uint64_t clusterId; bool stopUpdate; SDynViewVersion dynViewVer; SHashObj* userCache; // key:user, value:SCtgUserAuth SHashObj* dbCache; // key:dbname, value:SCtgDBCache - SCtgGrantCache grantCache; SCtgRentMgmt dbRent; SCtgRentMgmt stbRent; SCtgRentMgmt viewRent; @@ -381,7 +370,6 @@ typedef struct SCtgJob { int32_t tbCfgNum; int32_t svrVerNum; int32_t viewNum; - int32_t grantNum; } SCtgJob; typedef struct SCtgMsgCtx { @@ -560,15 +548,6 @@ typedef struct SCtgDropViewMetaMsg { uint64_t viewId; } SCtgDropViewMetaMsg; -typedef struct SCtgUpdateGrantInfoMsg { - SCatalog* pCtg; - SGrantHbRsp* pRsp; -} SCtgUpdateGrantInfoMsg; - -typedef struct SCtgDropGrantInfoMsg { - SCatalog* pCtg; -} SCtgDropGrantInfoMsg; - typedef struct SCtgCacheOperation { int32_t opId; void* data; @@ -931,7 +910,6 @@ int32_t ctgOpDropDbVgroup(SCtgCacheOperation* action); int32_t ctgOpDropStbMeta(SCtgCacheOperation* action); int32_t ctgOpDropTbMeta(SCtgCacheOperation* action); int32_t ctgOpDropViewMeta(SCtgCacheOperation* action); -int32_t ctgOpDropGrantInfo(SCtgCacheOperation* action); int32_t ctgOpUpdateUser(SCtgCacheOperation* action); int32_t ctgOpUpdateEpset(SCtgCacheOperation* operation); int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char* dbFName, SCtgDBCache** pCache); @@ -967,10 +945,8 @@ int32_t ctgUpdateRentStbVersion(SCatalog *pCtg, char *dbFName, char *tbName, uin SCtgTbCache *pCache); int32_t ctgUpdateRentViewVersion(SCatalog *pCtg, char *dbFName, char *viewName, uint64_t dbId, uint64_t viewId, SCtgViewCache *pCache); -// int32_t ctgUpdateRentGrantVersion(SCatalog* pCtg, int32_t grantId, SGrantHbRsp* pGrant); int32_t ctgUpdateTbMetaToCache(SCatalog* pCtg, STableMetaOutput* pOut, bool syncReq); int32_t ctgUpdateViewMetaToCache(SCatalog *pCtg, SViewMetaRsp *pRsp, bool syncReq); -int32_t ctgUpdateGrantInfoToCache(SCatalog *pCtg, SGrantHbRsp *pRsp, bool syncReq); int32_t ctgStartUpdateThread(); int32_t ctgRelaunchGetTbMetaTask(SCtgTask* pTask); void ctgReleaseVgInfoToCache(SCatalog* pCtg, SCtgDBCache* dbCache); @@ -980,7 +956,6 @@ int32_t ctgOpDropTbIndex(SCtgCacheOperation* operation); int32_t ctgOpUpdateTbIndex(SCtgCacheOperation* operation); int32_t ctgOpClearCache(SCtgCacheOperation* operation); int32_t ctgOpUpdateViewMeta(SCtgCacheOperation *operation); -int32_t ctgOpUpdateGrantInfo(SCtgCacheOperation *operation); int32_t ctgReadTbTypeFromCache(SCatalog* pCtg, char* dbFName, char* tableName, int32_t* tbType); int32_t ctgGetTbHashVgroupFromCache(SCatalog* pCtg, const SName* pTableName, SVgroupInfo** pVgroup); int32_t ctgGetViewsFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgViewsCtx *ctx, int32_t dbIdx, @@ -1040,12 +1015,10 @@ void ctgResetTbMetaTask(SCtgTask* pTask); void ctgFreeDbCache(SCtgDBCache* dbCache); int32_t ctgStbVersionSortCompare(const void* key1, const void* key2); int32_t ctgViewVersionSortCompare(const void* key1, const void* key2); -// int32_t ctgGrantVersionSortCompare(const void* key1, const void* key2); int32_t ctgDbCacheInfoSortCompare(const void* key1, const void* key2); int32_t ctgStbVersionSearchCompare(const void* key1, const void* key2); int32_t ctgDbCacheInfoSearchCompare(const void* key1, const void* key2); int32_t ctgViewVersionSearchCompare(const void* key1, const void* key2); -// int32_t ctgGrantVersionSearchCompare(const void* key1, const void* key2); void ctgFreeSTableMetaOutput(STableMetaOutput* pOutput); int32_t ctgUpdateMsgCtx(SCtgMsgCtx* pCtx, int32_t reqType, void* out, char* target); int32_t ctgAddMsgCtx(SArray* pCtxs, int32_t reqType, void* out, char* target); diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 3d256f07d3..5a53f167c4 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -353,39 +353,6 @@ _return: CTG_RET(code); } -int32_t ctgChkGrant(SCatalog* pCtg, EGrantType grant) { - int32_t code = 0; - int32_t flag = 0; - SCtgGrantCache* pCache = &pCtg->grantCache; - - CTG_LOCK(CTG_READ, &pCache->lock); - switch (grant) { - case TSDB_GRANT_ALL: { - flag = pCache->grantInfo.flags & GRANT_ALL_FLAG; - break; - } - case TSDB_GRANT_AUDIT: { - flag = pCache->grantInfo.flags & GRANT_AUDIT_FLAG; - break; - } - case TSDB_GRANT_CSV: { - flag = pCache->grantInfo.flags & GRANT_CSV_FLAG; - break; - } - case TSDB_GRANT_VIEW: { - flag = pCache->grantInfo.flags & GRANT_VIEW_FLAG; - break; - } - } - CTG_UNLOCK(CTG_READ, &pCache->lock); - - if (flag) code = TSDB_CODE_GRANT_EXPIRED; - -_return: - - CTG_RET(code); -} - int32_t ctgGetTbType(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTableName, int32_t* tbType) { char dbFName[TSDB_DB_FNAME_LEN]; tNameGetFullDbName(pTableName, dbFName); @@ -1546,24 +1513,6 @@ int32_t catalogGetExpiredViews(SCatalog* pCtg, SViewVersion** views, uint32_t* n CTG_API_LEAVE(ctgMetaRentGet(&pCtg->viewRent, (void**)views, num, sizeof(SViewVersion))); } -int32_t catalogGetExpiredGrants(SCatalog* pCtg, SGrantVersion** grants, uint32_t* num) { - CTG_API_ENTER(); - - if (NULL == pCtg || NULL == grants || NULL == num) { - CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); - } - - *num = 1; - *grants = taosMemoryCalloc(*num, sizeof(SGrantVersion)); - if (!(*grants)) { - ctgError("calloc %d grantVersion failed", *num); - CTG_API_LEAVE(TSDB_CODE_OUT_OF_MEMORY); - } - (*grants)[0].version = pCtg->grantCache.grantInfo.version; - - CTG_API_LEAVE(TSDB_CODE_SUCCESS); -} - int32_t catalogGetExpiredDBs(SCatalog* pCtg, SDbCacheInfo** dbs, uint32_t* num) { CTG_API_ENTER(); @@ -1726,21 +1675,6 @@ _return: CTG_API_LEAVE(code); } -int32_t catalogChkGrant(SCatalog* pCtg, EGrantType grant) { - CTG_API_ENTER(); - - if (NULL == pCtg) { - CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); - } - - int32_t code = 0; - CTG_ERR_JRET(ctgChkGrant(pCtg, grant)); - -_return: - - CTG_API_LEAVE(code); -} - int32_t catalogGetServerVersion(SCatalog* pCtg, SRequestConnInfo* pConn, char** pVersion) { CTG_API_ENTER(); @@ -1811,22 +1745,6 @@ _return: CTG_API_LEAVE(code); } -int32_t catalogUpdateGrantInfo(SCatalog* pCtg, SGrantHbRsp* pMsg) { - CTG_API_ENTER(); - - if (NULL == pCtg || NULL == pMsg) { - CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); - } - - int32_t code = 0; - CTG_ERR_JRET(ctgUpdateGrantInfoToCache(pCtg, pMsg, true)); - -_return: - - CTG_API_LEAVE(code); -} - - int32_t catalogAsyncUpdateViewMeta(SCatalog* pCtg, SViewMetaRsp* pMsg) { CTG_API_ENTER(); diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index ad48aa3fb1..08a8f348e9 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -32,8 +32,6 @@ SCtgOperation gCtgCacheOperation[CTG_OP_MAX] = {{CTG_OP_UPDATE_VGROUP, "update v {CTG_OP_DROP_TB_INDEX, "drop tbIndex", ctgOpDropTbIndex}, {CTG_OP_UPDATE_VIEW_META, "update viewMeta", ctgOpUpdateViewMeta}, {CTG_OP_DROP_VIEW_META, "drop viewMeta", ctgOpDropViewMeta}, - {CTG_OP_UPDATE_GRANT_INFO, "update grantInfo", ctgOpUpdateGrantInfo}, - {CTG_OP_DROP_GRANT_INFO, "drop grantInfo", ctgOpDropGrantInfo}, {CTG_OP_CLEAR_CACHE, "clear cache", ctgOpClearCache}}; SCtgCacheItemInfo gCtgStatItem[CTG_CI_MAX_VALUE] = { @@ -160,15 +158,6 @@ int32_t ctgGetDBCache(SCatalog *pCtg, const char *dbFName, SCtgDBCache **pCache) CTG_RET(ctgAcquireDBCacheImpl(pCtg, dbFName, pCache, false)); } -int32_t ctgAcquireGrantCache(SCatalog *pCtg, SCtgGrantCache **ppCache) { - CTG_LOCK(CTG_READ, &pCtg->grantCache.lock); - *ppCache = &pCtg->grantCache; - CTG_CACHE_HIT_INC(CTG_CI_GRANT_INFO, 1); - return TSDB_CODE_SUCCESS; -} - -void ctgReleaseGrantCache(SCatalog *pCtg, SCtgGrantCache *pCache) { CTG_UNLOCK(CTG_READ, &pCache->lock); } - void ctgReleaseVgInfoToCache(SCatalog *pCtg, SCtgDBCache *dbCache) { ctgRUnlockVgInfo(dbCache); ctgReleaseDBCache(pCtg, dbCache); @@ -1334,33 +1323,6 @@ _return: CTG_RET(code); } -int32_t ctgUpdateGrantInfoEnqueue(SCatalog *pCtg, SGrantHbRsp *pRsp, bool syncOp) { - int32_t code = 0; - SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); - op->opId = CTG_OP_UPDATE_GRANT_INFO; - op->syncOp = syncOp; - - SCtgUpdateGrantInfoMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateGrantInfoMsg)); - if (NULL == msg) { - ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateGrantInfoMsg)); - taosMemoryFree(op); - CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); - } - - msg->pCtg = pCtg; - msg->pRsp = pRsp; - - op->data = msg; - - CTG_ERR_RET(ctgEnqueue(pCtg, op)); - - return TSDB_CODE_SUCCESS; -_return: - - taosMemoryFree(pRsp); - CTG_RET(code); -} - int32_t ctgDropViewMetaEnqueue(SCatalog *pCtg, const char *dbFName, uint64_t dbId, const char *viewName, uint64_t viewId, bool syncOp) { int32_t code = 0; SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); @@ -1732,21 +1694,6 @@ _return: CTG_RET(code); } -int32_t ctgWriteGrantInfoToCache(SCatalog *pCtg, SGrantHbRsp *pRsp) { - int32_t code = TSDB_CODE_SUCCESS; - SCtgGrantCache *pCache = &pCtg->grantCache; - - CTG_LOCK(CTG_WRITE, &pCache->lock); - pCache->grantInfo = *pRsp; - CTG_UNLOCK(CTG_WRITE, &pCache->lock); - - ctgDebug("grant info updated to cache, flags:%u, version:%d", pRsp->flags, pRsp->version); - -_return: - - CTG_RET(code); -} - int32_t ctgUpdateTbMetaToCache(SCatalog *pCtg, STableMetaOutput *pOut, bool syncReq) { STableMetaOutput *pOutput = NULL; int32_t code = 0; @@ -1768,10 +1715,6 @@ int32_t ctgUpdateViewMetaToCache(SCatalog *pCtg, SViewMetaRsp *pRsp, bool syncRe CTG_RET(ctgUpdateViewMetaEnqueue(pCtg, pRsp, syncReq)); } -int32_t ctgUpdateGrantInfoToCache(SCatalog *pCtg, SGrantHbRsp *pRsp, bool syncReq) { - CTG_RET(ctgUpdateGrantInfoEnqueue(pCtg, pRsp, syncReq)); -} - void ctgClearAllHandles(void) { SCatalog *pCtg = NULL; @@ -2507,43 +2450,6 @@ _return: CTG_RET(code); } -int32_t ctgOpUpdateGrantInfo(SCtgCacheOperation *operation) { - int32_t code = 0; - SCtgUpdateGrantInfoMsg *msg = operation->data; - SCatalog *pCtg = msg->pCtg; - SGrantHbRsp *pRsp = msg->pRsp; - SCtgGrantCache *pGrantCache = NULL; - - taosMemoryFreeClear(msg); - - if (pCtg->stopUpdate) { - goto _return; - } - - code = ctgWriteGrantInfoToCache(pCtg, pRsp); - -_return: - - taosMemoryFree(pRsp); - CTG_RET(code); -} - -int32_t ctgOpDropGrantInfo(SCtgCacheOperation *operation) { - int32_t code = 0; - SCtgDropGrantInfoMsg *msg = operation->data; - SCatalog *pCtg = msg->pCtg; - - if (pCtg->stopUpdate) { - goto _return; - } - -_return: - - taosMemoryFreeClear(msg); - - CTG_RET(code); -} - void ctgClearFreeCache(SCtgCacheOperation *operation) { SCtgClearCacheMsg *msg = operation->data; SCatalog *pCtg = msg->pCtg; diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index a229d2658e..4ac8e0e541 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -2142,10 +2142,6 @@ static int32_t parseDataFromFileImpl(SInsertParseContext* pCxt, SVnodeModifyOpSt int32_t code = 0; int32_t numOfRows = 0; - // if ((code = catalogChkGrant(pCxt->pComCxt->pCatalog, TSDB_GRANT_CSV)) < 0) { - // return code; - // } - // init only for file if (NULL == pStmt->pTableCxtHashObj) { pStmt->pTableCxtHashObj = From 60985381620e4e3ba028850724254a1195855f2a Mon Sep 17 00:00:00 2001 From: kailixu Date: Sun, 4 Feb 2024 18:00:02 +0800 Subject: [PATCH 15/19] feat: support uniq grant --- include/common/tgrant.h | 8 +++--- include/libs/catalog/catalog.h | 7 +----- source/client/src/clientHb.c | 2 ++ source/common/src/tglobal.c | 6 ++--- source/common/src/tmsg.c | 2 +- source/dnode/mnode/impl/inc/mndGrant.h | 5 ++-- source/dnode/mnode/impl/src/mndProfile.c | 1 - source/libs/catalog/inc/catalogInt.h | 5 ++-- source/libs/catalog/src/catalog.c | 3 +++ source/libs/catalog/src/ctgCache.c | 1 + source/libs/catalog/src/ctgRent.c | 1 + source/libs/nodes/src/nodesCodeFuncs.c | 10 +++++--- source/libs/parser/inc/parTranslater.h | 2 -- source/libs/parser/src/parInsertSql.c | 11 +++----- source/libs/parser/src/parUtil.c | 2 -- source/libs/qworker/src/qwMsg.c | 32 ++++++++++++------------ 16 files changed, 48 insertions(+), 50 deletions(-) diff --git a/include/common/tgrant.h b/include/common/tgrant.h index 66408e9fa4..03c68ef59b 100644 --- a/include/common/tgrant.h +++ b/include/common/tgrant.h @@ -30,10 +30,10 @@ extern "C" { #define GRANT_HEART_BEAT_MIN 2 #define GRANT_ACTIVE_CODE "activeCode" -#define GRANT_ALL_FLAG (0x01) -#define GRANT_AUDIT_FLAG (0x02) -#define GRANT_CSV_FLAG (0x04) -#define GRANT_VIEW_FLAG (0x08) +#define GRANT_FLAG_ALL (0x01) +#define GRANT_FLAG_AUDIT (0x02) +#define GRANT_FLAG_CSV (0x04) +#define GRANT_FLAG_VIEW (0x08) typedef enum { TSDB_GRANT_ALL, diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 96a05ccbf9..1957bcbb24 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -25,7 +25,6 @@ extern "C" { #include "taosdef.h" #include "tarray.h" #include "tcommon.h" -#include "tgrant.h" #include "thash.h" #include "tmsg.h" #include "tname.h" @@ -146,10 +145,6 @@ typedef struct SSTableVersion { int32_t smaVer; } SSTableVersion; -typedef struct SGrantVersion { - int32_t version; -} SGrantVersion; - typedef struct SDbCacheInfo { char dbFName[TSDB_DB_FNAME_LEN]; int64_t dbId; @@ -363,7 +358,7 @@ int32_t catalogGetUdfInfo(SCatalog* pCtg, SRequestConnInfo* pConn, const char* f int32_t catalogChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, SUserAuthInfo *pAuth, SUserAuthRes* pRes); -int32_t catalogChkAuthFromCache(SCatalog* pCtg, SUserAuthInfo* pAuth, SUserAuthRes* pRes, bool* exists); +int32_t catalogChkAuthFromCache(SCatalog* pCtg, SUserAuthInfo *pAuth, SUserAuthRes* pRes, bool* exists); int32_t catalogUpdateUserAuthInfo(SCatalog* pCtg, SGetUserAuthRsp* pAuth); diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 53a64cec86..a146712cab 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -327,6 +327,7 @@ static int32_t hbProcessViewInfoRsp(void *value, int32_t valueLen, struct SCatal return TSDB_CODE_SUCCESS; } + static void hbProcessQueryRspKvs(int32_t kvNum, SArray* pKvs, struct SCatalog *pCatalog, SAppHbMgr *pAppHbMgr) { for (int32_t i = 0; i < kvNum; ++i) { SKv *kv = taosArrayGet(pKvs, i); @@ -865,6 +866,7 @@ int32_t hbGetExpiredViewInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, S return TSDB_CODE_SUCCESS; } + int32_t hbGetAppInfo(int64_t clusterId, SClientHbReq *req) { SAppHbReq *pApp = taosHashGet(clientHbMgr.appSummary, &clusterId, sizeof(clusterId)); if (NULL != pApp) { diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index d46b91276b..444a4c0ccc 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -1804,11 +1804,11 @@ void taosSetAllDebugFlag(int32_t flag) { int8_t taosGranted(int8_t type) { switch (type) { case TSDB_GRANT_ALL: - return atomic_load_8(&tsGrant) & GRANT_ALL_FLAG; + return atomic_load_8(&tsGrant) & GRANT_FLAG_ALL; case TSDB_GRANT_AUDIT: - return atomic_load_8(&tsGrant) & GRANT_AUDIT_FLAG; + return atomic_load_8(&tsGrant) & GRANT_FLAG_AUDIT; case TSDB_GRANT_VIEW: - return atomic_load_8(&tsGrant) & GRANT_VIEW_FLAG; + return atomic_load_8(&tsGrant) & GRANT_FLAG_VIEW; default: ASSERTS(0, "undefined grant type:%" PRIi8, type); break; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 785cb1878e..af1c1c4923 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -9333,4 +9333,4 @@ void tFreeSViewHbRsp(SViewHbRsp *pRsp) { } taosArrayDestroy(pRsp->pViewRsp); -} \ No newline at end of file +} diff --git a/source/dnode/mnode/impl/inc/mndGrant.h b/source/dnode/mnode/impl/inc/mndGrant.h index bda5cebf01..82b3260860 100644 --- a/source/dnode/mnode/impl/inc/mndGrant.h +++ b/source/dnode/mnode/impl/inc/mndGrant.h @@ -41,8 +41,9 @@ char **mergeActive); #endif - int32_t mndProcessConfigGrantReq(SMnode * pMnode, SRpcMsg * pReq, SMCfgClusterReq * pCfg); - int32_t mndProcessUpdGrantLog(SMnode * pMnode, SRpcMsg * pReq, SArray * pMachines, SGrantState * pState); + int32_t mndProcessConfigGrantReq(SMnode * pMnode, SRpcMsg * pReq, SMCfgClusterReq * pCfg); + int32_t mndProcessUpdGrantLog(SMnode * pMnode, SRpcMsg * pReq, SArray * pMachines, SGrantState * pState); + int32_t mndGrantGetLastState(SMnode * pMnode, SGrantState * pState); SGrantLogObj *mndAcquireGrant(SMnode * pMnode, void **ppIter); void mndReleaseGrant(SMnode * pMnode, SGrantLogObj * pGrant, void *pIter); diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index f42cdfa4f3..e0559b4c48 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -18,7 +18,6 @@ #include "audit.h" #include "mndDb.h" #include "mndDnode.h" -#include "mndGrant.h" #include "mndMnode.h" #include "mndPrivilege.h" #include "mndQnode.h" diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index a35eec0ec5..7474b3bf91 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -548,6 +548,7 @@ typedef struct SCtgDropViewMetaMsg { uint64_t viewId; } SCtgDropViewMetaMsg; + typedef struct SCtgCacheOperation { int32_t opId; void* data; @@ -943,8 +944,8 @@ void ctgRemoveStbRent(SCatalog *pCtg, SCtgDBCache *dbCache); void ctgRemoveViewRent(SCatalog *pCtg, SCtgDBCache *dbCache); int32_t ctgUpdateRentStbVersion(SCatalog *pCtg, char *dbFName, char *tbName, uint64_t dbId, uint64_t suid, SCtgTbCache *pCache); -int32_t ctgUpdateRentViewVersion(SCatalog *pCtg, char *dbFName, char *viewName, uint64_t dbId, uint64_t viewId, - SCtgViewCache *pCache); +int32_t ctgUpdateRentViewVersion(SCatalog* pCtg, char* dbFName, char* viewName, uint64_t dbId, uint64_t viewId, + SCtgViewCache* pCache); int32_t ctgUpdateTbMetaToCache(SCatalog* pCtg, STableMetaOutput* pOut, bool syncReq); int32_t ctgUpdateViewMetaToCache(SCatalog *pCtg, SViewMetaRsp *pRsp, bool syncReq); int32_t ctgStartUpdateThread(); diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 5a53f167c4..c2d88e5ce3 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -1513,6 +1513,7 @@ int32_t catalogGetExpiredViews(SCatalog* pCtg, SViewVersion** views, uint32_t* n CTG_API_LEAVE(ctgMetaRentGet(&pCtg->viewRent, (void**)views, num, sizeof(SViewVersion))); } + int32_t catalogGetExpiredDBs(SCatalog* pCtg, SDbCacheInfo** dbs, uint32_t* num) { CTG_API_ENTER(); @@ -1675,6 +1676,7 @@ _return: CTG_API_LEAVE(code); } + int32_t catalogGetServerVersion(SCatalog* pCtg, SRequestConnInfo* pConn, char** pVersion) { CTG_API_ENTER(); @@ -1745,6 +1747,7 @@ _return: CTG_API_LEAVE(code); } + int32_t catalogAsyncUpdateViewMeta(SCatalog* pCtg, SViewMetaRsp* pMsg) { CTG_API_ENTER(); diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 08a8f348e9..1b693b4e07 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -2450,6 +2450,7 @@ _return: CTG_RET(code); } + void ctgClearFreeCache(SCtgCacheOperation *operation) { SCtgClearCacheMsg *msg = operation->data; SCatalog *pCtg = msg->pCtg; diff --git a/source/libs/catalog/src/ctgRent.c b/source/libs/catalog/src/ctgRent.c index 0ac133b8e7..f5016e9fef 100755 --- a/source/libs/catalog/src/ctgRent.c +++ b/source/libs/catalog/src/ctgRent.c @@ -301,3 +301,4 @@ int32_t ctgUpdateRentViewVersion(SCatalog *pCtg, char *dbFName, char *viewName, return TSDB_CODE_SUCCESS; } + diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index b05c9b019e..7533d00704 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -3294,6 +3294,8 @@ static const char* jkSubplanTagIndexCond = "TagIndexCond"; static const char* jkSubplanShowRewrite = "ShowRewrite"; static const char* jkSubplanRowsThreshold = "RowThreshold"; static const char* jkSubplanDynamicRowsThreshold = "DyRowThreshold"; +static const char* jkSubplanIsView = "IsView"; +static const char* jkSubplanIsAudit = "IsAudit"; static int32_t subplanToJson(const void* pObj, SJson* pJson) { const SSubplan* pNode = (const SSubplan*)pObj; @@ -3333,10 +3335,10 @@ static int32_t subplanToJson(const void* pObj, SJson* pJson) { code = tjsonAddBoolToObject(pJson, jkSubplanShowRewrite, pNode->showRewrite); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddBoolToObject(pJson, jkSubplanShowRewrite, pNode->isView); + code = tjsonAddBoolToObject(pJson, jkSubplanIsView, pNode->isView); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddBoolToObject(pJson, jkSubplanShowRewrite, pNode->isAudit); + code = tjsonAddBoolToObject(pJson, jkSubplanIsAudit, pNode->isAudit); } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkSubplanRowsThreshold, pNode->rowsThreshold); @@ -3386,10 +3388,10 @@ static int32_t jsonToSubplan(const SJson* pJson, void* pObj) { code = tjsonGetBoolValue(pJson, jkSubplanShowRewrite, &pNode->showRewrite); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetBoolValue(pJson, jkSubplanShowRewrite, &pNode->isView); + code = tjsonGetBoolValue(pJson, jkSubplanIsView, &pNode->isView); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetBoolValue(pJson, jkSubplanShowRewrite, &pNode->isAudit); + code = tjsonGetBoolValue(pJson, jkSubplanIsAudit, &pNode->isAudit); } if (TSDB_CODE_SUCCESS == code) { code = tjsonGetIntValue(pJson, jkSubplanRowsThreshold, &pNode->rowsThreshold); diff --git a/source/libs/parser/inc/parTranslater.h b/source/libs/parser/inc/parTranslater.h index 00f37cb002..55ea71a368 100644 --- a/source/libs/parser/inc/parTranslater.h +++ b/source/libs/parser/inc/parTranslater.h @@ -43,8 +43,6 @@ typedef struct STranslateContext { bool createStream; bool stableQuery; bool showRewrite; - // bool isView; - // bool isAudit; SNode* pPrevRoot; SNode* pPostRoot; } STranslateContext; diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 4ac8e0e541..7d10d1f2df 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -2139,16 +2139,13 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt } static int32_t parseDataFromFileImpl(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SRowsDataContext rowsDataCxt) { - int32_t code = 0; - int32_t numOfRows = 0; - // init only for file if (NULL == pStmt->pTableCxtHashObj) { pStmt->pTableCxtHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); } - - code = parseCsvFile(pCxt, pStmt, rowsDataCxt, &numOfRows); + int32_t numOfRows = 0; + int32_t code = parseCsvFile(pCxt, pStmt, rowsDataCxt, &numOfRows); if (TSDB_CODE_SUCCESS == code) { pStmt->totalRowsNum += numOfRows; pStmt->totalTbNum += 1; @@ -2759,11 +2756,11 @@ static int32_t parseInsertSqlFromStart(SInsertParseContext* pCxt, SVnodeModifyOp } static int32_t parseInsertSqlFromCsv(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; SRowsDataContext rowsDataCxt; if (!pStmt->stbSyntax) { - STableDataCxt* pTableCxt = NULL; + STableDataCxt* pTableCxt = NULL; code = getTableDataCxt(pCxt, pStmt, &pTableCxt); rowsDataCxt.pTableDataCxt = pTableCxt; } else { diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 8aa37f5784..dfe33ce55e 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -192,8 +192,6 @@ static char* getSyntaxErrFormat(int32_t errCode) { return "Out of memory"; case TSDB_CODE_PAR_ORDERBY_AMBIGUOUS: return "ORDER BY \"%s\" is ambiguous"; - case TSDB_CODE_GRANT_EXPIRED: - return "License expired"; default: return "Unknown error"; } diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c index c9d3cb51e3..66ec460861 100644 --- a/source/libs/qworker/src/qwMsg.c +++ b/source/libs/qworker/src/qwMsg.c @@ -361,22 +361,22 @@ int32_t qWorkerPreprocessQueryMsg(void *qWorkerMgmt, SRpcMsg *pMsg, bool chkGran } if (chkGrant) { - if ((!TEST_SHOW_REWRITE_MASK(msg.msgMask)) && !taosGranted(TSDB_GRANT_ALL)) { - QW_ELOG("query failed cause of grant expired, msgMask:%d", msg.msgMask); - tFreeSSubQueryMsg(&msg); - QW_ERR_RET(TSDB_CODE_GRANT_EXPIRED); - } - - if ((TEST_VIEW_MASK(msg.msgMask)) && !taosGranted(TSDB_GRANT_VIEW)) { - QW_ELOG("query failed cause of view grant expired, msgMask:%d", msg.msgMask); - tFreeSSubQueryMsg(&msg); - QW_ERR_RET(TSDB_CODE_GRANT_EXPIRED); - } - - if ((TEST_AUDIT_MASK(msg.msgMask)) && !taosGranted(TSDB_GRANT_AUDIT)) { - QW_ELOG("query failed cause of audit grant expired, msgMask:%d", msg.msgMask); - tFreeSSubQueryMsg(&msg); - QW_ERR_RET(TSDB_CODE_GRANT_EXPIRED); + if ((!TEST_SHOW_REWRITE_MASK(msg.msgMask))) { + if (!taosGranted(TSDB_GRANT_ALL)) { + QW_ELOG("query failed cause of grant expired, msgMask:%d", msg.msgMask); + tFreeSSubQueryMsg(&msg); + QW_ERR_RET(TSDB_CODE_GRANT_EXPIRED); + } + if ((TEST_VIEW_MASK(msg.msgMask)) && !taosGranted(TSDB_GRANT_VIEW)) { + QW_ELOG("query failed cause of view grant expired, msgMask:%d", msg.msgMask); + tFreeSSubQueryMsg(&msg); + QW_ERR_RET(TSDB_CODE_GRANT_EXPIRED); + } + if ((TEST_AUDIT_MASK(msg.msgMask)) && !taosGranted(TSDB_GRANT_AUDIT)) { + QW_ELOG("query failed cause of audit grant expired, msgMask:%d", msg.msgMask); + tFreeSSubQueryMsg(&msg); + QW_ERR_RET(TSDB_CODE_GRANT_EXPIRED); + } } } From 021247c45bd7b88395184ab9b98b55ea4c9d6282 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 5 Feb 2024 06:51:44 +0800 Subject: [PATCH 16/19] feat: support uniq grant --- include/common/tgrant.h | 3 +-- include/libs/qcom/query.h | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/include/common/tgrant.h b/include/common/tgrant.h index 03c68ef59b..2fa6dde8f6 100644 --- a/include/common/tgrant.h +++ b/include/common/tgrant.h @@ -32,8 +32,7 @@ extern "C" { #define GRANT_ACTIVE_CODE "activeCode" #define GRANT_FLAG_ALL (0x01) #define GRANT_FLAG_AUDIT (0x02) -#define GRANT_FLAG_CSV (0x04) -#define GRANT_FLAG_VIEW (0x08) +#define GRANT_FLAG_VIEW (0x04) typedef enum { TSDB_GRANT_ALL, diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 4c33c36d1b..e80268bb5f 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -66,8 +66,8 @@ typedef enum { #define QUERY_RSP_POLICY_QUICK 1 #define QUERY_MSG_MASK_SHOW_REWRITE() (1 << 0) -#define QUERY_MSG_MASK_VIEW() (1 << 1) -#define QUERY_MSG_MASK_AUDIT() (1 << 2) +#define QUERY_MSG_MASK_AUDIT() (1 << 1) +#define QUERY_MSG_MASK_VIEW() (1 << 2) #define TEST_SHOW_REWRITE_MASK(m) (((m)&QUERY_MSG_MASK_SHOW_REWRITE()) != 0) #define TEST_VIEW_MASK(m) (((m)&QUERY_MSG_MASK_VIEW()) != 0) #define TEST_AUDIT_MASK(m) (((m)&QUERY_MSG_MASK_AUDIT()) != 0) From 205525dd652dbf89de91f47d63e780e75f0ee2b6 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 5 Feb 2024 07:03:51 +0800 Subject: [PATCH 17/19] feat: support uniq grant --- include/libs/qcom/query.h | 7 +++---- source/libs/catalog/inc/catalogInt.h | 4 ++-- source/libs/catalog/src/ctgRent.c | 2 -- source/libs/scheduler/src/schRemote.c | 1 - 4 files changed, 5 insertions(+), 9 deletions(-) diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index e80268bb5f..8d6cd6a3c0 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -68,10 +68,9 @@ typedef enum { #define QUERY_MSG_MASK_SHOW_REWRITE() (1 << 0) #define QUERY_MSG_MASK_AUDIT() (1 << 1) #define QUERY_MSG_MASK_VIEW() (1 << 2) -#define TEST_SHOW_REWRITE_MASK(m) (((m)&QUERY_MSG_MASK_SHOW_REWRITE()) != 0) -#define TEST_VIEW_MASK(m) (((m)&QUERY_MSG_MASK_VIEW()) != 0) -#define TEST_AUDIT_MASK(m) (((m)&QUERY_MSG_MASK_AUDIT()) != 0) - +#define TEST_SHOW_REWRITE_MASK(m) (((m) & QUERY_MSG_MASK_SHOW_REWRITE()) != 0) +#define TEST_AUDIT_MASK(m) (((m) & QUERY_MSG_MASK_AUDIT()) != 0) +#define TEST_VIEW_MASK(m) (((m) & QUERY_MSG_MASK_VIEW()) != 0) typedef struct STableComInfo { uint8_t numOfTags; // the number of tags in schema diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 7474b3bf91..f9f4ee7dfc 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -944,8 +944,8 @@ void ctgRemoveStbRent(SCatalog *pCtg, SCtgDBCache *dbCache); void ctgRemoveViewRent(SCatalog *pCtg, SCtgDBCache *dbCache); int32_t ctgUpdateRentStbVersion(SCatalog *pCtg, char *dbFName, char *tbName, uint64_t dbId, uint64_t suid, SCtgTbCache *pCache); -int32_t ctgUpdateRentViewVersion(SCatalog* pCtg, char* dbFName, char* viewName, uint64_t dbId, uint64_t viewId, - SCtgViewCache* pCache); +int32_t ctgUpdateRentViewVersion(SCatalog *pCtg, char *dbFName, char *viewName, uint64_t dbId, uint64_t viewId, + SCtgViewCache *pCache); int32_t ctgUpdateTbMetaToCache(SCatalog* pCtg, STableMetaOutput* pOut, bool syncReq); int32_t ctgUpdateViewMetaToCache(SCatalog *pCtg, SViewMetaRsp *pRsp, bool syncReq); int32_t ctgStartUpdateThread(); diff --git a/source/libs/catalog/src/ctgRent.c b/source/libs/catalog/src/ctgRent.c index f5016e9fef..a19eb19c02 100755 --- a/source/libs/catalog/src/ctgRent.c +++ b/source/libs/catalog/src/ctgRent.c @@ -300,5 +300,3 @@ int32_t ctgUpdateRentViewVersion(SCatalog *pCtg, char *dbFName, char *viewName, return TSDB_CODE_SUCCESS; } - - diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 1a9c37ffe9..1c0b31109e 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -1111,7 +1111,6 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, qMsg.msgMask = (pTask->plan->showRewrite) ? QUERY_MSG_MASK_SHOW_REWRITE() : 0; qMsg.msgMask |= (pTask->plan->isView) ? QUERY_MSG_MASK_VIEW() : 0; qMsg.msgMask |= (pTask->plan->isAudit) ? QUERY_MSG_MASK_AUDIT() : 0; - qMsg.taskType = TASK_TYPE_TEMP; qMsg.explain = SCH_IS_EXPLAIN_JOB(pJob); qMsg.needFetch = SCH_TASK_NEED_FETCH(pTask); From 7e5dc8a02425a7826ea7195a77c0a812ce509e30 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 5 Feb 2024 09:21:11 +0800 Subject: [PATCH 18/19] feat: support uniq grant --- source/dnode/mnode/impl/src/mndCluster.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index 36b079d347..f2b279276e 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -409,7 +409,7 @@ int32_t mndProcessConfigClusterReq(SRpcMsg *pReq) { } { // audit - auditRecord(pReq, pMnode->clusterId, "alterCluster", "", "", cfgReq.sql, MIN(cfgReq.sqlLen, GRANT_ACTIVE_HEAD_LEN << 1)); + auditRecord(pReq, pMnode->clusterId, "alterCluster", "", "", cfgReq.sql, TMIN(cfgReq.sqlLen, GRANT_ACTIVE_HEAD_LEN << 1)); } _exit: tFreeSMCfgClusterReq(&cfgReq); From 196256d9af4b2de3effba0c1bafd7bf6b025b5d0 Mon Sep 17 00:00:00 2001 From: kailixu Date: Mon, 5 Feb 2024 09:23:19 +0800 Subject: [PATCH 19/19] feat: support uniq grant --- source/dnode/mnode/impl/src/mndGrant.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndGrant.c b/source/dnode/mnode/impl/src/mndGrant.c index 0b85f8fd5a..2ec6e09d12 100644 --- a/source/dnode/mnode/impl/src/mndGrant.c +++ b/source/dnode/mnode/impl/src/mndGrant.c @@ -79,11 +79,6 @@ char *tGetMachineId() { return NULL; }; int32_t dmProcessGrantReq(void *pInfo, SRpcMsg *pMsg) { return TSDB_CODE_SUCCESS; } int32_t dmProcessGrantNotify(void *pInfo, SRpcMsg *pMsg) { return TSDB_CODE_SUCCESS; } int32_t mndProcessConfigGrantReq(SMnode *pMnode, SRpcMsg *pReq, SMCfgClusterReq *pCfg) { return 0; } -#else -#ifndef TD_UNIQ_GRANT -char *tGetMachineId() { return NULL; }; -int32_t mndProcessConfigGrantReq(SMnode *pMnode, SRpcMsg *pReq, SMCfgClusterReq *pCfg) { return 0; } -#endif #endif void mndGenerateMachineCode() { grantParseParameter(); } \ No newline at end of file