From 140a7458c3b8068ea68eb0407d66eaf036ac31be Mon Sep 17 00:00:00 2001 From: cpwu Date: Thu, 5 May 2022 20:08:23 +0800 Subject: [PATCH 01/26] fix join case --- tests/system-test/2-query/join.py | 144 +++++++++++++++++++++--------- 1 file changed, 103 insertions(+), 41 deletions(-) diff --git a/tests/system-test/2-query/join.py b/tests/system-test/2-query/join.py index b4878e42c9..c6431cdc8d 100644 --- a/tests/system-test/2-query/join.py +++ b/tests/system-test/2-query/join.py @@ -5,6 +5,7 @@ from util.sql import * from util.cases import * from util.dnodes import * +PRIMARY_COL = "ts" INT_COL = "c1" BINT_COL = "c2" @@ -18,9 +19,10 @@ BINARY_COL = "c8" NCHAR_COL = "c9" TS_COL = "c10" -UN_CHAR_COL = [INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, BOOL_COL, ] +NUM_COL = [ INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, ] CHAR_COL = [ BINARY_COL, NCHAR_COL, ] -TS_TYPE_COL = [TS_COL] +BOOLEAN_COL = [ BOOL_COL, ] +TS_TYPE_COL = [ TS_COL, ] class TDTestCase: @@ -28,50 +30,78 @@ class TDTestCase: tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) - def __length_condition(self): - length_condition = [] + def __query_condition(self,tbname): + query_condition = [] for char_col in CHAR_COL: - length_condition.extend( + query_condition.extend( ( - char_col, - f"upper( {char_col} )", + f"{tbname}.{char_col}", + f"upper( {tbname}.{char_col} )", ) ) - length_condition.extend( f"cast( {un_char_col} as binary(16) ) " for un_char_col in UN_CHAR_COL) - length_condition.extend( f"cast( {char_col} + {char_col_2} as binary(32) ) " for char_col_2 in CHAR_COL ) - length_condition.extend( f"cast( {char_col} + {un_char_col} as binary(32) ) " for un_char_col in UN_CHAR_COL ) + query_condition.extend( f"cast( {tbname}.{un_char_col} as binary(16) ) " for un_char_col in NUM_COL) + query_condition.extend( f"cast( {tbname}.{char_col} + {tbname}.{char_col_2} as binary(32) ) " for char_col_2 in CHAR_COL ) + query_condition.extend( f"cast( {tbname}.{char_col} + {tbname}.{un_char_col} as binary(32) ) " for un_char_col in NUM_COL ) + for num_col in NUM_COL: + query_condition.extend( + ( + f"{tbname}.{num_col}", + f"sin( {tbname}.{num_col} )" + ) + ) + query_condition.extend( f"{tbname}.{num_col} + {tbname}.{num_col_1} " for num_col_1 in NUM_COL ) - length_condition.append('''"test1234!@#$%^&*():'> 0 " - return "" + def __join_condition(self, tb_list, filter=PRIMARY_COL): + # sourcery skip: flip-comparison + if 1 == len(tb_list): + join_filter = f"{tb_list[0]}.{filter} = {tb_list[0]}.{filter} " + elif 2 == len(tb_list): + join_filter = f"{tb_list[0]}.{filter} = {tb_list[1]}.{filter} " + else: + join_filter = f"{tb_list[0]}.{filter} = {tb_list[1]}.{filter} " + for i in range(1, len(tb_list)-1 ): + join_filter += f"and {tb_list[i]}.{filter} = {tb_list[i+1]}.{filter}" - def __group_condition(self, col, having = ""): - return f" group by {col} having {having}" if having else f" group by {col} " + return join_filter - def __length_current_check(self, tbname): - length_condition = self.__length_condition() - for condition in length_condition: - where_condition = self.__where_condition(condition) - group_having = self.__group_condition(condition, having=f"{condition} is not null " ) - group_no_having= self.__group_condition(condition ) + def __where_condition(self, col, tbname): + if col in NUM_COL: + return f" abs( {tbname}.{col} ) >= 0" + elif col in CHAR_COL: + return f" lower( {tbname}.{col} ) is not null" + elif col in BOOLEAN_COL: + return f" {tbname}.{col} in (false, true) " + elif col in TS_TYPE_COL or col in PRIMARY_COL: + return f" abs( cast( {tbname}.{col} as bigint ) ) >= 0 " + else: + return "" + + def __group_condition(self, tbname, col, having = ""): + return f" group by {tbname}.{col} having {having}" if having else f" group by {tbname}.{col} " + + def __join_check(self, tblist, checkrows, join_flag=True): + query_conditions = self.__query_condition(tblist[0]) + join_condition = self.__join_condition(tb_list=tblist) if join_flag else " " + for condition in query_conditions: + where_condition = self.__where_condition(col=condition, tbname=tblist[0]) + group_having = self.__group_condition(tbname=tblist[0], col=condition, having=f"{condition} is not null " ) + group_no_having= self.__group_condition(tbname=tblist[0], col=condition ) groups = ["", group_having, group_no_having] - for group_condition in groups: - tdSql.query(f"select {condition} from {tbname} {where_condition} {group_condition} ") - datas = [tdSql.getData(i,0) for i in range(tdSql.queryRows)] - length_data = [ len(str(data)) if data else None for data in datas ] - tdSql.query(f"select length( {condition} ) from {tbname} {where_condition} {group_condition}") - for i in range(len(length_data)): - tdSql.checkData(i, 0, length_data[i] ) if length_data[i] else tdSql.checkData(i, 0, None) + sql = f"select {condition} from {tblist[0]},{tblist[1]} where {join_condition} and {where_condition} {group_condition}" + if len(tblist) == 2: + self.__join_current(sql, checkrows) + elif len(tblist) > 2 or len(tblist) < 1: + tdSql.error(sql=sql) - def __length_err_check(self,tbname): + def __join_err_check(self,tbname): sqls = [] - for un_char_col in UN_CHAR_COL: + for un_char_col in NUM_COL: sqls.extend( ( f"select length( {un_char_col} ) from {tbname} ", @@ -80,12 +110,12 @@ class TDTestCase: ) ) - sqls.extend( f"select length( {un_char_col} + {un_char_col_2} ) from {tbname} " for un_char_col_2 in UN_CHAR_COL ) + sqls.extend( f"select length( {un_char_col} + {un_char_col_2} ) from {tbname} " for un_char_col_2 in NUM_COL ) sqls.extend( f"select length( {un_char_col} + {ts_col} ) from {tbname} " for ts_col in TS_TYPE_COL ) sqls.extend( f"select {char_col} from {tbname} group by length( {char_col} ) " for char_col in CHAR_COL) sqls.extend( f"select length( {ts_col} ) from {tbname} " for ts_col in TS_TYPE_COL ) - sqls.extend( f"select length( {char_col} + {ts_col} ) from {tbname} " for char_col in UN_CHAR_COL for ts_col in TS_TYPE_COL) + sqls.extend( f"select length( {char_col} + {ts_col} ) from {tbname} " for char_col in NUM_COL for ts_col in TS_TYPE_COL) sqls.extend( f"select length( {char_col} + {char_col_2} ) from {tbname} " for char_col in CHAR_COL for char_col_2 in CHAR_COL ) sqls.extend( f"select upper({char_col}, 11) from {tbname} " for char_col in CHAR_COL ) sqls.extend( f"select upper({char_col}) from {tbname} interval(2d) sliding(1d)" for char_col in CHAR_COL ) @@ -101,15 +131,46 @@ class TDTestCase: return sqls + def __join_current(self, sql, checkrows): + tdSql.query(sql=sql) + tdSql.checkRows(checkrows) + + def __test_current(self): + # sourcery skip: extract-duplicate-method, inline-immediately-returned-variable tdLog.printNoPrefix("==========current sql condition check , must return query ok==========") - tbname = ["ct1", "ct2", "ct4", "t1"] - for tb in tbname: - self.__length_current_check(tb) - tdLog.printNoPrefix(f"==========current sql condition check in {tb} over==========") + tblist_1 = ["ct1", "ct2"] + self.__join_check(tblist_1, 1) + tdLog.printNoPrefix(f"==========current sql condition check in {tblist_1} over==========") + tblist_2 = ["ct2", "ct4"] + self.__join_check(tblist_2, self.rows - 3) + tdLog.printNoPrefix(f"==========current sql condition check in {tblist_2} over==========") + tblist_3 = ["t1", "ct4"] + self.__join_check(tblist_3, 1) + tdLog.printNoPrefix(f"==========current sql condition check in {tblist_3} over==========") + tblist_4 = ["t1", "ct1"] + self.__join_check(tblist_4, 1) + tdLog.printNoPrefix(f"==========current sql condition check in {tblist_4} over==========") def __test_error(self): + # sourcery skip: extract-duplicate-method, move-assign-in-block tdLog.printNoPrefix("==========err sql condition check , must return error==========") + err_list_1 = ["ct1","ct2", "ct4"] + err_list_2 = ["ct1","ct2", "t1"] + err_list_3 = ["ct1","ct4", "t1"] + err_list_4 = ["ct2","ct4", "t1"] + err_list_5 = ["ct1", "ct2","ct4", "t1"] + self.__join_check(err_list_1, -1) + tdLog.printNoPrefix(f"==========err sql condition check in {err_list_1} over==========") + self.__join_check(err_list_2, -1) + tdLog.printNoPrefix(f"==========err sql condition check in {err_list_2} over==========") + self.__join_check(err_list_3, -1) + tdLog.printNoPrefix(f"==========err sql condition check in {err_list_3} over==========") + self.__join_check(err_list_4, -1) + tdLog.printNoPrefix(f"==========err sql condition check in {err_list_4} over==========") + self.__join_check(err_list_5, -1) + tdLog.printNoPrefix(f"==========err sql condition check in {err_list_5} over==========") + tbname = ["ct1", "ct2", "ct4", "t1"] for tb in tbname: @@ -168,7 +229,7 @@ class TDTestCase: tdSql.execute( f'''insert into ct4 values ( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ( { now_time - rows * 3888000000+ 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time - rows * 3888000000 + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( { now_time + 5184000000}, {pow(2,31)-pow(2,15)}, {pow(2,63)-pow(2,30)}, 32767, 127, @@ -184,7 +245,7 @@ class TDTestCase: tdSql.execute( f'''insert into ct2 values ( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ( { now_time - rows * 3888000000+ 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( { now_time - rows * 3888000000 + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( { now_time + 5184000000 }, { -1 * pow(2,31) + pow(2,15) }, { -1 * pow(2,63) + pow(2,30) }, -32766, -126, @@ -228,7 +289,8 @@ class TDTestCase: self.__create_tb() tdLog.printNoPrefix("==========step2:insert data") - self.__insert_data(10) + self.rows = 10 + self.__insert_data(self.rows) tdLog.printNoPrefix("==========step3:all check") self.all_test() From 0bf8a8223dadaaa189c3c5141f12951e992a611d Mon Sep 17 00:00:00 2001 From: dapan Date: Thu, 5 May 2022 21:36:33 +0800 Subject: [PATCH 02/26] user auth --- include/common/tmsg.h | 2 + include/libs/catalog/catalog.h | 6 + source/libs/catalog/inc/catalogInt.h | 20 +++ source/libs/catalog/src/catalog.c | 229 +++++++++++++++++++++++++++ source/libs/qcom/src/querymsg.c | 36 +++++ 5 files changed, 293 insertions(+) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index a8053d8854..8741af54ec 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -424,7 +424,9 @@ int32_t tDeserializeSGetUserAuthReq(void* buf, int32_t bufLen, SGetUserAuthReq* typedef struct { char user[TSDB_USER_LEN]; + int32_t version; int8_t superAuth; + SHashObj* createdDbs; SHashObj* readDbs; SHashObj* writeDbs; } SGetUserAuthRsp; diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 30d1bd0a51..5fc9e08de4 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -40,6 +40,9 @@ enum { CTG_DBG_STB_RENT_NUM, }; +#define USER_AUTH_READ 1 +#define USER_AUTH_WRITE 2 +#define USER_AUTH_ALL 4 typedef struct SCatalogReq { SArray *pTableName; // element is SNAME @@ -57,6 +60,7 @@ typedef struct SMetaData { typedef struct SCatalogCfg { uint32_t maxTblCacheNum; uint32_t maxDBCacheNum; + uint32_t maxUserCacheNum; uint32_t dbRentSec; uint32_t stbRentSec; } SCatalogCfg; @@ -225,6 +229,8 @@ int32_t catalogGetIndexInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, int32_t catalogGetUdfInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* funcName, SFuncInfo** pInfo); +int32_t catalogGetUserDbAuth(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* user, const char* dbFName, int32_t* auth); + /** * Destroy catalog and relase all resources diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 918892b786..d6aa03aecb 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -54,6 +54,7 @@ enum { CTG_ACT_REMOVE_DB, CTG_ACT_REMOVE_STB, CTG_ACT_REMOVE_TBL, + CTG_ACT_UPDATE_USER, CTG_ACT_MAX }; @@ -95,8 +96,18 @@ typedef struct SCtgRentMgmt { SCtgRentSlot *slots; } SCtgRentMgmt; +typedef struct SCtgUserAuth { + int32_t version; + SRWLatch lock; + bool superUser; + SHashObj *createdDbs; + SHashObj *readDbs; + SHashObj *writeDbs; +} SCtgUserAuth; + typedef struct SCatalog { uint64_t clusterId; + SHashObj *userCache; //key:user, value:SCtgUserAuth SHashObj *dbCache; //key:dbname, value:SCtgDBCache SCtgRentMgmt dbRent; SCtgRentMgmt stbRent; @@ -124,6 +135,8 @@ typedef struct SCtgCacheStat { uint64_t vgMissNum; uint64_t tblHitNum; uint64_t tblMissNum; + uint64_t userHitNum; + uint64_t userMissNum; } SCtgCacheStat; typedef struct SCatalogStat { @@ -169,6 +182,11 @@ typedef struct SCtgRemoveTblMsg { uint64_t dbId; } SCtgRemoveTblMsg; +typedef struct SCtgUpdateUserMsg { + SCatalog* pCtg; + SGetUserAuthRsp userAuth; +} SCtgUpdateTblMsg; + typedef struct SCtgMetaAction { int32_t act; @@ -234,6 +252,8 @@ typedef struct SCtgAction { #define CTG_FLAG_SYS_DB 0x8 #define CTG_FLAG_FORCE_UPDATE 0x10 +#define CTG_FLAG_SET(_flag, _v) ((_flag) |= (_v)) + #define CTG_FLAG_IS_STB(_flag) ((_flag) & CTG_FLAG_STB) #define CTG_FLAG_IS_NOT_STB(_flag) ((_flag) & CTG_FLAG_NOT_STB) #define CTG_FLAG_IS_UNKNOWN_STB(_flag) ((_flag) & CTG_FLAG_UNKNOWN_STB) diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 6f1f34a57b..39a784d00c 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -51,6 +51,11 @@ SCtgAction gCtgAction[CTG_ACT_MAX] = {{ CTG_ACT_REMOVE_TBL, "remove tbMeta", ctgActRemoveTbl + }, + { + CTG_ACT_UPDATE_USER, + "update user", + ctgActUpdateUser } }; @@ -357,6 +362,30 @@ _return: CTG_RET(code); } +int32_t ctgPushUpdateUserMsgInQueue(SCatalog* pCtg, SGetUserAuthRsp *pAuth, bool syncReq) { + int32_t code = 0; + SCtgMetaAction action= {.act = CTG_ACT_UPDATE_USER, .syncReq = syncReq}; + SCtgUpdateUserMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateUserMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateUserMsg)); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + msg->pCtg = pCtg; + msg->userAuth = *pAuth; + + action.data = msg; + + CTG_ERR_JRET(ctgPushAction(pCtg, &action)); + + return TSDB_CODE_SUCCESS; + +_return: + + taosMemoryFreeClear(msg); + + CTG_RET(code); +} int32_t ctgAcquireVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache, bool *inCache) { CTG_LOCK(CTG_READ, &dbCache->vgLock); @@ -687,6 +716,43 @@ int32_t ctgGetUdfInfoFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEp return TSDB_CODE_SUCCESS; } +int32_t ctgGetUserDbAuthFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char *user, SGetUserAuthRsp *authRsp) { + char *msg = NULL; + int32_t msgLen = 0; + + ctgDebug("try to get user auth from mnode, user:%s", user); + + int32_t code = queryBuildMsg[TMSG_INDEX(TDMT_MND_GET_USER_AUTH)]((void *)user, &msg, 0, &msgLen); + if (code) { + ctgError("Build get user auth msg failed, code:%x, db:%s", code, user); + CTG_ERR_RET(code); + } + + SRpcMsg rpcMsg = { + .msgType = TDMT_MND_GET_USER_AUTH, + .pCont = msg, + .contLen = msgLen, + }; + + SRpcMsg rpcRsp = {0}; + + rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); + if (TSDB_CODE_SUCCESS != rpcRsp.code) { + ctgError("error rsp for get user auth, error:%s, user:%s", tstrerror(rpcRsp.code), user); + CTG_ERR_RET(rpcRsp.code); + } + + code = queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_GET_USER_AUTH)](authRsp, rpcRsp.pCont, rpcRsp.contLen); + if (code) { + ctgError("Process get user auth rsp failed, code:%x, user:%s", code, user); + CTG_ERR_RET(code); + } + + ctgDebug("Got user auth from mnode, user:%s", user); + + return TSDB_CODE_SUCCESS; +} + int32_t ctgIsTableMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName, int32_t *exist) { if (NULL == pCtg->dbCache) { @@ -859,6 +925,55 @@ int32_t ctgGetTableTypeFromCache(SCatalog* pCtg, const char* dbFName, const char return TSDB_CODE_SUCCESS; } +int32_t ctgGetUserDbAuthFromCache(SCatalog* pCtg, const char* user, const char* dbFName, bool *inCache, int32_t *auth) { + if (NULL == pCtg->userCache) { + ctgDebug("empty user auth cache, user:%s", user); + goto _return; + } + + SCtgUserAuth *pUser = (SCtgUserAuth *)taosHashGet(pCtg->userCache, user, strlen(user)); + if (NULL == pUser) { + ctgDebug("user not in cache, user:%s", user); + goto _return; + } + + *inCache = true; + + ctgDebug("Got user from cache, user:%s", user); + CTG_CACHE_STAT_ADD(userHitNum, 1); + + if (pUser->superUser) { + CTG_FLAG_SET(auth, USER_AUTH_ALL); + return TSDB_CODE_SUCCESS; + } + + CTG_LOCK(CTG_READ, &pUser->lock); + if (pUser->createdDbs && taosHashGet(pUser->createdDbs, dbFName, strlen(dbFName))) { + CTG_FLAG_SET(auth, USER_AUTH_ALL); + CTG_UNLOCK(CTG_READ, &pUser->lock); + return TSDB_CODE_SUCCESS; + } + + if (pUser->readDbs && taosHashGet(pUser->readDbs, dbFName, strlen(dbFName))) { + CTG_FLAG_SET(auth, USER_AUTH_READ); + } + + if (pUser->writeDbs && taosHashGet(pUser->writeDbs, dbFName, strlen(dbFName))) { + CTG_FLAG_SET(auth, USER_AUTH_WRITE); + } + + CTG_UNLOCK(CTG_READ, &pUser->lock); + + return TSDB_CODE_SUCCESS; + +_return: + + *inCache = false; + CTG_CACHE_STAT_ADD(userMissNum, 1); + + return TSDB_CODE_SUCCESS; +} + int32_t ctgGetTableMetaFromMnodeImpl(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, char *dbFName, char* tbName, STableMetaOutput* output) { SBuildTableMetaInput bInput = {.vgId = 0, .dbFName = dbFName, .tbName = tbName}; char *msg = NULL; @@ -1952,6 +2067,44 @@ _return: CTG_RET(code); } +int32_t ctgGetUserDbAuth(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* user, const char* dbFName, int32_t* auth) { + bool inCache = false; + int32_t code = 0; + *auth = 0; + + CTG_ERR_RET(ctgGetUserDbAuthFromCache(pCtg, user, dbFName, &inCache, auth)); + + if (inCache) { + return TSDB_CODE_SUCCESS; + } + + SGetUserAuthRsp authRsp = {0}; + CTG_ERR_RET(ctgGetUserDbAuthFromMnode(pCtg, pRpc, pMgmtEps, user, &authRsp)); + + if (authRsp.superAuth) { + CTG_FLAG_SET(auth, USER_AUTH_ALL); + goto _return; + } + + if (authRsp.createdDbs && taosHashGet(authRsp.createdDbs, dbFName, strlen(dbFName))) { + CTG_FLAG_SET(auth, USER_AUTH_ALL); + goto _return; + } + + if (authRsp.readDbs && taosHashGet(authRsp.readDbs, dbFName, strlen(dbFName))) { + CTG_FLAG_SET(auth, USER_AUTH_READ); + } + + if (authRsp.writeDbs && taosHashGet(authRsp.writeDbs, dbFName, strlen(dbFName))) { + CTG_FLAG_SET(auth, USER_AUTH_WRITE); + } + +_return: + + ctgPushUpdateUserMsgInQueue(pCtg, &authRsp, false); + + return TSDB_CODE_SUCCESS; +} int32_t ctgActUpdateVg(SCtgMetaAction *action) { @@ -2121,6 +2274,67 @@ _return: CTG_RET(code); } +int32_t ctgActUpdateUser(SCtgMetaAction *action) { + int32_t code = 0; + SCtgUpdateUserMsg *msg = action->data; + SCatalog* pCtg = msg->pCtg; + + if (NULL == pCtg->userCache) { + pCtg->userCache = taosHashInit(gCtgMgmt.cfg.maxUserCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); + if (NULL == pCtg->userCache) { + ctgError("taosHashInit %d user cache failed", gCtgMgmt.cfg.maxUserCacheNum); + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + } + + SCtgUserAuth *pUser = (SCtgUserAuth *)taosHashGet(pCtg->userCache, msg->userAuth.user, strlen(msg->userAuth.user)); + if (NULL == pUser) { + SCtgUserAuth userAuth = {0}; + + userAuth.version = msg->userAuth.version; + userAuth.superUser = msg->userAuth.superAuth; + userAuth.createdDbs = msg->userAuth.createdDbs; + userAuth.readDbs = msg->userAuth.readDbs; + userAuth.writeDbs = msg->userAuth.writeDbs; + + if (taosHashPut(pCtg->userCache, msg->userAuth.user, strlen(msg->userAuth.user), &userAuth, sizeof(userAuth))) { + ctgError("taosHashPut user %s to cache failed", msg->userAuth.user); + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + + return TSDB_CODE_SUCCESS; + } + + pUser->version = msg->userAuth.version; + + CTG_LOCK(CTG_WRITE, &pUser->lock); + + taosHashCleanup(pUser->createdDbs); + pUser->createdDbs = msg->userAuth.createdDbs; + msg->userAuth.createdDbs = NULL; + + taosHashCleanup(pUser->readDbs); + pUser->readDbs = msg->userAuth.readDbs; + msg->userAuth.readDbs = NULL; + + taosHashCleanup(pUser->writeDbs); + pUser->writeDbs = msg->userAuth.writeDbs; + msg->userAuth.writeDbs = NULL; + + CTG_UNLOCK(CTG_WRITE, &pUser->lock); + +_return: + + + taosHashCleanup(msg->userAuth.createdDbs); + taosHashCleanup(msg->userAuth.readDbs); + taosHashCleanup(msg->userAuth.writeDbs); + + taosMemoryFreeClear(msg); + + CTG_RET(code); +} + void* ctgUpdateThreadFunc(void* param) { setThreadName("catalog"); @@ -2880,6 +3094,21 @@ _return: CTG_API_LEAVE(code); } +int32_t catalogGetUserDbAuth(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* user, const char* dbFName, int32_t* auth) { + CTG_API_ENTER(); + + if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == user || NULL == dbFName || NULL == auth) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + + int32_t code = 0; + CTG_ERR_JRET(ctgGetUserDbAuth(pCtg, pRpc, pMgmtEps, user, dbFName, auth)); + +_return: + + CTG_API_LEAVE(code); +} + void catalogDestroy(void) { qInfo("start to destroy catalog"); diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 286e7d3d44..822c214fe5 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -181,6 +181,25 @@ int32_t queryBuildRetrieveFuncMsg(void *input, char **msg, int32_t msgSize, int3 return TSDB_CODE_SUCCESS; } +int32_t queryBuildGetUserAuthMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen) { + if (NULL == msg || NULL == msgLen) { + return TSDB_CODE_TSC_INVALID_INPUT; + } + + SGetUserAuthReq req = {0}; + strncpy(req.user, input, sizeof(req.user)); + + int32_t bufLen = tSerializeSGetUserAuthReq(NULL, 0, &req); + void *pBuf = rpcMallocCont(bufLen); + tSerializeSGetUserAuthReq(pBuf, bufLen, &req); + + *msg = pBuf; + *msgLen = bufLen; + + return TSDB_CODE_SUCCESS; +} + + int32_t queryProcessUseDBRsp(void *output, char *msg, int32_t msgSize) { SUseDbOutput *pOut = output; SUseDbRsp usedbRsp = {0}; @@ -419,6 +438,20 @@ int32_t queryProcessRetrieveFuncRsp(void *output, char *msg, int32_t msgSize) { return TSDB_CODE_SUCCESS; } +int32_t queryProcessGetUserAuthRsp(void *output, char *msg, int32_t msgSize) { + if (NULL == output || NULL == msg || msgSize <= 0) { + return TSDB_CODE_TSC_INVALID_INPUT; + } + + if (tDeserializeSGetUserAuthRsp(msg, msgSize, (SGetUserAuthRsp *)output) != 0) { + qError("tDeserializeSGetUserAuthRsp failed, msgSize:%d", msgSize); + return TSDB_CODE_INVALID_MSG; + } + + return TSDB_CODE_SUCCESS; +} + + void initQueryModuleMsgHandle() { queryBuildMsg[TMSG_INDEX(TDMT_VND_TABLE_META)] = queryBuildTableMetaReqMsg; queryBuildMsg[TMSG_INDEX(TDMT_MND_TABLE_META)] = queryBuildTableMetaReqMsg; @@ -427,6 +460,8 @@ void initQueryModuleMsgHandle() { queryBuildMsg[TMSG_INDEX(TDMT_MND_GET_DB_CFG)] = queryBuildGetDBCfgMsg; queryBuildMsg[TMSG_INDEX(TDMT_MND_GET_INDEX)] = queryBuildGetIndexMsg; queryBuildMsg[TMSG_INDEX(TDMT_MND_RETRIEVE_FUNC)] = queryBuildRetrieveFuncMsg; + queryBuildMsg[TMSG_INDEX(TDMT_MND_GET_USER_AUTH)] = queryBuildGetUserAuthMsg; + queryProcessMsgRsp[TMSG_INDEX(TDMT_VND_TABLE_META)] = queryProcessTableMetaRsp; queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_TABLE_META)] = queryProcessTableMetaRsp; @@ -435,6 +470,7 @@ void initQueryModuleMsgHandle() { queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_GET_DB_CFG)] = queryProcessGetDbCfgRsp; queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_GET_INDEX)] = queryProcessGetIndexRsp; queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_RETRIEVE_FUNC)] = queryProcessRetrieveFuncRsp; + queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_GET_USER_AUTH)] = queryProcessGetUserAuthRsp; } #pragma GCC diagnostic pop From 2383b316d7f067ca4c4c77109b5452d4e0c65c61 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 5 May 2022 21:52:18 +0800 Subject: [PATCH 03/26] refactor: adjust db codes --- source/dnode/mnode/impl/inc/mndDef.h | 1 + source/dnode/mnode/impl/inc/mndVgroup.h | 1 + source/dnode/mnode/impl/src/mndDb.c | 171 +++++++++--------------- source/dnode/mnode/impl/src/mndUser.c | 13 +- source/dnode/mnode/impl/src/mndVgroup.c | 145 ++++++++++++++------ 5 files changed, 176 insertions(+), 155 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 4a0df80358..81ebcf9082 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -257,6 +257,7 @@ typedef struct { int32_t acctId; SHashObj* readDbs; SHashObj* writeDbs; + SRWLatch lock; } SUserObj; typedef struct { diff --git a/source/dnode/mnode/impl/inc/mndVgroup.h b/source/dnode/mnode/impl/inc/mndVgroup.h index f42829eddf..8aecc22454 100644 --- a/source/dnode/mnode/impl/inc/mndVgroup.h +++ b/source/dnode/mnode/impl/inc/mndVgroup.h @@ -33,6 +33,7 @@ int32_t mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId); void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); void *mndBuildDropVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); +void *mndBuildAlterVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index dfad99bdfb..0c31e58145 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -47,13 +47,15 @@ static int32_t mndProcessGetDbCfgReq(SNodeMsg *pReq); static int32_t mndProcessGetIndexReq(SNodeMsg *pReq); int32_t mndInitDb(SMnode *pMnode) { - SSdbTable table = {.sdbType = SDB_DB, - .keyType = SDB_KEY_BINARY, - .encodeFp = (SdbEncodeFp)mndDbActionEncode, - .decodeFp = (SdbDecodeFp)mndDbActionDecode, - .insertFp = (SdbInsertFp)mndDbActionInsert, - .updateFp = (SdbUpdateFp)mndDbActionUpdate, - .deleteFp = (SdbDeleteFp)mndDbActionDelete}; + SSdbTable table = { + .sdbType = SDB_DB, + .keyType = SDB_KEY_BINARY, + .encodeFp = (SdbEncodeFp)mndDbActionEncode, + .decodeFp = (SdbDecodeFp)mndDbActionDecode, + .insertFp = (SdbInsertFp)mndDbActionInsert, + .updateFp = (SdbUpdateFp)mndDbActionUpdate, + .deleteFp = (SdbDeleteFp)mndDbActionDelete, + }; mndSetMsgHandle(pMnode, TDMT_MND_CREATE_DB, mndProcessCreateDbReq); mndSetMsgHandle(pMnode, TDMT_MND_ALTER_DB, mndProcessAlterDbReq); @@ -194,6 +196,7 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) { } SDB_GET_RESERVE(pRaw, dataPos, DB_RESERVE_SIZE, _OVER) + taosInitRWLatch(&pDb->lock); terrno = 0; @@ -222,17 +225,29 @@ static int32_t mndDbActionDelete(SSdb *pSdb, SDbObj *pDb) { static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOld, SDbObj *pNew) { mTrace("db:%s, perform update action, old row:%p new row:%p", pOld->name, pOld, pNew); taosWLockLatch(&pOld->lock); - SArray *pOldRetensions = pOld->cfg.pRetensions; pOld->updateTime = pNew->updateTime; pOld->cfgVersion = pNew->cfgVersion; pOld->vgVersion = pNew->vgVersion; - memcpy(&pOld->cfg, &pNew->cfg, sizeof(SDbCfg)); - pNew->cfg.pRetensions = pOldRetensions; + pOld->cfg.buffer = pNew->cfg.buffer; + pOld->cfg.pages = pNew->cfg.pages; + pOld->cfg.pageSize = pNew->cfg.pageSize; + pOld->cfg.daysPerFile = pNew->cfg.daysPerFile; + pOld->cfg.daysToKeep0 = pNew->cfg.daysToKeep0; + pOld->cfg.daysToKeep1 = pNew->cfg.daysToKeep1; + pOld->cfg.daysToKeep2 = pNew->cfg.daysToKeep2; + pOld->cfg.fsyncPeriod = pNew->cfg.fsyncPeriod; + pOld->cfg.walLevel = pNew->cfg.walLevel; + pOld->cfg.strict = pNew->cfg.strict; + pOld->cfg.cacheLastRow = pNew->cfg.cacheLastRow; + pOld->cfg.replications = pNew->cfg.replications; taosWUnLockLatch(&pOld->lock); return 0; } -static int32_t mndGetGlobalVgroupVersion(SMnode *pMnode) { return sdbGetTableVer(pMnode->pSdb, SDB_VGROUP); } +static inline int32_t mndGetGlobalVgroupVersion(SMnode *pMnode) { + SSdb *pSdb = pMnode->pSdb; + return sdbGetTableVer(pSdb, SDB_VGROUP); +} SDbObj *mndAcquireDb(SMnode *pMnode, const char *db) { SSdb *pSdb = pMnode->pSdb; @@ -638,69 +653,7 @@ static int32_t mndSetAlterDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *p return 0; } -void *mndBuildAlterVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen) { - SAlterVnodeReq alterReq = {0}; - alterReq.vgVersion = pVgroup->version; - alterReq.buffer = pDb->cfg.buffer; - alterReq.pages = pDb->cfg.pages; - alterReq.pageSize = pDb->cfg.pageSize; - alterReq.daysPerFile = pDb->cfg.daysPerFile; - alterReq.daysToKeep0 = pDb->cfg.daysToKeep0; - alterReq.daysToKeep1 = pDb->cfg.daysToKeep1; - alterReq.daysToKeep2 = pDb->cfg.daysToKeep2; - alterReq.fsyncPeriod = pDb->cfg.fsyncPeriod; - alterReq.walLevel = pDb->cfg.walLevel; - alterReq.strict = pDb->cfg.strict; - alterReq.cacheLastRow = pDb->cfg.cacheLastRow; - alterReq.replica = pVgroup->replica; - alterReq.selfIndex = -1; - - for (int32_t v = 0; v < pVgroup->replica; ++v) { - SReplica *pReplica = &alterReq.replicas[v]; - SVnodeGid *pVgid = &pVgroup->vnodeGid[v]; - SDnodeObj *pVgidDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); - if (pVgidDnode == NULL) { - return NULL; - } - - pReplica->id = pVgidDnode->id; - pReplica->port = pVgidDnode->port; - memcpy(pReplica->fqdn, pVgidDnode->fqdn, TSDB_FQDN_LEN); - mndReleaseDnode(pMnode, pVgidDnode); - - if (pDnode->id == pVgid->dnodeId) { - alterReq.selfIndex = v; - } - } - - if (alterReq.selfIndex == -1) { - terrno = TSDB_CODE_MND_APP_ERROR; - return NULL; - } - - int32_t contLen = tSerializeSAlterVnodeReq(NULL, 0, &alterReq); - if (contLen < 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - contLen += +sizeof(SMsgHead); - - void *pReq = taosMemoryMalloc(contLen); - if (pReq == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - - SMsgHead *pHead = pReq; - pHead->contLen = htonl(contLen); - pHead->vgId = htonl(pVgroup->vgId); - - tSerializeSAlterVnodeReq((char *)pReq + sizeof(SMsgHead), contLen, &alterReq); - *pContLen = contLen; - return pReq; -} - -static int32_t mndBuilAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) { +static int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) { for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { STransAction action = {0}; SVnodeGid *pVgid = pVgroup->vnodeGid + vn; @@ -736,7 +689,7 @@ static int32_t mndSetAlterDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj * if (pIter == NULL) break; if (pVgroup->dbUid == pNew->uid) { - if (mndBuilAlterVgroupAction(pMnode, pTrans, pNew, pVgroup) != 0) { + if (mndBuildAlterVgroupAction(pMnode, pTrans, pNew, pVgroup) != 0) { sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pVgroup); return -1; @@ -752,19 +705,19 @@ static int32_t mndSetAlterDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj * static int32_t mndAlterDb(SMnode *pMnode, SNodeMsg *pReq, SDbObj *pOld, SDbObj *pNew) { int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_ALTER_DB, &pReq->rpcMsg); - if (pTrans == NULL) goto UPDATE_DB_OVER; + if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to alter db:%s", pTrans->id, pOld->name); mndTransSetDbInfo(pTrans, pOld); - if (mndSetAlterDbRedoLogs(pMnode, pTrans, pOld, pNew) != 0) goto UPDATE_DB_OVER; - if (mndSetAlterDbCommitLogs(pMnode, pTrans, pOld, pNew) != 0) goto UPDATE_DB_OVER; - if (mndSetAlterDbRedoActions(pMnode, pTrans, pOld, pNew) != 0) goto UPDATE_DB_OVER; - if (mndTransPrepare(pMnode, pTrans) != 0) goto UPDATE_DB_OVER; + if (mndSetAlterDbRedoLogs(pMnode, pTrans, pOld, pNew) != 0) goto _OVER; + if (mndSetAlterDbCommitLogs(pMnode, pTrans, pOld, pNew) != 0) goto _OVER; + if (mndSetAlterDbRedoActions(pMnode, pTrans, pOld, pNew) != 0) goto _OVER; + if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; code = 0; -UPDATE_DB_OVER: +_OVER: mndTransDrop(pTrans); return code; } @@ -778,7 +731,7 @@ static int32_t mndProcessAlterDbReq(SNodeMsg *pReq) { if (tDeserializeSAlterDbReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &alterReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; - goto ALTER_DB_OVER; + goto _OVER; } mDebug("db:%s, start to alter", alterReq.db); @@ -786,24 +739,26 @@ static int32_t mndProcessAlterDbReq(SNodeMsg *pReq) { pDb = mndAcquireDb(pMnode, alterReq.db); if (pDb == NULL) { terrno = TSDB_CODE_MND_DB_NOT_EXIST; - goto ALTER_DB_OVER; + goto _OVER; } pUser = mndAcquireUser(pMnode, pReq->user); if (pUser == NULL) { - goto ALTER_DB_OVER; + goto _OVER; } if (mndCheckAlterDropCompactDbAuth(pUser, pDb) != 0) { - goto ALTER_DB_OVER; + goto _OVER; } SDbObj dbObj = {0}; memcpy(&dbObj, pDb, sizeof(SDbObj)); + dbObj.cfg.numOfRetensions = 0; + dbObj.cfg.pRetensions = NULL; code = mndSetDbCfgFromAlterDbReq(&dbObj, &alterReq); if (code != 0) { - goto ALTER_DB_OVER; + goto _OVER; } dbObj.cfgVersion++; @@ -811,7 +766,7 @@ static int32_t mndProcessAlterDbReq(SNodeMsg *pReq) { code = mndAlterDb(pMnode, pReq, pDb, &dbObj); if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; -ALTER_DB_OVER: +_OVER: if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("db:%s, failed to alter since %s", alterReq.db, terrstr()); } @@ -831,13 +786,13 @@ static int32_t mndProcessGetDbCfgReq(SNodeMsg *pReq) { if (tDeserializeSDbCfgReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &cfgReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; - goto GET_DB_CFG_OVER; + goto _OVER; } pDb = mndAcquireDb(pMnode, cfgReq.db); if (pDb == NULL) { terrno = TSDB_CODE_MND_DB_NOT_EXIST; - goto GET_DB_CFG_OVER; + goto _OVER; } cfgRsp.numOfVgroups = pDb->cfg.numOfVgroups; @@ -866,7 +821,7 @@ static int32_t mndProcessGetDbCfgReq(SNodeMsg *pReq) { if (pRsp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; code = -1; - goto GET_DB_CFG_OVER; + goto _OVER; } tSerializeSDbCfgRsp(pRsp, contLen, &cfgRsp); @@ -876,9 +831,9 @@ static int32_t mndProcessGetDbCfgReq(SNodeMsg *pReq) { code = 0; -GET_DB_CFG_OVER: +_OVER: - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0) { mError("db:%s, failed to get cfg since %s", cfgReq.db, terrstr()); } @@ -1097,7 +1052,8 @@ _OVER: return code; } -void mndGetDBTableNum(SDbObj *pDb, SMnode *pMnode, int32_t *num) { +static int32_t mndGetDBTableNum(SDbObj *pDb, SMnode *pMnode) { + int32_t numOfTables = 0; int32_t vindex = 0; SSdb *pSdb = pMnode->pSdb; @@ -1108,8 +1064,7 @@ void mndGetDBTableNum(SDbObj *pDb, SMnode *pMnode, int32_t *num) { if (pIter == NULL) break; if (pVgroup->dbUid == pDb->uid) { - *num += pVgroup->numOfTables / TSDB_TABLE_NUM_UNIT; - + numOfTables += pVgroup->numOfTables / TSDB_TABLE_NUM_UNIT; vindex++; } @@ -1117,6 +1072,7 @@ void mndGetDBTableNum(SDbObj *pDb, SMnode *pMnode, int32_t *num) { } sdbCancelFetch(pSdb, pIter); + return numOfTables; } static void mndBuildDBVgroupInfo(SDbObj *pDb, SMnode *pMnode, SArray *pVgList) { @@ -1170,8 +1126,7 @@ int32_t mndExtractDbInfo(SMnode *pMnode, SDbObj *pDb, SUseDbRsp *pRsp, const SUs return -1; } - int32_t numOfTable = 0; - mndGetDBTableNum(pDb, pMnode, &numOfTable); + int32_t numOfTable = mndGetDBTableNum(pDb, pMnode); if (pReq == NULL || pReq->vgVersion < pDb->vgVersion || pReq->dbId != pDb->uid || numOfTable != pReq->numOfTable) { mndBuildDBVgroupInfo(pDb, pMnode, pRsp->pVgroupInfos); @@ -1195,7 +1150,7 @@ static int32_t mndProcessUseDbReq(SNodeMsg *pReq) { if (tDeserializeSUseDbReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &usedbReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; - goto USE_DB_OVER; + goto _OVER; } char *p = strchr(usedbReq.db, '.'); @@ -1206,12 +1161,11 @@ static int32_t mndProcessUseDbReq(SNodeMsg *pReq) { usedbRsp.pVgroupInfos = taosArrayInit(10, sizeof(SVgroupInfo)); if (usedbRsp.pVgroupInfos == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - goto USE_DB_OVER; + goto _OVER; } mndBuildDBVgroupInfo(NULL, pMnode, usedbRsp.pVgroupInfos); usedbRsp.vgVersion = vgVersion++; - } else { usedbRsp.vgVersion = usedbReq.vgVersion; } @@ -1232,15 +1186,15 @@ static int32_t mndProcessUseDbReq(SNodeMsg *pReq) { } else { pUser = mndAcquireUser(pMnode, pReq->user); if (pUser == NULL) { - goto USE_DB_OVER; + goto _OVER; } if (mndCheckUseDbAuth(pUser, pDb) != 0) { - goto USE_DB_OVER; + goto _OVER; } if (mndExtractDbInfo(pMnode, pDb, &usedbRsp, &usedbReq) < 0) { - goto USE_DB_OVER; + goto _OVER; } code = 0; @@ -1252,7 +1206,7 @@ static int32_t mndProcessUseDbReq(SNodeMsg *pReq) { if (pRsp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; code = -1; - goto USE_DB_OVER; + goto _OVER; } tSerializeSUseDbRsp(pRsp, contLen, &usedbRsp); @@ -1260,7 +1214,7 @@ static int32_t mndProcessUseDbReq(SNodeMsg *pReq) { pReq->pRsp = pRsp; pReq->rspLen = contLen; -USE_DB_OVER: +_OVER: if (code != 0) { mError("db:%s, failed to process use db req since %s", usedbReq.db, terrstr()); } @@ -1298,8 +1252,7 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs, continue; } - int32_t numOfTable = 0; - mndGetDBTableNum(pDb, pMnode, &numOfTable); + int32_t numOfTable = mndGetDBTableNum(pDb, pMnode); if (pDbVgVersion->vgVersion >= pDb->vgVersion && numOfTable == pDbVgVersion->numOfTable) { mDebug("db:%s, version & numOfTable not changed", pDbVgVersion->dbFName); @@ -1514,9 +1467,6 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in pColInfo = taosArrayGet(pBlock->pDataBlock, cols); colDataAppend(pColInfo, rows, (const char *)b, false); } - - // pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity); - // *(int8_t *)pWrite = pDb->cfg.update; } static void setInformationSchemaDbCfg(SDbObj *pDbObj) { @@ -1544,7 +1494,6 @@ static void setPerfSchemaDbCfg(SDbObj *pDbObj) { static bool mndGetTablesOfDbFp(SMnode *pMnode, void *pObj, void *p1, void *p2, void *p3) { SVgObj *pVgroup = pObj; int32_t *numOfTables = p1; - *numOfTables += pVgroup->numOfTables; return true; } diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 5e15bdeb43..31e78e0ca0 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -186,6 +186,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { } SDB_GET_RESERVE(pRaw, dataPos, USER_RESERVE_SIZE, _OVER) + taosInitRWLatch(&pUser->lock); terrno = 0; @@ -228,11 +229,12 @@ static int32_t mndUserActionDelete(SSdb *pSdb, SUserObj *pUser) { static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pOld, SUserObj *pNew) { mTrace("user:%s, perform update action, old row:%p new row:%p", pOld->user, pOld, pNew); - memcpy(pOld->pass, pNew->pass, TSDB_PASSWORD_LEN); + taosWLockLatch(&pOld->lock); pOld->updateTime = pNew->updateTime; - + memcpy(pOld->pass, pNew->pass, TSDB_PASSWORD_LEN); TSWAP(pOld->readDbs, pNew->readDbs); TSWAP(pOld->writeDbs, pNew->writeDbs); + taosWUnLockLatch(&pOld->lock); return 0; } @@ -426,8 +428,12 @@ static int32_t mndProcessAlterUserReq(SNodeMsg *pReq) { } memcpy(&newUser, pUser, sizeof(SUserObj)); + + taosRLockLatch(&pUser->lock); newUser.readDbs = mndDupDbHash(pUser->readDbs); newUser.writeDbs = mndDupDbHash(pUser->writeDbs); + taosRUnLockLatch(&pUser->lock); + if (newUser.readDbs == NULL || newUser.writeDbs == NULL) { goto _OVER; } @@ -598,8 +604,11 @@ static int32_t mndProcessGetUserAuthReq(SNodeMsg *pReq) { memcpy(authRsp.user, pUser->user, TSDB_USER_LEN); authRsp.superAuth = pUser->superUser; + + taosRLockLatch(&pUser->lock); authRsp.readDbs = mndDupDbHash(pUser->readDbs); authRsp.writeDbs = mndDupDbHash(pUser->writeDbs); + taosRUnLockLatch(&pUser->lock); SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 31e4a8ea7d..8c57fb5dbe 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -34,9 +34,9 @@ static int32_t mndProcessAlterVnodeRsp(SNodeMsg *pRsp); static int32_t mndProcessDropVnodeRsp(SNodeMsg *pRsp); static int32_t mndProcessCompactVnodeRsp(SNodeMsg *pRsp); -static int32_t mndRetrieveVgroups(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows); +static int32_t mndRetrieveVgroups(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextVgroup(SMnode *pMnode, void *pIter); -static int32_t mndRetrieveVnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows); +static int32_t mndRetrieveVnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextVnode(SMnode *pMnode, void *pIter); int32_t mndInitVgroup(SMnode *pMnode) { @@ -46,7 +46,7 @@ int32_t mndInitVgroup(SMnode *pMnode) { .decodeFp = (SdbDecodeFp)mndVgroupActionDecode, .insertFp = (SdbInsertFp)mndVgroupActionInsert, .updateFp = (SdbUpdateFp)mndVgroupActionUpdate, - .deleteFp = (SdbDeleteFp)mndVgroupActionDelete}; + .deleteFp = (SdbDeleteFp)mndVgroupActionDelete,}; mndSetMsgHandle(pMnode, TDMT_DND_CREATE_VNODE_RSP, mndProcessCreateVnodeRsp); mndSetMsgHandle(pMnode, TDMT_VND_ALTER_VNODE_RSP, mndProcessAlterVnodeRsp); @@ -67,28 +67,28 @@ SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup) { terrno = TSDB_CODE_OUT_OF_MEMORY; SSdbRaw *pRaw = sdbAllocRaw(SDB_VGROUP, TSDB_VGROUP_VER_NUMBER, sizeof(SVgObj) + TSDB_VGROUP_RESERVE_SIZE); - if (pRaw == NULL) goto VG_ENCODE_OVER; + if (pRaw == NULL) goto _OVER; int32_t dataPos = 0; - SDB_SET_INT32(pRaw, dataPos, pVgroup->vgId, VG_ENCODE_OVER) - SDB_SET_INT64(pRaw, dataPos, pVgroup->createdTime, VG_ENCODE_OVER) - SDB_SET_INT64(pRaw, dataPos, pVgroup->updateTime, VG_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pVgroup->version, VG_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pVgroup->hashBegin, VG_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pVgroup->hashEnd, VG_ENCODE_OVER) - SDB_SET_BINARY(pRaw, dataPos, pVgroup->dbName, TSDB_DB_FNAME_LEN, VG_ENCODE_OVER) - SDB_SET_INT64(pRaw, dataPos, pVgroup->dbUid, VG_ENCODE_OVER) - SDB_SET_INT8(pRaw, dataPos, pVgroup->replica, VG_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, pVgroup->vgId, _OVER) + SDB_SET_INT64(pRaw, dataPos, pVgroup->createdTime, _OVER) + SDB_SET_INT64(pRaw, dataPos, pVgroup->updateTime, _OVER) + SDB_SET_INT32(pRaw, dataPos, pVgroup->version, _OVER) + SDB_SET_INT32(pRaw, dataPos, pVgroup->hashBegin, _OVER) + SDB_SET_INT32(pRaw, dataPos, pVgroup->hashEnd, _OVER) + SDB_SET_BINARY(pRaw, dataPos, pVgroup->dbName, TSDB_DB_FNAME_LEN, _OVER) + SDB_SET_INT64(pRaw, dataPos, pVgroup->dbUid, _OVER) + SDB_SET_INT8(pRaw, dataPos, pVgroup->replica, _OVER) for (int8_t i = 0; i < pVgroup->replica; ++i) { SVnodeGid *pVgid = &pVgroup->vnodeGid[i]; - SDB_SET_INT32(pRaw, dataPos, pVgid->dnodeId, VG_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, pVgid->dnodeId, _OVER) } - SDB_SET_RESERVE(pRaw, dataPos, TSDB_VGROUP_RESERVE_SIZE, VG_ENCODE_OVER) - SDB_SET_DATALEN(pRaw, dataPos, VG_ENCODE_OVER) + SDB_SET_RESERVE(pRaw, dataPos, TSDB_VGROUP_RESERVE_SIZE, _OVER) + SDB_SET_DATALEN(pRaw, dataPos, _OVER) terrno = 0; -VG_ENCODE_OVER: +_OVER: if (terrno != 0) { mError("vgId:%d, failed to encode to raw:%p since %s", pVgroup->vgId, pRaw, terrstr()); sdbFreeRaw(pRaw); @@ -103,41 +103,41 @@ SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw) { terrno = TSDB_CODE_OUT_OF_MEMORY; int8_t sver = 0; - if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto VG_DECODE_OVER; + if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER; if (sver != TSDB_VGROUP_VER_NUMBER) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; - goto VG_DECODE_OVER; + goto _OVER; } SSdbRow *pRow = sdbAllocRow(sizeof(SVgObj)); - if (pRow == NULL) goto VG_DECODE_OVER; + if (pRow == NULL) goto _OVER; SVgObj *pVgroup = sdbGetRowObj(pRow); - if (pVgroup == NULL) goto VG_DECODE_OVER; + if (pVgroup == NULL) goto _OVER; int32_t dataPos = 0; - SDB_GET_INT32(pRaw, dataPos, &pVgroup->vgId, VG_DECODE_OVER) - SDB_GET_INT64(pRaw, dataPos, &pVgroup->createdTime, VG_DECODE_OVER) - SDB_GET_INT64(pRaw, dataPos, &pVgroup->updateTime, VG_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pVgroup->version, VG_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pVgroup->hashBegin, VG_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pVgroup->hashEnd, VG_DECODE_OVER) - SDB_GET_BINARY(pRaw, dataPos, pVgroup->dbName, TSDB_DB_FNAME_LEN, VG_DECODE_OVER) - SDB_GET_INT64(pRaw, dataPos, &pVgroup->dbUid, VG_DECODE_OVER) - SDB_GET_INT8(pRaw, dataPos, &pVgroup->replica, VG_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &pVgroup->vgId, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pVgroup->createdTime, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pVgroup->updateTime, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pVgroup->version, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pVgroup->hashBegin, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pVgroup->hashEnd, _OVER) + SDB_GET_BINARY(pRaw, dataPos, pVgroup->dbName, TSDB_DB_FNAME_LEN, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pVgroup->dbUid, _OVER) + SDB_GET_INT8(pRaw, dataPos, &pVgroup->replica, _OVER) for (int8_t i = 0; i < pVgroup->replica; ++i) { SVnodeGid *pVgid = &pVgroup->vnodeGid[i]; - SDB_GET_INT32(pRaw, dataPos, &pVgid->dnodeId, VG_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &pVgid->dnodeId, _OVER) if (pVgroup->replica == 1) { pVgid->role = TAOS_SYNC_STATE_LEADER; } } - SDB_GET_RESERVE(pRaw, dataPos, TSDB_VGROUP_RESERVE_SIZE, VG_DECODE_OVER) + SDB_GET_RESERVE(pRaw, dataPos, TSDB_VGROUP_RESERVE_SIZE, _OVER) terrno = 0; -VG_DECODE_OVER: +_OVER: if (terrno != 0) { mError("vgId:%d, failed to decode from raw:%p since %s", pVgroup->vgId, pRaw, terrstr()); taosMemoryFreeClear(pRow); @@ -254,6 +254,68 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg return pReq; } +void *mndBuildAlterVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen) { + SAlterVnodeReq alterReq = {0}; + alterReq.vgVersion = pVgroup->version; + alterReq.buffer = pDb->cfg.buffer; + alterReq.pages = pDb->cfg.pages; + alterReq.pageSize = pDb->cfg.pageSize; + alterReq.daysPerFile = pDb->cfg.daysPerFile; + alterReq.daysToKeep0 = pDb->cfg.daysToKeep0; + alterReq.daysToKeep1 = pDb->cfg.daysToKeep1; + alterReq.daysToKeep2 = pDb->cfg.daysToKeep2; + alterReq.fsyncPeriod = pDb->cfg.fsyncPeriod; + alterReq.walLevel = pDb->cfg.walLevel; + alterReq.strict = pDb->cfg.strict; + alterReq.cacheLastRow = pDb->cfg.cacheLastRow; + alterReq.replica = pVgroup->replica; + alterReq.selfIndex = -1; + + for (int32_t v = 0; v < pVgroup->replica; ++v) { + SReplica *pReplica = &alterReq.replicas[v]; + SVnodeGid *pVgid = &pVgroup->vnodeGid[v]; + SDnodeObj *pVgidDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); + if (pVgidDnode == NULL) { + return NULL; + } + + pReplica->id = pVgidDnode->id; + pReplica->port = pVgidDnode->port; + memcpy(pReplica->fqdn, pVgidDnode->fqdn, TSDB_FQDN_LEN); + mndReleaseDnode(pMnode, pVgidDnode); + + if (pDnode->id == pVgid->dnodeId) { + alterReq.selfIndex = v; + } + } + + if (alterReq.selfIndex == -1) { + terrno = TSDB_CODE_MND_APP_ERROR; + return NULL; + } + + int32_t contLen = tSerializeSAlterVnodeReq(NULL, 0, &alterReq); + if (contLen < 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + contLen += +sizeof(SMsgHead); + + void *pReq = taosMemoryMalloc(contLen); + if (pReq == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + SMsgHead *pHead = pReq; + pHead->contLen = htonl(contLen); + pHead->vgId = htonl(pVgroup->vgId); + + tSerializeSAlterVnodeReq((char *)pReq + sizeof(SMsgHead), contLen, &alterReq); + *pContLen = contLen; + return pReq; +} + void *mndBuildDropVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen) { SDropVnodeReq dropReq = {0}; dropReq.dnodeId = pDnode->id; @@ -372,12 +434,12 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) { pVgroups = taosMemoryCalloc(pDb->cfg.numOfVgroups, sizeof(SVgObj)); if (pVgroups == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - goto ALLOC_VGROUP_OVER; + goto _OVER; } pArray = mndBuildDnodesArray(pMnode); if (pArray == NULL) { - goto ALLOC_VGROUP_OVER; + goto _OVER; } mDebug("db:%s, total %d dnodes used to create %d vgroups (%d vnodes)", pDb->name, (int32_t)taosArrayGetSize(pArray), @@ -410,7 +472,7 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) { if (mndGetAvailableDnode(pMnode, pVgroup, pArray) != 0) { terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES; - goto ALLOC_VGROUP_OVER; + goto _OVER; } allocedVgroups++; @@ -421,7 +483,7 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) { mDebug("db:%s, %d vgroups is alloced, replica:%d", pDb->name, pDb->cfg.numOfVgroups, pDb->cfg.replications); -ALLOC_VGROUP_OVER: +_OVER: if (code != 0) taosMemoryFree(pVgroups); taosArrayDestroy(pArray); return code; @@ -492,7 +554,7 @@ static int32_t mndGetVgroupMaxReplica(SMnode *pMnode, char *dbName, int8_t *pRep return 0; } -static int32_t mndRetrieveVgroups(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows) { +static int32_t mndRetrieveVgroups(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; @@ -533,14 +595,13 @@ static int32_t mndRetrieveVgroups(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* // default 3 replica for (int32_t i = 0; i < 3; ++i) { - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); if (i < pVgroup->replica) { colDataAppend(pColInfo, numOfRows, (const char *)&pVgroup->vnodeGid[i].dnodeId, false); char buf1[20] = {0}; const char *role = syncStr(pVgroup->vnodeGid[i].role); - STR_WITH_MAXSIZE_TO_VARSTR(buf1, role, pShow->pMeta->pSchemas[cols].bytes); + STR_WITH_MAXSIZE_TO_VARSTR(buf1, role, pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)buf1, false); @@ -597,13 +658,13 @@ int32_t mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId) { return numOfVnodes; } -static int32_t mndRetrieveVnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows) { +static int32_t mndRetrieveVnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; SVgObj *pVgroup = NULL; int32_t cols = 0; -// int32_t dnodeId = pShow->replica; + // int32_t dnodeId = pShow->replica; while (numOfRows < rows) { pShow->pIter = sdbFetch(pSdb, SDB_VGROUP, pShow->pIter, (void **)&pVgroup); From 315c9c37dc4251093c58b5e2903aad74937d80ca Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 5 May 2022 23:14:36 +0800 Subject: [PATCH 04/26] enh(rpc): taosd exited when fqdn is configed to invalid --- include/libs/transport/trpc.h | 1 + include/os/osSocket.h | 1 + source/dnode/mgmt/implement/src/dmTransport.c | 12 +++--- source/libs/transport/src/trans.c | 13 +++++- source/libs/transport/src/transSrv.c | 5 ++- source/os/src/osSocket.c | 42 +++++++++++++++++++ 6 files changed, 67 insertions(+), 7 deletions(-) diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 0e7d486eab..a7d1522d12 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -68,6 +68,7 @@ typedef int (*RpcAfp)(void *parent, char *tableId, char *spi, char *encrypt, cha typedef bool (*RpcRfp)(int32_t code); typedef struct SRpcInit { + char localFqdn[TSDB_FQDN_LEN]; uint16_t localPort; // local port char * label; // for debug purpose int numOfThreads; // number of threads to handle connections diff --git a/include/os/osSocket.h b/include/os/osSocket.h index 62c3771669..213a6930ee 100644 --- a/include/os/osSocket.h +++ b/include/os/osSocket.h @@ -161,6 +161,7 @@ int taosCreateSocketWithTimeOutOpt(uint32_t conn_timeout_sec); TdSocketPtr taosOpenUdpSocket(uint32_t localIp, uint16_t localPort); TdSocketPtr taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp); +bool taosValidIpAndPort(uint32_t ip, uint16_t port); TdSocketServerPtr taosOpenTcpServerSocket(uint32_t ip, uint16_t port); int32_t taosKeepTcpAlive(TdSocketPtr pSocket); TdSocketPtr taosAcceptTcpConnectSocket(TdSocketServerPtr pServerSocket, struct sockaddr *destAddr, int *addrLen); diff --git a/source/dnode/mgmt/implement/src/dmTransport.c b/source/dnode/mgmt/implement/src/dmTransport.c index 446894556e..114d7b6dfc 100644 --- a/source/dnode/mgmt/implement/src/dmTransport.c +++ b/source/dnode/mgmt/implement/src/dmTransport.c @@ -16,8 +16,8 @@ #define _DEFAULT_SOURCE #include "dmImp.h" -#define INTERNAL_USER "_dnd" -#define INTERNAL_CKEY "_key" +#define INTERNAL_USER "_dnd" +#define INTERNAL_CKEY "_key" #define INTERNAL_SECRET "_pwd" static void dmGetMnodeEpSet(SDnode *pDnode, SEpSet *pEpSet) { @@ -130,10 +130,10 @@ _OVER: } static void dmProcessMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - SDnodeTrans *pTrans = &pDnode->trans; + SDnodeTrans * pTrans = &pDnode->trans; tmsg_t msgType = pMsg->msgType; bool isReq = msgType & 1u; - SMsgHandle *pHandle = &pTrans->msgHandles[TMSG_INDEX(msgType)]; + SMsgHandle * pHandle = &pTrans->msgHandles[TMSG_INDEX(msgType)]; SMgmtWrapper *pWrapper = pHandle->pNdWrapper; if (msgType == TDMT_DND_SERVER_STATUS) { @@ -517,7 +517,7 @@ static inline int32_t dmRetrieveUserAuthInfo(SDnode *pDnode, char *user, char *s SAuthReq authReq = {0}; tstrncpy(authReq.user, user, TSDB_USER_LEN); int32_t contLen = tSerializeSAuthReq(NULL, 0, &authReq); - void *pReq = rpcMallocCont(contLen); + void * pReq = rpcMallocCont(contLen); tSerializeSAuthReq(pReq, contLen, &authReq); SRpcMsg rpcMsg = {.pCont = pReq, .contLen = contLen, .msgType = TDMT_MND_AUTH, .ahandle = (void *)9528}; @@ -547,6 +547,8 @@ static int32_t dmInitServer(SDnode *pDnode) { SDnodeTrans *pTrans = &pDnode->trans; SRpcInit rpcInit = {0}; + + strncpy(rpcInit.localFqdn, pDnode->data.localFqdn, strlen(pDnode->data.localFqdn)); rpcInit.localPort = pDnode->data.serverPort; rpcInit.label = "DND"; rpcInit.numOfThreads = tsNumOfRpcThreads; diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index f776fb3764..f8277c575e 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -46,9 +46,20 @@ void* rpcOpen(const SRpcInit* pInit) { pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; } + uint32_t ip = 0; + if (pInit->connType == TAOS_CONN_SERVER) { + ip = taosGetIpv4FromFqdn(pInit->localFqdn); + if (ip == 0xFFFFFFFF) { + tError("invalid fqdn: %s", pInit->localFqdn); + taosMemoryFree(pRpc); + return NULL; + } + } + pRpc->connType = pInit->connType; pRpc->idleTime = pInit->idleTime; - pRpc->tcphandle = (*taosInitHandle[pRpc->connType])(0, pInit->localPort, pRpc->label, pRpc->numOfThreads, NULL, pRpc); + pRpc->tcphandle = + (*taosInitHandle[pRpc->connType])(ip, pInit->localPort, pRpc->label, pRpc->numOfThreads, NULL, pRpc); if (pRpc->tcphandle == NULL) { taosMemoryFree(pRpc); return NULL; diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index 7378ca3241..e1b0871135 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -817,7 +817,6 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, srv->pipe[i] = (uv_pipe_t*)taosMemoryCalloc(2, sizeof(uv_pipe_t)); - uv_os_sock_t fds[2]; if (uv_socketpair(SOCK_STREAM, 0, fds, UV_NONBLOCK_PIPE, UV_NONBLOCK_PIPE) != 0) { goto End; @@ -841,6 +840,10 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, goto End; } } + if (false == taosValidIpAndPort(srv->ip, srv->port)) { + tError("failed to bind, reason: %s", strerror(errno)); + goto End; + } if (false == addHandleToAcceptloop(srv)) { goto End; } diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index 6aa8520082..8cac660039 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -638,6 +638,48 @@ int32_t taosKeepTcpAlive(TdSocketPtr pSocket) { return 0; } +bool taosValidIpAndPort(uint32_t ip, uint16_t port) { + struct sockaddr_in serverAdd; + SocketFd fd; + int32_t reuse; + + // printf("open tcp server socket:0x%x:%hu", ip, port); + + bzero((char *)&serverAdd, sizeof(serverAdd)); + serverAdd.sin_family = AF_INET; + serverAdd.sin_addr.s_addr = ip; + serverAdd.sin_port = (uint16_t)htons(port); + + if ((fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) <= 2) { + // printf("failed to open TCP socket: %d (%s)", errno, strerror(errno)); + taosCloseSocketNoCheck1(fd); + return false; + } + + TdSocketPtr pSocket = (TdSocketPtr)taosMemoryMalloc(sizeof(TdSocket)); + if (pSocket == NULL) { + taosCloseSocketNoCheck1(fd); + return false; + } + pSocket->refId = 0; + pSocket->fd = fd; + + /* set REUSEADDR option, so the portnumber can be re-used */ + reuse = 1; + if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) { + // printf("setsockopt SO_REUSEADDR failed: %d (%s)", errno, strerror(errno)); + taosCloseSocket(&pSocket); + return NULL; + } + /* bind socket to server address */ + if (bind(pSocket->fd, (struct sockaddr *)&serverAdd, sizeof(serverAdd)) < 0) { + // printf("bind tcp server socket failed, 0x%x:%hu(%s)", ip, port, strerror(errno)); + taosCloseSocket(&pSocket); + return false; + } + taosCloseSocket(&pSocket); + return true; +} TdSocketServerPtr taosOpenTcpServerSocket(uint32_t ip, uint16_t port) { struct sockaddr_in serverAdd; SocketFd fd; From 1d149e554041601d1e0d0cacc8e63ed2843ae8c1 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 6 May 2022 09:56:21 +0800 Subject: [PATCH 05/26] refactor: format sma code --- source/dnode/mnode/impl/inc/mndSma.h | 1 - source/dnode/mnode/impl/src/mndDb.c | 48 ------------------------ source/dnode/mnode/impl/src/mndSma.c | 49 ++++++++++++++++++++++++- source/dnode/mnode/impl/src/mndVgroup.c | 29 ++++++++------- 4 files changed, 63 insertions(+), 64 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndSma.h b/source/dnode/mnode/impl/inc/mndSma.h index 91c6e24e28..4a80f619d3 100644 --- a/source/dnode/mnode/impl/inc/mndSma.h +++ b/source/dnode/mnode/impl/inc/mndSma.h @@ -26,7 +26,6 @@ int32_t mndInitSma(SMnode *pMnode); void mndCleanupSma(SMnode *pMnode); SSmaObj *mndAcquireSma(SMnode *pMnode, char *smaName); void mndReleaseSma(SMnode *pMnode, SSmaObj *pSma); -int32_t mndProcessGetSmaReq(SMnode *pMnode, SUserIndexReq *indexReq, SUserIndexRsp *rsp, bool *exist); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 0c31e58145..af7efb5543 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -44,7 +44,6 @@ static int32_t mndProcessCompactDbReq(SNodeMsg *pReq); static int32_t mndRetrieveDbs(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rowsCapacity); static void mndCancelGetNextDb(SMnode *pMnode, void *pIter); static int32_t mndProcessGetDbCfgReq(SNodeMsg *pReq); -static int32_t mndProcessGetIndexReq(SNodeMsg *pReq); int32_t mndInitDb(SMnode *pMnode) { SSdbTable table = { @@ -63,7 +62,6 @@ int32_t mndInitDb(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_USE_DB, mndProcessUseDbReq); mndSetMsgHandle(pMnode, TDMT_MND_COMPACT_DB, mndProcessCompactDbReq); mndSetMsgHandle(pMnode, TDMT_MND_GET_DB_CFG, mndProcessGetDbCfgReq); - mndSetMsgHandle(pMnode, TDMT_MND_GET_INDEX, mndProcessGetIndexReq); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_DB, mndRetrieveDbs); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_DB, mndCancelGetNextDb); @@ -1543,49 +1541,3 @@ static void mndCancelGetNextDb(SMnode *pMnode, void *pIter) { SSdb *pSdb = pMnode->pSdb; sdbCancelFetch(pSdb, pIter); } - -static int32_t mndProcessGetIndexReq(SNodeMsg *pReq) { - SUserIndexReq indexReq = {0}; - SMnode *pMnode = pReq->pNode; - int32_t code = -1; - SUserIndexRsp rsp = {0}; - bool exist = false; - - if (tDeserializeSUserIndexReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &indexReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - goto _OVER; - } - - code = mndProcessGetSmaReq(pMnode, &indexReq, &rsp, &exist); - if (code) { - goto _OVER; - } - - if (!exist) { - // TODO GET INDEX FROM FULLTEXT - code = -1; - terrno = TSDB_CODE_MND_DB_INDEX_NOT_EXIST; - } else { - int32_t contLen = tSerializeSUserIndexRsp(NULL, 0, &rsp); - void *pRsp = rpcMallocCont(contLen); - if (pRsp == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - code = -1; - goto _OVER; - } - - tSerializeSUserIndexRsp(pRsp, contLen, &rsp); - - pReq->pRsp = pRsp; - pReq->rspLen = contLen; - - code = 0; - } - -_OVER: - if (code != 0) { - mError("failed to get index %s since %s", indexReq.indexFName, terrstr()); - } - - return code; -} diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 0e3e8b68cc..8619df978b 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -40,6 +40,7 @@ static int32_t mndProcessMCreateSmaReq(SNodeMsg *pReq); static int32_t mndProcessMDropSmaReq(SNodeMsg *pReq); static int32_t mndProcessVCreateSmaRsp(SNodeMsg *pRsp); static int32_t mndProcessVDropSmaRsp(SNodeMsg *pRsp); +static int32_t mndProcessGetSmaReq(SNodeMsg *pReq); static int32_t mndRetrieveSma(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextSma(SMnode *pMnode, void *pIter); @@ -56,6 +57,7 @@ int32_t mndInitSma(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_DROP_SMA, mndProcessMDropSmaReq); mndSetMsgHandle(pMnode, TDMT_VND_CREATE_SMA_RSP, mndProcessVCreateSmaRsp); mndSetMsgHandle(pMnode, TDMT_VND_DROP_SMA_RSP, mndProcessVDropSmaRsp); + mndSetMsgHandle(pMnode, TDMT_MND_GET_INDEX, mndProcessGetSmaReq); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_INDEX, mndRetrieveSma); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_INDEX, mndCancelGetNextSma); @@ -686,7 +688,7 @@ _OVER: return code; } -int32_t mndProcessGetSmaReq(SMnode *pMnode, SUserIndexReq *indexReq, SUserIndexRsp *rsp, bool *exist) { +static int32_t mndGetSma(SMnode *pMnode, SUserIndexReq *indexReq, SUserIndexRsp *rsp, bool *exist) { int32_t code = -1; SSmaObj *pSma = NULL; @@ -715,6 +717,51 @@ int32_t mndProcessGetSmaReq(SMnode *pMnode, SUserIndexReq *indexReq, SUserIndexR } mndReleaseSma(pMnode, pSma); + return code; +} + +static int32_t mndProcessGetSmaReq(SNodeMsg *pReq) { + SUserIndexReq indexReq = {0}; + SMnode *pMnode = pReq->pNode; + int32_t code = -1; + SUserIndexRsp rsp = {0}; + bool exist = false; + + if (tDeserializeSUserIndexReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &indexReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto _OVER; + } + + code = mndGetSma(pMnode, &indexReq, &rsp, &exist); + if (code) { + goto _OVER; + } + + if (!exist) { + // TODO GET INDEX FROM FULLTEXT + code = -1; + terrno = TSDB_CODE_MND_DB_INDEX_NOT_EXIST; + } else { + int32_t contLen = tSerializeSUserIndexRsp(NULL, 0, &rsp); + void *pRsp = rpcMallocCont(contLen); + if (pRsp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + code = -1; + goto _OVER; + } + + tSerializeSUserIndexRsp(pRsp, contLen, &rsp); + + pReq->pRsp = pRsp; + pReq->rspLen = contLen; + + code = 0; + } + +_OVER: + if (code != 0) { + mError("failed to get index %s since %s", indexReq.indexFName, terrstr()); + } return code; } diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 8c57fb5dbe..d1e4be1161 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -21,8 +21,8 @@ #include "mndShow.h" #include "mndTrans.h" -#define TSDB_VGROUP_VER_NUMBER 1 -#define TSDB_VGROUP_RESERVE_SIZE 64 +#define VGROUP_VER_NUMBER 1 +#define VGROUP_RESERVE_SIZE 64 static SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw); static int32_t mndVgroupActionInsert(SSdb *pSdb, SVgObj *pVgroup); @@ -40,13 +40,15 @@ static int32_t mndRetrieveVnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *p static void mndCancelGetNextVnode(SMnode *pMnode, void *pIter); int32_t mndInitVgroup(SMnode *pMnode) { - SSdbTable table = {.sdbType = SDB_VGROUP, - .keyType = SDB_KEY_INT32, - .encodeFp = (SdbEncodeFp)mndVgroupActionEncode, - .decodeFp = (SdbDecodeFp)mndVgroupActionDecode, - .insertFp = (SdbInsertFp)mndVgroupActionInsert, - .updateFp = (SdbUpdateFp)mndVgroupActionUpdate, - .deleteFp = (SdbDeleteFp)mndVgroupActionDelete,}; + SSdbTable table = { + .sdbType = SDB_VGROUP, + .keyType = SDB_KEY_INT32, + .encodeFp = (SdbEncodeFp)mndVgroupActionEncode, + .decodeFp = (SdbDecodeFp)mndVgroupActionDecode, + .insertFp = (SdbInsertFp)mndVgroupActionInsert, + .updateFp = (SdbUpdateFp)mndVgroupActionUpdate, + .deleteFp = (SdbDeleteFp)mndVgroupActionDelete, + }; mndSetMsgHandle(pMnode, TDMT_DND_CREATE_VNODE_RSP, mndProcessCreateVnodeRsp); mndSetMsgHandle(pMnode, TDMT_VND_ALTER_VNODE_RSP, mndProcessAlterVnodeRsp); @@ -66,7 +68,7 @@ void mndCleanupVgroup(SMnode *pMnode) {} SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup) { terrno = TSDB_CODE_OUT_OF_MEMORY; - SSdbRaw *pRaw = sdbAllocRaw(SDB_VGROUP, TSDB_VGROUP_VER_NUMBER, sizeof(SVgObj) + TSDB_VGROUP_RESERVE_SIZE); + SSdbRaw *pRaw = sdbAllocRaw(SDB_VGROUP, VGROUP_VER_NUMBER, sizeof(SVgObj) + VGROUP_RESERVE_SIZE); if (pRaw == NULL) goto _OVER; int32_t dataPos = 0; @@ -83,7 +85,7 @@ SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup) { SVnodeGid *pVgid = &pVgroup->vnodeGid[i]; SDB_SET_INT32(pRaw, dataPos, pVgid->dnodeId, _OVER) } - SDB_SET_RESERVE(pRaw, dataPos, TSDB_VGROUP_RESERVE_SIZE, _OVER) + SDB_SET_RESERVE(pRaw, dataPos, VGROUP_RESERVE_SIZE, _OVER) SDB_SET_DATALEN(pRaw, dataPos, _OVER) terrno = 0; @@ -105,7 +107,7 @@ SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw) { int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER; - if (sver != TSDB_VGROUP_VER_NUMBER) { + if (sver != VGROUP_VER_NUMBER) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; goto _OVER; } @@ -133,7 +135,7 @@ SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw) { pVgid->role = TAOS_SYNC_STATE_LEADER; } } - SDB_GET_RESERVE(pRaw, dataPos, TSDB_VGROUP_RESERVE_SIZE, _OVER) + SDB_GET_RESERVE(pRaw, dataPos, VGROUP_RESERVE_SIZE, _OVER) terrno = 0; @@ -664,7 +666,6 @@ static int32_t mndRetrieveVnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *p int32_t numOfRows = 0; SVgObj *pVgroup = NULL; int32_t cols = 0; - // int32_t dnodeId = pShow->replica; while (numOfRows < rows) { pShow->pIter = sdbFetch(pSdb, SDB_VGROUP, pShow->pIter, (void **)&pVgroup); From e33586922ab1975ba5690b072d1e491812ae3b01 Mon Sep 17 00:00:00 2001 From: cpwu Date: Fri, 6 May 2022 13:08:33 +0800 Subject: [PATCH 06/26] fix case --- tests/system-test/2-query/join.py | 74 +++++++++++++++++++------------ 1 file changed, 45 insertions(+), 29 deletions(-) diff --git a/tests/system-test/2-query/join.py b/tests/system-test/2-query/join.py index c6431cdc8d..d66a300afb 100644 --- a/tests/system-test/2-query/join.py +++ b/tests/system-test/2-query/join.py @@ -93,43 +93,48 @@ class TDTestCase: groups = ["", group_having, group_no_having] for group_condition in groups: sql = f"select {condition} from {tblist[0]},{tblist[1]} where {join_condition} and {where_condition} {group_condition}" + if not join_flag : + tdSql.error(sql=sql) + return if len(tblist) == 2: self.__join_current(sql, checkrows) - elif len(tblist) > 2 or len(tblist) < 1: + return + if len(tblist) > 2 or len(tblist) < 1: tdSql.error(sql=sql) + return - def __join_err_check(self,tbname): - sqls = [] + # def __join_err_check(self,tbname): + # sqls = [] - for un_char_col in NUM_COL: - sqls.extend( - ( - f"select length( {un_char_col} ) from {tbname} ", - f"select length(ceil( {un_char_col} )) from {tbname} ", - f"select {un_char_col} from {tbname} group by length( {un_char_col} ) ", - ) - ) + # for un_char_col in NUM_COL: + # sqls.extend( + # ( + # f"select length( {un_char_col} ) from {tbname} ", + # f"select length(ceil( {un_char_col} )) from {tbname} ", + # f"select {un_char_col} from {tbname} group by length( {un_char_col} ) ", + # ) + # ) - sqls.extend( f"select length( {un_char_col} + {un_char_col_2} ) from {tbname} " for un_char_col_2 in NUM_COL ) - sqls.extend( f"select length( {un_char_col} + {ts_col} ) from {tbname} " for ts_col in TS_TYPE_COL ) + # sqls.extend( f"select length( {un_char_col} + {un_char_col_2} ) from {tbname} " for un_char_col_2 in NUM_COL ) + # sqls.extend( f"select length( {un_char_col} + {ts_col} ) from {tbname} " for ts_col in TS_TYPE_COL ) - sqls.extend( f"select {char_col} from {tbname} group by length( {char_col} ) " for char_col in CHAR_COL) - sqls.extend( f"select length( {ts_col} ) from {tbname} " for ts_col in TS_TYPE_COL ) - sqls.extend( f"select length( {char_col} + {ts_col} ) from {tbname} " for char_col in NUM_COL for ts_col in TS_TYPE_COL) - sqls.extend( f"select length( {char_col} + {char_col_2} ) from {tbname} " for char_col in CHAR_COL for char_col_2 in CHAR_COL ) - sqls.extend( f"select upper({char_col}, 11) from {tbname} " for char_col in CHAR_COL ) - sqls.extend( f"select upper({char_col}) from {tbname} interval(2d) sliding(1d)" for char_col in CHAR_COL ) - sqls.extend( - ( - f"select length() from {tbname} ", - f"select length(*) from {tbname} ", - f"select length(ccccccc) from {tbname} ", - f"select length(111) from {tbname} ", - f"select length(c8, 11) from {tbname} ", - ) - ) + # sqls.extend( f"select {char_col} from {tbname} group by length( {char_col} ) " for char_col in CHAR_COL) + # sqls.extend( f"select length( {ts_col} ) from {tbname} " for ts_col in TS_TYPE_COL ) + # sqls.extend( f"select length( {char_col} + {ts_col} ) from {tbname} " for char_col in NUM_COL for ts_col in TS_TYPE_COL) + # sqls.extend( f"select length( {char_col} + {char_col_2} ) from {tbname} " for char_col in CHAR_COL for char_col_2 in CHAR_COL ) + # sqls.extend( f"select upper({char_col}, 11) from {tbname} " for char_col in CHAR_COL ) + # sqls.extend( f"select upper({char_col}) from {tbname} interval(2d) sliding(1d)" for char_col in CHAR_COL ) + # sqls.extend( + # ( + # f"select length() from {tbname} ", + # f"select length(*) from {tbname} ", + # f"select length(ccccccc) from {tbname} ", + # f"select length(111) from {tbname} ", + # f"select length(c8, 11) from {tbname} ", + # ) + # ) - return sqls + # return sqls def __join_current(self, sql, checkrows): tdSql.query(sql=sql) @@ -170,6 +175,17 @@ class TDTestCase: tdLog.printNoPrefix(f"==========err sql condition check in {err_list_4} over==========") self.__join_check(err_list_5, -1) tdLog.printNoPrefix(f"==========err sql condition check in {err_list_5} over==========") + self.__join_check(["ct2", "ct4"], -1, join_flag=False) + tdLog.printNoPrefix("==========err sql condition check in has no join condition over==========") + + tdSql.error( f"select c1, c2 from ct2, ct4 where ct2.{PRIMARY_COL}=ct4.{PRIMARY_COL}" ) + tdSql.error( f"select ct2.c1, ct2.c2 from ct2, ct4 where ct2.{INT_COL}=ct4.{INT_COL}" ) + tdSql.error( f"select ct2.c1, ct2.c2 from ct2, ct4 where ct2.{TS_COL}=ct4.{TS_COL}" ) + tdSql.error( f"select ct2.c1, ct2.c2 from ct2, ct4 where ct2.{PRIMARY_COL}=ct4.{TS_COL}" ) + tdSql.error( f"select ct2.c1, ct1.c2 from ct2, ct4 where ct2.{PRIMARY_COL}=ct4.{PRIMARY_COL}" ) + tdSql.error( f"select ct2.c1, ct4.c2 from ct2, ct4 where ct2.{PRIMARY_COL}=ct4.{PRIMARY_COL} and c1 is not null " ) + tdSql.error( f"select ct2.c1, ct4.c2 from ct2, ct4 where ct2.{PRIMARY_COL}=ct4.{PRIMARY_COL} and ct1.c1 is not null " ) + tbname = ["ct1", "ct2", "ct4", "t1"] From c97518a81eff7f3524d1197b0ab39cd9158b5ca0 Mon Sep 17 00:00:00 2001 From: cpwu Date: Fri, 6 May 2022 13:41:46 +0800 Subject: [PATCH 07/26] fix case --- tests/system-test/2-query/join.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/system-test/2-query/join.py b/tests/system-test/2-query/join.py index d66a300afb..733f15bfc6 100644 --- a/tests/system-test/2-query/join.py +++ b/tests/system-test/2-query/join.py @@ -227,18 +227,18 @@ class TDTestCase: now_time = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000) for i in range(rows): tdSql.execute( - f"insert into ct1 values ( { now_time - i * 1000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar{i}', { now_time + 1 * i } )" + f"insert into ct1 values ( { now_time - i * 1000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )" ) tdSql.execute( - f"insert into ct4 values ( { now_time - i * 7776000000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar{i}', { now_time + 1 * i } )" + f"insert into ct4 values ( { now_time - i * 7776000000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )" ) tdSql.execute( - f"insert into ct2 values ( { now_time - i * 7776000000 }, {-i}, {-11111 * i}, {-111 * i % 32767 }, {-11 * i % 127}, {-1.11*i}, {-1100.0011*i}, {i%2}, 'binary{i}', 'nchar{i}', { now_time + 1 * i } )" + f"insert into ct2 values ( { now_time - i * 7776000000 }, {-i}, {-11111 * i}, {-111 * i % 32767 }, {-11 * i % 127}, {-1.11*i}, {-1100.0011*i}, {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i } )" ) tdSql.execute( f'''insert into ct1 values - ( { now_time - rows * 5 }, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', { now_time + 8 } ) - ( { now_time + 10000 }, { rows }, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', { now_time + 9 } ) + ( { now_time - rows * 5 }, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar_测试_0', { now_time + 8 } ) + ( { now_time + 10000 }, { rows }, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar_测试_9', { now_time + 9 } ) ''' ) @@ -249,11 +249,11 @@ class TDTestCase: ( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( { now_time + 5184000000}, {pow(2,31)-pow(2,15)}, {pow(2,63)-pow(2,30)}, 32767, 127, - { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_limit-1", { now_time - 86400000} + { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000} ) ( { now_time + 2592000000 }, {pow(2,31)-pow(2,16)}, {pow(2,63)-pow(2,31)}, 32766, 126, - { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_limit-2", { now_time - 172800000} + { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000} ) ''' ) @@ -265,11 +265,11 @@ class TDTestCase: ( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( { now_time + 5184000000 }, { -1 * pow(2,31) + pow(2,15) }, { -1 * pow(2,63) + pow(2,30) }, -32766, -126, - { -1 * 3.2 * pow(10,38) }, { -1.2 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_limit-1", { now_time - 86400000 } + { -1 * 3.2 * pow(10,38) }, { -1.2 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000 } ) ( { now_time + 2592000000 }, { -1 * pow(2,31) + pow(2,16) }, { -1 * pow(2,63) + pow(2,31) }, -32767, -127, - { - 3.3 * pow(10,38) }, { -1.3 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_limit-2", { now_time - 172800000 } + { - 3.3 * pow(10,38) }, { -1.3 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000 } ) ''' ) @@ -277,7 +277,7 @@ class TDTestCase: for i in range(rows): insert_data = f'''insert into t1 values ( { now_time - i * 3600000 }, {i}, {i * 11111}, { i % 32767 }, { i % 127}, { i * 1.11111 }, { i * 1000.1111 }, { i % 2}, - "binary_{i}", "nchar_{i}", { now_time - 1000 * i } ) + "binary_{i}", "nchar_测试_{i}", { now_time - 1000 * i } ) ''' tdSql.execute(insert_data) tdSql.execute( @@ -287,12 +287,12 @@ class TDTestCase: ( { now_time - rows * 3600000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ( { now_time + 7200000 }, { pow(2,31) - pow(2,15) }, { pow(2,63) - pow(2,30) }, 32767, 127, { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 }, - "binary_limit-1", "nchar_limit-1", { now_time - 86400000 } + "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000 } ) ( { now_time + 3600000 } , { pow(2,31) - pow(2,16) }, { pow(2,63) - pow(2,31) }, 32766, 126, { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 }, - "binary_limit-2", "nchar_limit-2", { now_time - 172800000 } + "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000 } ) ''' ) From 3f29e09b41a46ce3d9ccd757413648aed87d18f7 Mon Sep 17 00:00:00 2001 From: cpwu Date: Fri, 6 May 2022 13:52:17 +0800 Subject: [PATCH 08/26] fix case --- tests/system-test/2-query/join.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/system-test/2-query/join.py b/tests/system-test/2-query/join.py index 733f15bfc6..7ed614c7a5 100644 --- a/tests/system-test/2-query/join.py +++ b/tests/system-test/2-query/join.py @@ -72,11 +72,11 @@ class TDTestCase: if col in NUM_COL: return f" abs( {tbname}.{col} ) >= 0" elif col in CHAR_COL: - return f" lower( {tbname}.{col} ) is not null" + return f" lower( {tbname}.{col} ) like 'bina%' or lower( {tbname}.{col} ) like '_cha%' " elif col in BOOLEAN_COL: return f" {tbname}.{col} in (false, true) " elif col in TS_TYPE_COL or col in PRIMARY_COL: - return f" abs( cast( {tbname}.{col} as bigint ) ) >= 0 " + return f" cast( {tbname}.{col} as binary(16) ) is not null " else: return "" @@ -92,7 +92,11 @@ class TDTestCase: group_no_having= self.__group_condition(tbname=tblist[0], col=condition ) groups = ["", group_having, group_no_having] for group_condition in groups: - sql = f"select {condition} from {tblist[0]},{tblist[1]} where {join_condition} and {where_condition} {group_condition}" + if where_condition: + sql = f" select {condition} from {tblist[0]},{tblist[1]} where {join_condition} and {where_condition} {group_condition} " + else: + sql = f" select {condition} from {tblist[0]},{tblist[1]} where {join_condition} {group_condition} " + if not join_flag : tdSql.error(sql=sql) return From d2ef75b578718f9aa4a8b302dadc08339b3b21e1 Mon Sep 17 00:00:00 2001 From: cpwu Date: Fri, 6 May 2022 13:52:59 +0800 Subject: [PATCH 09/26] fix case --- tests/system-test/2-query/join.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/2-query/join.py b/tests/system-test/2-query/join.py index 7ed614c7a5..c263a0b412 100644 --- a/tests/system-test/2-query/join.py +++ b/tests/system-test/2-query/join.py @@ -51,7 +51,7 @@ class TDTestCase: ) query_condition.extend( f"{tbname}.{num_col} + {tbname}.{num_col_1} " for num_col_1 in NUM_COL ) - query_condition.append('''"test1234!@#$%^&*():'> Date: Fri, 6 May 2022 13:57:45 +0800 Subject: [PATCH 10/26] fix case --- tests/system-test/2-query/join.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/2-query/join.py b/tests/system-test/2-query/join.py index c263a0b412..7744360e65 100644 --- a/tests/system-test/2-query/join.py +++ b/tests/system-test/2-query/join.py @@ -101,7 +101,7 @@ class TDTestCase: tdSql.error(sql=sql) return if len(tblist) == 2: - self.__join_current(sql, checkrows) + self.__join_current(sql, checkrows + 2 ) if where_condition else self.__join_current(sql, checkrows + 5 ) return if len(tblist) > 2 or len(tblist) < 1: tdSql.error(sql=sql) From 01ea52e33d26dcdecfba35ab4354755488bf60d3 Mon Sep 17 00:00:00 2001 From: cpwu Date: Fri, 6 May 2022 13:59:19 +0800 Subject: [PATCH 11/26] fix case --- tests/system-test/2-query/join.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/system-test/2-query/join.py b/tests/system-test/2-query/join.py index 7744360e65..1ab8fe6dd8 100644 --- a/tests/system-test/2-query/join.py +++ b/tests/system-test/2-query/join.py @@ -99,13 +99,10 @@ class TDTestCase: if not join_flag : tdSql.error(sql=sql) - return if len(tblist) == 2: self.__join_current(sql, checkrows + 2 ) if where_condition else self.__join_current(sql, checkrows + 5 ) - return if len(tblist) > 2 or len(tblist) < 1: tdSql.error(sql=sql) - return # def __join_err_check(self,tbname): # sqls = [] From 8ee2ae550f4c02436123457729ee4ebe2b604537 Mon Sep 17 00:00:00 2001 From: cpwu Date: Fri, 6 May 2022 14:06:46 +0800 Subject: [PATCH 12/26] fix case --- tests/system-test/2-query/join.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/system-test/2-query/join.py b/tests/system-test/2-query/join.py index 1ab8fe6dd8..aea454b9b7 100644 --- a/tests/system-test/2-query/join.py +++ b/tests/system-test/2-query/join.py @@ -100,7 +100,10 @@ class TDTestCase: if not join_flag : tdSql.error(sql=sql) if len(tblist) == 2: - self.__join_current(sql, checkrows + 2 ) if where_condition else self.__join_current(sql, checkrows + 5 ) + if "ct1" in tblist or "t1" in tblist: + self.__join_current(sql, checkrows) + else: + self.__join_current(sql, checkrows + 2 ) if where_condition else self.__join_current(sql, checkrows + 5 ) if len(tblist) > 2 or len(tblist) < 1: tdSql.error(sql=sql) From f755df6176c2593fb7c198c0468bba583a39dd99 Mon Sep 17 00:00:00 2001 From: cpwu Date: Fri, 6 May 2022 14:10:00 +0800 Subject: [PATCH 13/26] fix case --- tests/system-test/2-query/join.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/2-query/join.py b/tests/system-test/2-query/join.py index aea454b9b7..ebaa2e8b9f 100644 --- a/tests/system-test/2-query/join.py +++ b/tests/system-test/2-query/join.py @@ -81,7 +81,7 @@ class TDTestCase: return "" def __group_condition(self, tbname, col, having = ""): - return f" group by {tbname}.{col} having {having}" if having else f" group by {tbname}.{col} " + return f" group by {col} having {having}" if having else f" group by {col} " def __join_check(self, tblist, checkrows, join_flag=True): query_conditions = self.__query_condition(tblist[0]) From c77b20687c30805e0e86e767bf16961cbfda2f84 Mon Sep 17 00:00:00 2001 From: dapan Date: Fri, 6 May 2022 14:13:56 +0800 Subject: [PATCH 14/26] user auth --- include/common/tmsg.h | 13 ++- include/libs/catalog/catalog.h | 19 ++- source/client/src/clientHb.c | 77 ++++++++++++ source/common/src/tmsg.c | 142 +++++++++++++++++++---- source/dnode/mnode/impl/inc/mndDef.h | 1 + source/dnode/mnode/impl/inc/mndUser.h | 1 + source/dnode/mnode/impl/src/mndProfile.c | 10 ++ source/dnode/mnode/impl/src/mndUser.c | 128 +++++++++++++++++--- source/libs/catalog/inc/catalogInt.h | 2 +- source/libs/catalog/src/catalog.c | 80 ++++++++++--- 10 files changed, 404 insertions(+), 69 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 8741af54ec..ed744fa315 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -73,7 +73,8 @@ typedef uint16_t tmsg_t; enum { CONN_TYPE__QUERY = 1, CONN_TYPE__TMQ, CONN_TYPE__MAX }; enum { - HEARTBEAT_KEY_DBINFO = 1, + HEARTBEAT_KEY_USER_AUTHINFO = 1, + HEARTBEAT_KEY_DBINFO, HEARTBEAT_KEY_STBINFO, HEARTBEAT_KEY_MQ_TMP, }; @@ -669,10 +670,20 @@ typedef struct { SArray* pArray; // Array of SUseDbRsp } SUseDbBatchRsp; + int32_t tSerializeSUseDbBatchRsp(void* buf, int32_t bufLen, SUseDbBatchRsp* pRsp); int32_t tDeserializeSUseDbBatchRsp(void* buf, int32_t bufLen, SUseDbBatchRsp* pRsp); void tFreeSUseDbBatchRsp(SUseDbBatchRsp* pRsp); +typedef struct { + SArray* pArray; // Array of SGetUserAuthRsp +} SUserAuthBatchRsp; + +int32_t tSerializeSUserAuthBatchRsp(void* buf, int32_t bufLen, SUserAuthBatchRsp* pRsp); +int32_t tDeserializeSUserAuthBatchRsp(void* buf, int32_t bufLen, SUserAuthBatchRsp* pRsp); +void tFreeSUserAuthBatchRsp(SUserAuthBatchRsp* pRsp); + + typedef struct { char db[TSDB_DB_FNAME_LEN]; } SCompactDbReq; diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 5fc9e08de4..04a24c4f32 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -40,9 +40,11 @@ enum { CTG_DBG_STB_RENT_NUM, }; -#define USER_AUTH_READ 1 -#define USER_AUTH_WRITE 2 -#define USER_AUTH_ALL 4 +typedef enum { + AUTH_TYPE_READ = 1, + AUTH_TYPE_WRITE, + AUTH_TYPE_OTHER, +} AUTH_TYPE; typedef struct SCatalogReq { SArray *pTableName; // element is SNAME @@ -81,6 +83,11 @@ typedef struct SDbVgVersion { int32_t numOfTable; // unit is TSDB_TABLE_NUM_UNIT } SDbVgVersion; +typedef struct SUserAuthVersion { + char user[TSDB_USER_LEN]; + int32_t version; +} SUserAuthVersion; + typedef SDbCfgRsp SDbCfgInfo; typedef SUserIndexRsp SIndexInfo; @@ -223,13 +230,17 @@ int32_t catalogGetExpiredSTables(SCatalog* pCatalog, SSTableMetaVersion **stable int32_t catalogGetExpiredDBs(SCatalog* pCatalog, SDbVgVersion **dbs, uint32_t *num); +int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion **users, uint32_t *num); + int32_t catalogGetDBCfg(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName, SDbCfgInfo* pDbCfg); int32_t catalogGetIndexInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* indexName, SIndexInfo* pInfo); int32_t catalogGetUdfInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* funcName, SFuncInfo** pInfo); -int32_t catalogGetUserDbAuth(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* user, const char* dbFName, int32_t* auth); +int32_t catalogChkAuth(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* user, const char* dbFName, AUTH_TYPE type, bool *pass); + +int32_t catalogUpdateUserAuthInfo(SCatalog* pCtg, SGetUserAuthRsp* pAuth); /** diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index b11a49fa1a..fc39e80c1e 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -28,6 +28,27 @@ static int32_t hbMqHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq static int32_t hbMqHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { return 0; } +static int32_t hbProcessUserAuthInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) { + int32_t code = 0; + + SUserAuthBatchRsp batchRsp = {0}; + if (tDeserializeSUserAuthBatchRsp(value, valueLen, &batchRsp) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + int32_t numOfBatchs = taosArrayGetSize(batchRsp.pArray); + for (int32_t i = 0; i < numOfBatchs; ++i) { + SGetUserAuthRsp *rsp = taosArrayGet(batchRsp.pArray, i); + tscDebug("hb user auth rsp, user:%s, version:%d", rsp->user, rsp->version); + + catalogUpdateUserAuthInfo(pCatalog, rsp); + } + + tFreeSUserAuthBatchRsp(&batchRsp); + return TSDB_CODE_SUCCESS; +} + static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) { int32_t code = 0; @@ -148,6 +169,24 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { for (int32_t i = 0; i < kvNum; ++i) { SKv *kv = taosArrayGet(pRsp->info, i); switch (kv->key) { + case HEARTBEAT_KEY_USER_AUTHINFO: { + if (kv->valueLen <= 0 || NULL == kv->value) { + tscError("invalid hb user auth info, len:%d, value:%p", kv->valueLen, kv->value); + break; + } + + int64_t *clusterId = (int64_t *)info->param; + struct SCatalog *pCatalog = NULL; + + int32_t code = catalogGetHandle(*clusterId, &pCatalog); + if (code != TSDB_CODE_SUCCESS) { + tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", *clusterId, tstrerror(code)); + break; + } + + hbProcessUserAuthInfoRsp(kv->value, kv->valueLen, pCatalog); + break; + } case HEARTBEAT_KEY_DBINFO: { if (kv->valueLen <= 0 || NULL == kv->value) { tscError("invalid hb db info, len:%d, value:%p", kv->valueLen, kv->value); @@ -327,6 +366,39 @@ int32_t hbGetQueryBasicInfo(SClientHbKey *connKey, SClientHbReq *req) { return TSDB_CODE_SUCCESS; } +int32_t hbGetExpiredUserInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SClientHbReq *req) { + SUserAuthVersion *users = NULL; + uint32_t userNum = 0; + int32_t code = 0; + + code = catalogGetExpiredUsers(pCatalog, &users, &userNum); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + + if (userNum <= 0) { + return TSDB_CODE_SUCCESS; + } + + for (int32_t i = 0; i < userNum; ++i) { + SUserAuthVersion *user = &users[i]; + user->version = htonl(user->version); + } + + SKv kv = { + .key = HEARTBEAT_KEY_USER_AUTHINFO, + .valueLen = sizeof(SUserAuthVersion) * userNum, + .value = users, + }; + + tscDebug("hb got %d expired users, valueLen:%d", userNum, kv.valueLen); + + taosHashPut(req->info, &kv.key, sizeof(kv.key), &kv, sizeof(kv)); + + return TSDB_CODE_SUCCESS; +} + + int32_t hbGetExpiredDBInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SClientHbReq *req) { SDbVgVersion *dbs = NULL; uint32_t dbNum = 0; @@ -407,6 +479,11 @@ int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req hbGetQueryBasicInfo(connKey, req); + code = hbGetExpiredUserInfo(connKey, pCatalog, req); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + code = hbGetExpiredDBInfo(connKey, pCatalog, req); if (TSDB_CODE_SUCCESS != code) { return code; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 6235dcf895..f49d0b02f6 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -1145,31 +1145,47 @@ int32_t tDeserializeSGetUserAuthReq(void *buf, int32_t bufLen, SGetUserAuthReq * return 0; } -int32_t tSerializeSGetUserAuthRsp(void *buf, int32_t bufLen, SGetUserAuthRsp *pRsp) { - SCoder encoder = {0}; - tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); - - if (tStartEncode(&encoder) < 0) return -1; - if (tEncodeCStr(&encoder, pRsp->user) < 0) return -1; - if (tEncodeI8(&encoder, pRsp->superAuth) < 0) return -1; +int32_t tSerializeSGetUserAuthRspImpl(SCoder *pEncoder, SGetUserAuthRsp *pRsp) { + if (tEncodeCStr(pEncoder, pRsp->user) < 0) return -1; + if (tEncodeI8(pEncoder, pRsp->superAuth) < 0) return -1; + if (tEncodeI32(pEncoder, pRsp->version) < 0) return -1; + int32_t numOfCreatedDbs = taosHashGetSize(pRsp->createdDbs); int32_t numOfReadDbs = taosHashGetSize(pRsp->readDbs); int32_t numOfWriteDbs = taosHashGetSize(pRsp->writeDbs); - if (tEncodeI32(&encoder, numOfReadDbs) < 0) return -1; - if (tEncodeI32(&encoder, numOfWriteDbs) < 0) return -1; + if (tEncodeI32(pEncoder, numOfCreatedDbs) < 0) return -1; + if (tEncodeI32(pEncoder, numOfReadDbs) < 0) return -1; + if (tEncodeI32(pEncoder, numOfWriteDbs) < 0) return -1; - char *db = taosHashIterate(pRsp->readDbs, NULL); + char *db = taosHashIterate(pRsp->createdDbs, NULL); while (db != NULL) { - if (tEncodeCStr(&encoder, db) < 0) return -1; + if (tEncodeCStr(pEncoder, db) < 0) return -1; + db = taosHashIterate(pRsp->createdDbs, db); + } + + db = taosHashIterate(pRsp->readDbs, NULL); + while (db != NULL) { + if (tEncodeCStr(pEncoder, db) < 0) return -1; db = taosHashIterate(pRsp->readDbs, db); } db = taosHashIterate(pRsp->writeDbs, NULL); while (db != NULL) { - if (tEncodeCStr(&encoder, db) < 0) return -1; + if (tEncodeCStr(pEncoder, db) < 0) return -1; db = taosHashIterate(pRsp->writeDbs, db); } + return 0; +} + +int32_t tSerializeSGetUserAuthRsp(void *buf, int32_t bufLen, SGetUserAuthRsp *pRsp) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + + if (tSerializeSGetUserAuthRspImpl(&encoder, pRsp) < 0) return -1; + tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -1177,39 +1193,58 @@ int32_t tSerializeSGetUserAuthRsp(void *buf, int32_t bufLen, SGetUserAuthRsp *pR return tlen; } -int32_t tDeserializeSGetUserAuthRsp(void *buf, int32_t bufLen, SGetUserAuthRsp *pRsp) { - pRsp->readDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - pRsp->writeDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); +int32_t tDeserializeSGetUserAuthRspImpl(SCoder *pDecoder, SGetUserAuthRsp *pRsp) { + pRsp->createdDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + pRsp->readDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + pRsp->writeDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); if (pRsp->readDbs == NULL || pRsp->writeDbs == NULL) { return -1; } - SCoder decoder = {0}; - tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); - - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeCStrTo(&decoder, pRsp->user) < 0) return -1; - if (tDecodeI8(&decoder, &pRsp->superAuth) < 0) return -1; + if (tDecodeCStrTo(pDecoder, pRsp->user) < 0) return -1; + if (tDecodeI8(pDecoder, &pRsp->superAuth) < 0) return -1; + if (tDecodeI32(pDecoder, &pRsp->version) < 0) return -1; + int32_t numOfCreatedDbs = 0; int32_t numOfReadDbs = 0; int32_t numOfWriteDbs = 0; - if (tDecodeI32(&decoder, &numOfReadDbs) < 0) return -1; - if (tDecodeI32(&decoder, &numOfWriteDbs) < 0) return -1; + if (tDecodeI32(pDecoder, &numOfCreatedDbs) < 0) return -1; + if (tDecodeI32(pDecoder, &numOfReadDbs) < 0) return -1; + if (tDecodeI32(pDecoder, &numOfWriteDbs) < 0) return -1; + + for (int32_t i = 0; i < numOfCreatedDbs; ++i) { + char db[TSDB_DB_FNAME_LEN] = {0}; + if (tDecodeCStrTo(pDecoder, db) < 0) return -1; + int32_t len = strlen(db) + 1; + taosHashPut(pRsp->createdDbs, db, len, db, len); + } for (int32_t i = 0; i < numOfReadDbs; ++i) { char db[TSDB_DB_FNAME_LEN] = {0}; - if (tDecodeCStrTo(&decoder, db) < 0) return -1; + if (tDecodeCStrTo(pDecoder, db) < 0) return -1; int32_t len = strlen(db) + 1; taosHashPut(pRsp->readDbs, db, len, db, len); } for (int32_t i = 0; i < numOfWriteDbs; ++i) { char db[TSDB_DB_FNAME_LEN] = {0}; - if (tDecodeCStrTo(&decoder, db) < 0) return -1; + if (tDecodeCStrTo(pDecoder, db) < 0) return -1; int32_t len = strlen(db) + 1; taosHashPut(pRsp->writeDbs, db, len, db, len); } + return 0; +} + + +int32_t tDeserializeSGetUserAuthRsp(void *buf, int32_t bufLen, SGetUserAuthRsp *pRsp) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + + if (tDeserializeSGetUserAuthRspImpl(&decoder, pRsp) < 0) return -1; + tEndDecode(&decoder); tCoderClear(&decoder); @@ -1217,6 +1252,7 @@ int32_t tDeserializeSGetUserAuthRsp(void *buf, int32_t bufLen, SGetUserAuthRsp * } void tFreeSGetUserAuthRsp(SGetUserAuthRsp *pRsp) { + taosHashCleanup(pRsp->createdDbs); taosHashCleanup(pRsp->readDbs); taosHashCleanup(pRsp->writeDbs); } @@ -2036,6 +2072,62 @@ void tFreeSUseDbBatchRsp(SUseDbBatchRsp *pRsp) { taosArrayDestroy(pRsp->pArray); } +int32_t tSerializeSUserAuthBatchRsp(void* buf, int32_t bufLen, SUserAuthBatchRsp* pRsp){ + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + + int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); + if (tEncodeI32(&encoder, numOfBatch) < 0) return -1; + for (int32_t i = 0; i < numOfBatch; ++i) { + SGetUserAuthRsp *pUserAuthRsp = taosArrayGet(pRsp->pArray, i); + if (tSerializeSGetUserAuthRspImpl(&encoder, pUserAuthRsp) < 0) return -1; + } + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSUserAuthBatchRsp(void* buf, int32_t bufLen, SUserAuthBatchRsp* pRsp){ + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + + int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); + if (tDecodeI32(&decoder, &numOfBatch) < 0) return -1; + + pRsp->pArray = taosArrayInit(numOfBatch, sizeof(SGetUserAuthRsp)); + if (pRsp->pArray == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + for (int32_t i = 0; i < numOfBatch; ++i) { + SGetUserAuthRsp rsp = {0}; + if (tDeserializeSGetUserAuthRspImpl(&decoder, &rsp) < 0) return -1; + taosArrayPush(pRsp->pArray, &rsp); + } + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + +void tFreeSUserAuthBatchRsp(SUserAuthBatchRsp* pRsp){ + int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); + for (int32_t i = 0; i < numOfBatch; ++i) { + SGetUserAuthRsp *pUserAuthRsp = taosArrayGet(pRsp->pArray, i); + tFreeSGetUserAuthRsp(pUserAuthRsp); + } + + taosArrayDestroy(pRsp->pArray); +} + + int32_t tSerializeSDbCfgReq(void *buf, int32_t bufLen, SDbCfgReq *pReq) { SCoder encoder = {0}; tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index d516b0bf26..0ec8e5bd29 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -255,6 +255,7 @@ typedef struct { int64_t updateTime; int8_t superUser; int32_t acctId; + int32_t authVersion; SHashObj* readDbs; SHashObj* writeDbs; } SUserObj; diff --git a/source/dnode/mnode/impl/inc/mndUser.h b/source/dnode/mnode/impl/inc/mndUser.h index b3eb7f2f95..2140d0fa67 100644 --- a/source/dnode/mnode/impl/inc/mndUser.h +++ b/source/dnode/mnode/impl/inc/mndUser.h @@ -29,6 +29,7 @@ void mndReleaseUser(SMnode *pMnode, SUserObj *pUser); // for trans test SSdbRaw *mndUserActionEncode(SUserObj *pUser); +int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_t numOfUses, void **ppRsp, int32_t *pRspLen); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 59d07fd4aa..2de337537f 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -403,6 +403,16 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb SKv *kv = pIter; switch (kv->key) { + case HEARTBEAT_KEY_USER_AUTHINFO: { + void * rspMsg = NULL; + int32_t rspLen = 0; + mndValidateUserAuthInfo(pMnode, kv->value, kv->valueLen / sizeof(SUserAuthVersion), &rspMsg, &rspLen); + if (rspMsg && rspLen > 0) { + SKv kv1 = {.key = HEARTBEAT_KEY_USER_AUTHINFO, .valueLen = rspLen, .value = rspMsg}; + taosArrayPush(hbRsp.info, &kv1); + } + break; + } case HEARTBEAT_KEY_DBINFO: { void * rspMsg = NULL; int32_t rspLen = 0; diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 5e15bdeb43..d2a9151167 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -451,13 +451,16 @@ static int32_t mndProcessAlterUserReq(SNodeMsg *pReq) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; } + newUser.authVersion++; } else if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_READ_DB) { if (taosHashRemove(newUser.readDbs, alterReq.dbname, len) != 0) { terrno = TSDB_CODE_MND_DB_NOT_EXIST; goto _OVER; } + newUser.authVersion++; } else if (alterReq.alterType == TSDB_ALTER_USER_CLEAR_READ_DB) { taosHashClear(newUser.readDbs); + newUser.authVersion++; } else if (alterReq.alterType == TSDB_ALTER_USER_ADD_WRITE_DB) { if (pDb == NULL) { terrno = TSDB_CODE_MND_DB_NOT_EXIST; @@ -467,13 +470,16 @@ static int32_t mndProcessAlterUserReq(SNodeMsg *pReq) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; } + newUser.authVersion++; } else if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_WRITE_DB) { if (taosHashRemove(newUser.writeDbs, alterReq.dbname, len) != 0) { terrno = TSDB_CODE_MND_DB_NOT_EXIST; goto _OVER; } + newUser.authVersion++; } else if (alterReq.alterType == TSDB_ALTER_USER_CLEAR_WRITE_DB) { taosHashClear(newUser.writeDbs); + newUser.authVersion++; } else { terrno = TSDB_CODE_MND_INVALID_ALTER_OPER; goto _OVER; @@ -576,6 +582,36 @@ _OVER: return code; } +static int32_t mndSetUserAuthRsp(SMnode *pMnode, SUserObj *pUser, SGetUserAuthRsp *pRsp) { + memcpy(pRsp->user, pUser->user, TSDB_USER_LEN); + pRsp->superAuth = pUser->superUser; + pRsp->version = pUser->authVersion; + pRsp->readDbs = mndDupDbHash(pUser->readDbs); + pRsp->writeDbs = mndDupDbHash(pUser->writeDbs); + pRsp->createdDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + if (NULL == pRsp->createdDbs) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; + while (1) { + SDbObj *pDb = NULL; + pIter = sdbFetch(pSdb, SDB_DB, pIter, (void **)&pDb); + if (pIter == NULL) break; + + if (strcmp(pDb->createUser, pUser->user) == 0) { + int32_t len = strlen(pDb->name) + 1; + taosHashPut(pRsp->createdDbs, pDb->name, len, pDb->name, len); + } + + sdbRelease(pSdb, pDb); + } + + return 0; +} + static int32_t mndProcessGetUserAuthReq(SNodeMsg *pReq) { SMnode *pMnode = pReq->pNode; int32_t code = -1; @@ -596,25 +632,9 @@ static int32_t mndProcessGetUserAuthReq(SNodeMsg *pReq) { goto _OVER; } - memcpy(authRsp.user, pUser->user, TSDB_USER_LEN); - authRsp.superAuth = pUser->superUser; - authRsp.readDbs = mndDupDbHash(pUser->readDbs); - authRsp.writeDbs = mndDupDbHash(pUser->writeDbs); - - SSdb *pSdb = pMnode->pSdb; - void *pIter = NULL; - while (1) { - SDbObj *pDb = NULL; - pIter = sdbFetch(pSdb, SDB_DB, pIter, (void **)&pDb); - if (pIter == NULL) break; - - if (strcmp(pDb->createUser, pUser->user) == 0) { - int32_t len = strlen(pDb->name) + 1; - taosHashPut(authRsp.readDbs, pDb->name, len, pDb->name, len); - taosHashPut(authRsp.writeDbs, pDb->name, len, pDb->name, len); - } - - sdbRelease(pSdb, pDb); + code = mndSetUserAuthRsp(pMnode, pUser, &authRsp); + if (code) { + goto _OVER; } int32_t contLen = tSerializeSGetUserAuthRsp(NULL, 0, &authRsp); @@ -631,6 +651,7 @@ static int32_t mndProcessGetUserAuthReq(SNodeMsg *pReq) { code = 0; _OVER: + mndReleaseUser(pMnode, pUser); tFreeSGetUserAuthRsp(&authRsp); @@ -681,3 +702,72 @@ static void mndCancelGetNextUser(SMnode *pMnode, void *pIter) { SSdb *pSdb = pMnode->pSdb; sdbCancelFetch(pSdb, pIter); } + +int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_t numOfUses, void **ppRsp, int32_t *pRspLen) { + SUserAuthBatchRsp batchRsp = {0}; + batchRsp.pArray = taosArrayInit(numOfUses, sizeof(SGetUserAuthRsp)); + if (batchRsp.pArray == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + int32_t code = 0; + for (int32_t i = 0; i < numOfUses; ++i) { + SUserObj *pUser = mndAcquireUser(pMnode, pUsers[i].user); + if (pUser == NULL) { + mError("user:%s, failed to auth user since %s", pUsers[i].user, terrstr()); + continue; + } + + if (pUser->authVersion <= pUsers[i].version) { + mndReleaseUser(pMnode, pUser); + continue; + } + + SGetUserAuthRsp rsp = {0}; + code = mndSetUserAuthRsp(pMnode, pUser, &rsp); + if (code) { + mndReleaseUser(pMnode, pUser); + tFreeSGetUserAuthRsp(&rsp); + goto _OVER; + } + + + taosArrayPush(batchRsp.pArray, &rsp); + mndReleaseUser(pMnode, pUser); + } + + if (taosArrayGetSize(batchRsp.pArray) <= 0) { + *ppRsp = NULL; + *pRspLen = 0; + + tFreeSUserAuthBatchRsp(&batchRsp); + return 0; + } + + int32_t rspLen = tSerializeSUserAuthBatchRsp(NULL, 0, &batchRsp); + void *pRsp = taosMemoryMalloc(rspLen); + if (pRsp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + tFreeSUserAuthBatchRsp(&batchRsp); + return -1; + } + tSerializeSUserAuthBatchRsp(pRsp, rspLen, &batchRsp); + + *ppRsp = pRsp; + *pRspLen = rspLen; + + tFreeSUserAuthBatchRsp(&batchRsp); + return 0; + +_OVER: + + *ppRsp = NULL; + *pRspLen = 0; + + tFreeSUserAuthBatchRsp(&batchRsp); + return code; +} + + + diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index d6aa03aecb..3e8528e3d9 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -185,7 +185,7 @@ typedef struct SCtgRemoveTblMsg { typedef struct SCtgUpdateUserMsg { SCatalog* pCtg; SGetUserAuthRsp userAuth; -} SCtgUpdateTblMsg; +} SCtgUpdateUserMsg; typedef struct SCtgMetaAction { diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 39a784d00c..f485f85809 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -24,6 +24,7 @@ int32_t ctgActUpdateTbl(SCtgMetaAction *action); int32_t ctgActRemoveDB(SCtgMetaAction *action); int32_t ctgActRemoveStb(SCtgMetaAction *action); int32_t ctgActRemoveTbl(SCtgMetaAction *action); +int32_t ctgActUpdateUser(SCtgMetaAction *action); extern SCtgDebug gCTGDebug; SCatalogMgmt gCtgMgmt = {0}; @@ -382,6 +383,7 @@ int32_t ctgPushUpdateUserMsgInQueue(SCatalog* pCtg, SGetUserAuthRsp *pAuth, bool _return: + tFreeSGetUserAuthRsp(pAuth); taosMemoryFreeClear(msg); CTG_RET(code); @@ -925,7 +927,7 @@ int32_t ctgGetTableTypeFromCache(SCatalog* pCtg, const char* dbFName, const char return TSDB_CODE_SUCCESS; } -int32_t ctgGetUserDbAuthFromCache(SCatalog* pCtg, const char* user, const char* dbFName, bool *inCache, int32_t *auth) { +int32_t ctgChkAuthFromCache(SCatalog* pCtg, const char* user, const char* dbFName, AUTH_TYPE type, bool *inCache, bool *pass) { if (NULL == pCtg->userCache) { ctgDebug("empty user auth cache, user:%s", user); goto _return; @@ -943,23 +945,23 @@ int32_t ctgGetUserDbAuthFromCache(SCatalog* pCtg, const char* user, const char* CTG_CACHE_STAT_ADD(userHitNum, 1); if (pUser->superUser) { - CTG_FLAG_SET(auth, USER_AUTH_ALL); + *pass = true; return TSDB_CODE_SUCCESS; } CTG_LOCK(CTG_READ, &pUser->lock); if (pUser->createdDbs && taosHashGet(pUser->createdDbs, dbFName, strlen(dbFName))) { - CTG_FLAG_SET(auth, USER_AUTH_ALL); + *pass = true; CTG_UNLOCK(CTG_READ, &pUser->lock); return TSDB_CODE_SUCCESS; } - if (pUser->readDbs && taosHashGet(pUser->readDbs, dbFName, strlen(dbFName))) { - CTG_FLAG_SET(auth, USER_AUTH_READ); + if (pUser->readDbs && taosHashGet(pUser->readDbs, dbFName, strlen(dbFName)) && type == AUTH_TYPE_READ) { + *pass = true; } - if (pUser->writeDbs && taosHashGet(pUser->writeDbs, dbFName, strlen(dbFName))) { - CTG_FLAG_SET(auth, USER_AUTH_WRITE); + if (pUser->writeDbs && taosHashGet(pUser->writeDbs, dbFName, strlen(dbFName)) && type == AUTH_TYPE_WRITE) { + *pass = true; } CTG_UNLOCK(CTG_READ, &pUser->lock); @@ -2067,12 +2069,13 @@ _return: CTG_RET(code); } -int32_t ctgGetUserDbAuth(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* user, const char* dbFName, int32_t* auth) { +int32_t ctgChkAuth(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* user, const char* dbFName, AUTH_TYPE type, bool *pass) { bool inCache = false; int32_t code = 0; - *auth = 0; - CTG_ERR_RET(ctgGetUserDbAuthFromCache(pCtg, user, dbFName, &inCache, auth)); + *pass = false; + + CTG_ERR_RET(ctgChkAuthFromCache(pCtg, user, dbFName, type, &inCache, pass)); if (inCache) { return TSDB_CODE_SUCCESS; @@ -2082,21 +2085,21 @@ int32_t ctgGetUserDbAuth(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, con CTG_ERR_RET(ctgGetUserDbAuthFromMnode(pCtg, pRpc, pMgmtEps, user, &authRsp)); if (authRsp.superAuth) { - CTG_FLAG_SET(auth, USER_AUTH_ALL); + *pass = true; goto _return; } if (authRsp.createdDbs && taosHashGet(authRsp.createdDbs, dbFName, strlen(dbFName))) { - CTG_FLAG_SET(auth, USER_AUTH_ALL); + *pass = true; goto _return; } - if (authRsp.readDbs && taosHashGet(authRsp.readDbs, dbFName, strlen(dbFName))) { - CTG_FLAG_SET(auth, USER_AUTH_READ); + if (authRsp.readDbs && taosHashGet(authRsp.readDbs, dbFName, strlen(dbFName)) && type == AUTH_TYPE_READ) { + *pass = true; } - if (authRsp.writeDbs && taosHashGet(authRsp.writeDbs, dbFName, strlen(dbFName))) { - CTG_FLAG_SET(auth, USER_AUTH_WRITE); + if (authRsp.writeDbs && taosHashGet(authRsp.writeDbs, dbFName, strlen(dbFName)) && type == AUTH_TYPE_WRITE) { + *pass = true; } _return: @@ -3050,6 +3053,35 @@ int32_t catalogGetExpiredDBs(SCatalog* pCtg, SDbVgVersion **dbs, uint32_t *num) CTG_API_LEAVE(ctgMetaRentGet(&pCtg->dbRent, (void **)dbs, num, sizeof(SDbVgVersion))); } +int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion **users, uint32_t *num) { + CTG_API_ENTER(); + + if (NULL == pCtg || NULL == users || NULL == num) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + + *num = taosHashGetSize(pCtg->userCache); + if (*num > 0) { + *users = taosMemoryCalloc(*num, sizeof(SUserAuthVersion)); + if (NULL == *users) { + ctgError("calloc %d userAuthVersion failed", *num); + CTG_API_LEAVE(TSDB_CODE_OUT_OF_MEMORY); + } + } + + uint32_t i = 0; + SCtgUserAuth *pAuth = taosHashIterate(pCtg->userCache, NULL); + while (pAuth != NULL) { + void *key = taosHashGetKey(pAuth, NULL); + strncpy((*users)[i].user, key, sizeof((*users)[i].user)); + (*users)[i].version = pAuth->version; + pAuth = taosHashIterate(pCtg->userCache, pAuth); + } + + CTG_API_LEAVE(TSDB_CODE_SUCCESS); +} + + int32_t catalogGetDBCfg(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName, SDbCfgInfo* pDbCfg) { CTG_API_ENTER(); @@ -3094,21 +3126,31 @@ _return: CTG_API_LEAVE(code); } -int32_t catalogGetUserDbAuth(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* user, const char* dbFName, int32_t* auth) { +int32_t catalogChkAuth(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* user, const char* dbFName, AUTH_TYPE type, bool *pass) { CTG_API_ENTER(); - if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == user || NULL == dbFName || NULL == auth) { + if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == user || NULL == dbFName || NULL == pass) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } int32_t code = 0; - CTG_ERR_JRET(ctgGetUserDbAuth(pCtg, pRpc, pMgmtEps, user, dbFName, auth)); + CTG_ERR_JRET(ctgChkAuth(pCtg, pRpc, pMgmtEps, user, dbFName, type, pass)); _return: CTG_API_LEAVE(code); } +int32_t catalogUpdateUserAuthInfo(SCatalog* pCtg, SGetUserAuthRsp* pAuth) { + CTG_API_ENTER(); + + if (NULL == pCtg || NULL == pAuth) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + + CTG_API_LEAVE(ctgPushUpdateUserMsgInQueue(pCtg, pAuth, false)); +} + void catalogDestroy(void) { qInfo("start to destroy catalog"); From cbd55c47a5e2caaf02cc18bb5d9835e1440a52c8 Mon Sep 17 00:00:00 2001 From: cpwu Date: Fri, 6 May 2022 14:21:10 +0800 Subject: [PATCH 15/26] fix case --- tests/system-test/2-query/join.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/system-test/2-query/join.py b/tests/system-test/2-query/join.py index ebaa2e8b9f..015538ec1e 100644 --- a/tests/system-test/2-query/join.py +++ b/tests/system-test/2-query/join.py @@ -102,8 +102,10 @@ class TDTestCase: if len(tblist) == 2: if "ct1" in tblist or "t1" in tblist: self.__join_current(sql, checkrows) + elif where_condition: + self.__join_current(sql, checkrows + 5 ) else: - self.__join_current(sql, checkrows + 2 ) if where_condition else self.__join_current(sql, checkrows + 5 ) + self.__join_current(sql, checkrows + 2 ) if len(tblist) > 2 or len(tblist) < 1: tdSql.error(sql=sql) From f2eca15fe2f716fc8ea3a8086252db5d97c00105 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 6 May 2022 14:27:49 +0800 Subject: [PATCH 16/26] enh(rpc): validate fqdn --- tools/shell/src/shellNettest.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tools/shell/src/shellNettest.c b/tools/shell/src/shellNettest.c index c8ec31c48b..dfdb4951ad 100644 --- a/tools/shell/src/shellNettest.c +++ b/tools/shell/src/shellNettest.c @@ -21,7 +21,7 @@ static void shellWorkAsClient() { SRpcInit rpcInit = {0}; SEpSet epSet = {.inUse = 0, .numOfEps = 1}; SRpcMsg rpcRsp = {0}; - void *clientRpc = NULL; + void * clientRpc = NULL; char pass[TSDB_PASSWORD_LEN + 1] = {0}; taosEncryptPass_c((uint8_t *)("_pwd"), strlen("_pwd"), pass); @@ -111,11 +111,16 @@ void shellNettestHandler(int32_t signum, void *sigInfo, void *context) { shellEx static void shellWorkAsServer() { SShellArgs *pArgs = &shell.args; + // char fqdn[TSDB_FQDN_LEN] = {0}; + /// tstrncpy(fqdn, pArgs->host, TSDB_FQDN_LEN); + // strtok(fqdn, ":"); + if (pArgs->port == 0) { pArgs->port = tsServerPort; } SRpcInit rpcInit = {0}; + memcpy(rpcInit.localFqdn, tsLocalFqdn, strlen(tsLocalFqdn)); rpcInit.localPort = pArgs->port; rpcInit.label = "CHK"; rpcInit.numOfThreads = tsNumOfRpcThreads; From 78c617a9f1a4d800b591593dc0cb37447fcad2a6 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 6 May 2022 14:30:07 +0800 Subject: [PATCH 17/26] enh(rpc): validate fqdn --- tools/shell/src/shellNettest.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tools/shell/src/shellNettest.c b/tools/shell/src/shellNettest.c index dfdb4951ad..9b68beb4e1 100644 --- a/tools/shell/src/shellNettest.c +++ b/tools/shell/src/shellNettest.c @@ -111,10 +111,6 @@ void shellNettestHandler(int32_t signum, void *sigInfo, void *context) { shellEx static void shellWorkAsServer() { SShellArgs *pArgs = &shell.args; - // char fqdn[TSDB_FQDN_LEN] = {0}; - /// tstrncpy(fqdn, pArgs->host, TSDB_FQDN_LEN); - // strtok(fqdn, ":"); - if (pArgs->port == 0) { pArgs->port = tsServerPort; } From 6e27e06da2acb846bdd5d04e09887fa21a6f14f9 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 6 May 2022 14:30:40 +0800 Subject: [PATCH 18/26] refactor: delete false positive logs --- source/dnode/mnode/impl/src/mndTrans.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index f3438bd0dc..43e2735984 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -1050,7 +1050,7 @@ static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pA static int32_t mndTransExecuteRedoActions(SMnode *pMnode, STrans *pTrans) { int32_t code = mndTransExecuteActions(pMnode, pTrans, pTrans->redoActions); - if (code != 0) { + if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("failed to execute redoActions since %s", terrstr()); } return code; @@ -1058,7 +1058,7 @@ static int32_t mndTransExecuteRedoActions(SMnode *pMnode, STrans *pTrans) { static int32_t mndTransExecuteUndoActions(SMnode *pMnode, STrans *pTrans) { int32_t code = mndTransExecuteActions(pMnode, pTrans, pTrans->undoActions); - if (code != 0) { + if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("failed to execute undoActions since %s", terrstr()); } return code; From 699c8a0461a89b1e343290500f51c2b9d1f8b103 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 6 May 2022 14:42:06 +0800 Subject: [PATCH 19/26] enh(rpc): validate fqdn --- source/libs/transport/src/trans.c | 1 + source/libs/transport/src/transSrv.c | 2 +- tools/shell/src/shellNettest.c | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index f8277c575e..846cf6f967 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -51,6 +51,7 @@ void* rpcOpen(const SRpcInit* pInit) { ip = taosGetIpv4FromFqdn(pInit->localFqdn); if (ip == 0xFFFFFFFF) { tError("invalid fqdn: %s", pInit->localFqdn); + terrno = TSDB_CODE_RPC_FQDN_ERROR; taosMemoryFree(pRpc); return NULL; } diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index e1b0871135..ad3f520210 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -841,7 +841,7 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, } } if (false == taosValidIpAndPort(srv->ip, srv->port)) { - tError("failed to bind, reason: %s", strerror(errno)); + tError("failed to bind, reason: %s", terrstr()); goto End; } if (false == addHandleToAcceptloop(srv)) { diff --git a/tools/shell/src/shellNettest.c b/tools/shell/src/shellNettest.c index 9b68beb4e1..345b85d896 100644 --- a/tools/shell/src/shellNettest.c +++ b/tools/shell/src/shellNettest.c @@ -127,7 +127,7 @@ static void shellWorkAsServer() { void *serverRpc = rpcOpen(&rpcInit); if (serverRpc == NULL) { - printf("failed to init net test server since %s", terrstr()); + printf("failed to init net test server since %s\n", terrstr()); } else { printf("network test server is initialized, port:%u\n", pArgs->port); taosSetSignal(SIGTERM, shellNettestHandler); From c4b4c008c3bc2b8ece89ecf31e6f70b83d8f178a Mon Sep 17 00:00:00 2001 From: cpwu Date: Fri, 6 May 2022 15:00:23 +0800 Subject: [PATCH 20/26] fix case --- tests/system-test/2-query/join.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/system-test/2-query/join.py b/tests/system-test/2-query/join.py index 015538ec1e..57e433804d 100644 --- a/tests/system-test/2-query/join.py +++ b/tests/system-test/2-query/join.py @@ -103,9 +103,9 @@ class TDTestCase: if "ct1" in tblist or "t1" in tblist: self.__join_current(sql, checkrows) elif where_condition: - self.__join_current(sql, checkrows + 5 ) - else: self.__join_current(sql, checkrows + 2 ) + else: + self.__join_current(sql, checkrows + 5 ) if len(tblist) > 2 or len(tblist) < 1: tdSql.error(sql=sql) @@ -154,7 +154,7 @@ class TDTestCase: self.__join_check(tblist_1, 1) tdLog.printNoPrefix(f"==========current sql condition check in {tblist_1} over==========") tblist_2 = ["ct2", "ct4"] - self.__join_check(tblist_2, self.rows - 3) + self.__join_check(tblist_2, self.rows) tdLog.printNoPrefix(f"==========current sql condition check in {tblist_2} over==========") tblist_3 = ["t1", "ct4"] self.__join_check(tblist_3, 1) From 1436796da72e4775fcce02cefef2984fcb1b30c6 Mon Sep 17 00:00:00 2001 From: cpwu Date: Fri, 6 May 2022 15:20:26 +0800 Subject: [PATCH 21/26] fix case --- tests/system-test/2-query/join.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/2-query/join.py b/tests/system-test/2-query/join.py index 57e433804d..61c0ecbf2c 100644 --- a/tests/system-test/2-query/join.py +++ b/tests/system-test/2-query/join.py @@ -102,7 +102,7 @@ class TDTestCase: if len(tblist) == 2: if "ct1" in tblist or "t1" in tblist: self.__join_current(sql, checkrows) - elif where_condition: + elif where_condition or "null" in groups: self.__join_current(sql, checkrows + 2 ) else: self.__join_current(sql, checkrows + 5 ) From a1b3d0254aea1313a40801b32f8262b56beaf524 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Fri, 6 May 2022 15:36:10 +0800 Subject: [PATCH 22/26] feat: extract row version from trow --- include/common/trow.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/common/trow.h b/include/common/trow.h index 8b1d612433..f87ea1f009 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -141,7 +141,7 @@ typedef struct { /// row total length uint32_t len; /// row version - uint64_t ver; + // uint64_t ver; /// the inline data, maybe a tuple or a k-v tuple char data[]; } STSRow; @@ -176,7 +176,7 @@ typedef struct { #define TD_ROW_DATA(r) ((r)->data) #define TD_ROW_LEN(r) ((r)->len) #define TD_ROW_KEY(r) ((r)->ts) -#define TD_ROW_VER(r) ((r)->ver) +// #define TD_ROW_VER(r) ((r)->ver) #define TD_ROW_KEY_ADDR(r) (r) // N.B. If without STSchema, getExtendedRowSize() is used to get the rowMaxBytes and From 71188483ec62d4a9f4885700bfa19d14f849b4ad Mon Sep 17 00:00:00 2001 From: slzhou Date: Fri, 6 May 2022 15:42:25 +0800 Subject: [PATCH 23/26] fix: create function bufSize default value 0 --- source/dnode/mnode/impl/src/mndFunc.c | 2 +- source/dnode/mnode/impl/test/func/func.cpp | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index b9331e6e03..3ac2951b6f 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -314,7 +314,7 @@ static int32_t mndProcessCreateFuncReq(SNodeMsg *pReq) { goto _OVER; } - if (createReq.bufSize <= 0 || createReq.bufSize > TSDB_FUNC_BUF_SIZE) { + if (createReq.bufSize < 0 || createReq.bufSize > TSDB_FUNC_BUF_SIZE) { terrno = TSDB_CODE_MND_INVALID_FUNC_BUFSIZE; goto _OVER; } diff --git a/source/dnode/mnode/impl/test/func/func.cpp b/source/dnode/mnode/impl/test/func/func.cpp index 0473fa375e..c8f832160b 100644 --- a/source/dnode/mnode/impl/test/func/func.cpp +++ b/source/dnode/mnode/impl/test/func/func.cpp @@ -24,6 +24,7 @@ class MndTestFunc : public ::testing::Test { void SetCode(SCreateFuncReq* pReq, const char* pCode, int32_t size); void SetComment(SCreateFuncReq* pReq, const char* pComment); + void SetBufSize(SCreateFuncReq* pReq, int32_t size); }; Testbase MndTestFunc::test; @@ -40,6 +41,10 @@ void MndTestFunc::SetComment(SCreateFuncReq* pReq, const char* pComment) { strcpy(pReq->pComment, pComment); } +void MndTestFunc::SetBufSize(SCreateFuncReq* pReq, int32_t size) { + pReq->bufSize = size; +} + TEST_F(MndTestFunc, 01_Show_Func) { test.SendShowReq(TSDB_MGMT_TABLE_FUNC, "user_functions", ""); EXPECT_EQ(test.GetShowRows(), 0); @@ -96,6 +101,7 @@ TEST_F(MndTestFunc, 02_Create_Func) { strcpy(createReq.name, "f1"); SetCode(&createReq, "code1", 6); SetComment(&createReq, "comment1"); + SetBufSize(&createReq, -1); int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); From f4f4bccdbb29c9c7bca06b57727acaa90a7ccdd5 Mon Sep 17 00:00:00 2001 From: cpwu Date: Fri, 6 May 2022 15:45:16 +0800 Subject: [PATCH 24/26] fix case --- tests/system-test/2-query/join.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/2-query/join.py b/tests/system-test/2-query/join.py index 61c0ecbf2c..83a33cc6ea 100644 --- a/tests/system-test/2-query/join.py +++ b/tests/system-test/2-query/join.py @@ -102,7 +102,7 @@ class TDTestCase: if len(tblist) == 2: if "ct1" in tblist or "t1" in tblist: self.__join_current(sql, checkrows) - elif where_condition or "null" in groups: + elif where_condition or "not null" in groups: self.__join_current(sql, checkrows + 2 ) else: self.__join_current(sql, checkrows + 5 ) From b6b3247d41b270dd83b89bccb8356147fdb5c97a Mon Sep 17 00:00:00 2001 From: cpwu Date: Fri, 6 May 2022 16:26:13 +0800 Subject: [PATCH 25/26] fix case --- tests/system-test/2-query/join.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/2-query/join.py b/tests/system-test/2-query/join.py index 83a33cc6ea..2d91f754af 100644 --- a/tests/system-test/2-query/join.py +++ b/tests/system-test/2-query/join.py @@ -102,7 +102,7 @@ class TDTestCase: if len(tblist) == 2: if "ct1" in tblist or "t1" in tblist: self.__join_current(sql, checkrows) - elif where_condition or "not null" in groups: + elif where_condition or "not null" in group_condition: self.__join_current(sql, checkrows + 2 ) else: self.__join_current(sql, checkrows + 5 ) From 43fd14759b82a919e6bd3cb5e516364faf8eef34 Mon Sep 17 00:00:00 2001 From: cpwu Date: Fri, 6 May 2022 16:35:45 +0800 Subject: [PATCH 26/26] fix case --- tests/system-test/2-query/join.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/system-test/2-query/join.py b/tests/system-test/2-query/join.py index 2d91f754af..a39bc21946 100644 --- a/tests/system-test/2-query/join.py +++ b/tests/system-test/2-query/join.py @@ -104,6 +104,8 @@ class TDTestCase: self.__join_current(sql, checkrows) elif where_condition or "not null" in group_condition: self.__join_current(sql, checkrows + 2 ) + elif group_condition: + self.__join_current(sql, checkrows + 3 ) else: self.__join_current(sql, checkrows + 5 ) if len(tblist) > 2 or len(tblist) < 1: