diff --git a/cmake/cmake.version b/cmake/cmake.version index 27e0f1d68a..0e4785f643 100644 --- a/cmake/cmake.version +++ b/cmake/cmake.version @@ -2,7 +2,7 @@ IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "3.1.2.0.alpha") + SET(TD_VER_NUMBER "3.2.0.0.alpha") ENDIF () IF (DEFINED VERCOMPATIBLE) diff --git a/docs/en/12-taos-sql/24-show.md b/docs/en/12-taos-sql/24-show.md index 7a58343f24..9e2897160c 100644 --- a/docs/en/12-taos-sql/24-show.md +++ b/docs/en/12-taos-sql/24-show.md @@ -384,7 +384,7 @@ Shows information about all vgroups in the current database. ## SHOW VNODES ```sql -SHOW VNODES {dnode_id | dnode_endpoint}; +SHOW VNODES [ON DNODE dnode_id]; ``` Shows information about all vnodes in the system or about the vnodes for a specified dnode. diff --git a/docs/zh/12-taos-sql/24-show.md b/docs/zh/12-taos-sql/24-show.md index b7ca9493d4..197a7c78d6 100644 --- a/docs/zh/12-taos-sql/24-show.md +++ b/docs/zh/12-taos-sql/24-show.md @@ -327,7 +327,7 @@ SHOW [db_name.]VGROUPS; ## SHOW VNODES ```sql -SHOW VNODES {dnode_id | dnode_endpoint}; +SHOW VNODES [ON DNODE dnode_id]; ``` 显示当前系统中所有 VNODE 或某个 DNODE 的 VNODE 的信息。 diff --git a/include/client/taos.h b/include/client/taos.h index a640e12ac5..dac8e61542 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -125,6 +125,7 @@ typedef enum { typedef enum { TAOS_NOTIFY_PASSVER = 0, + TAOS_NOTIFY_WHITELIST_VER = 1 } TAOS_NOTIFY_TYPE; #define RET_MSG_LENGTH 1024 @@ -236,6 +237,9 @@ DLL_EXPORT void taos_set_hb_quit(int8_t quitByKill); DLL_EXPORT int taos_set_notify_cb(TAOS *taos, __taos_notify_fn_t fp, void *param, int type); +typedef void (*__taos_async_whitelist_fn_t)(void *param, int code, TAOS *taos, int numOfWhiteLists, uint64_t* pWhiteLists); +DLL_EXPORT void taos_fetch_whitelist_a(TAOS *taos, __taos_async_whitelist_fn_t fp, void *param); + /* --------------------------schemaless INTERFACE------------------------------- */ DLL_EXPORT TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision); diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 244894b59b..b2aff6fd7e 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -28,6 +28,7 @@ typedef struct SBlockOrderInfo { bool nullFirst; int32_t order; int32_t slotId; + void* compFn; SColumnInfoData* pColData; } SBlockOrderInfo; @@ -82,6 +83,15 @@ static FORCE_INLINE bool colDataIsNull_s(const SColumnInfoData* pColumnInfoData, } } +static FORCE_INLINE bool colDataIsNull_t(const SColumnInfoData* pColumnInfoData, uint32_t row, bool isVarType) { + if (!pColumnInfoData->hasNull) return false; + if (isVarType) { + return colDataIsNull_var(pColumnInfoData, row); + } else { + return pColumnInfoData->nullbitmap ? colDataIsNull_f(pColumnInfoData->nullbitmap, row) : false; + } +} + static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, uint32_t totalRows, uint32_t row, SColumnDataAgg* pColAgg) { if (!pColumnInfoData->hasNull) { @@ -210,6 +220,10 @@ double blockDataGetSerialRowSize(const SSDataBlock* pBlock); size_t blockDataGetSerialMetaSize(uint32_t numOfCols); int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo); +/** + * @brief find how many rows already in order start from first row + */ +int32_t blockDataGetSortedRows(SSDataBlock* pDataBlock, SArray* pOrderInfo); int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows, bool clearPayload); int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows); diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 2be6aca97e..a0af96853e 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -849,6 +849,7 @@ typedef struct { int32_t authVer; char sVer[TSDB_VERSION_LEN]; char sDetailVer[128]; + int64_t whiteListVer; } SConnectRsp; int32_t tSerializeSConnectRsp(void* buf, int32_t bufLen, SConnectRsp* pRsp); @@ -875,11 +876,17 @@ typedef struct { int32_t tSerializeSDropUserReq(void* buf, int32_t bufLen, SDropUserReq* pReq); int32_t tDeserializeSDropUserReq(void* buf, int32_t bufLen, SDropUserReq* pReq); -typedef struct SIpV4Range { - uint32_t ip; - uint32_t mask; +typedef struct SIpV4Range{ + uint32_t ip; + uint32_t mask; } SIpV4Range; +typedef struct { + int32_t num; + SIpV4Range pIpRange[]; +} SIpWhiteList; + +SIpWhiteList* cloneIpWhiteList(SIpWhiteList* pIpWhiteList); typedef struct { int8_t createType; int8_t superUser; // denote if it is a super user or not @@ -895,6 +902,30 @@ int32_t tSerializeSCreateUserReq(void* buf, int32_t bufLen, SCreateUserReq* pReq int32_t tDeserializeSCreateUserReq(void* buf, int32_t bufLen, SCreateUserReq* pReq); void tFreeSCreateUserReq(SCreateUserReq* pReq); +typedef struct { + int64_t ver; + char user[TSDB_USER_LEN]; + int32_t numOfRange; + SIpV4Range* pIpRanges; +} SUpdateUserIpWhite; +typedef struct { + int64_t ver; + int numOfUser; + SUpdateUserIpWhite* pUserIpWhite; +} SUpdateIpWhite; + +int32_t tSerializeSUpdateIpWhite(void* buf, int32_t bufLen, SUpdateIpWhite* pReq); +int32_t tDeserializeSUpdateIpWhite(void* buf, int32_t bufLen, SUpdateIpWhite* pReq); +void tFreeSUpdateIpWhiteReq(SUpdateIpWhite* pReq); +SUpdateIpWhite* cloneSUpdateIpWhiteReq(SUpdateIpWhite* pReq); + +typedef struct { + int64_t ipWhiteVer; +} SRetrieveIpWhiteReq; + +int32_t tSerializeRetrieveIpWhite(void* buf, int32_t bufLen, SRetrieveIpWhiteReq* pReq); +int32_t tDeserializeRetrieveIpWhite(void* buf, int32_t bufLen, SRetrieveIpWhiteReq* pReq); + typedef struct { int8_t alterType; int8_t superUser; @@ -935,12 +966,30 @@ typedef struct { SHashObj* readTbs; SHashObj* writeTbs; SHashObj* useDbs; + int64_t whiteListVer; } SGetUserAuthRsp; int32_t tSerializeSGetUserAuthRsp(void* buf, int32_t bufLen, SGetUserAuthRsp* pRsp); int32_t tDeserializeSGetUserAuthRsp(void* buf, int32_t bufLen, SGetUserAuthRsp* pRsp); void tFreeSGetUserAuthRsp(SGetUserAuthRsp* pRsp); +typedef struct { + char user[TSDB_USER_LEN]; +} SGetUserWhiteListReq; + +int32_t tSerializeSGetUserWhiteListReq(void* buf, int32_t bufLen, SGetUserWhiteListReq* pReq); +int32_t tDeserializeSGetUserWhiteListReq(void* buf, int32_t bufLen, SGetUserWhiteListReq* pReq); + +typedef struct { + char user[TSDB_USER_LEN]; + int32_t numWhiteLists; + SIpV4Range* pWhiteLists; +} SGetUserWhiteListRsp; + +int32_t tSerializeSGetUserWhiteListRsp(void* buf, int32_t bufLen, SGetUserWhiteListRsp* pRsp); +int32_t tDeserializeSGetUserWhiteListRsp(void* buf, int32_t bufLen, SGetUserWhiteListRsp* pRsp); +void tFreeSGetUserWhiteListRsp(SGetUserWhiteListRsp* pRsp); + /* * for client side struct, only column id, type, bytes are necessary * But for data in vnode side, we need all the following information. @@ -1455,6 +1504,7 @@ typedef struct { SClusterCfg clusterCfg; SArray* pVloads; // array of SVnodeLoad int32_t statusSeq; + int64_t ipWhiteVer; } SStatusReq; int32_t tSerializeSStatusReq(void* buf, int32_t bufLen, SStatusReq* pReq); @@ -1485,6 +1535,7 @@ typedef struct { SDnodeCfg dnodeCfg; SArray* pDnodeEps; // Array of SDnodeEp int32_t statusSeq; + int64_t ipWhiteVer; } SStatusRsp; int32_t tSerializeSStatusRsp(void* buf, int32_t bufLen, SStatusRsp* pRsp); diff --git a/include/common/tmsgcb.h b/include/common/tmsgcb.h index 9b709272b2..311bffb7da 100644 --- a/include/common/tmsgcb.h +++ b/include/common/tmsgcb.h @@ -52,6 +52,7 @@ typedef struct { void* data; void* mgmt; void* clientRpc; + void* serverRpc; PutToQueueFp putToQueueFp; GetQueueSizeFp qsizeFp; SendReqFp sendReqFp; diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index fb2c780724..dd9a1af67c 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -177,6 +177,8 @@ enum { // WARN: new msg should be appended to segment tail TD_DEF_MSG_TYPE(TDMT_MND_UPTIME_TIMER, "uptime-timer", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_TMQ_LOST_CONSUMER_CLEAR, "lost-consumer-clear", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_STREAM_HEARTBEAT, "stream-heartbeat", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_RETRIEVE_IP_WHITE, "retrieve-ip-white", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_GET_USER_WHITELIST, "get-user-whitelist", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_MAX_MSG, "mnd-max", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_BALANCE_VGROUP_LEADER, "balance-vgroup-leader", NULL, NULL) diff --git a/include/common/ttokendef.h b/include/common/ttokendef.h index 944880511b..f1c22f750f 100644 --- a/include/common/ttokendef.h +++ b/include/common/ttokendef.h @@ -366,6 +366,7 @@ #define TK_NK_BIN 605 // bin format data 0b111 #define TK_BATCH_SCAN 606 #define TK_NO_BATCH_SCAN 607 +#define TK_SORT_FOR_GROUP 608 #define TK_NK_NIL 65535 diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 9d2247f479..885e9e5a30 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -136,6 +136,8 @@ typedef struct SAggLogicNode { bool hasTimeLineFunc; bool onlyHasKeepOrderFunc; bool hasGroupKeyOptimized; + bool isGroupTb; + bool isPartTb; // true if partition keys has tbname } SAggLogicNode; typedef struct SProjectLogicNode { @@ -221,6 +223,7 @@ typedef struct SMergeLogicNode { int32_t srcGroupId; bool groupSort; bool ignoreGroupId; + bool inputWithGroupId; } SMergeLogicNode; typedef enum EWindowType { @@ -263,6 +266,7 @@ typedef struct SWindowLogicNode { int8_t igExpired; int8_t igCheckUpdate; EWindowAlgorithm windowAlgo; + bool isPartTb; } SWindowLogicNode; typedef struct SFillLogicNode { @@ -279,8 +283,9 @@ typedef struct SSortLogicNode { SLogicNode node; SNodeList* pSortKeys; bool groupSort; - int64_t maxRows; bool skipPKSortOpt; + bool calcGroupId; + bool excludePkCol; // exclude PK ts col when calc group id } SSortLogicNode; typedef struct SPartitionLogicNode { @@ -288,6 +293,10 @@ typedef struct SPartitionLogicNode { SNodeList* pPartitionKeys; SNodeList* pTags; SNode* pSubtable; + + bool needBlockOutputTsOrder; // if true, partition output block will have ts order maintained + int32_t pkTsColId; + uint64_t pkTsColTbId; } SPartitionLogicNode; typedef enum ESubplanType { @@ -527,6 +536,7 @@ typedef struct SMergePhysiNode { int32_t srcGroupId; bool groupSort; bool ignoreGroupId; + bool inputWithGroupId; } SMergePhysiNode; typedef struct SWindowPhysiNode { @@ -603,6 +613,8 @@ typedef struct SSortPhysiNode { SNodeList* pExprs; // these are expression list of order_by_clause and parameter expression of aggregate function SNodeList* pSortKeys; // element is SOrderByExprNode, and SOrderByExprNode::pExpr is SColumnNode SNodeList* pTargets; + bool calcGroupId; + bool excludePkCol; } SSortPhysiNode; typedef SSortPhysiNode SGroupSortPhysiNode; @@ -612,6 +624,9 @@ typedef struct SPartitionPhysiNode { SNodeList* pExprs; // these are expression list of partition_by_clause SNodeList* pPartitionKeys; SNodeList* pTargets; + + bool needBlockOutputTsOrder; + int32_t tsSlotId; } SPartitionPhysiNode; typedef struct SStreamPartitionPhysiNode { diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index d7d45c57ad..bb6713dc83 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -119,6 +119,7 @@ typedef struct SLeftValueNode { typedef enum EHintOption { HINT_NO_BATCH_SCAN = 1, HINT_BATCH_SCAN, + HINT_SORT_FOR_GROUP, } EHintOption; typedef struct SHintNode { diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index e5955aad54..83dbf2c4af 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -37,6 +37,7 @@ typedef struct { int64_t applyIndex; uint64_t applyTerm; char user[TSDB_USER_LEN]; + } SRpcConnInfo; typedef struct SRpcHandleInfo { @@ -60,6 +61,8 @@ typedef struct SRpcHandleInfo { STraceId traceId; SRpcConnInfo conn; + int8_t forbiddenIp; + } SRpcHandleInfo; typedef struct SRpcMsg { @@ -162,6 +165,11 @@ int rpcSendRequestWithCtx(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int rpcSendRecv(void *shandle, SEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp); int rpcSetDefaultAddr(void *thandle, const char *ip, const char *fqdn); void *rpcAllocHandle(); +void rpcSetIpWhite(void *thandl, void *arg); + +int32_t rpcUtilSIpRangeToStr(SIpV4Range *pRange, char *buf); + +int32_t rpcUtilSWhiteListToStr(SIpWhiteList *pWhiteList, char **ppBuf); #ifdef __cplusplus } diff --git a/include/util/taoserror.h b/include/util/taoserror.h index b64c015284..39bf2b5681 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -123,6 +123,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_INVALID_DATA_FMT TAOS_DEF_ERROR_CODE(0, 0x0132) #define TSDB_CODE_INVALID_CFG_VALUE TAOS_DEF_ERROR_CODE(0, 0x0133) +#define TSDB_CODE_IP_NOT_IN_WHITE_LIST TAOS_DEF_ERROR_CODE(0, 0x0134) + //client #define TSDB_CODE_TSC_INVALID_OPERATION TAOS_DEF_ERROR_CODE(0, 0x0200) #define TSDB_CODE_TSC_INVALID_QHANDLE TAOS_DEF_ERROR_CODE(0, 0x0201) @@ -245,6 +247,10 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_AUTH_FAILURE TAOS_DEF_ERROR_CODE(0, 0x0357) #define TSDB_CODE_MND_USER_NOT_AVAILABLE TAOS_DEF_ERROR_CODE(0, 0x0358) #define TSDB_CODE_MND_PRIVILEDGE_EXIST TAOS_DEF_ERROR_CODE(0, 0x0359) +#define TSDB_CODE_MND_USER_HOST_EXIST TAOS_DEF_ERROR_CODE(0, 0x035A) +#define TSDB_CODE_MND_USER_HOST_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x035B) +#define TSDB_CODE_MND_TOO_MANY_USER_HOST TAOS_DEF_ERROR_CODE(0, 0x035C) +#define TSDB_CODE_MND_USER_LOCAL_HOST_NOT_DROP TAOS_DEF_ERROR_CODE(0, 0x035D) // mnode-stable-part1 #define TSDB_CODE_MND_STB_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0360) diff --git a/include/util/tdef.h b/include/util/tdef.h index 3bfa136d3e..91133a05f8 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -184,7 +184,7 @@ typedef enum ELogicConditionType { #define TSDB_UNI_LEN 24 #define TSDB_USER_LEN TSDB_UNI_LEN -#define TSDB_POINTER_PRINT_BYTES 18 // 0x1122334455667788 +#define TSDB_POINTER_PRINT_BYTES 18 // 0x1122334455667788 // ACCOUNT is a 32 bit positive integer // this is the length of its string representation, including the terminator zero #define TSDB_ACCT_ID_LEN 11 @@ -202,6 +202,7 @@ typedef enum ELogicConditionType { #define TSDB_DB_NAME_LEN 65 #define TSDB_DB_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN + TSDB_NAME_DELIMITER_LEN) #define TSDB_PRIVILEDGE_CONDITION_LEN 48 * 1024 +#define TSDB_PRIVILEDGE_HOST_LEN 48 * 1024 #define TSDB_FUNC_NAME_LEN 65 #define TSDB_FUNC_COMMENT_LEN 1024 * 1024 @@ -417,7 +418,7 @@ typedef enum ELogicConditionType { #define TSDB_EXPLAIN_RESULT_ROW_SIZE (16 * 1024) #define TSDB_EXPLAIN_RESULT_COLUMN_NAME "QUERY_PLAN" -#define TSDB_MAX_FIELD_LEN 65519 // 16384:65519 +#define TSDB_MAX_FIELD_LEN 65519 // 16384:65519 #define TSDB_MAX_BINARY_LEN TSDB_MAX_FIELD_LEN // 16384-8:65519 #define TSDB_MAX_NCHAR_LEN TSDB_MAX_FIELD_LEN // 16384-8:65519 #define TSDB_MAX_GEOMETRY_LEN TSDB_MAX_FIELD_LEN // 16384-8:65519 diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index 408a5664a8..19efa7b169 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -33,17 +33,14 @@ adapterName="taosadapter" benchmarkName="taosBenchmark" dumpName="taosdump" demoName="taosdemo" -xname="taosx" clientName2="taos" serverName2="${clientName2}d" configFile2="${clientName2}.cfg" productName2="TDengine" emailName2="taosdata.com" -xname2="${clientName2}x" adapterName2="${clientName2}adapter" -explorerName="${clientName2}-explorer" benchmarkName2="${clientName2}Benchmark" demoName2="${clientName2}demo" dumpName2="${clientName2}dump" @@ -218,8 +215,6 @@ function install_bin() { ${csudo}rm -f ${bin_link_dir}/${demoName2} || : ${csudo}rm -f ${bin_link_dir}/${benchmarkName2} || : ${csudo}rm -f ${bin_link_dir}/${dumpName2} || : - ${csudo}rm -f ${bin_link_dir}/${xname2} || : - ${csudo}rm -f ${bin_link_dir}/${explorerName} || : ${csudo}rm -f ${bin_link_dir}/set_core || : ${csudo}rm -f ${bin_link_dir}/TDinsight.sh || : @@ -233,8 +228,6 @@ function install_bin() { [ -x ${install_main_dir}/bin/${benchmarkName2} ] && ${csudo}ln -sf ${install_main_dir}/bin/${benchmarkName2} ${bin_link_dir}/${demoName2} || : [ -x ${install_main_dir}/bin/${benchmarkName2} ] && ${csudo}ln -sf ${install_main_dir}/bin/${benchmarkName2} ${bin_link_dir}/${benchmarkName2} || : [ -x ${install_main_dir}/bin/${dumpName2} ] && ${csudo}ln -sf ${install_main_dir}/bin/${dumpName2} ${bin_link_dir}/${dumpName2} || : - [ -x ${install_main_dir}/bin/${xname2} ] && ${csudo}ln -sf ${install_main_dir}/bin/${xname2} ${bin_link_dir}/${xname2} || : - [ -x ${install_main_dir}/bin/${explorerName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${explorerName} ${bin_link_dir}/${explorerName} || : [ -x ${install_main_dir}/bin/TDinsight.sh ] && ${csudo}ln -sf ${install_main_dir}/bin/TDinsight.sh ${bin_link_dir}/TDinsight.sh || : if [ "$clientName2" == "${clientName}" ]; then [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || : @@ -703,26 +696,6 @@ function clean_service_on_systemd() { # if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then # ${csudo}rm -f ${service_config_dir}/${serverName2}.service # fi - x_service_config="${service_config_dir}/${xName2}.service" - if [ -e "$x_service_config" ]; then - if systemctl is-active --quiet ${xName2}; then - echo "${productName2} ${xName2} is running, stopping it..." - ${csudo}systemctl stop ${xName2} &>/dev/null || echo &>/dev/null - fi - ${csudo}systemctl disable ${xName2} &>/dev/null || echo &>/dev/null - ${csudo}rm -f ${x_service_config} - fi - - explorer_service_config="${service_config_dir}/${explorerName2}.service" - if [ -e "$explorer_service_config" ]; then - if systemctl is-active --quiet ${explorerName2}; then - echo "${productName2} ${explorerName2} is running, stopping it..." - ${csudo}systemctl stop ${explorerName2} &>/dev/null || echo &>/dev/null - fi - ${csudo}systemctl disable ${explorerName2} &>/dev/null || echo &>/dev/null - ${csudo}rm -f ${explorer_service_config} - ${csudo}rm -f /etc/${clientName2}/explorer.toml - fi } function install_service_on_systemd() { diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index aa7caaaba3..9448634c5b 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -135,6 +135,12 @@ typedef struct { __taos_notify_fn_t fp; } SPassInfo; +typedef struct { + int64_t ver; + void* param; + __taos_notify_fn_t fp; +} SWhiteListInfo; + typedef struct STscObj { char user[TSDB_USER_LEN]; char pass[TSDB_PASSWORD_LEN]; @@ -152,6 +158,7 @@ typedef struct STscObj { SAppInstInfo* pAppInfo; SHashObj* pRequests; SPassInfo passInfo; + SWhiteListInfo whiteListInfo; } STscObj; typedef struct STscDbg { diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 54e3a6ee48..e64f0d779c 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -116,6 +116,19 @@ static int32_t hbUpdateUserAuthInfo(SAppHbMgr *pAppHbMgr, SUserAuthBatchRsp *bat atomic_load_32(&passInfo->ver), pTscObj->id); } } + + if (pTscObj->whiteListInfo.fp) { + SWhiteListInfo *whiteListInfo = &pTscObj->whiteListInfo; + int64_t oldVer = atomic_load_64(&whiteListInfo->ver); + if (oldVer < pRsp->whiteListVer) { + atomic_store_64(&whiteListInfo->ver, pRsp->whiteListVer); + if (whiteListInfo->fp) { + (*whiteListInfo->fp)(whiteListInfo->param, &pRsp->whiteListVer, TAOS_NOTIFY_WHITELIST_VER); + } + tscDebug("update whitelist version of user %s from %"PRId64" to %"PRId64", tscRid:%" PRIi64, pRsp->user, oldVer, + atomic_load_64(&whiteListInfo->ver), pTscObj->id); + } + } releaseTscObj(pReq->connKey.tscRid); } } diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index ac7a6e6646..7902d6029e 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -140,6 +140,13 @@ int taos_set_notify_cb(TAOS *taos, __taos_notify_fn_t fp, void *param, int type) taosThreadMutexUnlock(&pObj->mutex); break; } + case TAOS_NOTIFY_WHITELIST_VER: { + taosThreadMutexLock(&pObj->mutex); + pObj->whiteListInfo.fp = fp; + pObj->whiteListInfo.param = param; + taosThreadMutexUnlock(&pObj->mutex); + break; + } default: { terrno = TSDB_CODE_INVALID_PARA; releaseTscObj(*(int64_t *)taos); @@ -151,6 +158,113 @@ int taos_set_notify_cb(TAOS *taos, __taos_notify_fn_t fp, void *param, int type) return 0; } +typedef struct SFetchWhiteListInfo{ + int64_t connId; + __taos_async_whitelist_fn_t userCbFn; + void* userParam; +} SFetchWhiteListInfo; + +int32_t fetchWhiteListCallbackFn(void* param, SDataBuf* pMsg, int32_t code) { + SFetchWhiteListInfo* pInfo = (SFetchWhiteListInfo*)param; + TAOS* taos = &pInfo->connId; + if (code != TSDB_CODE_SUCCESS) { + pInfo->userCbFn(pInfo->userParam, code, taos, 0, NULL); + taosMemoryFree(pMsg->pData); + taosMemoryFree(pMsg->pEpSet); + taosMemoryFree(pInfo); + return code; + } + + SGetUserWhiteListRsp wlRsp; + tDeserializeSGetUserWhiteListRsp(pMsg->pData, pMsg->len, &wlRsp); + + uint64_t* pWhiteLists = taosMemoryMalloc(wlRsp.numWhiteLists * sizeof(uint64_t)); + if (pWhiteLists == NULL) { + taosMemoryFree(pMsg->pData); + taosMemoryFree(pMsg->pEpSet); + taosMemoryFree(pInfo); + tFreeSGetUserWhiteListRsp(&wlRsp); + } + + for (int i = 0; i < wlRsp.numWhiteLists; ++i) { + pWhiteLists[i] = ((uint64_t)wlRsp.pWhiteLists[i].mask << 32) | wlRsp.pWhiteLists[i].ip; + } + + pInfo->userCbFn(pInfo->userParam, code, taos, wlRsp.numWhiteLists, pWhiteLists); + + taosMemoryFree(pWhiteLists); + taosMemoryFree(pMsg->pData); + taosMemoryFree(pMsg->pEpSet); + taosMemoryFree(pInfo); + tFreeSGetUserWhiteListRsp(&wlRsp); + return code; +} + +void taos_fetch_whitelist_a(TAOS *taos, __taos_async_whitelist_fn_t fp, void *param) { + if (NULL == taos) { + fp(param, TSDB_CODE_INVALID_PARA, taos, 0, NULL); + return; + } + + int64_t connId = *(int64_t*)taos; + + STscObj *pTsc = acquireTscObj(connId); + if (NULL == pTsc) { + fp(param, TSDB_CODE_TSC_DISCONNECTED, taos, 0, NULL); + return; + } + + SGetUserWhiteListReq req; + memcpy(req.user, pTsc->user, TSDB_USER_LEN); + int32_t msgLen = tSerializeSGetUserWhiteListReq(NULL, 0, &req); + void* pReq = taosMemoryMalloc(msgLen); + if (pReq == NULL) { + fp(param, TSDB_CODE_OUT_OF_MEMORY, taos, 0, NULL); + releaseTscObj(connId); + return; + } + + if (tSerializeSGetUserWhiteListReq(pReq, msgLen, &req) < 0) { + fp(param, TSDB_CODE_INVALID_PARA, taos, 0, NULL); + taosMemoryFree(pReq); + releaseTscObj(connId); + return; + } + + SFetchWhiteListInfo* pParam = taosMemoryMalloc(sizeof(SFetchWhiteListInfo)); + if (pParam == NULL) { + fp(param, TSDB_CODE_OUT_OF_MEMORY, taos, 0, NULL); + taosMemoryFree(pReq); + releaseTscObj(connId); + return; + } + + pParam->connId = connId; + pParam->userCbFn = fp; + pParam->userParam = param; + SMsgSendInfo* pSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); + if (pSendInfo == NULL) { + fp(param, TSDB_CODE_OUT_OF_MEMORY, taos, 0, NULL); + taosMemoryFree(pParam); + taosMemoryFree(pReq); + releaseTscObj(connId); + return; + } + + pSendInfo->msgInfo = (SDataBuf){.pData = pReq, .len = msgLen, .handle = NULL}; + pSendInfo->requestId = generateRequestId(); + pSendInfo->requestObjRefId = 0; + pSendInfo->param = pParam; + pSendInfo->fp = fetchWhiteListCallbackFn; + pSendInfo->msgType = TDMT_MND_GET_USER_WHITELIST; + + int64_t transportId = 0; + SEpSet epSet = getEpSet_s(&pTsc->pAppInfo->mgmtEp); + asyncSendMsgToServer(pTsc->pAppInfo->pTransporter, &epSet, &transportId, pSendInfo); + releaseTscObj(connId); + return; +} + void taos_close_internal(void *taos) { if (taos == NULL) { return; diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 9f9809b227..8027a61b8f 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -139,6 +139,7 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) { pTscObj->connType = connectRsp.connType; pTscObj->passInfo.ver = connectRsp.passVer; pTscObj->authVer = connectRsp.authVer; + pTscObj->whiteListInfo.ver = connectRsp.whiteListVer; hbRegisterConn(pTscObj->pAppInfo->pAppHbMgr, pTscObj->id, connectRsp.clusterId, connectRsp.connType); diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 66a498d15c..107da6830e 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -221,6 +221,7 @@ static const SSysDbTableSchema userUsersSchema[] = { {.name = "enable", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = false}, {.name = "sysinfo", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = false}, {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false}, + {.name = "allowed_host", .bytes = TSDB_PRIVILEDGE_HOST_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, }; GRANTS_SCHEMA; @@ -308,7 +309,7 @@ static const SSysDbTableSchema userUserPrivilegesSchema[] = { {.name = "privilege", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, {.name = "db_name", .bytes = TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, {.name = "table_name", .bytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, - {.name = "condition", .bytes = TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "condition", .bytes = TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, }; static const SSysTableMeta infosMeta[] = { @@ -326,7 +327,7 @@ static const SSysTableMeta infosMeta[] = { {TSDB_INS_TABLE_TAGS, userTagsSchema, tListLen(userTagsSchema), false}, {TSDB_INS_TABLE_COLS, userColsSchema, tListLen(userColsSchema), false}, // {TSDB_INS_TABLE_TABLE_DISTRIBUTED, userTblDistSchema, tListLen(userTblDistSchema)}, - {TSDB_INS_TABLE_USERS, userUsersSchema, tListLen(userUsersSchema), false}, + {TSDB_INS_TABLE_USERS, userUsersSchema, tListLen(userUsersSchema), true}, {TSDB_INS_TABLE_LICENCES, grantsSchema, tListLen(grantsSchema), true}, {TSDB_INS_TABLE_VGROUPS, vgroupsSchema, tListLen(vgroupsSchema), true}, {TSDB_INS_TABLE_CONFIGS, configSchema, tListLen(configSchema), false}, @@ -336,7 +337,7 @@ static const SSysTableMeta infosMeta[] = { {TSDB_INS_TABLE_STREAMS, streamSchema, tListLen(streamSchema), false}, {TSDB_INS_TABLE_STREAM_TASKS, streamTaskSchema, tListLen(streamTaskSchema), false}, {TSDB_INS_TABLE_VNODES, vnodesSchema, tListLen(vnodesSchema), true}, - {TSDB_INS_TABLE_USER_PRIVILEGES, userUserPrivilegesSchema, tListLen(userUserPrivilegesSchema), false}, + {TSDB_INS_TABLE_USER_PRIVILEGES, userUserPrivilegesSchema, tListLen(userUserPrivilegesSchema), true}, }; static const SSysDbTableSchema connectionsSchema[] = { diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 53646b84b3..330eb4ae30 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -933,7 +933,13 @@ int32_t dataBlockCompar(const void* p1, const void* p2, const void* param) { return 0; } } - __compar_fn_t fn = getKeyComparFunc(pColInfoData->info.type, pOrder->order); + + __compar_fn_t fn; + if (pOrder->compFn) { + fn = pOrder->compFn; + } else { + fn = getKeyComparFunc(pColInfoData->info.type, pOrder->order); + } int ret = fn(left1, right1); if (ret == 0) { @@ -1099,6 +1105,7 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) { for (int32_t i = 0; i < taosArrayGetSize(helper.orderInfo); ++i) { struct SBlockOrderInfo* pInfo = taosArrayGet(helper.orderInfo, i); pInfo->pColData = taosArrayGet(pDataBlock->pDataBlock, pInfo->slotId); + pInfo->compFn = getKeyComparFunc(pInfo->pColData->info.type, pInfo->order); } terrno = 0; @@ -2509,3 +2516,20 @@ void trimDataBlock(SSDataBlock* pBlock, int32_t totalRows, const bool* pBoolList int32_t blockGetEncodeSize(const SSDataBlock* pBlock) { return blockDataGetSerialMetaSize(taosArrayGetSize(pBlock->pDataBlock)) + blockDataGetSize(pBlock); } + +int32_t blockDataGetSortedRows(SSDataBlock* pDataBlock, SArray* pOrderInfo) { + if (!pDataBlock || !pOrderInfo) return 0; + for (int32_t i = 0; i < taosArrayGetSize(pOrderInfo); ++i) { + SBlockOrderInfo* pOrder = taosArrayGet(pOrderInfo, i); + pOrder->pColData = taosArrayGet(pDataBlock->pDataBlock, pOrder->slotId); + pOrder->compFn = getKeyComparFunc(pOrder->pColData->info.type, pOrder->order); + } + SSDataBlockSortHelper sortHelper = {.orderInfo = pOrderInfo, .pDataBlock = pDataBlock}; + int32_t rowIdx = 0, nextRowIdx = 1; + for (; rowIdx < pDataBlock->info.rows && nextRowIdx < pDataBlock->info.rows; ++rowIdx, ++nextRowIdx) { + if (dataBlockCompar(&nextRowIdx, &rowIdx, &sortHelper) < 0) { + break; + } + } + return nextRowIdx; +} diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 49b1e4f6f4..b4ce55b71d 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -1116,6 +1116,8 @@ int32_t tSerializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) { if (tEncodeI64(&encoder, reserved) < 0) return -1; if (tEncodeI64(&encoder, reserved) < 0) return -1; } + + if (tEncodeI64(&encoder, pReq->ipWhiteVer) < 0) return -1; tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -1226,6 +1228,10 @@ int32_t tDeserializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) { if (tDecodeI64(&decoder, &reserved) < 0) return -1; } } + if (!tDecodeIsEnd(&decoder)) { + if (tDecodeI64(&decoder, &pReq->ipWhiteVer) < 0) return -1; + } + tEndDecode(&decoder); tDecoderClear(&decoder); return 0; @@ -1258,6 +1264,8 @@ int32_t tSerializeSStatusRsp(void *buf, int32_t bufLen, SStatusRsp *pRsp) { } if (tEncodeI32(&encoder, pRsp->statusSeq) < 0) return -1; + + if (tEncodeI64(&encoder, pRsp->ipWhiteVer) < 0) return -1; tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -1300,6 +1308,10 @@ int32_t tDeserializeSStatusRsp(void *buf, int32_t bufLen, SStatusRsp *pRsp) { } if (tDecodeI32(&decoder, &pRsp->statusSeq) < 0) return -1; + + if (!tDecodeIsEnd(&decoder)) { + if (tDecodeI64(&decoder, &pRsp->ipWhiteVer) < 0) return -1; + } tEndDecode(&decoder); tDecoderClear(&decoder); return 0; @@ -1371,6 +1383,15 @@ int32_t tDeserializeSDropUserReq(void *buf, int32_t bufLen, SDropUserReq *pReq) return 0; } +SIpWhiteList *cloneIpWhiteList(SIpWhiteList *pIpWhiteList) { + if (pIpWhiteList == NULL) return NULL; + + int32_t sz = sizeof(SIpWhiteList) + pIpWhiteList->num * sizeof(SIpV4Range); + SIpWhiteList *pNew = taosMemoryCalloc(1, sz); + memcpy(pNew, pIpWhiteList, sz); + return pNew; +} + int32_t tSerializeSCreateUserReq(void *buf, int32_t bufLen, SCreateUserReq *pReq) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -1414,12 +1435,123 @@ int32_t tDeserializeSCreateUserReq(void *buf, int32_t bufLen, SCreateUserReq *pR } tEndDecode(&decoder); + tDecoderClear(&decoder); return 0; } void tFreeSCreateUserReq(SCreateUserReq *pReq) { taosMemoryFree(pReq->pIpRanges); } +int32_t tSerializeSUpdateIpWhite(void *buf, int32_t bufLen, SUpdateIpWhite *pReq) { + // impl later + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI64(&encoder, pReq->ver) < 0) return -1; + if (tEncodeI32(&encoder, pReq->numOfUser) < 0) return -1; + for (int i = 0; i < pReq->numOfUser; i++) { + SUpdateUserIpWhite *pUser = &(pReq->pUserIpWhite[i]); + + if (tEncodeI64(&encoder, pUser->ver) < 0) return -1; + if (tEncodeCStr(&encoder, pUser->user) < 0) return -1; + if (tEncodeI32(&encoder, pUser->numOfRange) < 0) return -1; + for (int j = 0; j < pUser->numOfRange; j++) { + SIpV4Range *pRange = &pUser->pIpRanges[j]; + if (tEncodeU32(&encoder, pRange->ip) < 0) return -1; + if (tEncodeU32(&encoder, pRange->mask) < 0) return -1; + } + } + + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + return tlen; +} +int32_t tDeserializeSUpdateIpWhite(void *buf, int32_t bufLen, SUpdateIpWhite *pReq) { + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, bufLen); + + if (tStartDecode(&decoder) < 0) return -1; + // impl later + if (tDecodeI64(&decoder, &pReq->ver) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->numOfUser) < 0) return -1; + + pReq->pUserIpWhite = taosMemoryCalloc(1, sizeof(SUpdateUserIpWhite) * pReq->numOfUser); + for (int i = 0; i < pReq->numOfUser; i++) { + SUpdateUserIpWhite *pUserWhite = &pReq->pUserIpWhite[i]; + if (tDecodeI64(&decoder, &pUserWhite->ver) < 0) return -1; + if (tDecodeCStrTo(&decoder, pUserWhite->user) < 0) return -1; + if (tDecodeI32(&decoder, &pUserWhite->numOfRange) < 0) return -1; + + pUserWhite->pIpRanges = taosMemoryCalloc(1, pUserWhite->numOfRange * sizeof(SIpV4Range)); + for (int j = 0; j < pUserWhite->numOfRange; j++) { + SIpV4Range *pRange = &pUserWhite->pIpRanges[j]; + if (tDecodeU32(&decoder, &pRange->ip) < 0) return -1; + if (tDecodeU32(&decoder, &pRange->mask) < 0) return -1; + } + } + + tEndDecode(&decoder); + tDecoderClear(&decoder); + return 0; +} +void tFreeSUpdateIpWhiteReq(SUpdateIpWhite *pReq) { + for (int i = 0; i < pReq->numOfUser; i++) { + SUpdateUserIpWhite *pUserWhite = &pReq->pUserIpWhite[i]; + taosMemoryFree(pUserWhite->pIpRanges); + } + taosMemoryFree(pReq->pUserIpWhite); + // impl later + return; +} +SUpdateIpWhite *cloneSUpdateIpWhiteReq(SUpdateIpWhite *pReq) { + SUpdateIpWhite *pClone = taosMemoryCalloc(1, sizeof(SUpdateIpWhite)); + + pClone->numOfUser = pReq->numOfUser; + pClone->ver = pReq->ver; + pClone->pUserIpWhite = taosMemoryCalloc(1, sizeof(SUpdateUserIpWhite) * pReq->numOfUser); + + for (int i = 0; i < pReq->numOfUser; i++) { + SUpdateUserIpWhite *pNew = &pClone->pUserIpWhite[i]; + SUpdateUserIpWhite *pOld = &pReq->pUserIpWhite[i]; + + pNew->ver = pOld->ver; + memcpy(pNew->user, pOld->user, strlen(pOld->user)); + pNew->numOfRange = pOld->numOfRange; + + int32_t sz = pOld->numOfRange * sizeof(SIpV4Range); + pNew->pIpRanges = taosMemoryCalloc(1, sz); + memcpy(pNew->pIpRanges, pOld->pIpRanges, sz); + } + return pClone; +} +int32_t tSerializeRetrieveIpWhite(void *buf, int32_t bufLen, SRetrieveIpWhiteReq *pReq) { + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + if (tStartEncode(&encoder) < 0) return -1; + + if (tEncodeI64(&encoder, pReq->ipWhiteVer) < 0) { + return -1; + } + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + return tlen; +} +int32_t tDeserializeRetrieveIpWhite(void *buf, int32_t bufLen, SRetrieveIpWhiteReq *pReq) { + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, bufLen); + + if (tStartDecode(&decoder) < 0) return -1; + // impl later + if (tDecodeI64(&decoder, &pReq->ipWhiteVer) < 0) return -1; + tEndDecode(&decoder); + tDecoderClear(&decoder); + return 0; +} + int32_t tSerializeSAlterUserReq(void *buf, int32_t bufLen, SAlterUserReq *pReq) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -1599,7 +1731,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp) // since 3.0.7.0 if (tEncodeI32(pEncoder, pRsp->passVer) < 0) return -1; - + if (tEncodeI64(pEncoder, pRsp->whiteListVer) < 0) return -1; return 0; } @@ -1731,6 +1863,11 @@ int32_t tDeserializeSGetUserAuthRspImpl(SDecoder *pDecoder, SGetUserAuthRsp *pRs } else { pRsp->passVer = 0; } + if (!tDecodeIsEnd(pDecoder)) { + if (tDecodeI64(pDecoder, &pRsp->whiteListVer) < 0) goto _err; + } else { + pRsp->whiteListVer = 0; + } } return 0; _err: @@ -1769,6 +1906,73 @@ void tFreeSGetUserAuthRsp(SGetUserAuthRsp *pRsp) { taosHashCleanup(pRsp->useDbs); } +int32_t tSerializeSGetUserWhiteListReq(void *buf, int32_t bufLen, SGetUserWhiteListReq *pReq) { + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->user) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSGetUserWhiteListReq(void *buf, int32_t bufLen, SGetUserWhiteListReq *pReq) { + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, bufLen); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1; + tEndDecode(&decoder); + + tDecoderClear(&decoder); + return 0; +} + +int32_t tSerializeSGetUserWhiteListRsp(void* buf, int32_t bufLen, SGetUserWhiteListRsp* pRsp) { + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pRsp->user) < 0) return -1; + if (tEncodeI32(&encoder, pRsp->numWhiteLists) < 0) return -1; + for (int i = 0; i < pRsp->numWhiteLists; ++i) { + if (tEncodeU32(&encoder, pRsp->pWhiteLists[i].ip) < 0) return -1; + if (tEncodeU32(&encoder, pRsp->pWhiteLists[i].mask) < 0) return -1; + } + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSGetUserWhiteListRsp(void* buf, int32_t bufLen, SGetUserWhiteListRsp* pRsp) { + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, bufLen); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeCStrTo(&decoder, pRsp->user) < 0) return -1; + + if (tDecodeI32(&decoder, &pRsp->numWhiteLists) < 0) return -1; + pRsp->pWhiteLists = taosMemoryMalloc(pRsp->numWhiteLists * sizeof(SIpV4Range)); + if (pRsp->pWhiteLists == NULL) return -1; + for (int32_t i = 0; i < pRsp->numWhiteLists; ++i) { + if (tDecodeU32(&decoder, &(pRsp->pWhiteLists[i].ip)) < 0) return -1; + if (tDecodeU32(&decoder, &(pRsp->pWhiteLists[i].mask)) < 0) return -1; + } + + tEndDecode(&decoder); + tDecoderClear(&decoder); + return 0; +} + +void tFreeSGetUserWhiteListRsp(SGetUserWhiteListRsp* pRsp) { + taosMemoryFree(pRsp->pWhiteLists); +} + int32_t tSerializeSCreateDropMQSNodeReq(void *buf, int32_t bufLen, SMCreateQnodeReq *pReq) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -4235,6 +4439,7 @@ int32_t tSerializeSConnectRsp(void *buf, int32_t bufLen, SConnectRsp *pRsp) { if (tEncodeCStr(&encoder, pRsp->sDetailVer) < 0) return -1; if (tEncodeI32(&encoder, pRsp->passVer) < 0) return -1; if (tEncodeI32(&encoder, pRsp->authVer) < 0) return -1; + if (tEncodeI64(&encoder, pRsp->whiteListVer) < 0) return -1; tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -4271,6 +4476,11 @@ int32_t tDeserializeSConnectRsp(void *buf, int32_t bufLen, SConnectRsp *pRsp) { pRsp->authVer = 0; } + if (!tDecodeIsEnd(&decoder)) { + if (tDecodeI64(&decoder, &pRsp->whiteListVer) < 0) return -1; + } else { + pRsp->whiteListVer = 0; + } tEndDecode(&decoder); tDecoderClear(&decoder); diff --git a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h index 6c40fa808f..35947c001b 100644 --- a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h +++ b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h @@ -23,22 +23,22 @@ extern "C" { #endif typedef struct SDnodeMgmt { - SDnodeData *pData; - SMsgCb msgCb; - const char *path; - const char *name; - TdThread statusThread; - TdThread monitorThread; - TdThread crashReportThread; - SSingleWorker mgmtWorker; - ProcessCreateNodeFp processCreateNodeFp; + SDnodeData *pData; + SMsgCb msgCb; + const char *path; + const char *name; + TdThread statusThread; + TdThread monitorThread; + TdThread crashReportThread; + SSingleWorker mgmtWorker; + ProcessCreateNodeFp processCreateNodeFp; ProcessAlterNodeTypeFp processAlterNodeTypeFp; - ProcessDropNodeFp processDropNodeFp; - SendMonitorReportFp sendMonitorReportFp; - GetVnodeLoadsFp getVnodeLoadsFp; - GetMnodeLoadsFp getMnodeLoadsFp; - GetQnodeLoadsFp getQnodeLoadsFp; - int32_t statusSeq; + ProcessDropNodeFp processDropNodeFp; + SendMonitorReportFp sendMonitorReportFp; + GetVnodeLoadsFp getVnodeLoadsFp; + GetMnodeLoadsFp getMnodeLoadsFp; + GetQnodeLoadsFp getQnodeLoadsFp; + int32_t statusSeq; } SDnodeMgmt; // dmHandle.c diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index e6b7c0cb68..a56387079f 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -30,7 +30,36 @@ static void dmUpdateDnodeCfg(SDnodeMgmt *pMgmt, SDnodeCfg *pCfg) { taosThreadRwlockUnlock(&pMgmt->pData->lock); } } +static void dmMayShouldUpdateIpWhiteList(SDnodeMgmt *pMgmt, int64_t ver) { + dDebug("ip-white-list on dnode ver: %" PRId64 ", status ver: %" PRId64 "", pMgmt->pData->ipWhiteVer, ver); + if (pMgmt->pData->ipWhiteVer == ver) { + if (ver == 0) { + dDebug("disable ip-white-list on dnode ver: %" PRId64 ", status ver: %" PRId64 "", pMgmt->pData->ipWhiteVer, ver); + rpcSetIpWhite(pMgmt->msgCb.serverRpc, NULL); + // pMgmt->ipWhiteVer = ver; + } + return; + } + int64_t oldVer = pMgmt->pData->ipWhiteVer; + // pMgmt->ipWhiteVer = ver; + SRetrieveIpWhiteReq req = {.ipWhiteVer = oldVer}; + int32_t contLen = tSerializeRetrieveIpWhite(NULL, 0, &req); + void *pHead = rpcMallocCont(contLen); + tSerializeRetrieveIpWhite(pHead, contLen, &req); + + SRpcMsg rpcMsg = {.pCont = pHead, + .contLen = contLen, + .msgType = TDMT_MND_RETRIEVE_IP_WHITE, + .info.ahandle = (void *)0x9527, + .info.refId = 0, + .info.noResp = 0}; + SEpSet epset = {0}; + + dmGetMnodeEpSet(pMgmt->pData, &epset); + + rpcSendRequest(pMgmt->msgCb.clientRpc, &epset, &rpcMsg, NULL); +} static void dmProcessStatusRsp(SDnodeMgmt *pMgmt, SRpcMsg *pRsp) { const STraceId *trace = &pRsp->info.traceId; dGTrace("status rsp received from mnode, statusSeq:%d code:0x%x", pMgmt->statusSeq, pRsp->code); @@ -55,6 +84,7 @@ static void dmProcessStatusRsp(SDnodeMgmt *pMgmt, SRpcMsg *pRsp) { dmUpdateDnodeCfg(pMgmt, &statusRsp.dnodeCfg); dmUpdateEps(pMgmt->pData, statusRsp.pDnodeEps); } + dmMayShouldUpdateIpWhiteList(pMgmt, statusRsp.ipWhiteVer); } tFreeSStatusRsp(&statusRsp); } @@ -111,6 +141,7 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { pMgmt->statusSeq++; req.statusSeq = pMgmt->statusSeq; + req.ipWhiteVer = pMgmt->pData->ipWhiteVer; int32_t contLen = tSerializeSStatusReq(NULL, 0, &req); void *pHead = rpcMallocCont(contLen); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c index 09783a5ea9..e754b08daf 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c @@ -55,6 +55,7 @@ static int32_t dmOpenMgmt(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { pMgmt->getMnodeLoadsFp = pInput->getMnodeLoadsFp; pMgmt->getQnodeLoadsFp = pInput->getQnodeLoadsFp; + // pMgmt->pData->ipWhiteVer = 0; if (dmStartWorker(pMgmt) != 0) { return -1; } diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index 4c43326959..42eb89b5fb 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -161,6 +161,9 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_MND_PAUSE_STREAM, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_RESUME_STREAM, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + + if (dmSetMgmtHandle(pArray, TDMT_MND_RETRIEVE_IP_WHITE, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_GET_USER_WHITELIST, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_GET_INDEX, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_GET_TABLE_INDEX, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_TMQ_CREATE_TOPIC, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index 665f86034d..dc48ff71f8 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -63,6 +63,29 @@ static void dmConvertErrCode(tmsg_t msgType) { terrno = TSDB_CODE_VND_STOPPED; } } +static void dmUpdateRpcIpWhite(SDnodeData *pData, void *pTrans, SRpcMsg *pRpc) { + SUpdateIpWhite ipWhite = {0}; // aosMemoryCalloc(1, sizeof(SUpdateIpWhite)); + tDeserializeSUpdateIpWhite(pRpc->pCont, pRpc->contLen, &ipWhite); + + rpcSetIpWhite(pTrans, &ipWhite); + pData->ipWhiteVer = ipWhite.ver; + + tFreeSUpdateIpWhiteReq(&ipWhite); + + rpcFreeCont(pRpc->pCont); +} +static bool dmIsForbiddenIp(int8_t forbidden, char *user, uint32_t clientIp) { + if (forbidden) { + SIpV4Range range = {.ip = clientIp, .mask = 32}; + char buf[36] = {0}; + + rpcUtilSIpRangeToStr(&range, buf); + dError("User:%s host:%s not in ip white list", user, buf); + return true; + } else { + return false; + } +} static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) { SDnodeTrans *pTrans = &pDnode->trans; int32_t code = -1; @@ -81,6 +104,12 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) { goto _OVER; } + bool isForbidden = dmIsForbiddenIp(pRpc->info.forbiddenIp, pRpc->info.conn.user, pRpc->info.conn.clientIp); + if (isForbidden) { + terrno = TSDB_CODE_IP_NOT_IN_WHITE_LIST; + goto _OVER; + } + switch (pRpc->msgType) { case TDMT_DND_NET_TEST: dmProcessNetTestReq(pDnode, pRpc); @@ -97,6 +126,10 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) { dmSetMnodeEpSet(&pDnode->data, pEpSet); } break; + case TDMT_MND_RETRIEVE_IP_WHITE_RSP: { + dmUpdateRpcIpWhite(&pDnode->data, pTrans->serverRpc, pRpc); + return; + } break; default: break; } @@ -372,6 +405,7 @@ void dmCleanupServer(SDnode *pDnode) { SMsgCb dmGetMsgcb(SDnode *pDnode) { SMsgCb msgCb = { .clientRpc = pDnode->trans.clientRpc, + .serverRpc = pDnode->trans.serverRpc, .sendReqFp = dmSendReq, .sendRspFp = dmSendRsp, .registerBrokenLinkArgFp = dmRegisterBrokenLinkArg, diff --git a/source/dnode/mgmt/node_util/inc/dmUtil.h b/source/dnode/mgmt/node_util/inc/dmUtil.h index 32c3d22506..ad87bc91c6 100644 --- a/source/dnode/mgmt/node_util/inc/dmUtil.h +++ b/source/dnode/mgmt/node_util/inc/dmUtil.h @@ -107,6 +107,7 @@ typedef struct { TdThreadRwlock lock; SMsgCb msgCb; bool validMnodeEps; + int64_t ipWhiteVer; } SDnodeData; typedef struct { diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 90d54e7f58..844e69e659 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -277,18 +277,21 @@ typedef struct { } SAcctObj; typedef struct { - char user[TSDB_USER_LEN]; - char pass[TSDB_PASSWORD_LEN]; - char acct[TSDB_USER_LEN]; - int64_t createdTime; - int64_t updateTime; - int8_t superUser; - int8_t sysInfo; - int8_t enable; - int8_t reserve; - int32_t acctId; - int32_t authVersion; - int32_t passVersion; + char user[TSDB_USER_LEN]; + char pass[TSDB_PASSWORD_LEN]; + char acct[TSDB_USER_LEN]; + int64_t createdTime; + int64_t updateTime; + int8_t superUser; + int8_t sysInfo; + int8_t enable; + int8_t reserve; + int32_t acctId; + int32_t authVersion; + int32_t passVersion; + int64_t ipWhiteListVer; + SIpWhiteList* pIpWhiteList; + SHashObj* readDbs; SHashObj* writeDbs; SHashObj* topics; diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index ec83f8a7e6..72f9ec1735 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -130,6 +130,7 @@ typedef struct SMnode { SGrantInfo grant; MndMsgFp msgFp[TDMT_MAX]; SMsgCb msgCb; + int64_t ipWhiteVer; } SMnode; void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp); @@ -140,6 +141,8 @@ bool mndGetRestored(SMnode *pMnode); void mndSetStop(SMnode *pMnode); bool mndGetStop(SMnode *pMnode); +SArray *mndGetAllDnodeFqdns(SMnode *pMnode); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/inc/mndPrivilege.h b/source/dnode/mnode/impl/inc/mndPrivilege.h index dfde2f671e..3aa8a3d2d5 100644 --- a/source/dnode/mnode/impl/inc/mndPrivilege.h +++ b/source/dnode/mnode/impl/inc/mndPrivilege.h @@ -33,6 +33,9 @@ int32_t mndCheckTopicPrivilegeByName(SMnode *pMnode, const char *user, EOperType int32_t mndCheckShowPrivilege(SMnode *pMnode, const char *user, EShowType showType, const char *dbname); int32_t mndCheckAlterUserPrivilege(SUserObj *pOperUser, SUserObj *pUser, SAlterUserReq *pAlter); int32_t mndSetUserAuthRsp(SMnode *pMnode, SUserObj *pUser, SGetUserAuthRsp *pRsp); +int32_t mndSetUserWhiteListRsp(SMnode* pMnode, SUserObj* pUser, SGetUserWhiteListRsp* pWhiteListRsp); +int32_t mndEnableIpWhiteList(SMnode *pMnode); +int32_t mndFetchIpWhiteList(SIpWhiteList *ipList, char **buf); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndUser.h b/source/dnode/mnode/impl/inc/mndUser.h index 8b930e7f18..fab8ee4707 100644 --- a/source/dnode/mnode/impl/inc/mndUser.h +++ b/source/dnode/mnode/impl/inc/mndUser.h @@ -23,6 +23,10 @@ extern "C" { #endif +enum { + IP_WHITE_ADD, + IP_WHITE_DROP, +}; int32_t mndInitUser(SMnode *pMnode); void mndCleanupUser(SMnode *pMnode); SUserObj *mndAcquireUser(SMnode *pMnode, const char *userName); @@ -38,8 +42,15 @@ int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int3 int32_t mndUserRemoveDb(SMnode *pMnode, STrans *pTrans, char *db); int32_t mndUserRemoveTopic(SMnode *pMnode, STrans *pTrans, char *topic); -int32_t mndUserDupObj(SUserObj *pUser, SUserObj *pNew); -void mndUserFreeObj(SUserObj *pUser); +int32_t mndUserDupObj(SUserObj *pUser, SUserObj *pNew); +void mndUserFreeObj(SUserObj *pUser); + +int64_t mndGetIpWhiteVer(SMnode *pMnode); + +void mndUpdateIpWhite(SMnode *pMnode, char *user, char *fqdn, int8_t type, int8_t lock); + +int32_t mndRefreshUserIpWhiteList(SMnode *pMnode); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index a11bd2556e..f38cf70dc6 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -51,6 +51,12 @@ enum { DND_CONN_ACTIVE_CODE, }; +enum { + DND_CREATE, + DND_ADD, + DND_DROP, +}; + static int32_t mndCreateDefaultDnode(SMnode *pMnode); static SSdbRaw *mndDnodeActionEncode(SDnodeObj *pDnode); static SSdbRow *mndDnodeActionDecode(SSdbRaw *pRaw); @@ -103,7 +109,10 @@ int32_t mndInitDnode(SMnode *pMnode) { return sdbSetTable(pMnode->pSdb, table); } -void mndCleanupDnode(SMnode *pMnode) {} +SIpWhiteList *mndCreateIpWhiteOfDnode(SMnode *pMnode); +SIpWhiteList *mndAddIpWhiteOfDnode(SIpWhiteList *pIpWhiteList, char *fqdn); +SIpWhiteList *mndRmIpWhiteOfDnode(SIpWhiteList *pIpWhiteList, char *fqdn); +void mndCleanupDnode(SMnode *pMnode) {} static int32_t mndCreateDefaultDnode(SMnode *pMnode) { int32_t code = -1; @@ -130,6 +139,7 @@ static int32_t mndCreateDefaultDnode(SMnode *pMnode) { if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; code = 0; + mndUpdateIpWhite(pMnode, TSDB_DEFAULT_USER, dnodeObj.fqdn, IP_WHITE_ADD, 1); _OVER: mndTransDrop(pTrans); @@ -518,13 +528,16 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { } } + pMnode->ipWhiteVer = mndGetIpWhiteVer(pMnode); + int64_t dnodeVer = sdbGetTableVer(pMnode->pSdb, SDB_DNODE) + sdbGetTableVer(pMnode->pSdb, SDB_MNODE); int64_t curMs = taosGetTimestampMs(); bool online = mndIsDnodeOnline(pDnode, curMs); bool dnodeChanged = (statusReq.dnodeVer == 0) || (statusReq.dnodeVer != dnodeVer); bool reboot = (pDnode->rebootTime != statusReq.rebootTime); bool supportVnodesChanged = pDnode->numOfSupportVnodes != statusReq.numOfSupportVnodes; - bool needCheck = !online || dnodeChanged || reboot || supportVnodesChanged; + bool needCheck = + !online || dnodeChanged || reboot || supportVnodesChanged || pMnode->ipWhiteVer != statusReq.ipWhiteVer; const STraceId *trace = &pReq->info.traceId; mGTrace("dnode:%d, status received, accessTimes:%d check:%d online:%d reboot:%d changed:%d statusSeq:%d", pDnode->id, @@ -645,6 +658,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { } mndGetDnodeEps(pMnode, statusRsp.pDnodeEps); + statusRsp.ipWhiteVer = pMnode->ipWhiteVer; int32_t contLen = tSerializeSStatusRsp(NULL, 0, &statusRsp); void *pHead = rpcMallocCont(contLen); @@ -691,6 +705,7 @@ static int32_t mndCreateDnode(SMnode *pMnode, SRpcMsg *pReq, SCreateDnodeReq *pC if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; code = 0; + mndUpdateIpWhite(pMnode, TSDB_DEFAULT_USER, dnodeObj.fqdn, IP_WHITE_ADD, 1); _OVER: mndTransDrop(pTrans); sdbFreeRaw(pRaw); @@ -983,6 +998,7 @@ static int32_t mndDropDnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SM if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; + mndUpdateIpWhite(pMnode, TSDB_DEFAULT_USER, pDnode->fqdn, IP_WHITE_DROP, 1); code = 0; _OVER: @@ -1455,3 +1471,19 @@ _err: terrno = TSDB_CODE_INVALID_CFG; return -1; } + +SArray *mndGetAllDnodeFqdns(SMnode *pMnode) { + SDnodeObj *pObj = NULL; + void *pIter = NULL; + SSdb *pSdb = pMnode->pSdb; + SArray *fqdns = taosArrayInit(4, sizeof(void *)); + while (1) { + pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pObj); + if (pIter == NULL) break; + + char *fqdn = taosStrdup(pObj->fqdn); + taosArrayPush(fqdns, &fqdn); + sdbRelease(pSdb, pObj); + } + return fqdns; +} diff --git a/source/dnode/mnode/impl/src/mndInfoSchema.c b/source/dnode/mnode/impl/src/mndInfoSchema.c index 82294ac7bf..f17df28129 100644 --- a/source/dnode/mnode/impl/src/mndInfoSchema.c +++ b/source/dnode/mnode/impl/src/mndInfoSchema.c @@ -76,12 +76,19 @@ int32_t mndBuildInsTableSchema(SMnode *pMnode, const char *dbFName, const char * } STableMetaRsp *pMeta = taosHashGet(pMnode->infosMeta, tbName, strlen(tbName)); - if (NULL == pMeta || (!sysinfo && pMeta->sysInfo)) { + if (NULL == pMeta) { mError("invalid information schema table name:%s", tbName); terrno = TSDB_CODE_MND_INVALID_SYS_TABLENAME; return -1; } + if (!sysinfo && pMeta->sysInfo) { + mError("no permission to get schema of table name:%s", tbName); + terrno = TSDB_CODE_PAR_PERMISSION_DENIED; + return -1; + } + + *pRsp = *pMeta; pRsp->pSchemas = taosMemoryCalloc(pMeta->numOfColumns, sizeof(SSchema)); diff --git a/source/dnode/mnode/impl/src/mndPrivilege.c b/source/dnode/mnode/impl/src/mndPrivilege.c index bec516b1ee..2769c3ac4c 100644 --- a/source/dnode/mnode/impl/src/mndPrivilege.c +++ b/source/dnode/mnode/impl/src/mndPrivilege.c @@ -28,10 +28,33 @@ int32_t mndCheckDbPrivilege(SMnode *pMnode, const char *user, EOperType operType int32_t mndCheckDbPrivilegeByName(SMnode *pMnode, const char *user, EOperType operType, const char *dbname) { return 0; } + int32_t mndCheckTopicPrivilege(SMnode *pMnode, const char *user, EOperType operType, SMqTopicObj *pTopic) { return 0; } int32_t mndCheckTopicPrivilegeByName(SMnode *pMnode, const char *user, EOperType operType, const char *topicName) { return 0; } + + +// TODO: for community version use the commented version +int32_t mndSetUserWhiteListRsp(SMnode *pMnode, SUserObj *pUser, SGetUserWhiteListRsp *pWhiteListRsp) { + memcpy(pWhiteListRsp->user, pUser->user, TSDB_USER_LEN); + pWhiteListRsp->numWhiteLists = 1; + pWhiteListRsp->pWhiteLists = taosMemoryMalloc(pWhiteListRsp->numWhiteLists * sizeof(SIpV4Range)); + if (pWhiteListRsp->pWhiteLists == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + memset(pWhiteListRsp->pWhiteLists, 0, pWhiteListRsp->numWhiteLists * sizeof(SIpV4Range)); + // pWhiteListRsp->numWhiteLists = pUser->pIpWhiteList->num; + // pWhiteListRsp->pWhiteLists = taosMemoryMalloc(pWhiteListRsp->numWhiteLists * sizeof(SIpV4Range)); + // if (pWhiteListRsp->pWhiteLists == NULL) { + // return TSDB_CODE_OUT_OF_MEMORY; + // } + // memcpy(pWhiteListRsp->pWhiteLists, pUser->pIpWhiteList->pIpRange, + // pWhiteListRsp->numWhiteLists * sizeof(SIpV4Range)); + + return 0; +} + int32_t mndSetUserAuthRsp(SMnode *pMnode, SUserObj *pUser, SGetUserAuthRsp *pRsp) { memcpy(pRsp->user, pUser->user, TSDB_USER_LEN); pRsp->superAuth = 1; @@ -39,6 +62,14 @@ int32_t mndSetUserAuthRsp(SMnode *pMnode, SUserObj *pUser, SGetUserAuthRsp *pRsp pRsp->sysInfo = pUser->sysInfo; pRsp->version = pUser->authVersion; pRsp->passVer = pUser->passVersion; + pRsp->whiteListVer = pUser->ipWhiteListVer; return 0; } -#endif \ No newline at end of file + +int32_t mndEnableIpWhiteList(SMnode *pMnode) { return 1; } + +int32_t mndFetchIpWhiteList(SIpWhiteList *ipList, char **buf) { + *buf = NULL; + return 0; +} +#endif diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 9847024bee..6f67778615 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -290,6 +290,7 @@ _CONNECT: connectRsp.svrTimestamp = taosGetTimestampSec(); connectRsp.passVer = pUser->passVersion; connectRsp.authVer = pUser->authVersion; + connectRsp.whiteListVer = pUser->ipWhiteListVer; strcpy(connectRsp.sVer, version); snprintf(connectRsp.sDetailVer, sizeof(connectRsp.sDetailVer), "ver:%s\nbuild:%s\ngitinfo:%s", version, buildinfo, diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 14be05d973..d0a242c8ea 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "mndStb.h" +#include "audit.h" #include "mndDb.h" #include "mndDnode.h" #include "mndIndex.h" @@ -31,7 +32,6 @@ #include "mndUser.h" #include "mndVgroup.h" #include "tname.h" -#include "audit.h" #define STB_VER_NUMBER 1 #define STB_RESERVE_SIZE 64 @@ -858,7 +858,19 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat } return 0; } +static int32_t mndGenIdxNameForFirstTag(char *fullname, char *dbname, char *tagname) { + char randStr[24] = {0}; + int8_t start = 8; + int8_t end = sizeof(randStr) - 1; + // gen rand str len [base:end] + // note: ignore rand performance issues + int64_t len = taosRand() % (end - start + 1) + start; + + taosRandStr2(randStr, len); + sprintf(fullname, "%s.%s_%s", dbname, tagname, randStr); + return 0; +} static int32_t mndCreateStb(SMnode *pMnode, SRpcMsg *pReq, SMCreateStbReq *pCreate, SDbObj *pDb) { SStbObj stbObj = {0}; int32_t code = -1; @@ -871,11 +883,8 @@ static int32_t mndCreateStb(SMnode *pMnode, SRpcMsg *pReq, SMCreateStbReq *pCrea mInfo("trans:%d, used to create stb:%s", pTrans->id, pCreate->name); if (mndBuildStbFromReq(pMnode, &stbObj, pCreate, pDb) != 0) goto _OVER; - char randStr[24] = {0}; - taosRandStr2(randStr, tListLen(randStr) - 1); SSchema *pSchema = &(stbObj.pTags[0]); - sprintf(fullIdxName, "%s.%s_%s", pDb->name, pSchema->name, randStr); - + mndGenIdxNameForFirstTag(fullIdxName, pDb->name, pSchema->name); SSIdx idx = {0}; if (mndAcquireGlobalIdx(pMnode, fullIdxName, SDB_IDX, &idx) == 0 && idx.pIdx != NULL) { terrno = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST; @@ -1066,78 +1075,75 @@ static int32_t mndBuildStbFromAlter(SStbObj *pStb, SStbObj *pDst, SMCreateStbReq return TSDB_CODE_SUCCESS; } -static char* mndAuditFieldTypeStr(int32_t type){ - switch (type) - { - case TSDB_DATA_TYPE_NULL: - return "null"; - case TSDB_DATA_TYPE_BOOL: - return "bool"; - case TSDB_DATA_TYPE_TINYINT: - return "tinyint"; - case TSDB_DATA_TYPE_SMALLINT: - return "smallint"; - case TSDB_DATA_TYPE_INT: - return "int"; - case TSDB_DATA_TYPE_BIGINT: - return "bigint"; - case TSDB_DATA_TYPE_FLOAT: - return "float"; - case TSDB_DATA_TYPE_DOUBLE: - return "double"; - case TSDB_DATA_TYPE_VARCHAR: - return "varchar"; - case TSDB_DATA_TYPE_TIMESTAMP: - return "timestamp"; - case TSDB_DATA_TYPE_NCHAR: - return "nchar"; - case TSDB_DATA_TYPE_UTINYINT: - return "utinyint"; - case TSDB_DATA_TYPE_USMALLINT: - return "usmallint"; - case TSDB_DATA_TYPE_UINT: - return "uint"; - case TSDB_DATA_TYPE_UBIGINT: - return "ubigint"; - case TSDB_DATA_TYPE_JSON: - return "json"; - case TSDB_DATA_TYPE_VARBINARY: - return "varbinary"; - case TSDB_DATA_TYPE_DECIMAL: - return "decimal"; - case TSDB_DATA_TYPE_BLOB: - return "blob"; - case TSDB_DATA_TYPE_MEDIUMBLOB: - return "mediumblob"; - case TSDB_DATA_TYPE_GEOMETRY: - return "geometry"; +static char *mndAuditFieldTypeStr(int32_t type) { + switch (type) { + case TSDB_DATA_TYPE_NULL: + return "null"; + case TSDB_DATA_TYPE_BOOL: + return "bool"; + case TSDB_DATA_TYPE_TINYINT: + return "tinyint"; + case TSDB_DATA_TYPE_SMALLINT: + return "smallint"; + case TSDB_DATA_TYPE_INT: + return "int"; + case TSDB_DATA_TYPE_BIGINT: + return "bigint"; + case TSDB_DATA_TYPE_FLOAT: + return "float"; + case TSDB_DATA_TYPE_DOUBLE: + return "double"; + case TSDB_DATA_TYPE_VARCHAR: + return "varchar"; + case TSDB_DATA_TYPE_TIMESTAMP: + return "timestamp"; + case TSDB_DATA_TYPE_NCHAR: + return "nchar"; + case TSDB_DATA_TYPE_UTINYINT: + return "utinyint"; + case TSDB_DATA_TYPE_USMALLINT: + return "usmallint"; + case TSDB_DATA_TYPE_UINT: + return "uint"; + case TSDB_DATA_TYPE_UBIGINT: + return "ubigint"; + case TSDB_DATA_TYPE_JSON: + return "json"; + case TSDB_DATA_TYPE_VARBINARY: + return "varbinary"; + case TSDB_DATA_TYPE_DECIMAL: + return "decimal"; + case TSDB_DATA_TYPE_BLOB: + return "blob"; + case TSDB_DATA_TYPE_MEDIUMBLOB: + return "mediumblob"; + case TSDB_DATA_TYPE_GEOMETRY: + return "geometry"; - default: - return "error"; + default: + return "error"; } } -static void mndAuditFieldStr(char* detail, SArray *arr, int32_t len, int32_t max){ +static void mndAuditFieldStr(char *detail, SArray *arr, int32_t len, int32_t max) { int32_t detialLen = strlen(detail); int32_t fieldLen = 0; for (int32_t i = 0; i < len; ++i) { SField *pField = taosArrayGet(arr, i); - char field[TSDB_COL_NAME_LEN + 20] = {0}; + char field[TSDB_COL_NAME_LEN + 20] = {0}; fieldLen = strlen(", "); - if(detialLen > 0 && detialLen < max-fieldLen-1) { + if (detialLen > 0 && detialLen < max - fieldLen - 1) { strcat(detail, ", "); detialLen += fieldLen; - } - else{ + } else { break; } sprintf(field, "%s:%s", pField->name, mndAuditFieldTypeStr(pField->type)); fieldLen = strlen(field); - if(detialLen < max-fieldLen-1) { + if (detialLen < max - fieldLen - 1) { strcat(detail, field); detialLen += fieldLen; - } - else{ + } else { break; } } @@ -1252,14 +1258,17 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) { if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; char detail[AUDIT_DETAIL_MAX] = {0}; - sprintf(detail, "colVer:%d, delay1:%" PRId64 ", delay2:%" PRId64 ", deleteMark1:%" PRId64 ", " - "deleteMark2:%" PRId64 ", igExists:%d, numOfColumns:%d, numOfFuncs:%d, numOfTags:%d, " - "source:%d, suid:%" PRId64 ", tagVer:%d, ttl:%d, " + sprintf(detail, + "colVer:%d, delay1:%" PRId64 ", delay2:%" PRId64 ", deleteMark1:%" PRId64 + ", " + "deleteMark2:%" PRId64 + ", igExists:%d, numOfColumns:%d, numOfFuncs:%d, numOfTags:%d, " + "source:%d, suid:%" PRId64 + ", tagVer:%d, ttl:%d, " "watermark1:%" PRId64 ", watermark2:%" PRId64, - createReq.colVer, createReq.delay1, createReq.delay2, createReq.deleteMark1, - createReq.deleteMark2, createReq.igExists, createReq.numOfColumns, createReq.numOfFuncs, createReq.numOfTags, - createReq.source, createReq.suid, createReq.tagVer, createReq.ttl, - createReq.watermark1, createReq.watermark2); + createReq.colVer, createReq.delay1, createReq.delay2, createReq.deleteMark1, createReq.deleteMark2, + createReq.igExists, createReq.numOfColumns, createReq.numOfFuncs, createReq.numOfTags, createReq.source, + createReq.suid, createReq.tagVer, createReq.ttl, createReq.watermark1, createReq.watermark2); mndAuditFieldStr(detail, createReq.pColumns, createReq.numOfColumns, AUDIT_DETAIL_MAX); mndAuditFieldStr(detail, createReq.pTags, createReq.numOfTags, AUDIT_DETAIL_MAX); @@ -2338,8 +2347,7 @@ static int32_t mndProcessAlterStbReq(SRpcMsg *pReq) { if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; char detail[2000] = {0}; - sprintf(detail, "alterType:%d, numOfFields:%d, ttl:%d" , - alterReq.alterType, alterReq.numOfFields, alterReq.ttl); + sprintf(detail, "alterType:%d, numOfFields:%d, ttl:%d", alterReq.alterType, alterReq.numOfFields, alterReq.ttl); SName name = {0}; tNameFromString(&name, alterReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); @@ -2608,8 +2616,7 @@ static int32_t mndProcessDropStbReq(SRpcMsg *pReq) { if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; char detail[2000] = {0}; - sprintf(detail, "igNotExists:%d, source:%d" , - dropReq.igNotExists, dropReq.source); + sprintf(detail, "igNotExists:%d, source:%d", dropReq.igNotExists, dropReq.source); SName name = {0}; tNameFromString(&name, dropReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); @@ -3369,7 +3376,7 @@ static int32_t buildSysDbColsInfo(SSDataBlock *p, int8_t buildWhichDBs, char *tb return p->info.rows; } -static int8_t determineBuildColForWhichDBs(const char* db) { +static int8_t determineBuildColForWhichDBs(const char *db) { int8_t buildWhichDBs; if (!db[0]) buildWhichDBs = BUILD_COL_FOR_ALL_DB; @@ -3387,11 +3394,11 @@ static int8_t determineBuildColForWhichDBs(const char* db) { } static int32_t mndRetrieveStbCol(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { - uint8_t buildWhichDBs; + uint8_t buildWhichDBs; SMnode *pMnode = pReq->info.node; SSdb *pSdb = pMnode->pSdb; SStbObj *pStb = NULL; - int32_t numOfRows = 0; + int32_t numOfRows = 0; buildWhichDBs = determineBuildColForWhichDBs(pShow->db); diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index a601d3c121..5759737a6a 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -17,6 +17,7 @@ #include "mndSync.h" #include "mndCluster.h" #include "mndTrans.h" +#include "mndUser.h" static int32_t mndSyncEqCtrlMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) { if (pMsg == NULL || pMsg->pCont == NULL) { @@ -167,7 +168,7 @@ int32_t mndProcessWriteMsg(SMnode *pMnode, SRpcMsg *pMsg, SFsmCbMeta *pMeta) { SSdbRaw *pRaw = pMsg->pCont; STrans *pTrans = NULL; int32_t code = -1; - int32_t transId = sdbGetIdFromRaw(pMnode->pSdb, pRaw); + int32_t transId = sdbGetIdFromRaw(pMnode->pSdb, pRaw); if (transId <= 0) { mError("trans:%d, invalid commit msg, cache transId:%d seq:%" PRId64, transId, pMgmt->transId, pMgmt->transSeq); @@ -304,6 +305,7 @@ void mndRestoreFinish(const SSyncFSM *pFsm, const SyncIndex commitIdx) { } else { mInfo("vgId:1, sync restore finished"); } + mndRefreshUserIpWhiteList(pMnode); ASSERT(commitIdx == mndSyncAppliedIndex(pFsm)); } diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index aa916a130c..a82623ae1b 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -14,7 +14,10 @@ */ #define _DEFAULT_SOURCE +// clang-format off +#include #include "mndUser.h" +#include "audit.h" #include "mndDb.h" #include "mndPrivilege.h" #include "mndShow.h" @@ -22,11 +25,22 @@ #include "mndTopic.h" #include "mndTrans.h" #include "tbase64.h" -#include "audit.h" -#define USER_VER_NUMBER 4 +// clang-format on + +#define USER_VER_NUMBER 5 #define USER_RESERVE_SIZE 64 +static SIpWhiteList *createDefaultIpWhiteList(); +SIpWhiteList *createIpWhiteList(void *buf, int32_t len); +static bool updateIpWhiteList(SIpWhiteList *pOld, SIpWhiteList *pNew); +static bool isIpWhiteListEqual(SIpWhiteList *a, SIpWhiteList *b); +static bool isIpRangeEqual(SIpV4Range *a, SIpV4Range *b); + +void destroyIpWhiteTab(SHashObj *pIpWhiteTab); + +#define MND_MAX_USE_HOST (TSDB_PRIVILEDGE_HOST_LEN / 24) + static int32_t mndCreateDefaultUsers(SMnode *pMnode); static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw); static int32_t mndUserActionInsert(SSdb *pSdb, SUserObj *pUser); @@ -37,12 +51,358 @@ static int32_t mndProcessCreateUserReq(SRpcMsg *pReq); static int32_t mndProcessAlterUserReq(SRpcMsg *pReq); static int32_t mndProcessDropUserReq(SRpcMsg *pReq); static int32_t mndProcessGetUserAuthReq(SRpcMsg *pReq); +static int32_t mndProcessGetUserWhiteListReq(SRpcMsg *pReq); static int32_t mndRetrieveUsers(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextUser(SMnode *pMnode, void *pIter); static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextPrivileges(SMnode *pMnode, void *pIter); +SHashObj *mndFetchAllIpWhite(SMnode *pMnode); +static int32_t mndProcesSRetrieveIpWhiteReq(SRpcMsg *pReq); + +void ipWhiteMgtUpdateAll(SMnode *pMnode); +typedef struct { + SHashObj *pIpWhiteTab; + int64_t ver; + TdThreadRwlock rw; +} SIpWhiteMgt; + +static SIpWhiteMgt ipWhiteMgt; + +const static SIpV4Range defaultIpRange = {.ip = 16777343, .mask = 32}; + +void ipWhiteMgtInit() { + ipWhiteMgt.pIpWhiteTab = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), 1, HASH_ENTRY_LOCK); + ipWhiteMgt.ver = 0; + taosThreadRwlockInit(&ipWhiteMgt.rw, NULL); +} +void ipWhiteMgtCleanup() { + destroyIpWhiteTab(ipWhiteMgt.pIpWhiteTab); + taosThreadRwlockDestroy(&ipWhiteMgt.rw); +} + +int32_t ipWhiteMgtUpdate(char *user, SIpWhiteList *pNew) { + bool update = true; + taosThreadRwlockWrlock(&ipWhiteMgt.rw); + SIpWhiteList **ppList = taosHashGet(ipWhiteMgt.pIpWhiteTab, user, strlen(user)); + + if (ppList == NULL || *ppList == NULL) { + SIpWhiteList *p = cloneIpWhiteList(pNew); + taosHashPut(ipWhiteMgt.pIpWhiteTab, user, strlen(user), &p, sizeof(void *)); + } else { + SIpWhiteList *pOld = *ppList; + if (isIpWhiteListEqual(pOld, pNew)) { + update = false; + } else { + taosMemoryFree(pOld); + SIpWhiteList *p = cloneIpWhiteList(pNew); + taosHashPut(ipWhiteMgt.pIpWhiteTab, user, strlen(user), &p, sizeof(void *)); + } + } + if (update) ipWhiteMgt.ver++; + + taosThreadRwlockUnlock(&ipWhiteMgt.rw); + return 0; +} +int32_t ipWhiteMgtRemove(char *user) { + bool update = true; + taosThreadRwlockWrlock(&ipWhiteMgt.rw); + SIpWhiteList **ppList = taosHashGet(ipWhiteMgt.pIpWhiteTab, user, strlen(user)); + if (ppList == NULL || *ppList == NULL) { + update = false; + } else { + taosMemoryFree(*ppList); + taosHashRemove(ipWhiteMgt.pIpWhiteTab, user, strlen(user)); + } + + if (update) ipWhiteMgt.ver++; + taosThreadRwlockUnlock(&ipWhiteMgt.rw); + return 0; +} + +bool isRangeInWhiteList(SIpWhiteList *pList, SIpV4Range *range) { + for (int i = 0; i < pList->num; i++) { + if (isIpRangeEqual(&pList->pIpRange[i], range)) { + return true; + } + } + return false; +} +int32_t ipWhiteUpdateForAllUser(SIpWhiteList *pList) { + taosThreadRwlockWrlock(&ipWhiteMgt.rw); + + SHashObj *pIpWhiteTab = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), 1, HASH_ENTRY_LOCK); + void *pIter = taosHashIterate(ipWhiteMgt.pIpWhiteTab, NULL); + + while (pIter) { + SIpWhiteList *p = *(SIpWhiteList **)pIter; + SIpWhiteList *clone = cloneIpWhiteList(pList); + int32_t idx = 0; + for (int i = 0; i < pList->num; i++) { + SIpV4Range *e = &pList->pIpRange[i]; + if (!isRangeInWhiteList(p, e)) { + clone->pIpRange[idx] = *e; + idx++; + } + } + clone->num = idx; + + SIpWhiteList *val = NULL; + if (clone->num != 0) { + int32_t sz = clone->num + p->num; + val = taosMemoryCalloc(1, sizeof(SIpWhiteList) + sz * sizeof(SIpV4Range)); + memcpy(val->pIpRange, p->pIpRange, sizeof(SIpV4Range) * p->num); + memcpy(((char *)val->pIpRange) + sizeof(SIpV4Range) * p->num, (char *)clone->pIpRange, + sizeof(SIpV4Range) * clone->num); + + } else { + val = cloneIpWhiteList(p); + } + taosMemoryFree(clone); + + size_t klen; + void *key = taosHashGetKey(pIter, &klen); + taosHashPut(pIpWhiteTab, key, klen, val, sizeof(void *)); + } + + destroyIpWhiteTab(ipWhiteMgt.pIpWhiteTab); + + ipWhiteMgt.pIpWhiteTab = pIpWhiteTab; + ipWhiteMgt.ver++; + taosThreadRwlockUnlock(&ipWhiteMgt.rw); + return 0; +} + +void ipWhiteMgtUpdateAll(SMnode *pMnode) { + ipWhiteMgt.ver++; + SHashObj *pNew = mndFetchAllIpWhite(pMnode); + SHashObj *pOld = ipWhiteMgt.pIpWhiteTab; + + ipWhiteMgt.pIpWhiteTab = pNew; + + destroyIpWhiteTab(pOld); +} +void ipWhiteMgtUpdate2(SMnode *pMnode) { + taosThreadRwlockWrlock(&ipWhiteMgt.rw); + + ipWhiteMgtUpdateAll(pMnode); + + taosThreadRwlockUnlock(&ipWhiteMgt.rw); +} + +int64_t mndGetIpWhiteVer(SMnode *pMnode) { + int64_t ver = 0; + taosThreadRwlockWrlock(&ipWhiteMgt.rw); + if (ipWhiteMgt.ver == 0) { + // user and dnode r + ipWhiteMgtUpdateAll(pMnode); + ipWhiteMgt.ver = taosGetTimestampMs(); + } + ver = ipWhiteMgt.ver; + taosThreadRwlockUnlock(&ipWhiteMgt.rw); + mDebug("ip-white-list on mnode ver: %" PRId64 "", ver); + + if (mndEnableIpWhiteList(pMnode) == 0 || tsEnableWhiteList == false) { + return 0; + } + return ver; +} + +bool mndUpdateIpWhiteImpl(SHashObj *pIpWhiteTab, char *user, char *fqdn, int8_t type) { + bool update = false; + SIpV4Range range = {.ip = taosGetIpv4FromFqdn(fqdn), .mask = 32}; + mDebug("ip-white-list may update for user: %s, fqdn: %s", user, fqdn); + SIpWhiteList **ppList = taosHashGet(pIpWhiteTab, user, strlen(user)); + SIpWhiteList *pList = NULL; + if (ppList != NULL && *ppList != NULL) { + pList = *ppList; + } + + if (type == IP_WHITE_ADD) { + if (pList == NULL) { + SIpWhiteList *pNewList = taosMemoryCalloc(1, sizeof(SIpWhiteList) + sizeof(SIpV4Range)); + memcpy(pNewList->pIpRange, &range, sizeof(SIpV4Range)); + pNewList->num = 1; + + taosHashPut(pIpWhiteTab, user, strlen(user), &pNewList, sizeof(void *)); + update = true; + } else { + if (!isRangeInWhiteList(pList, &range)) { + int32_t sz = sizeof(SIpWhiteList) + sizeof(SIpV4Range) * (pList->num + 1); + SIpWhiteList *pNewList = taosMemoryCalloc(1, sz); + memcpy(pNewList->pIpRange, pList->pIpRange, sizeof(SIpV4Range) * (pList->num)); + pNewList->pIpRange[pList->num].ip = range.ip; + pNewList->pIpRange[pList->num].mask = range.mask; + + pNewList->num = pList->num + 1; + + taosHashPut(pIpWhiteTab, user, strlen(user), &pNewList, sizeof(void *)); + taosMemoryFree(pList); + update = true; + } + } + } else if (type == IP_WHITE_DROP) { + if (pList != NULL) { + if (isRangeInWhiteList(pList, &range)) { + if (pList->num == 1) { + taosHashRemove(pIpWhiteTab, user, strlen(user)); + taosMemoryFree(pList); + } else { + int32_t idx = 0; + int32_t sz = sizeof(SIpWhiteList) + sizeof(SIpV4Range) * (pList->num - 1); + SIpWhiteList *pNewList = taosMemoryCalloc(1, sz); + for (int i = 0; i < pList->num; i++) { + SIpV4Range *e = &pList->pIpRange[i]; + if (!isIpRangeEqual(e, &range)) { + pNewList->pIpRange[idx].ip = e->ip; + pNewList->pIpRange[idx].mask = e->mask; + idx++; + } + } + pNewList->num = idx; + taosHashPut(pIpWhiteTab, user, strlen(user), &pNewList, sizeof(void *)); + taosMemoryFree(pList); + } + update = true; + } + } + } + if (update) { + mDebug("ip-white-list update for user: %s, fqdn: %s", user, fqdn); + } + + return update; +} + +int32_t mndRefreshUserIpWhiteList(SMnode *pMnode) { + taosThreadRwlockWrlock(&ipWhiteMgt.rw); + + ipWhiteMgtUpdateAll(pMnode); + ipWhiteMgt.ver = taosGetTimestampMs(); + taosThreadRwlockUnlock(&ipWhiteMgt.rw); + + return 0; +} +void mndUpdateIpWhite(SMnode *pMnode, char *user, char *fqdn, int8_t type, int8_t lock) { + if (lock) { + taosThreadRwlockWrlock(&ipWhiteMgt.rw); + if (ipWhiteMgt.ver == 0) { + ipWhiteMgtUpdateAll(pMnode); + ipWhiteMgt.ver = taosGetTimestampMs(); + mInfo("ip-white-list, user: %" PRId64 "", ipWhiteMgt.ver); + } + } + + bool update = mndUpdateIpWhiteImpl(ipWhiteMgt.pIpWhiteTab, user, fqdn, type); + if (update) ipWhiteMgt.ver++; + + if (lock) taosThreadRwlockUnlock(&ipWhiteMgt.rw); +} +int64_t ipWhiteMgtFillMsg(SUpdateIpWhite *pUpdate) { + int64_t ver = 0; + taosThreadRwlockWrlock(&ipWhiteMgt.rw); + ver = ipWhiteMgt.ver; + int32_t num = taosHashGetSize(ipWhiteMgt.pIpWhiteTab); + + pUpdate->pUserIpWhite = taosMemoryCalloc(1, num * sizeof(SUpdateUserIpWhite)); + + void *pIter = taosHashIterate(ipWhiteMgt.pIpWhiteTab, NULL); + int32_t i = 0; + while (pIter) { + SUpdateUserIpWhite *pUser = &pUpdate->pUserIpWhite[i]; + SIpWhiteList *list = *(SIpWhiteList **)pIter; + + size_t klen; + char *key = taosHashGetKey(pIter, &klen); + if (list->num != 0) { + pUser->ver = ver; + memcpy(pUser->user, key, klen); + pUser->numOfRange = list->num; + pUser->pIpRanges = taosMemoryCalloc(1, list->num * sizeof(SIpV4Range)); + memcpy(pUser->pIpRanges, list->pIpRange, list->num * sizeof(SIpV4Range)); + i++; + } + pIter = taosHashIterate(ipWhiteMgt.pIpWhiteTab, pIter); + } + pUpdate->numOfUser = i; + pUpdate->ver = ver; + + taosThreadRwlockUnlock(&ipWhiteMgt.rw); + return 0; +} + +void destroyIpWhiteTab(SHashObj *pIpWhiteTab) { + if (pIpWhiteTab == NULL) return; + + void *pIter = taosHashIterate(pIpWhiteTab, NULL); + while (pIter) { + SIpWhiteList *list = *(SIpWhiteList **)pIter; + taosMemoryFree(list); + pIter = taosHashIterate(pIpWhiteTab, pIter); + } + + taosHashCleanup(pIpWhiteTab); +} +SHashObj *mndFetchAllIpWhite(SMnode *pMnode) { + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; + SHashObj *pIpWhiteTab = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), 1, HASH_ENTRY_LOCK); + + SArray *pUserNames = taosArrayInit(8, sizeof(void *)); + while (1) { + SUserObj *pUser = NULL; + pIter = sdbFetch(pSdb, SDB_USER, pIter, (void **)&pUser); + if (pIter == NULL) break; + + SIpWhiteList *pWhiteList = cloneIpWhiteList(pUser->pIpWhiteList); + taosHashPut(pIpWhiteTab, pUser->user, strlen(pUser->user), &pWhiteList, sizeof(void *)); + + char *name = taosStrdup(pUser->user); + taosArrayPush(pUserNames, &name); + + sdbRelease(pSdb, pUser); + } + + bool found = false; + for (int i = 0; i < taosArrayGetSize(pUserNames); i++) { + char *name = taosArrayGetP(pUserNames, i); + if (strlen(name) == strlen(TSDB_DEFAULT_USER) && strncmp(name, TSDB_DEFAULT_USER, strlen(TSDB_DEFAULT_USER)) == 0) { + found = true; + break; + } + } + if (found == false) { + char *name = taosStrdup(TSDB_DEFAULT_USER); + taosArrayPush(pUserNames, &name); + } + + SArray *fqdns = mndGetAllDnodeFqdns(pMnode); + + for (int i = 0; i < taosArrayGetSize(fqdns); i++) { + char *fqdn = taosArrayGetP(fqdns, i); + + for (int j = 0; j < taosArrayGetSize(pUserNames); j++) { + char *name = taosArrayGetP(pUserNames, j); + mndUpdateIpWhiteImpl(pIpWhiteTab, name, fqdn, IP_WHITE_ADD); + } + } + + for (int i = 0; i < taosArrayGetSize(fqdns); i++) { + char *fqdn = taosArrayGetP(fqdns, i); + taosMemoryFree(fqdn); + } + taosArrayDestroy(fqdns); + + for (int i = 0; i < taosArrayGetSize(pUserNames); i++) { + taosMemoryFree(taosArrayGetP(pUserNames, i)); + } + taosArrayDestroy(pUserNames); + + return pIpWhiteTab; +} int32_t mndInitUser(SMnode *pMnode) { + ipWhiteMgtInit(); + SSdbTable table = { .sdbType = SDB_USER, .keyType = SDB_KEY_BINARY, @@ -58,6 +418,9 @@ int32_t mndInitUser(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_ALTER_USER, mndProcessAlterUserReq); mndSetMsgHandle(pMnode, TDMT_MND_DROP_USER, mndProcessDropUserReq); mndSetMsgHandle(pMnode, TDMT_MND_GET_USER_AUTH, mndProcessGetUserAuthReq); + mndSetMsgHandle(pMnode, TDMT_MND_GET_USER_WHITELIST, mndProcessGetUserWhiteListReq); + + mndSetMsgHandle(pMnode, TDMT_MND_RETRIEVE_IP_WHITE, mndProcesSRetrieveIpWhiteReq); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_USER, mndRetrieveUsers); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_USER, mndCancelGetNextUser); @@ -66,7 +429,133 @@ int32_t mndInitUser(SMnode *pMnode) { return sdbSetTable(pMnode->pSdb, table); } -void mndCleanupUser(SMnode *pMnode) {} +void mndCleanupUser(SMnode *pMnode) { ipWhiteMgtCleanup(); } + +static void ipRangeToStr(SIpV4Range *range, char *buf) { + struct in_addr addr; + addr.s_addr = range->ip; + + uv_inet_ntop(AF_INET, &addr, buf, 32); + if (range->mask != 32) { + sprintf(buf + strlen(buf), "/%d", range->mask); + } + return; +} +static bool isDefaultRange(SIpV4Range *pRange) { + static SIpV4Range val = {.ip = 16777343, .mask = 32}; + return pRange->ip == val.ip && pRange->mask == val.mask; +} +static int32_t ipRangeListToStr(SIpV4Range *range, int32_t num, char *buf) { + int32_t len = 0; + for (int i = 0; i < num; i++) { + char tbuf[36] = {0}; + SIpV4Range *pRange = &range[i]; + + ipRangeToStr(&range[i], tbuf); + len += sprintf(buf + len, "%s,", tbuf); + } + if (len > 0) buf[len - 1] = 0; + return len; +} + +static bool isIpRangeEqual(SIpV4Range *a, SIpV4Range *b) { + // equal or not + return a->ip == b->ip && a->mask == b->mask; +} +static bool isRangeInIpWhiteList(SIpWhiteList *pList, SIpV4Range *tgt) { + for (int i = 0; i < pList->num; i++) { + if (isIpRangeEqual(&pList->pIpRange[i], tgt)) return true; + } + return false; +} +static bool isIpWhiteListEqual(SIpWhiteList *a, SIpWhiteList *b) { + if (a->num != b->num) { + return false; + } + for (int i = 0; i < a->num; i++) { + if (!isIpRangeEqual(&a->pIpRange[i], &b->pIpRange[i])) { + return false; + } + } + return true; +} +int32_t convertIpWhiteListToStr(SIpWhiteList *pList, char **buf) { + if (pList->num == 0) { + *buf = NULL; + return 0; + } + *buf = taosMemoryCalloc(1, pList->num * 36); + int32_t len = ipRangeListToStr(pList->pIpRange, pList->num, *buf); + if (len == 0) { + taosMemoryFree(*buf); + return 0; + } + return strlen(*buf); +} +int32_t tSerializeIpWhiteList(void *buf, int32_t len, SIpWhiteList *pList) { + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, len); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI32(&encoder, pList->num) < 0) return -1; + + for (int i = 0; i < pList->num; i++) { + SIpV4Range *pRange = &(pList->pIpRange[i]); + if (tEncodeU32(&encoder, pRange->ip) < 0) return -1; + if (tEncodeU32(&encoder, pRange->mask) < 0) return -1; + } + + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + return tlen; +} + +int32_t tDerializeIpWhileList(void *buf, int32_t len, SIpWhiteList *pList) { + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, len); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI32(&decoder, &pList->num) < 0) return -1; + + for (int i = 0; i < pList->num; i++) { + SIpV4Range *pRange = &(pList->pIpRange[i]); + if (tDecodeU32(&decoder, &pRange->ip) < 0) return -1; + if (tDecodeU32(&decoder, &pRange->mask) < 0) return -1; + } + tEndDecode(&decoder); + tDecoderClear(&decoder); + + return 0; +} +SIpWhiteList *createIpWhiteList(void *buf, int32_t len) { + int32_t num = 0; + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, len); + + if (tStartDecode(&decoder) < 0) return NULL; + if (tDecodeI32(&decoder, &num) < 0) return NULL; + tEndDecode(&decoder); + tDecoderClear(&decoder); + + SIpWhiteList *p = taosMemoryCalloc(1, sizeof(SIpWhiteList) + num * sizeof(SIpV4Range)); + tDerializeIpWhileList(buf, len, p); + return p; +} + +static SIpWhiteList *createDefaultIpWhiteList() { + SIpWhiteList *pWhiteList = taosMemoryCalloc(1, sizeof(SIpWhiteList) + sizeof(SIpV4Range) * 1); + pWhiteList->num = 1; + SIpV4Range *range = &(pWhiteList->pIpRange[0]); + + struct in_addr addr; + if (uv_inet_pton(AF_INET, "127.0.0.1", &addr) == 0) { + range->ip = addr.s_addr; + range->mask = 32; + } + return pWhiteList; +} static int32_t mndCreateDefaultUser(SMnode *pMnode, char *acct, char *user, char *pass) { SUserObj userObj = {0}; @@ -77,13 +566,14 @@ static int32_t mndCreateDefaultUser(SMnode *pMnode, char *acct, char *user, char userObj.updateTime = userObj.createdTime; userObj.sysInfo = 1; userObj.enable = 1; - + userObj.ipWhiteListVer = taosGetTimestampMs(); + userObj.pIpWhiteList = createDefaultIpWhiteList(); if (strcmp(user, TSDB_DEFAULT_USER) == 0) { userObj.superUser = 1; } SSdbRaw *pRaw = mndUserActionEncode(&userObj); - if (pRaw == NULL) return -1; + if (pRaw == NULL) goto _ERROR; (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY); mInfo("user:%s, will be created when deploying, raw:%p", userObj.user, pRaw); @@ -92,25 +582,29 @@ static int32_t mndCreateDefaultUser(SMnode *pMnode, char *acct, char *user, char if (pTrans == NULL) { sdbFreeRaw(pRaw); mError("user:%s, failed to create since %s", userObj.user, terrstr()); - return -1; + goto _ERROR; } mInfo("trans:%d, used to create user:%s", pTrans->id, userObj.user); if (mndTransAppendCommitlog(pTrans, pRaw) != 0) { mError("trans:%d, failed to commit redo log since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); - return -1; + goto _ERROR; } (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY); if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); - return -1; + goto _ERROR; } mndTransDrop(pTrans); + taosMemoryFree(userObj.pIpWhiteList); return 0; +_ERROR: + taosMemoryFree(userObj.pIpWhiteList); + return -1; } static int32_t mndCreateDefaultUsers(SMnode *pMnode) { @@ -124,6 +618,8 @@ static int32_t mndCreateDefaultUsers(SMnode *pMnode) { SSdbRaw *mndUserActionEncode(SUserObj *pUser) { terrno = TSDB_CODE_OUT_OF_MEMORY; + int32_t ipWhiteReserve = + pUser->pIpWhiteList ? (sizeof(SIpV4Range) * pUser->pIpWhiteList->num + sizeof(SIpWhiteList) + 4) : 16; int32_t numOfReadDbs = taosHashGetSize(pUser->readDbs); int32_t numOfWriteDbs = taosHashGetSize(pUser->writeDbs); int32_t numOfReadStbs = taosHashGetSize(pUser->readTbs); @@ -131,7 +627,8 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) { int32_t numOfTopics = taosHashGetSize(pUser->topics); int32_t numOfUseDbs = taosHashGetSize(pUser->useDbs); int32_t size = sizeof(SUserObj) + USER_RESERVE_SIZE + - (numOfReadDbs + numOfWriteDbs + numOfUseDbs) * TSDB_DB_FNAME_LEN + numOfTopics * TSDB_TOPIC_FNAME_LEN; + (numOfReadDbs + numOfWriteDbs + numOfUseDbs) * TSDB_DB_FNAME_LEN + numOfTopics * TSDB_TOPIC_FNAME_LEN + + ipWhiteReserve; char *stb = taosHashIterate(pUser->readTbs, NULL); while (stb != NULL) { @@ -241,6 +738,18 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) { useDb = taosHashIterate(pUser->useDbs, useDb); } + // save white list + int32_t num = pUser->pIpWhiteList->num; + int32_t tlen = sizeof(SIpWhiteList) + num * sizeof(SIpV4Range) + 4; + char *buf = taosMemoryCalloc(1, tlen); + int32_t len = tSerializeIpWhiteList(buf, tlen, pUser->pIpWhiteList); + + SDB_SET_INT32(pRaw, dataPos, len, _OVER); + SDB_SET_BINARY(pRaw, dataPos, buf, len, _OVER); + taosMemoryFree(buf); + + SDB_SET_INT64(pRaw, dataPos, pUser->ipWhiteListVer, _OVER); + SDB_SET_RESERVE(pRaw, dataPos, USER_RESERVE_SIZE, _OVER) SDB_SET_DATALEN(pRaw, dataPos, _OVER) @@ -398,6 +907,25 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { taosMemoryFree(key); } } + // decoder white list + if (sver >= 5) { + int32_t len = 0; + SDB_GET_INT32(pRaw, dataPos, &len, _OVER); + + char *buf = taosMemoryMalloc(len); + if (buf == NULL) goto _OVER; + SDB_GET_BINARY(pRaw, dataPos, buf, len, _OVER); + + pUser->pIpWhiteList = createIpWhiteList(buf, len); + taosMemoryFree(buf); + + SDB_GET_INT64(pRaw, dataPos, &pUser->ipWhiteListVer, _OVER); + } + + if (pUser->pIpWhiteList == NULL) { + pUser->pIpWhiteList = createDefaultIpWhiteList(); + pUser->ipWhiteListVer = taosGetTimestampMs(); + } SDB_GET_RESERVE(pRaw, dataPos, USER_RESERVE_SIZE, _OVER) taosInitRWLatch(&pUser->lock); @@ -414,6 +942,7 @@ _OVER: taosHashCleanup(pUser->readTbs); taosHashCleanup(pUser->writeTbs); taosHashCleanup(pUser->useDbs); + taosMemoryFreeClear(pUser->pIpWhiteList); } taosMemoryFreeClear(pRow); return NULL; @@ -501,6 +1030,8 @@ int32_t mndUserDupObj(SUserObj *pUser, SUserObj *pNew) { pNew->writeTbs = mndDupTableHash(pUser->writeTbs); pNew->topics = mndDupTopicHash(pUser->topics); pNew->useDbs = mndDupUseDbHash(pUser->useDbs); + pNew->pIpWhiteList = cloneIpWhiteList(pUser->pIpWhiteList); + taosRUnLockLatch(&pUser->lock); if (pNew->readDbs == NULL || pNew->writeDbs == NULL || pNew->topics == NULL) { @@ -516,6 +1047,7 @@ void mndUserFreeObj(SUserObj *pUser) { taosHashCleanup(pUser->readTbs); taosHashCleanup(pUser->writeTbs); taosHashCleanup(pUser->useDbs); + taosMemoryFreeClear(pUser->pIpWhiteList); pUser->readDbs = NULL; pUser->writeDbs = NULL; pUser->topics = NULL; @@ -545,6 +1077,12 @@ static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pOld, SUserObj *pNew) { TSWAP(pOld->readTbs, pNew->readTbs); TSWAP(pOld->writeTbs, pNew->writeTbs); TSWAP(pOld->useDbs, pNew->useDbs); + + int32_t sz = sizeof(SIpWhiteList) + pNew->pIpWhiteList->num * sizeof(SIpV4Range); + pOld->pIpWhiteList = taosMemoryRealloc(pOld->pIpWhiteList, sz); + memcpy(pOld->pIpWhiteList, pNew->pIpWhiteList, sz); + pOld->ipWhiteListVer = pNew->ipWhiteListVer; + taosWUnLockLatch(&pOld->lock); return 0; @@ -579,9 +1117,50 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate userObj.sysInfo = pCreate->sysInfo; userObj.enable = pCreate->enable; + if (pCreate->numIpRanges == 0) { + userObj.pIpWhiteList = createDefaultIpWhiteList(); + + } else { + SHashObj *pUniqueTab = taosHashInit(64, MurmurHash3_32, false, HASH_NO_LOCK); + int32_t dummpy = 0; + for (int i = 0; i < pCreate->numIpRanges; i++) { + SIpV4Range range = {.ip = pCreate->pIpRanges[i].ip, .mask = pCreate->pIpRanges[i].mask}; + taosHashPut(pUniqueTab, &range, sizeof(range), &dummpy, sizeof(dummpy)); + } + taosHashPut(pUniqueTab, &defaultIpRange, sizeof(defaultIpRange), &dummpy, sizeof(dummpy)); + + if (taosHashGetSize(pUniqueTab) > MND_MAX_USE_HOST) { + terrno = TSDB_CODE_MND_TOO_MANY_USER_HOST; + taosHashCleanup(pUniqueTab); + return terrno; + } + + int32_t numOfRanges = taosHashGetSize(pUniqueTab); + SIpWhiteList *p = taosMemoryCalloc(1, sizeof(SIpWhiteList) + numOfRanges * sizeof(SIpV4Range)); + void *pIter = taosHashIterate(pUniqueTab, NULL); + int32_t i = 0; + while (pIter) { + size_t len = 0; + SIpV4Range *key = taosHashGetKey(pIter, &len); + p->pIpRange[i].ip = key->ip; + p->pIpRange[i].mask = key->mask; + pIter = taosHashIterate(pUniqueTab, pIter); + + i++; + } + + taosHashCleanup(pUniqueTab); + p->num = numOfRanges; + userObj.pIpWhiteList = p; + } + + userObj.ipWhiteListVer = taosGetTimestampMs(); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "create-user"); if (pTrans == NULL) { mError("user:%s, failed to create since %s", pCreate->user, terrstr()); + + taosMemoryFree(userObj.pIpWhiteList); return -1; } mInfo("trans:%d, used to create user:%s", pTrans->id, pCreate->user); @@ -590,18 +1169,24 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { mError("trans:%d, failed to commit redo log since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); - return -1; + goto _OVER; } (void)sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY); if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); - return -1; + goto _OVER; } + ipWhiteMgtUpdate(userObj.user, userObj.pIpWhiteList); + taosMemoryFree(userObj.pIpWhiteList); mndTransDrop(pTrans); return 0; +_OVER: + taosMemoryFree(userObj.pIpWhiteList); + + return -1; } static int32_t mndProcessCreateUserReq(SRpcMsg *pReq) { @@ -631,7 +1216,7 @@ static int32_t mndProcessCreateUserReq(SRpcMsg *pReq) { goto _OVER; } - if (strlen(createReq.pass) >= TSDB_PASSWORD_LEN){ + if (strlen(createReq.pass) >= TSDB_PASSWORD_LEN) { terrno = TSDB_CODE_PAR_NAME_OR_PASSWD_TOO_LONG; goto _OVER; } @@ -657,8 +1242,8 @@ static int32_t mndProcessCreateUserReq(SRpcMsg *pReq) { if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; char detail[1000] = {0}; - sprintf(detail, "createType:%d, enable:%d, superUser:%d, sysInfo:%d", - createReq.createType, createReq.enable, createReq.superUser, createReq.sysInfo); + sprintf(detail, "createType:%d, enable:%d, superUser:%d, sysInfo:%d", createReq.createType, createReq.enable, + createReq.superUser, createReq.sysInfo); auditRecord(pReq, pMnode->clusterId, "createUser", createReq.user, "", detail); @@ -673,6 +1258,73 @@ _OVER: return code; } +int32_t mndProcessGetUserWhiteListReq(SRpcMsg *pReq) { + SMnode *pMnode = pReq->info.node; + int32_t code = -1; + SUserObj *pUser = NULL; + SGetUserWhiteListReq wlReq = {0}; + SGetUserWhiteListRsp wlRsp = {0}; + + if (tDeserializeSGetUserWhiteListReq(pReq->pCont, pReq->contLen, &wlReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto _OVER; + } + mTrace("user: %s, start to get whitelist", wlReq.user); + + pUser = mndAcquireUser(pMnode, wlReq.user); + if (pUser == NULL) { + terrno = TSDB_CODE_MND_USER_NOT_EXIST; + goto _OVER; + } + + code = mndSetUserWhiteListRsp(pMnode, pUser, &wlRsp); + if (code) { + goto _OVER; + } + int32_t contLen = tSerializeSGetUserWhiteListRsp(NULL, 0, &wlRsp); + void *pRsp = rpcMallocCont(contLen); + if (pRsp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _OVER; + } + + tSerializeSGetUserWhiteListRsp(pRsp, contLen, &wlRsp); + + pReq->info.rsp = pRsp; + pReq->info.rspLen = contLen; + code = 0; +_OVER: + mndReleaseUser(pMnode, pUser); + tFreeSGetUserWhiteListRsp(&wlRsp); + return code; +} + +int32_t mndProcesSRetrieveIpWhiteReq(SRpcMsg *pReq) { + // impl later + SRetrieveIpWhiteReq req = {0}; + if (tDeserializeRetrieveIpWhite(pReq->pCont, pReq->contLen, &req) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto _OVER; + } + + SUpdateIpWhite ipWhite = {0}; + ipWhiteMgtFillMsg(&ipWhite); + + int32_t len = tSerializeSUpdateIpWhite(NULL, 0, &ipWhite); + + void *pRsp = rpcMallocCont(len); + tSerializeSUpdateIpWhite(pRsp, len, &ipWhite); + + pReq->info.rsp = pRsp; + pReq->info.rspLen = len; + //} + tFreeSUpdateIpWhiteReq(&ipWhite); + + return 0; +_OVER: + return -1; +} + static int32_t mndAlterUser(SMnode *pMnode, SUserObj *pOld, SUserObj *pNew, SRpcMsg *pReq) { STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "alter-user"); if (pTrans == NULL) { @@ -694,7 +1346,7 @@ static int32_t mndAlterUser(SMnode *pMnode, SUserObj *pOld, SUserObj *pNew, SRpc mndTransDrop(pTrans); return -1; } - + ipWhiteMgtUpdate(pNew->user, pNew->pIpWhiteList); mndTransDrop(pTrans); return 0; } @@ -791,62 +1443,62 @@ static int32_t mndRemoveTablePriviledge(SMnode *pMnode, SHashObj *hash, SHashObj return 0; } -static char* mndUserAuditTypeStr(int32_t type){ - if(type == TSDB_ALTER_USER_PASSWD){ +static char *mndUserAuditTypeStr(int32_t type) { + if (type == TSDB_ALTER_USER_PASSWD) { return "changePassword"; } - if(type == TSDB_ALTER_USER_SUPERUSER){ + if (type == TSDB_ALTER_USER_SUPERUSER) { return "changeSuperUser"; } - if(type == TSDB_ALTER_USER_ADD_READ_DB){ + if (type == TSDB_ALTER_USER_ADD_READ_DB) { return "addReadToDB"; } - if(type == TSDB_ALTER_USER_ADD_READ_DB){ + if (type == TSDB_ALTER_USER_ADD_READ_DB) { return "addReadToDB"; } - if(type == TSDB_ALTER_USER_REMOVE_READ_DB){ + if (type == TSDB_ALTER_USER_REMOVE_READ_DB) { return "removeReadFromDB"; } - if(type == TSDB_ALTER_USER_ADD_WRITE_DB){ + if (type == TSDB_ALTER_USER_ADD_WRITE_DB) { return "addWriteToDB"; } - if(type == TSDB_ALTER_USER_REMOVE_WRITE_DB){ + if (type == TSDB_ALTER_USER_REMOVE_WRITE_DB) { return "removeWriteFromDB"; } - if(type == TSDB_ALTER_USER_ADD_ALL_DB){ + if (type == TSDB_ALTER_USER_ADD_ALL_DB) { return "addToAllDB"; } - if(type == TSDB_ALTER_USER_REMOVE_ALL_DB){ + if (type == TSDB_ALTER_USER_REMOVE_ALL_DB) { return "removeFromAllDB"; } - if(type == TSDB_ALTER_USER_ENABLE){ + if (type == TSDB_ALTER_USER_ENABLE) { return "enableUser"; } - if(type == TSDB_ALTER_USER_SYSINFO){ + if (type == TSDB_ALTER_USER_SYSINFO) { return "userSysInfo"; } - if(type == TSDB_ALTER_USER_ADD_SUBSCRIBE_TOPIC){ + if (type == TSDB_ALTER_USER_ADD_SUBSCRIBE_TOPIC) { return "addSubscribeTopic"; } - if(type == TSDB_ALTER_USER_REMOVE_SUBSCRIBE_TOPIC){ + if (type == TSDB_ALTER_USER_REMOVE_SUBSCRIBE_TOPIC) { return "removeSubscribeTopic"; } - if(type == TSDB_ALTER_USER_ADD_READ_TABLE){ + if (type == TSDB_ALTER_USER_ADD_READ_TABLE) { return "addReadToTable"; } - if(type == TSDB_ALTER_USER_REMOVE_READ_TABLE){ + if (type == TSDB_ALTER_USER_REMOVE_READ_TABLE) { return "removeReadFromTable"; } - if(type == TSDB_ALTER_USER_ADD_WRITE_TABLE){ + if (type == TSDB_ALTER_USER_ADD_WRITE_TABLE) { return "addWriteToTable"; } - if(type == TSDB_ALTER_USER_REMOVE_WRITE_TABLE){ + if (type == TSDB_ALTER_USER_REMOVE_WRITE_TABLE) { return "removeWriteFromTable"; } - if(type == TSDB_ALTER_USER_ADD_ALL_TABLE){ + if (type == TSDB_ALTER_USER_ADD_ALL_TABLE) { return "addToAllTable"; } - if(type == TSDB_ALTER_USER_REMOVE_ALL_TABLE){ + if (type == TSDB_ALTER_USER_REMOVE_ALL_TABLE) { return "removeFromAllTable"; } return "error"; @@ -874,7 +1526,7 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) { goto _OVER; } - if (TSDB_ALTER_USER_PASSWD == alterReq.alterType && + if (TSDB_ALTER_USER_PASSWD == alterReq.alterType && (alterReq.pass[0] == 0 || strlen(alterReq.pass) >= TSDB_PASSWORD_LEN)) { terrno = TSDB_CODE_MND_INVALID_PASS_FORMAT; goto _OVER; @@ -1007,11 +1659,13 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) { if (mndTablePriviledge(pMnode, newUser.writeTbs, newUser.useDbs, &alterReq, pSdb) != 0) goto _OVER; } - if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_READ_TABLE || alterReq.alterType == TSDB_ALTER_USER_REMOVE_ALL_TABLE) { + if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_READ_TABLE || + alterReq.alterType == TSDB_ALTER_USER_REMOVE_ALL_TABLE) { if (mndRemoveTablePriviledge(pMnode, newUser.readTbs, newUser.useDbs, &alterReq, pSdb) != 0) goto _OVER; } - if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_WRITE_TABLE || alterReq.alterType == TSDB_ALTER_USER_REMOVE_ALL_TABLE) { + if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_WRITE_TABLE || + alterReq.alterType == TSDB_ALTER_USER_REMOVE_ALL_TABLE) { if (mndRemoveTablePriviledge(pMnode, newUser.writeTbs, newUser.useDbs, &alterReq, pSdb) != 0) goto _OVER; } @@ -1035,50 +1689,135 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) { taosHashRemove(newUser.topics, alterReq.objname, len); } + if (alterReq.alterType == TSDB_ALTER_USER_ADD_WHITE_LIST) { + taosMemoryFreeClear(newUser.pIpWhiteList); + + int32_t num = pUser->pIpWhiteList->num + alterReq.numIpRanges; + + SIpWhiteList *pNew = taosMemoryCalloc(1, sizeof(SIpWhiteList) + sizeof(SIpV4Range) * num); + int32_t idx = pUser->pIpWhiteList->num; + + bool exist = false; + memcpy(pNew->pIpRange, pUser->pIpWhiteList->pIpRange, sizeof(SIpV4Range) * idx); + for (int i = 0; i < alterReq.numIpRanges; i++) { + SIpV4Range *range = &(alterReq.pIpRanges[i]); + if (!isRangeInIpWhiteList(pUser->pIpWhiteList, range)) { + // already exist, just ignore; + memcpy(&pNew->pIpRange[idx], range, sizeof(SIpV4Range)); + idx++; + continue; + } else { + exist = true; + } + } + if (exist) { + taosMemoryFree(pNew); + terrno = TSDB_CODE_MND_USER_HOST_EXIST; + code = terrno; + goto _OVER; + } + pNew->num = idx; + newUser.pIpWhiteList = pNew; + newUser.ipWhiteListVer = pUser->ipWhiteListVer + 1; + + if (pNew->num > MND_MAX_USE_HOST) { + terrno = TSDB_CODE_MND_TOO_MANY_USER_HOST; + code = terrno; + goto _OVER; + } + } + if (alterReq.alterType == TSDB_ALTER_USER_DROP_WHITE_LIST) { + taosMemoryFreeClear(newUser.pIpWhiteList); + + int32_t num = pUser->pIpWhiteList->num; + SIpWhiteList *pNew = taosMemoryCalloc(1, sizeof(SIpWhiteList) + sizeof(SIpV4Range) * num); + bool noexist = true; + bool localHost = false; + + if (pUser->pIpWhiteList->num > 0) { + int idx = 0; + for (int i = 0; i < pUser->pIpWhiteList->num; i++) { + SIpV4Range *oldRange = &pUser->pIpWhiteList->pIpRange[i]; + bool found = false; + for (int j = 0; j < alterReq.numIpRanges; j++) { + SIpV4Range *range = &alterReq.pIpRanges[j]; + if (isDefaultRange(range)) { + localHost = true; + break; + } + if (isIpRangeEqual(oldRange, range)) { + found = true; + break; + } + } + if (localHost) break; + + if (found == false) { + memcpy(&pNew->pIpRange[idx], oldRange, sizeof(SIpV4Range)); + idx++; + } else { + noexist = false; + } + } + pNew->num = idx; + newUser.pIpWhiteList = pNew; + newUser.ipWhiteListVer = pUser->ipWhiteListVer + 1; + + } else { + pNew->num = 0; + newUser.pIpWhiteList = pNew; + newUser.ipWhiteListVer = pUser->ipWhiteListVer + 1; + } + + if (localHost) { + terrno = TSDB_CODE_MND_USER_LOCAL_HOST_NOT_DROP; + code = terrno; + goto _OVER; + } + if (noexist) { + terrno = TSDB_CODE_MND_USER_HOST_NOT_EXIST; + code = terrno; + goto _OVER; + } + } + code = mndAlterUser(pMnode, pUser, &newUser, pReq); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; char detail[1000] = {0}; sprintf(detail, "alterType:%s, enable:%d, superUser:%d, sysInfo:%d, tabName:%s, password:", - mndUserAuditTypeStr(alterReq.alterType), alterReq.enable, alterReq.superUser, alterReq.sysInfo, alterReq.tabName); + mndUserAuditTypeStr(alterReq.alterType), alterReq.enable, alterReq.superUser, alterReq.sysInfo, + alterReq.tabName); - if(alterReq.alterType == TSDB_ALTER_USER_PASSWD){ + if (alterReq.alterType == TSDB_ALTER_USER_PASSWD) { sprintf(detail, "alterType:%s, enable:%d, superUser:%d, sysInfo:%d, tabName:%s, password:xxx", mndUserAuditTypeStr(alterReq.alterType), alterReq.enable, alterReq.superUser, alterReq.sysInfo, alterReq.tabName); auditRecord(pReq, pMnode->clusterId, "alterUser", alterReq.user, "", detail); - } - else if(alterReq.alterType == TSDB_ALTER_USER_SUPERUSER || - alterReq.alterType == TSDB_ALTER_USER_ENABLE || - alterReq.alterType == TSDB_ALTER_USER_SYSINFO){ + } else if (alterReq.alterType == TSDB_ALTER_USER_SUPERUSER || alterReq.alterType == TSDB_ALTER_USER_ENABLE || + alterReq.alterType == TSDB_ALTER_USER_SYSINFO) { auditRecord(pReq, pMnode->clusterId, "alterUser", alterReq.user, "", detail); - } - else if(alterReq.alterType == TSDB_ALTER_USER_ADD_READ_DB|| - alterReq.alterType == TSDB_ALTER_USER_ADD_WRITE_DB|| - alterReq.alterType == TSDB_ALTER_USER_ADD_ALL_DB|| - alterReq.alterType == TSDB_ALTER_USER_ADD_READ_TABLE|| - alterReq.alterType == TSDB_ALTER_USER_ADD_WRITE_TABLE|| - alterReq.alterType == TSDB_ALTER_USER_ADD_ALL_TABLE){ - if (strcmp(alterReq.objname, "1.*") != 0){ + } else if (alterReq.alterType == TSDB_ALTER_USER_ADD_READ_DB || alterReq.alterType == TSDB_ALTER_USER_ADD_WRITE_DB || + alterReq.alterType == TSDB_ALTER_USER_ADD_ALL_DB || alterReq.alterType == TSDB_ALTER_USER_ADD_READ_TABLE || + alterReq.alterType == TSDB_ALTER_USER_ADD_WRITE_TABLE || + alterReq.alterType == TSDB_ALTER_USER_ADD_ALL_TABLE) { + if (strcmp(alterReq.objname, "1.*") != 0) { SName name = {0}; tNameFromString(&name, alterReq.objname, T_NAME_ACCT | T_NAME_DB); auditRecord(pReq, pMnode->clusterId, "GrantPrivileges", alterReq.user, name.dbname, detail); - }else{ + } else { auditRecord(pReq, pMnode->clusterId, "GrantPrivileges", alterReq.user, "*", detail); } - } - else if(alterReq.alterType == TSDB_ALTER_USER_ADD_SUBSCRIBE_TOPIC){ + } else if (alterReq.alterType == TSDB_ALTER_USER_ADD_SUBSCRIBE_TOPIC) { auditRecord(pReq, pMnode->clusterId, "GrantPrivileges", alterReq.user, alterReq.objname, detail); - } - else if(alterReq.alterType == TSDB_ALTER_USER_REMOVE_SUBSCRIBE_TOPIC){ + } else if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_SUBSCRIBE_TOPIC) { auditRecord(pReq, pMnode->clusterId, "RevokePrivileges", alterReq.user, alterReq.objname, detail); - } - else{ - if (strcmp(alterReq.objname, "1.*") != 0){ + } else { + if (strcmp(alterReq.objname, "1.*") != 0) { SName name = {0}; tNameFromString(&name, alterReq.objname, T_NAME_ACCT | T_NAME_DB); auditRecord(pReq, pMnode->clusterId, "RevokePrivileges", alterReq.user, name.dbname, detail); - }else{ + } else { auditRecord(pReq, pMnode->clusterId, "RevokePrivileges", alterReq.user, "*", detail); } } @@ -1117,6 +1856,7 @@ static int32_t mndDropUser(SMnode *pMnode, SRpcMsg *pReq, SUserObj *pUser) { mndTransDrop(pTrans); return -1; } + ipWhiteMgtRemove(pUser->user); mndTransDrop(pTrans); return 0; @@ -1243,6 +1983,26 @@ static int32_t mndRetrieveUsers(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl pColInfo = taosArrayGet(pBlock->pDataBlock, cols); colDataSetVal(pColInfo, numOfRows, (const char *)&pUser->createdTime, false); + cols++; + + char *buf = NULL; + int32_t tlen = convertIpWhiteListToStr(pUser->pIpWhiteList, &buf); + // int32_t tlen = mndFetchIpWhiteList(pUser->pIpWhiteList, &buf); + if (tlen != 0) { + char *varstr = taosMemoryCalloc(1, VARSTR_HEADER_SIZE + tlen); + varDataSetLen(varstr, tlen); + memcpy(varDataVal(varstr), buf, tlen); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols); + colDataSetVal(pColInfo, numOfRows, (const char *)varstr, false); + + taosMemoryFree(varstr); + taosMemoryFree(buf); + } else { + pColInfo = taosArrayGet(pBlock->pDataBlock, cols); + colDataSetVal(pColInfo, numOfRows, (const char *)NULL, true); + } + numOfRows++; sdbRelease(pSdb, pUser); } @@ -1293,9 +2053,9 @@ static void mndLoopHash(SHashObj *hash, char *priType, SSDataBlock *pBlock, int3 if (strcmp("t", value) != 0) { SNode *pAst = NULL; int32_t sqlLen = 0; - size_t bufSz = strlen(value) + 1; - char* sql = taosMemoryMalloc(bufSz + 1); - char* obj = taosMemoryMalloc(TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE); + size_t bufSz = strlen(value) + 1; + char *sql = taosMemoryMalloc(bufSz + 1); + char *obj = taosMemoryMalloc(TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE); if (sql != NULL && obj != NULL && nodesStringToNode(value, &pAst) == 0) { nodesNodeToSQL(pAst, sql, bufSz, &sqlLen); @@ -1312,7 +2072,7 @@ static void mndLoopHash(SHashObj *hash, char *priType, SSDataBlock *pBlock, int3 taosMemoryFree(obj); taosMemoryFree(sql); } else { - char* condition = taosMemoryMalloc(TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE); + char *condition = taosMemoryMalloc(TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE); STR_WITH_MAXSIZE_TO_VARSTR(condition, "", pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, *numOfRows, (const char *)condition, false); @@ -1332,7 +2092,7 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock int32_t cols = 0; char *pWrite; - bool fetchNextUser = pShow->restore ? false : true; + bool fetchNextUser = pShow->restore ? false : true; pShow->restore = false; while (numOfRows < rows) { @@ -1354,8 +2114,10 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock int32_t numOfReadTbs = taosHashGetSize(pUser->readTbs); int32_t numOfWriteTbs = taosHashGetSize(pUser->writeTbs); if (numOfRows + numOfReadDbs + numOfWriteDbs + numOfTopics + numOfReadTbs + numOfWriteTbs >= rows) { - mInfo("will restore. current num of rows: %d, read dbs %d, write dbs %d, topics %d, read tables %d, write tables %d", - numOfRows, numOfReadDbs, numOfWriteDbs, numOfTopics, numOfReadTbs, numOfWriteTbs); + mInfo( + "will restore. current num of rows: %d, read dbs %d, write dbs %d, topics %d, read tables %d, write tables " + "%d", + numOfRows, numOfReadDbs, numOfWriteDbs, numOfTopics, numOfReadTbs, numOfWriteTbs); pShow->restore = true; sdbRelease(pSdb, pUser); break; @@ -1383,7 +2145,7 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)tableName, false); - char* condition = taosMemoryMalloc(TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE); + char *condition = taosMemoryMalloc(TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE); STR_WITH_MAXSIZE_TO_VARSTR(condition, "", pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)condition, false); @@ -1418,7 +2180,7 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)tableName, false); - char* condition = taosMemoryMalloc(TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE); + char *condition = taosMemoryMalloc(TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE); STR_WITH_MAXSIZE_TO_VARSTR(condition, "", pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)condition, false); @@ -1454,7 +2216,7 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)tableName, false); - char* condition = taosMemoryMalloc(TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE); + char *condition = taosMemoryMalloc(TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE); STR_WITH_MAXSIZE_TO_VARSTR(condition, "", pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)condition, false); @@ -1492,7 +2254,7 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)tableName, false); - char* condition = taosMemoryMalloc(TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE); + char *condition = taosMemoryMalloc(TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE); STR_WITH_MAXSIZE_TO_VARSTR(condition, "", pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)condition, false); diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 6ff33c4f6e..56d8b1ea45 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -1948,6 +1948,10 @@ static int32_t vnodeProcessDropIndexReq(SVnode *pVnode, int64_t ver, void *pReq, extern int32_t vnodeProcessCompactVnodeReqImpl(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessCompactVnodeReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) { + if (!pVnode->restored) { + vInfo("vgId:%d, ignore compact req during restoring. ver:%" PRId64, TD_VID(pVnode), ver); + return 0; + } return vnodeProcessCompactVnodeReqImpl(pVnode, ver, pReq, len, pRsp); } diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 5e2ca462f7..740ff7b0dc 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -196,4 +196,22 @@ void updateTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pWin, int64_t SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTables, SArray* pUidTagList, void* pVnode, SStorageAPI* pStorageAPI); + +/** + * @brief build a tuple into keyBuf + * @param [out] keyBuf the output buf + * @param [in] pSortGroupCols the cols to build + * @param [in] pBlock block the tuple in + */ +int32_t buildKeys(char* keyBuf, const SArray* pSortGroupCols, const SSDataBlock* pBlock, int32_t rowIndex); + +int32_t compKeys(const SArray* pSortGroupCols, const char* oldkeyBuf, int32_t oldKeysLen, const SSDataBlock* pDataBlock, + int32_t rowIndex); + +uint64_t calcGroupId(char *pData, int32_t len); + +SNodeList* makeColsNodeArrFromSortKeys(SNodeList* pSortKeys); + +int32_t extractKeysLen(const SArray* keys); + #endif // TDENGINE_EXECUTIL_H diff --git a/source/libs/executor/inc/tsort.h b/source/libs/executor/inc/tsort.h index 3180173ca7..365acf2bff 100644 --- a/source/libs/executor/inc/tsort.h +++ b/source/libs/executor/inc/tsort.h @@ -194,6 +194,16 @@ void tsortSetClosed(SSortHandle* pHandle); void tsortSetSingleTableMerge(SSortHandle* pHandle); void tsortSetAbortCheckFn(SSortHandle* pHandle, bool (*checkFn)(void* param), void* param); +/** + * @brief comp the tuple with keyBuf, if not equal, new keys will be built in keyBuf, newLen will be stored in keyLen + * @param [in] pSortCols cols to comp and build + * @param [in, out] pass in the old keys, if comp not equal, new keys will be built in it. + * @param [in, out] keyLen the old keysLen, if comp not equal, new keysLen will be stored in it. + * @param [in] the tuple to comp with + * @retval 0 if comp equal, 1 if not + */ +int32_t tsortCompAndBuildKeys(const SArray* pSortCols, char* keyBuf, int32_t* keyLen, const STupleHandle* pTuple); + #ifdef __cplusplus } #endif diff --git a/source/libs/executor/src/aggregateoperator.c b/source/libs/executor/src/aggregateoperator.c index f6a8c6689f..5e649af47e 100644 --- a/source/libs/executor/src/aggregateoperator.c +++ b/source/libs/executor/src/aggregateoperator.c @@ -303,6 +303,7 @@ static int32_t createDataBlockForEmptyInput(SOperatorInfo* pOperator, SSDataBloc SOperatorInfo* downstream = pOperator->pDownstream[0]; if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_PARTITION || + downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_SORT || (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN && ((STableScanInfo*)downstream->info)->hasGroupByTag == true)) { return TSDB_CODE_SUCCESS; diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 52631cd0db..753d3e680c 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -2261,3 +2261,104 @@ void updateTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pWin, int64_t ts[3] = pWin->skey; // window start key ts[4] = pWin->ekey + delta; // window end key } + +int32_t compKeys(const SArray* pSortGroupCols, const char* oldkeyBuf, int32_t oldKeysLen, const SSDataBlock* pBlock, int32_t rowIndex) { + SColumnDataAgg* pColAgg = NULL; + const char* isNull = oldkeyBuf; + const char* p = oldkeyBuf + sizeof(int8_t) * pSortGroupCols->size; + + for (int32_t i = 0; i < pSortGroupCols->size; ++i) { + const SColumn* pCol = (SColumn*)TARRAY_GET_ELEM(pSortGroupCols, i); + const SColumnInfoData* pColInfoData = TARRAY_GET_ELEM(pBlock->pDataBlock, pCol->slotId); + if (pBlock->pBlockAgg) pColAgg = pBlock->pBlockAgg[pCol->slotId]; + + if (colDataIsNull(pColInfoData, pBlock->info.rows, rowIndex, pColAgg)) { + if (isNull[i] != 1) return 1; + } else { + if (isNull[i] != 0) return 1; + const char* val = colDataGetData(pColInfoData, rowIndex); + if (pCol->type == TSDB_DATA_TYPE_JSON) { + int32_t len = getJsonValueLen(val); + if (memcmp(p, val, len) != 0) return 1; + p += len; + } else if (IS_VAR_DATA_TYPE(pCol->type)) { + if (memcmp(p, val, varDataTLen(val)) != 0) return 1; + p += varDataTLen(val); + } else { + if (0 != memcmp(p, val, pCol->bytes)) return 1; + p += pCol->bytes; + } + } + } + if ((int32_t)(p - oldkeyBuf) != oldKeysLen) return 1; + return 0; +} + +int32_t buildKeys(char* keyBuf, const SArray* pSortGroupCols, const SSDataBlock* pBlock, + int32_t rowIndex) { + uint32_t colNum = pSortGroupCols->size; + SColumnDataAgg* pColAgg = NULL; + char* isNull = keyBuf; + char* p = keyBuf + sizeof(int8_t) * colNum; + + for (int32_t i = 0; i < colNum; ++i) { + const SColumn* pCol = (SColumn*)TARRAY_GET_ELEM(pSortGroupCols, i); + const SColumnInfoData* pColInfoData = TARRAY_GET_ELEM(pBlock->pDataBlock, pCol->slotId); + if (pCol->slotId > pBlock->pDataBlock->size) continue; + + if (pBlock->pBlockAgg) pColAgg = pBlock->pBlockAgg[pCol->slotId]; + + if (colDataIsNull(pColInfoData, pBlock->info.rows, rowIndex, pColAgg)) { + isNull[i] = 1; + } else { + isNull[i] = 0; + const char* val = colDataGetData(pColInfoData, rowIndex); + if (pCol->type == TSDB_DATA_TYPE_JSON) { + int32_t len = getJsonValueLen(val); + memcpy(p, val, len); + p += len; + } else if (IS_VAR_DATA_TYPE(pCol->type)) { + varDataCopy(p, val); + p += varDataTLen(val); + } else { + memcpy(p, val, pCol->bytes); + p += pCol->bytes; + } + } + } + return (int32_t)(p - keyBuf); +} + +uint64_t calcGroupId(char* pData, int32_t len) { + T_MD5_CTX context; + tMD5Init(&context); + tMD5Update(&context, (uint8_t*)pData, len); + tMD5Final(&context); + + // NOTE: only extract the initial 8 bytes of the final MD5 digest + uint64_t id = 0; + memcpy(&id, context.digest, sizeof(uint64_t)); + if (0 == id) memcpy(&id, context.digest + 8, sizeof(uint64_t)); + return id; +} + +SNodeList* makeColsNodeArrFromSortKeys(SNodeList* pSortKeys) { + SNode* node; + SNodeList* ret = NULL; + FOREACH(node, pSortKeys) { + SOrderByExprNode* pSortKey = (SOrderByExprNode*)node; + nodesListMakeAppend(&ret, pSortKey->pExpr); + } + return ret; +} + +int32_t extractKeysLen(const SArray* keys) { + int32_t len = 0; + int32_t keyNum = taosArrayGetSize(keys); + for (int32_t i = 0; i < keyNum; ++i) { + SColumn* pCol = (SColumn*)taosArrayGet(keys, i); + len += pCol->bytes; + } + len += sizeof(int8_t) * keyNum; //null flag + return len; +} diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index fb2204eae8..1060dd4f0e 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -56,6 +56,10 @@ typedef struct SPartitionOperatorInfo { int32_t groupIndex; // group index int32_t pageIndex; // page index of current group SExprSupp scalarSup; + + int32_t remainRows; + int32_t orderedRows; + SArray* pOrderInfoArr; } SPartitionOperatorInfo; static void* getCurrentDataGroupInfo(const SPartitionOperatorInfo* pInfo, SDataGroupInfo** pGroupInfo, int32_t len); @@ -635,20 +639,6 @@ void* getCurrentDataGroupInfo(const SPartitionOperatorInfo* pInfo, SDataGroupInf return pPage; } -uint64_t calcGroupId(char* pData, int32_t len) { - T_MD5_CTX context; - tMD5Init(&context); - tMD5Update(&context, (uint8_t*)pData, len); - tMD5Final(&context); - - // NOTE: only extract the initial 8 bytes of the final MD5 digest - uint64_t id = 0; - memcpy(&id, context.digest, sizeof(uint64_t)); - if (0 == id) - memcpy(&id, context.digest + 8, sizeof(uint64_t)); - return id; -} - int32_t* setupColumnOffset(const SSDataBlock* pBlock, int32_t rowCapacity) { size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); int32_t* offset = taosMemoryCalloc(numOfCols, sizeof(int32_t)); @@ -699,37 +689,49 @@ static SSDataBlock* buildPartitionResult(SOperatorInfo* pOperator) { SPartitionOperatorInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SDataGroupInfo* pGroupInfo = - (pInfo->groupIndex != -1) ? taosArrayGet(pInfo->sortedGroupArray, pInfo->groupIndex) : NULL; - if (pInfo->groupIndex == -1 || pInfo->pageIndex >= taosArrayGetSize(pGroupInfo->pPageList)) { - // try next group data - ++pInfo->groupIndex; - if (pInfo->groupIndex >= taosArrayGetSize(pInfo->sortedGroupArray)) { - setOperatorCompleted(pOperator); - clearPartitionOperator(pInfo); - return NULL; + if (pInfo->remainRows == 0) { + blockDataCleanup(pInfo->binfo.pRes); + SDataGroupInfo* pGroupInfo = + (pInfo->groupIndex != -1) ? taosArrayGet(pInfo->sortedGroupArray, pInfo->groupIndex) : NULL; + if (pInfo->groupIndex == -1 || pInfo->pageIndex >= taosArrayGetSize(pGroupInfo->pPageList)) { + // try next group data + ++pInfo->groupIndex; + if (pInfo->groupIndex >= taosArrayGetSize(pInfo->sortedGroupArray)) { + setOperatorCompleted(pOperator); + clearPartitionOperator(pInfo); + return NULL; + } + + pGroupInfo = taosArrayGet(pInfo->sortedGroupArray, pInfo->groupIndex); + pInfo->pageIndex = 0; } - pGroupInfo = taosArrayGet(pInfo->sortedGroupArray, pInfo->groupIndex); - pInfo->pageIndex = 0; + int32_t* pageId = taosArrayGet(pGroupInfo->pPageList, pInfo->pageIndex); + void* page = getBufPage(pInfo->pBuf, *pageId); + if (page == NULL) { + qError("failed to get buffer, code:%s, %s", tstrerror(terrno), GET_TASKID(pTaskInfo)); + T_LONG_JMP(pTaskInfo->env, terrno); + } + + blockDataEnsureCapacity(pInfo->binfo.pRes, pInfo->rowCapacity); + blockDataFromBuf1(pInfo->binfo.pRes, page, pInfo->rowCapacity); + + pInfo->pageIndex += 1; + releaseBufPage(pInfo->pBuf, page); + pInfo->binfo.pRes->info.id.groupId = pGroupInfo->groupId; + pInfo->binfo.pRes->info.dataLoad = 1; + pInfo->orderedRows = 0; } - int32_t* pageId = taosArrayGet(pGroupInfo->pPageList, pInfo->pageIndex); - void* page = getBufPage(pInfo->pBuf, *pageId); - if (page == NULL) { - qError("failed to get buffer, code:%s, %s", tstrerror(terrno), GET_TASKID(pTaskInfo)); - T_LONG_JMP(pTaskInfo->env, terrno); + if (pInfo->pOrderInfoArr) { + pInfo->binfo.pRes->info.rows += pInfo->remainRows; + blockDataTrimFirstRows(pInfo->binfo.pRes, pInfo->orderedRows); + pInfo->orderedRows = blockDataGetSortedRows(pInfo->binfo.pRes, pInfo->pOrderInfoArr); + pInfo->remainRows = pInfo->binfo.pRes->info.rows - pInfo->orderedRows; + pInfo->binfo.pRes->info.rows = pInfo->orderedRows; } - blockDataEnsureCapacity(pInfo->binfo.pRes, pInfo->rowCapacity); - blockDataFromBuf1(pInfo->binfo.pRes, page, pInfo->rowCapacity); - - pInfo->pageIndex += 1; - releaseBufPage(pInfo->pBuf, page); - - pInfo->binfo.pRes->info.dataLoad = 1; blockDataUpdateTsWindow(pInfo->binfo.pRes, 0); - pInfo->binfo.pRes->info.id.groupId = pGroupInfo->groupId; pOperator->resultInfo.totalRows += pInfo->binfo.pRes->info.rows; return pInfo->binfo.pRes; @@ -746,7 +748,6 @@ static SSDataBlock* hashPartition(SOperatorInfo* pOperator) { SSDataBlock* pRes = pInfo->binfo.pRes; if (pOperator->status == OP_RES_TO_RETURN) { - blockDataCleanup(pRes); return buildPartitionResult(pOperator); } @@ -829,6 +830,7 @@ static void destroyPartitionOperatorInfo(void* param) { cleanupExprSupp(&pInfo->scalarSup); destroyDiskbasedBuf(pInfo->pBuf); + taosArrayDestroy(pInfo->pOrderInfoArr); taosMemoryFreeClear(param); } @@ -846,6 +848,17 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartition SExprInfo* pExprInfo = createExprInfo(pPartNode->pTargets, NULL, &numOfCols); pInfo->pGroupCols = makeColumnArrayFromList(pPartNode->pPartitionKeys); + if (pPartNode->needBlockOutputTsOrder) { + SBlockOrderInfo order = {.order = ORDER_ASC, .pColData = NULL, .nullFirst = false, .slotId = pPartNode->tsSlotId}; + pInfo->pOrderInfoArr = taosArrayInit(1, sizeof(SBlockOrderInfo)); + if (!pInfo->pOrderInfoArr) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + pTaskInfo->code = terrno; + goto _error; + } + taosArrayPush(pInfo->pOrderInfoArr, &order); + } + if (pPartNode->pExprs != NULL) { int32_t num = 0; SExprInfo* pExprInfo1 = createExprInfo(pPartNode->pExprs, NULL, &num); diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index a59a7bb1ea..ccef6640be 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -19,18 +19,28 @@ #include "querytask.h" #include "tdatablock.h" +typedef struct SSortOpGroupIdCalc { + STupleHandle* pSavedTuple; + SArray* pSortColsArr; + char* keyBuf; + int32_t lastKeysLen; // default to be 0 + uint64_t lastGroupId; + bool excludePKCol; +} SSortOpGroupIdCalc; + typedef struct SSortOperatorInfo { - SOptrBasicInfo binfo; - uint32_t sortBufSize; // max buffer size for in-memory sort - SArray* pSortInfo; - SSortHandle* pSortHandle; - SColMatchInfo matchInfo; - int32_t bufPageSize; - int64_t startTs; // sort start time - uint64_t sortElapsed; // sort elapsed time, time to flush to disk not included. - SLimitInfo limitInfo; - uint64_t maxTupleLength; - int64_t maxRows; + SOptrBasicInfo binfo; + uint32_t sortBufSize; // max buffer size for in-memory sort + SArray* pSortInfo; + SSortHandle* pSortHandle; + SColMatchInfo matchInfo; + int32_t bufPageSize; + int64_t startTs; // sort start time + uint64_t sortElapsed; // sort elapsed time, time to flush to disk not included. + SLimitInfo limitInfo; + uint64_t maxTupleLength; + int64_t maxRows; + SSortOpGroupIdCalc* pGroupIdCalc; } SSortOperatorInfo; static SSDataBlock* doSort(SOperatorInfo* pOperator); @@ -40,6 +50,8 @@ static int32_t getExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain static void destroySortOperatorInfo(void* param); static int32_t calcSortOperMaxTupleLength(SSortOperatorInfo* pSortOperInfo, SNodeList* pSortKeys); +static void destroySortOpGroupIdCalc(SSortOpGroupIdCalc* pCalc); + // todo add limit/offset impl SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* pSortNode, SExecTaskInfo* pTaskInfo) { SSortOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSortOperatorInfo)); @@ -78,6 +90,34 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* pInfo->binfo.pRes = createDataBlockFromDescNode(pDescNode); pInfo->pSortInfo = createSortInfo(pSortNode->pSortKeys); + + if (pSortNode->calcGroupId) { + int32_t keyLen; + SSortOpGroupIdCalc* pGroupIdCalc = pInfo->pGroupIdCalc = taosMemoryCalloc(1, sizeof(SSortOpGroupIdCalc)); + if (!pGroupIdCalc) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _error; + } + SNodeList* pSortColsNodeArr = makeColsNodeArrFromSortKeys(pSortNode->pSortKeys); + if (!pSortColsNodeArr) code = TSDB_CODE_OUT_OF_MEMORY; + if (TSDB_CODE_SUCCESS == code) { + pGroupIdCalc->pSortColsArr = makeColumnArrayFromList(pSortColsNodeArr); + if (!pGroupIdCalc->pSortColsArr) code = TSDB_CODE_OUT_OF_MEMORY; + nodesClearList(pSortColsNodeArr); + } + if (TSDB_CODE_SUCCESS == code) { + // PK ts col should always at last, see partColOptCreateSort + if (pSortNode->excludePkCol) taosArrayPop(pGroupIdCalc->pSortColsArr); + keyLen = extractKeysLen(pGroupIdCalc->pSortColsArr); + } + if (TSDB_CODE_SUCCESS == code) { + pGroupIdCalc->lastKeysLen = 0; + pGroupIdCalc->keyBuf = taosMemoryCalloc(1, keyLen); + if (!pGroupIdCalc->keyBuf) code = TSDB_CODE_OUT_OF_MEMORY; + } + } + if (code != TSDB_CODE_SUCCESS) goto _error; + pInfo->binfo.inputTsOrder = pSortNode->node.inputTsOrder; pInfo->binfo.outputTsOrder = pSortNode->node.outputTsOrder; initLimitInfo(pSortNode->node.pLimit, pSortNode->node.pSlimit, &pInfo->limitInfo); @@ -129,6 +169,52 @@ void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle) { pBlock->info.rows += 1; } +/** + * @brief get next tuple with group id attached, here assume that all tuples are sorted by group keys + * @param [in, out] pBlock the output block, the group id will be saved in it + * @retval NULL if next group tuple arrived and this new group tuple will be saved in pInfo.pSavedTuple + * @retval NULL if no more tuples + */ +static STupleHandle* nextTupleWithGroupId(SSortHandle* pHandle, SSortOperatorInfo* pInfo, SSDataBlock* pBlock) { + STupleHandle* retTuple = pInfo->pGroupIdCalc->pSavedTuple; + if (!retTuple) { + retTuple = tsortNextTuple(pHandle); + } + + if (retTuple) { + int32_t newGroup; + if (pInfo->pGroupIdCalc->pSavedTuple) { + newGroup = true; + pInfo->pGroupIdCalc->pSavedTuple = NULL; + } else { + newGroup = tsortCompAndBuildKeys(pInfo->pGroupIdCalc->pSortColsArr, pInfo->pGroupIdCalc->keyBuf, + &pInfo->pGroupIdCalc->lastKeysLen, retTuple); + } + bool emptyBlock = pBlock->info.rows == 0; + if (newGroup) { + if (!emptyBlock) { + // new group arrived, and we have already copied some tuples for cur group, save the new group tuple, return + // NULL. Note that the keyBuf and lastKeysLen has been updated to new value + pInfo->pGroupIdCalc->pSavedTuple = retTuple; + retTuple = NULL; + } else { + // new group with empty block + pInfo->pGroupIdCalc->lastGroupId = pBlock->info.id.groupId = + calcGroupId(pInfo->pGroupIdCalc->keyBuf, pInfo->pGroupIdCalc->lastKeysLen); + } + } else { + if (emptyBlock) { + // new block but not new group, assign last group id to it + pBlock->info.id.groupId = pInfo->pGroupIdCalc->lastGroupId; + } else { + // not new group and not empty block and ret NOT NULL, just return the tuple + } + } + } + + return retTuple; +} + SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity, SArray* pColMatchInfo, SSortOperatorInfo* pInfo) { blockDataCleanup(pDataBlock); @@ -140,8 +226,13 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i blockDataEnsureCapacity(p, capacity); + STupleHandle* pTupleHandle; while (1) { - STupleHandle* pTupleHandle = tsortNextTuple(pHandle); + if (pInfo->pGroupIdCalc) { + pTupleHandle = nextTupleWithGroupId(pHandle, pInfo, p); + } else { + pTupleHandle = tsortNextTuple(pHandle); + } if (pTupleHandle == NULL) { break; } @@ -168,6 +259,7 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i pDataBlock->info.dataLoad = 1; pDataBlock->info.rows = p->info.rows; pDataBlock->info.scanFlag = p->info.scanFlag; + pDataBlock->info.id.groupId = p->info.id.groupId; } blockDataDestroy(p); @@ -281,6 +373,7 @@ void destroySortOperatorInfo(void* param) { tsortDestroySortHandle(pInfo->pSortHandle); taosArrayDestroy(pInfo->pSortInfo); taosArrayDestroy(pInfo->matchInfo.pList); + destroySortOpGroupIdCalc(pInfo->pGroupIdCalc); taosMemoryFreeClear(param); } @@ -309,6 +402,14 @@ static int32_t calcSortOperMaxTupleLength(SSortOperatorInfo* pSortOperInfo, SNod return TSDB_CODE_SUCCESS; } +static void destroySortOpGroupIdCalc(SSortOpGroupIdCalc* pCalc) { + if (pCalc) { + taosArrayDestroy(pCalc->pSortColsArr); + taosMemoryFree(pCalc->keyBuf); + taosMemoryFree(pCalc); + } +} + //===================================================================================== // Group Sort Operator typedef enum EChildOperatorStatus { CHILD_OP_NEW_GROUP, CHILD_OP_SAME_GROUP, CHILD_OP_FINISHED } EChildOperatorStatus; @@ -591,6 +692,7 @@ typedef struct SMultiwayMergeOperatorInfo { bool ignoreGroupId; uint64_t groupId; STupleHandle* prefetchedTuple; + bool inputWithGroupId; } SMultiwayMergeOperatorInfo; int32_t openMultiwayMergeOperator(SOperatorInfo* pOperator) { @@ -641,7 +743,7 @@ static void doGetSortedBlockData(SMultiwayMergeOperatorInfo* pInfo, SSortHandle* while (1) { STupleHandle* pTupleHandle = NULL; - if (pInfo->groupSort) { + if (pInfo->groupSort || pInfo->inputWithGroupId) { if (pInfo->prefetchedTuple == NULL) { pTupleHandle = tsortNextTuple(pHandle); } else { @@ -662,7 +764,7 @@ static void doGetSortedBlockData(SMultiwayMergeOperatorInfo* pInfo, SSortHandle* break; } - if (pInfo->groupSort) { + if (pInfo->groupSort || pInfo->inputWithGroupId) { uint64_t tupleGroupId = tsortGetGroupId(pTupleHandle); if (pInfo->groupId == 0 || pInfo->groupId == tupleGroupId) { appendOneRowToDataBlock(p, pTupleHandle); @@ -842,6 +944,7 @@ SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** downStreams, size pInfo->sortBufSize = pInfo->bufPageSize * (numStreams + 1); // one additional is reserved for merged result. pInfo->binfo.inputTsOrder = pMergePhyNode->node.inputTsOrder; pInfo->binfo.outputTsOrder = pMergePhyNode->node.outputTsOrder; + pInfo->inputWithGroupId = pMergePhyNode->inputWithGroupId; setOperatorInfo(pOperator, "MultiwayMergeOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE, false, OP_NOT_OPENED, pInfo, pTaskInfo); pOperator->fpSet = createOperatorFpSet(openMultiwayMergeOperator, doMultiwayMerge, NULL, diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 287c824540..1ebc7ad3c6 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -25,6 +25,7 @@ #include "tsort.h" #include "tutil.h" #include "tsimplehash.h" +#include "executil.h" struct STupleHandle { SSDataBlock* pBlock; @@ -615,48 +616,62 @@ int32_t msortComparFn(const void* pLeft, const void* pRight, void* param) { int ret = pParam->cmpFn(left1, right1); return ret; } else { + bool isVarType; for (int32_t i = 0; i < pInfo->size; ++i) { SBlockOrderInfo* pOrder = TARRAY_GET_ELEM(pInfo, i); SColumnInfoData* pLeftColInfoData = TARRAY_GET_ELEM(pLeftBlock->pDataBlock, pOrder->slotId); SColumnInfoData* pRightColInfoData = TARRAY_GET_ELEM(pRightBlock->pDataBlock, pOrder->slotId); + isVarType = IS_VAR_DATA_TYPE(pLeftColInfoData->info.type); - bool leftNull = false; - if (pLeftColInfoData->hasNull) { - if (pLeftBlock->pBlockAgg == NULL) { - leftNull = colDataIsNull_s(pLeftColInfoData, pLeftSource->src.rowIndex); - } else { - leftNull = colDataIsNull(pLeftColInfoData, pLeftBlock->info.rows, pLeftSource->src.rowIndex, - pLeftBlock->pBlockAgg[i]); + if (pLeftColInfoData->hasNull || pRightColInfoData->hasNull) { + bool leftNull = false; + if (pLeftColInfoData->hasNull) { + if (pLeftBlock->pBlockAgg == NULL) { + leftNull = colDataIsNull_t(pLeftColInfoData, pLeftSource->src.rowIndex, isVarType); + } else { + leftNull = colDataIsNull(pLeftColInfoData, pLeftBlock->info.rows, pLeftSource->src.rowIndex, + pLeftBlock->pBlockAgg[i]); + } + } + + bool rightNull = false; + if (pRightColInfoData->hasNull) { + if (pRightBlock->pBlockAgg == NULL) { + rightNull = colDataIsNull_t(pRightColInfoData, pRightSource->src.rowIndex, isVarType); + } else { + rightNull = colDataIsNull(pRightColInfoData, pRightBlock->info.rows, pRightSource->src.rowIndex, + pRightBlock->pBlockAgg[i]); + } + } + + if (leftNull && rightNull) { + continue; // continue to next slot + } + + if (rightNull) { + return pOrder->nullFirst ? 1 : -1; + } + + if (leftNull) { + return pOrder->nullFirst ? -1 : 1; } } - bool rightNull = false; - if (pRightColInfoData->hasNull) { - if (pRightBlock->pBlockAgg == NULL) { - rightNull = colDataIsNull_s(pRightColInfoData, pRightSource->src.rowIndex); - } else { - rightNull = colDataIsNull(pRightColInfoData, pRightBlock->info.rows, pRightSource->src.rowIndex, - pRightBlock->pBlockAgg[i]); - } + void* left1, *right1; + if (isVarType) { + left1 = colDataGetVarData(pLeftColInfoData, pLeftSource->src.rowIndex); + right1 = colDataGetVarData(pRightColInfoData, pRightSource->src.rowIndex); + } else { + left1 = colDataGetNumData(pLeftColInfoData, pLeftSource->src.rowIndex); + right1 = colDataGetNumData(pRightColInfoData, pRightSource->src.rowIndex); } - if (leftNull && rightNull) { - continue; // continue to next slot + __compar_fn_t fn = pOrder->compFn; + if (!fn) { + fn = getKeyComparFunc(pLeftColInfoData->info.type, pOrder->order); + pOrder->compFn = fn; } - if (rightNull) { - return pOrder->nullFirst ? 1 : -1; - } - - if (leftNull) { - return pOrder->nullFirst ? -1 : 1; - } - - void* left1 = colDataGetData(pLeftColInfoData, pLeftSource->src.rowIndex); - void* right1 = colDataGetData(pRightColInfoData, pRightSource->src.rowIndex); - - __compar_fn_t fn = getKeyComparFunc(pLeftColInfoData->info.type, pOrder->order); - int ret = fn(left1, right1); if (ret == 0) { continue; @@ -1567,3 +1582,15 @@ SSortExecInfo tsortGetSortExecInfo(SSortHandle* pHandle) { return info; } + +int32_t tsortCompAndBuildKeys(const SArray* pSortCols, char* keyBuf, int32_t* keyLen, + const STupleHandle* pTuple) { + int32_t ret; + if (0 == compKeys(pSortCols, keyBuf, *keyLen, pTuple->pBlock, pTuple->rowIndex)) { + ret = 0; + } else { + *keyLen = buildKeys(keyBuf, pSortCols, pTuple->pBlock, pTuple->rowIndex); + ret = 1; + } + return ret; +} diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index c62190b68a..3d48036095 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -444,6 +444,7 @@ static int32_t logicAggCopy(const SAggLogicNode* pSrc, SAggLogicNode* pDst) { CLONE_NODE_LIST_FIELD(pGroupKeys); CLONE_NODE_LIST_FIELD(pAggFuncs); COPY_SCALAR_FIELD(hasGroupKeyOptimized); + COPY_SCALAR_FIELD(isPartTb); return TSDB_CODE_SUCCESS; } @@ -489,6 +490,7 @@ static int32_t logicMergeCopy(const SMergeLogicNode* pSrc, SMergeLogicNode* pDst COPY_SCALAR_FIELD(srcGroupId); COPY_SCALAR_FIELD(groupSort); COPY_SCALAR_FIELD(ignoreGroupId); + COPY_SCALAR_FIELD(inputWithGroupId); return TSDB_CODE_SUCCESS; } @@ -531,6 +533,8 @@ static int32_t logicSortCopy(const SSortLogicNode* pSrc, SSortLogicNode* pDst) { COPY_BASE_OBJECT_FIELD(node, logicNodeCopy); CLONE_NODE_LIST_FIELD(pSortKeys); COPY_SCALAR_FIELD(groupSort); + COPY_SCALAR_FIELD(calcGroupId); + COPY_SCALAR_FIELD(excludePkCol); return TSDB_CODE_SUCCESS; } @@ -539,6 +543,9 @@ static int32_t logicPartitionCopy(const SPartitionLogicNode* pSrc, SPartitionLog CLONE_NODE_LIST_FIELD(pPartitionKeys); CLONE_NODE_LIST_FIELD(pTags); CLONE_NODE_FIELD(pSubtable); + COPY_SCALAR_FIELD(needBlockOutputTsOrder); + COPY_SCALAR_FIELD(pkTsColId); + COPY_SCALAR_FIELD(pkTsColTbId); return TSDB_CODE_SUCCESS; } @@ -678,6 +685,8 @@ static int32_t physiPartitionCopy(const SPartitionPhysiNode* pSrc, SPartitionPhy CLONE_NODE_LIST_FIELD(pExprs); CLONE_NODE_LIST_FIELD(pPartitionKeys); CLONE_NODE_LIST_FIELD(pTargets); + COPY_SCALAR_FIELD(needBlockOutputTsOrder); + COPY_SCALAR_FIELD(tsSlotId); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index c72b03817b..c2acf0dbdf 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -2271,6 +2271,7 @@ static const char* jkMergePhysiPlanNumOfChannels = "NumOfChannels"; static const char* jkMergePhysiPlanSrcGroupId = "SrcGroupId"; static const char* jkMergePhysiPlanGroupSort = "GroupSort"; static const char* jkMergePhysiPlanIgnoreGroupID = "IgnoreGroupID"; +static const char* jkMergePhysiPlanInputWithGroupId = "InputWithGroupId"; static int32_t physiMergeNodeToJson(const void* pObj, SJson* pJson) { const SMergePhysiNode* pNode = (const SMergePhysiNode*)pObj; @@ -2294,6 +2295,9 @@ static int32_t physiMergeNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddBoolToObject(pJson, jkMergePhysiPlanIgnoreGroupID, pNode->ignoreGroupId); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkMergePhysiPlanInputWithGroupId, pNode->inputWithGroupId); + } return code; } @@ -2327,7 +2331,8 @@ static int32_t jsonToPhysiMergeNode(const SJson* pJson, void* pObj) { static const char* jkSortPhysiPlanExprs = "Exprs"; static const char* jkSortPhysiPlanSortKeys = "SortKeys"; static const char* jkSortPhysiPlanTargets = "Targets"; -static const char* jkSortPhysiPlanMaxRows = "MaxRows"; +static const char* jkSortPhysiPlanCalcGroupIds = "CalcGroupIds"; +static const char* jkSortPhysiPlanExcludePKCol = "ExcludePKCol"; static int32_t physiSortNodeToJson(const void* pObj, SJson* pJson) { const SSortPhysiNode* pNode = (const SSortPhysiNode*)pObj; @@ -2342,6 +2347,12 @@ static int32_t physiSortNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = nodeListToJson(pJson, jkSortPhysiPlanTargets, pNode->pTargets); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkSortPhysiPlanCalcGroupIds, pNode->calcGroupId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddBoolToObject(pJson, jkSortPhysiPlanExcludePKCol, pNode->excludePkCol); + } return code; } @@ -2359,6 +2370,12 @@ static int32_t jsonToPhysiSortNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeList(pJson, jkSortPhysiPlanTargets, &pNode->pTargets); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkSortPhysiPlanCalcGroupIds, &pNode->calcGroupId); + } + if (TSDB_CODE_SUCCESS == code) { + code= tjsonGetBoolValue(pJson, jkSortPhysiPlanExcludePKCol, &pNode->excludePkCol); + } return code; } @@ -2644,6 +2661,8 @@ static int32_t jsonToPhysiEventWindowNode(const SJson* pJson, void* pObj) { static const char* jkPartitionPhysiPlanExprs = "Exprs"; static const char* jkPartitionPhysiPlanPartitionKeys = "PartitionKeys"; static const char* jkPartitionPhysiPlanTargets = "Targets"; +static const char* jkPartitionPhysiPlanNeedBlockOutputTsOrder = "NeedBlockOutputTsOrder"; +static const char* jkPartitionPhysiPlanTsSlotId = "tsSlotId"; static int32_t physiPartitionNodeToJson(const void* pObj, SJson* pJson) { const SPartitionPhysiNode* pNode = (const SPartitionPhysiNode*)pObj; @@ -2658,6 +2677,12 @@ static int32_t physiPartitionNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = nodeListToJson(pJson, jkPartitionPhysiPlanTargets, pNode->pTargets); } + if (TSDB_CODE_SUCCESS == code) { + tjsonAddBoolToObject(pJson, jkPartitionPhysiPlanNeedBlockOutputTsOrder, pNode->needBlockOutputTsOrder); + } + if (TSDB_CODE_SUCCESS == code) { + tjsonAddIntegerToObject(pJson, jkPartitionPhysiPlanTsSlotId, pNode->tsSlotId); + } return code; } @@ -2675,6 +2700,12 @@ static int32_t jsonToPhysiPartitionNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeList(pJson, jkPartitionPhysiPlanTargets, &pNode->pTargets); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBoolValue(pJson, jkPartitionPhysiPlanNeedBlockOutputTsOrder, &pNode->needBlockOutputTsOrder); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkPartitionPhysiPlanTsSlotId, &pNode->tsSlotId); + } return code; } diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c index bc037f05ec..99100b2a1d 100644 --- a/source/libs/nodes/src/nodesMsgFuncs.c +++ b/source/libs/nodes/src/nodesMsgFuncs.c @@ -2682,6 +2682,7 @@ enum { PHY_MERGE_CODE_SRC_GROUP_ID, PHY_MERGE_CODE_GROUP_SORT, PHY_MERGE_CODE_IGNORE_GROUP_ID, + PHY_MERGE_CODE_INPUT_WITH_GROUP_ID, }; static int32_t physiMergeNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { @@ -2706,6 +2707,9 @@ static int32_t physiMergeNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { if (TSDB_CODE_SUCCESS == code) { code = tlvEncodeBool(pEncoder, PHY_MERGE_CODE_IGNORE_GROUP_ID, pNode->ignoreGroupId); } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeBool(pEncoder, PHY_MERGE_CODE_INPUT_WITH_GROUP_ID, pNode->inputWithGroupId); + } return code; } @@ -2738,6 +2742,9 @@ static int32_t msgToPhysiMergeNode(STlvDecoder* pDecoder, void* pObj) { case PHY_MERGE_CODE_IGNORE_GROUP_ID: code = tlvDecodeBool(pTlv, &pNode->ignoreGroupId); break; + case PHY_MERGE_CODE_INPUT_WITH_GROUP_ID: + code = tlvDecodeBool(pTlv, &pNode->inputWithGroupId); + break; default: break; } @@ -2746,7 +2753,14 @@ static int32_t msgToPhysiMergeNode(STlvDecoder* pDecoder, void* pObj) { return code; } -enum { PHY_SORT_CODE_BASE_NODE = 1, PHY_SORT_CODE_EXPR, PHY_SORT_CODE_SORT_KEYS, PHY_SORT_CODE_TARGETS }; +enum { + PHY_SORT_CODE_BASE_NODE = 1, + PHY_SORT_CODE_EXPR, + PHY_SORT_CODE_SORT_KEYS, + PHY_SORT_CODE_TARGETS, + PHY_SORT_CODE_CALC_GROUPID, + PHY_SORT_CODE_EXCLUDE_PK_COL +}; static int32_t physiSortNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { const SSortPhysiNode* pNode = (const SSortPhysiNode*)pObj; @@ -2761,6 +2775,12 @@ static int32_t physiSortNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { if (TSDB_CODE_SUCCESS == code) { code = tlvEncodeObj(pEncoder, PHY_SORT_CODE_TARGETS, nodeListToMsg, pNode->pTargets); } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeBool(pEncoder, PHY_SORT_CODE_CALC_GROUPID, pNode->calcGroupId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeBool(pEncoder, PHY_SORT_CODE_EXCLUDE_PK_COL, pNode->excludePkCol); + } return code; } @@ -2784,6 +2804,11 @@ static int32_t msgToPhysiSortNode(STlvDecoder* pDecoder, void* pObj) { case PHY_SORT_CODE_TARGETS: code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pTargets); break; + case PHY_SORT_CODE_CALC_GROUPID: + code = tlvDecodeBool(pTlv, &pNode->calcGroupId); + break; + case PHY_SORT_CODE_EXCLUDE_PK_COL: + code = tlvDecodeBool(pTlv, &pNode->excludePkCol); default: break; } @@ -3142,7 +3167,14 @@ static int32_t msgToPhysiEventWindowNode(STlvDecoder* pDecoder, void* pObj) { return code; } -enum { PHY_PARTITION_CODE_BASE_NODE = 1, PHY_PARTITION_CODE_EXPR, PHY_PARTITION_CODE_KEYS, PHY_PARTITION_CODE_TARGETS }; +enum { + PHY_PARTITION_CODE_BASE_NODE = 1, + PHY_PARTITION_CODE_EXPR, + PHY_PARTITION_CODE_KEYS, + PHY_PARTITION_CODE_TARGETS, + PHY_PARTITION_CODE_HAS_OUTPUT_TS_ORDER, + PHY_PARTITION_CODE_TS_SLOTID +}; static int32_t physiPartitionNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { const SPartitionPhysiNode* pNode = (const SPartitionPhysiNode*)pObj; @@ -3157,6 +3189,12 @@ static int32_t physiPartitionNodeToMsg(const void* pObj, STlvEncoder* pEncoder) if (TSDB_CODE_SUCCESS == code) { code = tlvEncodeObj(pEncoder, PHY_PARTITION_CODE_TARGETS, nodeListToMsg, pNode->pTargets); } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeBool(pEncoder, PHY_PARTITION_CODE_HAS_OUTPUT_TS_ORDER, pNode->needBlockOutputTsOrder); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI32(pEncoder, PHY_PARTITION_CODE_TS_SLOTID, pNode->tsSlotId); + } return code; } @@ -3180,6 +3218,12 @@ static int32_t msgToPhysiPartitionNode(STlvDecoder* pDecoder, void* pObj) { case PHY_PARTITION_CODE_TARGETS: code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pTargets); break; + case PHY_PARTITION_CODE_HAS_OUTPUT_TS_ORDER: + code = tlvDecodeBool(pTlv, &pNode->needBlockOutputTsOrder); + break; + case PHY_PARTITION_CODE_TS_SLOTID: + code = tlvDecodeI32(pTlv, &pNode->tsSlotId); + break; default: break; } diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 2a3235b4f5..68865110f7 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -13,8 +13,8 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#include #include +#include #include "parAst.h" #include "parUtil.h" @@ -359,6 +359,9 @@ bool addHintNodeToList(SAstCreateContext* pCxt, SNodeList** ppHintList, EHintOpt } break; } + case HINT_SORT_FOR_GROUP: + if (paramNum > 0) return true; + break; default: return true; } @@ -421,6 +424,14 @@ SNodeList* createHintNodeList(SAstCreateContext* pCxt, const SToken* pLiteral) { } opt = HINT_NO_BATCH_SCAN; break; + case TK_SORT_FOR_GROUP: + lastComma = false; + if (0 != opt || inParamList) { + quit = true; + break; + } + opt = HINT_SORT_FOR_GROUP; + break; case TK_NK_LP: lastComma = false; if (0 == opt || inParamList) { @@ -1655,8 +1666,8 @@ SNode* createShowTableTagsStmt(SAstCreateContext* pCxt, SNode* pTbName, SNode* p static int32_t getIpV4RangeFromWhitelistItem(char* ipRange, SIpV4Range* pIpRange) { int32_t code = TSDB_CODE_SUCCESS; - char* ipCopy = taosStrdup(ipRange); - char* slash = strchr(ipCopy, '/'); + char* ipCopy = taosStrdup(ipRange); + char* slash = strchr(ipCopy, '/'); if (slash) { *slash = '\0'; struct in_addr addr; @@ -1664,11 +1675,9 @@ static int32_t getIpV4RangeFromWhitelistItem(char* ipRange, SIpV4Range* pIpRange int prefix = atoi(slash + 1); if (prefix < 0 || prefix > 32) { code = TSDB_CODE_PAR_INVALID_IP_RANGE; - } else { + } else { pIpRange->ip = addr.s_addr; - uint32_t mask = (1 << (32 - prefix)) - 1; - mask = htonl(~mask); - pIpRange->mask = mask; + pIpRange->mask = prefix; code = TSDB_CODE_SUCCESS; } } else { @@ -1678,7 +1687,7 @@ static int32_t getIpV4RangeFromWhitelistItem(char* ipRange, SIpV4Range* pIpRange struct in_addr addr; if (uv_inet_pton(AF_INET, ipCopy, &addr) == 0) { pIpRange->ip = addr.s_addr; - pIpRange->mask = 0xFFFFFFFF; + pIpRange->mask = 32; code = TSDB_CODE_SUCCESS; } else { code = TSDB_CODE_PAR_INVALID_IP_RANGE; @@ -1686,7 +1695,7 @@ static int32_t getIpV4RangeFromWhitelistItem(char* ipRange, SIpV4Range* pIpRange } taosMemoryFreeClear(ipCopy); - return code; + return code; } static int32_t fillIpRangesFromWhiteList(SAstCreateContext* pCxt, SNodeList* pIpRangesNodeList, SIpV4Range* pIpRanges) { @@ -1758,7 +1767,7 @@ SNode* createAlterUserStmt(SAstCreateContext* pCxt, SToken* pUserName, int8_t al pStmt->alterType = alterType; switch (alterType) { case TSDB_ALTER_USER_PASSWD: { - char password[TSDB_USET_PASSWORD_LEN] = {0}; + char password[TSDB_USET_PASSWORD_LEN] = {0}; SToken* pVal = pAlterInfo; if (!checkPassword(pCxt, pVal, password)) { nodesDestroyNode((SNode*)pStmt); @@ -1777,7 +1786,7 @@ SNode* createAlterUserStmt(SAstCreateContext* pCxt, SToken* pUserName, int8_t al pStmt->sysinfo = taosStr2Int8(pVal->z, NULL, 10); break; } - case TSDB_ALTER_USER_ADD_WHITE_LIST: + case TSDB_ALTER_USER_ADD_WHITE_LIST: case TSDB_ALTER_USER_DROP_WHITE_LIST: { SNodeList* pIpRangesNodeList = pAlterInfo; pStmt->pNodeListIpRanges = pIpRangesNodeList; diff --git a/source/libs/parser/src/parTokenizer.c b/source/libs/parser/src/parTokenizer.c index 5e230a9116..cbb3b1952b 100644 --- a/source/libs/parser/src/parTokenizer.c +++ b/source/libs/parser/src/parTokenizer.c @@ -207,6 +207,7 @@ static SKeyword keywordTable[] = { {"SMALLINT", TK_SMALLINT}, {"SNODE", TK_SNODE}, {"SNODES", TK_SNODES}, + {"SORT_FOR_GROUP", TK_SORT_FOR_GROUP}, {"SOFFSET", TK_SOFFSET}, {"SPLIT", TK_SPLIT}, {"STABLE", TK_STABLE}, diff --git a/source/libs/planner/inc/planInt.h b/source/libs/planner/inc/planInt.h index d8fb0def5f..83a4e9ced8 100644 --- a/source/libs/planner/inc/planInt.h +++ b/source/libs/planner/inc/planInt.h @@ -44,12 +44,15 @@ int32_t splitLogicPlan(SPlanContext* pCxt, SLogicSubplan* pLogicSubplan); int32_t scaleOutLogicPlan(SPlanContext* pCxt, SLogicSubplan* pLogicSubplan, SQueryLogicPlan** pLogicPlan); int32_t createPhysiPlan(SPlanContext* pCxt, SQueryLogicPlan* pLogicPlan, SQueryPlan** pPlan, SArray* pExecNodeList); -bool getBatchScanOptionFromHint(SNodeList* pList); +bool getBatchScanOptionFromHint(SNodeList* pList); +bool getSortForGroupOptHint(SNodeList* pList); SLogicNode* getLogicNodeRootNode(SLogicNode* pCurr); -int32_t collectTableAliasFromNodes(SNode* pNode, SSHashObj** ppRes); -bool isPartTableAgg(SAggLogicNode* pAgg); -bool isPartTagAgg(SAggLogicNode* pAgg); -bool isPartTableWinodw(SWindowLogicNode* pWindow); +int32_t collectTableAliasFromNodes(SNode* pNode, SSHashObj** ppRes); +bool isPartTableAgg(SAggLogicNode* pAgg); +bool isPartTagAgg(SAggLogicNode* pAgg); +bool isPartTableWinodw(SWindowLogicNode* pWindow); +bool keysHasCol(SNodeList* pKeys); +bool keysHasTbname(SNodeList* pKeys); #define CLONE_LIMIT 1 #define CLONE_SLIMIT 1 << 1 diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 96d253494d..45b2a73822 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -741,6 +741,9 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, } nodesDestroyList(pOutputGroupKeys); + pAgg->isGroupTb = pAgg->pGroupKeys ? keysHasTbname(pAgg->pGroupKeys) : 0; + pAgg->isPartTb = pSelect->pPartitionByList ? keysHasTbname(pSelect->pPartitionByList) : 0; + if (TSDB_CODE_SUCCESS == code) { *pLogicNode = (SLogicNode*)pAgg; } else { @@ -962,6 +965,7 @@ static int32_t createWindowLogicNodeByInterval(SLogicPlanContext* pCxt, SInterva nodesDestroyNode((SNode*)pWindow); return TSDB_CODE_OUT_OF_MEMORY; } + pWindow->isPartTb = pSelect->pPartitionByList ? keysHasTbname(pSelect->pPartitionByList) : 0; return createWindowLogicNodeFinalize(pCxt, pSelect, pWindow, pLogicNode); } @@ -993,7 +997,6 @@ static int32_t createWindowLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSele if (NULL == pSelect->pWindow) { return TSDB_CODE_SUCCESS; } - switch (nodeType(pSelect->pWindow)) { case QUERY_NODE_STATE_WINDOW: return createWindowLogicNodeByState(pCxt, (SStateWindowNode*)pSelect->pWindow, pSelect, pLogicNode); @@ -1260,6 +1263,15 @@ static int32_t createPartitionLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pS } } + if (keysHasCol(pPartition->pPartitionKeys) && pSelect->pWindow && + nodeType(pSelect->pWindow) == QUERY_NODE_INTERVAL_WINDOW) { + pPartition->needBlockOutputTsOrder = true; + SIntervalWindowNode* pInterval = (SIntervalWindowNode*)pSelect->pWindow; + SColumnNode* pTsCol = (SColumnNode*)pInterval->pCol; + pPartition->pkTsColId = pTsCol->colId; + pPartition->pkTsColTbId = pTsCol->tableId; + } + if (TSDB_CODE_SUCCESS == code && NULL != pSelect->pTags) { pPartition->pTags = nodesCloneList(pSelect->pTags); if (NULL == pPartition->pTags) { diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 7b2cd71677..c81ca0b3d1 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1661,22 +1661,6 @@ static int32_t smaIndexOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSub return smaIndexOptimizeImpl(pCxt, pLogicSubplan, pScan); } -static EDealRes partTagsOptHasColImpl(SNode* pNode, void* pContext) { - if (QUERY_NODE_COLUMN == nodeType(pNode)) { - if (COLUMN_TYPE_TAG != ((SColumnNode*)pNode)->colType && COLUMN_TYPE_TBNAME != ((SColumnNode*)pNode)->colType) { - *(bool*)pContext = true; - return DEAL_RES_END; - } - } - return DEAL_RES_CONTINUE; -} - -static bool planOptNodeListHasCol(SNodeList* pKeys) { - bool hasCol = false; - nodesWalkExprs(pKeys, partTagsOptHasColImpl, &hasCol); - return hasCol; -} - static EDealRes partTagsOptHasTbname(SNode* pNode, void* pContext) { if (QUERY_NODE_COLUMN == nodeType(pNode)) { if (COLUMN_TYPE_TBNAME == ((SColumnNode*)pNode)->colType) { @@ -1755,7 +1739,7 @@ static bool partTagsOptMayBeOptimized(SLogicNode* pNode) { return false; } - return !planOptNodeListHasCol(partTagsGetPartKeys(pNode)) && partTagsOptAreSupportedFuncs(partTagsGetFuncs(pNode)); + return !keysHasCol(partTagsGetPartKeys(pNode)) && partTagsOptAreSupportedFuncs(partTagsGetFuncs(pNode)); } static int32_t partTagsOptRebuildTbanme(SNodeList* pPartKeys) { @@ -2042,6 +2026,7 @@ static int32_t eliminateProjOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* } int32_t code = replaceLogicNode(pLogicSubplan, (SLogicNode*)pProjectNode, pChild); + if (pProjectNode->node.pHint && !pChild->pHint) TSWAP(pProjectNode->node.pHint, pChild->pHint); if (TSDB_CODE_SUCCESS == code) { NODES_CLEAR_LIST(pProjectNode->node.pChildren); nodesDestroyNode((SNode*)pProjectNode); @@ -2734,7 +2719,7 @@ static bool tagScanOptShouldBeOptimized(SLogicNode* pNode) { } SAggLogicNode* pAgg = (SAggLogicNode*)(pNode->pParent); - if (NULL == pAgg->pGroupKeys || NULL != pAgg->pAggFuncs || planOptNodeListHasCol(pAgg->pGroupKeys) || + if (NULL == pAgg->pGroupKeys || NULL != pAgg->pAggFuncs || keysHasCol(pAgg->pGroupKeys) || !planOptNodeListHasTbname(pAgg->pGroupKeys)) { return false; } @@ -2853,13 +2838,14 @@ static bool pushDownLimitTo(SLogicNode* pNodeWithLimit, SLogicNode* pNodeLimitPu cloneLimit(pNodeWithLimit, pNodeLimitPushTo, CLONE_LIMIT_SLIMIT); return true; } + case QUERY_NODE_LOGIC_PLAN_SORT: + if (((SSortLogicNode*)pNodeLimitPushTo)->calcGroupId) break; + // fall through case QUERY_NODE_LOGIC_PLAN_FILL: - case QUERY_NODE_LOGIC_PLAN_SORT: { cloneLimit(pNodeWithLimit, pNodeLimitPushTo, CLONE_LIMIT_SLIMIT); SNode* pChild = NULL; FOREACH(pChild, pNodeLimitPushTo->pChildren) { pushDownLimitHow(pNodeLimitPushTo, (SLogicNode*)pChild); } return true; - } case QUERY_NODE_LOGIC_PLAN_AGG: { if (nodeType(pNodeWithLimit) == QUERY_NODE_LOGIC_PLAN_PROJECT && (isPartTagAgg((SAggLogicNode*)pNodeLimitPushTo) || isPartTableAgg((SAggLogicNode*)pNodeLimitPushTo))) { @@ -3587,6 +3573,96 @@ static int32_t stableJoinOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicS return stbJoinOptRewriteStableJoin(pCxt, pNode, pLogicSubplan); } +static bool partColOptShouldBeOptimized(SLogicNode* pNode) { + if (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode)) { + SPartitionLogicNode* pPartition = (SPartitionLogicNode*)pNode; + if (keysHasCol(pPartition->pPartitionKeys)) return true; + } + return false; +} + +static SSortLogicNode* partColOptCreateSort(SPartitionLogicNode* pPartition) { + SNode* node; + int32_t code = TSDB_CODE_SUCCESS; + SSortLogicNode* pSort = (SSortLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_SORT); + if (pSort) { + bool alreadyPartByPKTs = false; + pSort->groupSort = false; + FOREACH(node, pPartition->pPartitionKeys) { + SOrderByExprNode* pOrder = (SOrderByExprNode*)nodesMakeNode(QUERY_NODE_ORDER_BY_EXPR); + if (QUERY_NODE_COLUMN == nodeType(node) && ((SColumnNode*)node)->colId == pPartition->pkTsColId && + ((SColumnNode*)node)->tableId == pPartition->pkTsColTbId) + alreadyPartByPKTs = true; + if (!pOrder) { + code = TSDB_CODE_OUT_OF_MEMORY; + } else { + nodesListMakeAppend(&pSort->pSortKeys, (SNode*)pOrder); + pOrder->order = ORDER_ASC; + pOrder->pExpr = nodesCloneNode(node); + if (!pOrder->pExpr) code = TSDB_CODE_OUT_OF_MEMORY; + } + } + + if (pPartition->needBlockOutputTsOrder && !alreadyPartByPKTs) { + SOrderByExprNode* pOrder = (SOrderByExprNode*)nodesMakeNode(QUERY_NODE_ORDER_BY_EXPR); + if (!pOrder) { + code = TSDB_CODE_OUT_OF_MEMORY; + } else { + pSort->excludePkCol = true; + nodesListMakeAppend(&pSort->pSortKeys, (SNode*)pOrder); + pOrder->order = ORDER_ASC; + pOrder->pExpr = 0; + FOREACH(node, pPartition->node.pTargets) { + if (nodeType(node) == QUERY_NODE_COLUMN) { + SColumnNode* pCol = (SColumnNode*)node; + if (pCol->colId == pPartition->pkTsColId && pCol->tableId == pPartition->pkTsColTbId) { + pOrder->pExpr = nodesCloneNode((SNode*)pCol); + break; + } + } + } + if (!pOrder->pExpr) { + code = TSDB_CODE_PAR_INTERNAL_ERROR; + } + } + } + } + if (code != TSDB_CODE_SUCCESS) { + nodesDestroyNode((SNode*)pSort); + pSort = NULL; + } + return pSort; +} + +static int32_t partitionColsOpt(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { + int32_t code = TSDB_CODE_SUCCESS; + SPartitionLogicNode* pNode = (SPartitionLogicNode*)optFindPossibleNode(pLogicSubplan->pNode, partColOptShouldBeOptimized); + if (NULL == pNode) return TSDB_CODE_SUCCESS; + + SLogicNode* pRootNode = getLogicNodeRootNode((SLogicNode*)pNode); + if (!pRootNode->pHint || !getSortForGroupOptHint(pRootNode->pHint)) { + return code; + } + + // replace with sort node + SSortLogicNode* pSort = partColOptCreateSort(pNode); + if (!pSort) { + // if sort create failed, we eat the error, skip the optimization + code = TSDB_CODE_SUCCESS; + } else { + TSWAP(pSort->node.pChildren, pNode->node.pChildren); + TSWAP(pSort->node.pTargets, pNode->node.pTargets); + optResetParent((SLogicNode*)pSort); + pSort->calcGroupId = true; + code = replaceLogicNode(pLogicSubplan, (SLogicNode*)pNode, (SLogicNode*)pSort); + if (code == TSDB_CODE_SUCCESS) { + pCxt->optimized = true; + } else { + nodesDestroyNode((SNode*)pSort); + } + } + return code; +} // clang-format off static const SOptimizeRule optimizeRuleSet[] = { @@ -3606,6 +3682,7 @@ static const SOptimizeRule optimizeRuleSet[] = { {.pName = "TableCountScan", .optimizeFunc = tableCountScanOptimize}, {.pName = "EliminateProject", .optimizeFunc = eliminateProjOptimize}, {.pName = "EliminateSetOperator", .optimizeFunc = eliminateSetOpOptimize}, + {.pName = "PartitionCols", .optimizeFunc = partitionColsOpt}, }; // clang-format on diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 5628f5cf0f..d55e80a23d 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -1749,6 +1749,8 @@ static int32_t createSortPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren SNodeList* pPrecalcExprs = NULL; SNodeList* pSortKeys = NULL; int32_t code = rewritePrecalcExprs(pCxt, pSortLogicNode->pSortKeys, &pPrecalcExprs, &pSortKeys); + pSort->calcGroupId = pSortLogicNode->calcGroupId; + pSort->excludePkCol = pSortLogicNode->excludePkCol; SDataBlockDescNode* pChildTupe = (((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc); // push down expression to pOutputDataBlockDesc of child node @@ -1797,6 +1799,7 @@ static int32_t createPartitionPhysiNodeImpl(SPhysiPlanContext* pCxt, SNodeList* SNodeList* pPrecalcExprs = NULL; SNodeList* pPartitionKeys = NULL; int32_t code = rewritePrecalcExprs(pCxt, pPartLogicNode->pPartitionKeys, &pPrecalcExprs, &pPartitionKeys); + pPart->needBlockOutputTsOrder = pPartLogicNode->needBlockOutputTsOrder; SDataBlockDescNode* pChildTupe = (((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc); // push down expression to pOutputDataBlockDesc of child node @@ -1818,6 +1821,22 @@ static int32_t createPartitionPhysiNodeImpl(SPhysiPlanContext* pCxt, SNodeList* } } + if (pPart->needBlockOutputTsOrder) { + SNode* node; + bool found = false; + FOREACH(node, pPartLogicNode->node.pTargets) { + if (nodeType(node) == QUERY_NODE_COLUMN) { + SColumnNode* pCol = (SColumnNode*)node; + if (pCol->tableId == pPartLogicNode->pkTsColTbId && pCol->colId == pPartLogicNode->pkTsColId) { + pPart->tsSlotId = pCol->slotId; + found = true; + break; + } + } + } + if (!found) code = TSDB_CODE_PLAN_INTERNAL_ERROR; + } + if (TSDB_CODE_SUCCESS == code) { code = setConditionsSlotId(pCxt, (const SLogicNode*)pPartLogicNode, (SPhysiNode*)pPart); } @@ -1944,6 +1963,7 @@ static int32_t createMergePhysiNode(SPhysiPlanContext* pCxt, SMergeLogicNode* pM pMerge->srcGroupId = pMergeLogicNode->srcGroupId; pMerge->groupSort = pMergeLogicNode->groupSort; pMerge->ignoreGroupId = pMergeLogicNode->ignoreGroupId; + pMerge->inputWithGroupId = pMergeLogicNode->inputWithGroupId; int32_t code = addDataBlockSlots(pCxt, pMergeLogicNode->pInputs, pMerge->node.pOutputDataBlockDesc); diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 8b7f20f5cf..881cc02062 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -244,7 +244,12 @@ static bool stbSplHasMultiTbScan(bool streamQuery, SLogicNode* pNode) { } pChild = nodesListGetNode(((SLogicNode*)pChild)->pChildren, 0); } - return (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pChild) && stbSplIsMultiTbScan(streamQuery, (SScanLogicNode*)pChild)); + if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pChild) && stbSplIsMultiTbScan(streamQuery, (SScanLogicNode*)pChild)) { + return true; + } else if (QUERY_NODE_LOGIC_PLAN_SORT == nodeType(pChild)) { + return stbSplHasMultiTbScan(streamQuery, (SLogicNode*)pChild); + } + return false; } static bool stbSplIsMultiTbScanChild(bool streamQuery, SLogicNode* pNode) { @@ -519,6 +524,11 @@ static int32_t stbSplRewriteFromMergeNode(SMergeLogicNode* pMerge, SLogicNode* p } break; } + case QUERY_NODE_LOGIC_PLAN_SORT: { + SSortLogicNode* pSort = (SSortLogicNode*)pNode; + if (pSort->calcGroupId) pMerge->inputWithGroupId = true; + break; + } default: break; } diff --git a/source/libs/planner/src/planUtil.c b/source/libs/planner/src/planUtil.c index dcdc402c8b..1c7c937b7f 100644 --- a/source/libs/planner/src/planUtil.c +++ b/source/libs/planner/src/planUtil.c @@ -358,12 +358,12 @@ static bool stbNotSystemScan(SLogicNode* pNode) { } } -static bool stbHasPartTbname(SNodeList* pPartKeys) { - if (NULL == pPartKeys) { +bool keysHasTbname(SNodeList* pKeys) { + if (NULL == pKeys) { return false; } SNode* pPartKey = NULL; - FOREACH(pPartKey, pPartKeys) { + FOREACH(pPartKey, pKeys) { if (QUERY_NODE_GROUPING_SET == nodeType(pPartKey)) { pPartKey = nodesListGetNode(((SGroupingSetNode*)pPartKey)->pParameterList, 0); } @@ -390,10 +390,10 @@ bool isPartTableAgg(SAggLogicNode* pAgg) { return false; } if (NULL != pAgg->pGroupKeys) { - return stbHasPartTbname(pAgg->pGroupKeys) && + return (pAgg->isGroupTb || keysHasTbname(pAgg->pGroupKeys)) && stbNotSystemScan((SLogicNode*)nodesListGetNode(pAgg->node.pChildren, 0)); } - return stbHasPartTbname(stbGetPartKeys((SLogicNode*)nodesListGetNode(pAgg->node.pChildren, 0))); + return pAgg->isPartTb || keysHasTbname(stbGetPartKeys((SLogicNode*)nodesListGetNode(pAgg->node.pChildren, 0))); } static bool stbHasPartTag(SNodeList* pPartKeys) { @@ -430,6 +430,17 @@ bool getBatchScanOptionFromHint(SNodeList* pList) { return batchScan; } +bool getSortForGroupOptHint(SNodeList* pList) { + SNode* pNode; + FOREACH(pNode, pList) { + SHintNode* pHint = (SHintNode*)pNode; + if (pHint->option == HINT_SORT_FOR_GROUP) { + return true; + } + } + return false; +} + int32_t collectTableAliasFromNodes(SNode* pNode, SSHashObj** ppRes) { int32_t code = TSDB_CODE_SUCCESS; SLogicNode* pCurr = (SLogicNode*)pNode; @@ -467,7 +478,7 @@ bool isPartTagAgg(SAggLogicNode* pAgg) { } bool isPartTableWinodw(SWindowLogicNode* pWindow) { - return stbHasPartTbname(stbGetPartKeys((SLogicNode*)nodesListGetNode(pWindow->node.pChildren, 0))); + return pWindow->isPartTb || keysHasTbname(stbGetPartKeys((SLogicNode*)nodesListGetNode(pWindow->node.pChildren, 0))); } bool cloneLimit(SLogicNode* pParent, SLogicNode* pChild, uint8_t cloneWhat) { @@ -490,3 +501,19 @@ bool cloneLimit(SLogicNode* pParent, SLogicNode* pChild, uint8_t cloneWhat) { } return cloned; } + +static EDealRes partTagsOptHasColImpl(SNode* pNode, void* pContext) { + if (QUERY_NODE_COLUMN == nodeType(pNode)) { + if (COLUMN_TYPE_TAG != ((SColumnNode*)pNode)->colType && COLUMN_TYPE_TBNAME != ((SColumnNode*)pNode)->colType) { + *(bool*)pContext = true; + return DEAL_RES_END; + } + } + return DEAL_RES_CONTINUE; +} + +bool keysHasCol(SNodeList* pKeys) { + bool hasCol = false; + nodesWalkExprs(pKeys, partTagsOptHasColImpl, &hasCol); + return hasCol; +} diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 8a80d74c63..19f4ebbeea 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -1712,25 +1712,24 @@ int streamStateGetCfIdx(SStreamState* pState, const char* funcName) { if (pState != NULL && idx != -1) { SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper; rocksdb_column_family_handle_t* cf = NULL; - taosThreadRwlockRdlock(&wrapper->rwLock); + taosThreadRwlockWrlock(&wrapper->rwLock); cf = wrapper->pHandle[idx]; - taosThreadRwlockUnlock(&wrapper->rwLock); if (cf == NULL) { char buf[128] = {0}; GEN_COLUMN_FAMILY_NAME(buf, wrapper->idstr, ginitDict[idx].key); char* err = NULL; - taosThreadRwlockWrlock(&wrapper->rwLock); cf = rocksdb_create_column_family(wrapper->rocksdb, wrapper->cfOpts[idx], buf, &err); if (err != NULL) { idx = -1; qError("failed to to open cf, %p %s_%s, reason:%s", pState, wrapper->idstr, funcName, err); taosMemoryFree(err); } else { + qDebug("succ to to open cf, %p %s_%s", pState, wrapper->idstr, funcName); wrapper->pHandle[idx] = cf; } - taosThreadRwlockUnlock(&wrapper->rwLock); } + taosThreadRwlockUnlock(&wrapper->rwLock); } return idx; diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 17ef6ce530..dc8b7a3245 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -29,6 +29,8 @@ extern "C" { #include "ttrace.h" #include "tutil.h" +typedef bool (*FilteFunc)(void* arg); + typedef void* queue[2]; /* Private macros. */ #define QUEUE_NEXT(q) (*(queue**)&((*(q))[0])) @@ -303,11 +305,12 @@ void transUnrefCliHandle(void* handle); int transReleaseCliHandle(void* handle); int transReleaseSrvHandle(void* handle); -int transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pMsg, STransCtx* pCtx); -int transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pMsg, STransMsg* pRsp); -int transSendResponse(const STransMsg* msg); -int transRegisterMsg(const STransMsg* msg); -int transSetDefaultAddr(void* shandle, const char* ip, const char* fqdn); +int transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pMsg, STransCtx* pCtx); +int transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pMsg, STransMsg* pRsp); +int transSendResponse(const STransMsg* msg); +int transRegisterMsg(const STransMsg* msg); +int transSetDefaultAddr(void* shandle, const char* ip, const char* fqdn); +void transSetIpWhiteList(void* shandle, void* arg, FilteFunc* func); int transSockInfo2Str(struct sockaddr* sockname, char* dst); @@ -434,6 +437,23 @@ int32_t transGetRefMgt(); int32_t transGetInstMgt(); void transHttpEnvDestroy(); + +typedef struct { + uint32_t netmask; + uint32_t address; + uint32_t network; + uint32_t broadcast; + char info[32]; + int8_t type; +} SubnetUtils; + +int32_t subnetInit(SubnetUtils* pUtils, SIpV4Range* pRange); +int32_t subnetCheckIp(SubnetUtils* pUtils, uint32_t ip); +int32_t subnetDebugInfoToBuf(SubnetUtils* pUtils, char* buf); + +int32_t transUtilSIpRangeToStr(SIpV4Range* pRange, char* buf); +int32_t transUtilSWhiteListToStr(SIpWhiteList* pWhiteList, char** ppBuf); + #ifdef __cplusplus } #endif diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index ed94521df0..e9aa62eded 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -178,13 +178,21 @@ void rpcUnrefHandle(void* handle, int8_t type) { (*taosUnRefHandle[type])(handle int rpcRegisterBrokenLinkArg(SRpcMsg* msg) { return transRegisterMsg(msg); } int rpcReleaseHandle(void* handle, int8_t type) { return (*transReleaseHandle[type])(handle); } +// client only int rpcSetDefaultAddr(void* thandle, const char* ip, const char* fqdn) { // later return transSetDefaultAddr(thandle, ip, fqdn); } +// server only +void rpcSetIpWhite(void* thandle, void* arg) { transSetIpWhiteList(thandle, arg, NULL); } void* rpcAllocHandle() { return (void*)transAllocHandle(); } +int32_t rpcUtilSIpRangeToStr(SIpV4Range* pRange, char* buf) { return transUtilSIpRangeToStr(pRange, buf); } +int32_t rpcUtilSWhiteListToStr(SIpWhiteList* pWhiteList, char** ppBuf) { + return transUtilSWhiteListToStr(pWhiteList, ppBuf); +} + int32_t rpcInit() { transInit(); return 0; diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 5e602b1ea2..3dc59a93ee 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -654,3 +654,106 @@ void transDestoryExHandle(void* handle) { } taosMemoryFree(handle); } + +// void subnetIp2int(const char* const ip_addr, uint8_t* dst) { +// char ip_addr_cpy[20]; +// char ip[5]; + +// tstrncpy(ip_addr_cpy, ip_addr, sizeof(ip_addr_cpy)); + +// char *s_start, *s_end; +// s_start = ip_addr_cpy; +// s_end = ip_addr_cpy; + +// int32_t k = 0; + +// for (k = 0; *s_start != '\0'; s_start = s_end) { +// for (s_end = s_start; *s_end != '.' && *s_end != '\0'; s_end++) { +// } +// if (*s_end == '.') { +// *s_end = '\0'; +// s_end++; +// } +// dst[k++] = (char)atoi(s_start); +// } +// } + +uint32_t subnetIpRang2Int(SIpV4Range* pRange) { + uint32_t ip = pRange->ip; + return ((ip & 0xFF) << 24) | ((ip & 0xFF00) << 8) | ((ip & 0xFF0000) >> 8) | ((ip >> 24) & 0xFF); +} +int32_t subnetInit(SubnetUtils* pUtils, SIpV4Range* pRange) { + if (pRange->mask == 32) { + pUtils->type = 0; + pUtils->address = pRange->ip; + return 0; + } + pUtils->address = subnetIpRang2Int(pRange); + + for (int i = 0; i < pRange->mask; i++) { + pUtils->netmask |= (1 << (31 - i)); + } + + pUtils->network = pUtils->address & pUtils->netmask; + pUtils->broadcast = (pUtils->network) | (pUtils->netmask ^ 0xFFFFFFFF); + pUtils->type = (pRange->mask == 32 ? 0 : 1); + + return 0; +} +int32_t subnetDebugInfoToBuf(SubnetUtils* pUtils, char* buf) { + sprintf(buf, "raw: %s, address: %d, netmask:%d, network:%d, broadcast:%d", pUtils->info, pUtils->address, + pUtils->netmask, pUtils->network, pUtils->broadcast); + return 0; +} +int32_t subnetCheckIp(SubnetUtils* pUtils, uint32_t ip) { + // impl later + if (pUtils == NULL) return false; + if (pUtils->type == 0) { + return pUtils->address == ip; + } else { + SIpV4Range range = {.ip = ip, .mask = 32}; + + uint32_t t = subnetIpRang2Int(&range); + return t >= pUtils->network && t <= pUtils->broadcast; + } +} + +int32_t transUtilSIpRangeToStr(SIpV4Range* pRange, char* buf) { + int32_t len = 0; + + struct in_addr addr; + addr.s_addr = pRange->ip; + + uv_inet_ntop(AF_INET, &addr, buf, 32); + + len = strlen(buf); + + if (pRange->mask != 32) { + len += sprintf(buf + len, "/%d", pRange->mask); + } + buf[len] = 0; + return len; +} + +int32_t transUtilSWhiteListToStr(SIpWhiteList* pList, char** ppBuf) { + if (pList->num == 0) { + *ppBuf = NULL; + return 0; + } + int32_t len = 0; + char* pBuf = taosMemoryCalloc(1, pList->num * 36); + + for (int i = 0; i < pList->num; i++) { + SIpV4Range* pRange = &pList->pIpRange[i]; + + char tbuf[32] = {0}; + int tlen = transUtilSIpRangeToStr(pRange, tbuf); + len += sprintf(pBuf + len, "%s,", tbuf); + } + if (len > 0) { + pBuf[len - 1] = 0; + } + + *ppBuf = pBuf; + return len; +} diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 3117bbf00e..fd00c6b773 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -43,6 +43,7 @@ typedef struct SSvrConn { ConnStatus status; + uint32_t serverIp; uint32_t clientIp; uint16_t port; @@ -55,6 +56,8 @@ typedef struct SSvrConn { char user[TSDB_UNI_LEN]; // user ID for the link char secret[TSDB_PASSWORD_LEN]; char ckey[TSDB_PASSWORD_LEN]; // ciphering key + + int64_t whiteListVer; } SSvrConn; typedef struct SSvrMsg { @@ -62,8 +65,22 @@ typedef struct SSvrMsg { STransMsg msg; queue q; STransMsgType type; + + void* arg; + FilteFunc func; + } SSvrMsg; +typedef struct { + int64_t ver; + SIpWhiteList* pList; + // SArray* list; + +} SWhiteUserList; +typedef struct { + SHashObj* pList; + int64_t ver; +} SIpWhiteListTab; typedef struct SWorkThrd { TdThread thread; uv_connect_t connect_req; @@ -77,6 +94,10 @@ typedef struct SWorkThrd { queue conn; void* pTransInst; bool quit; + + SIpWhiteListTab* pWhiteList; + int64_t whiteListVer; + int8_t enableIpWhiteList; } SWorkThrd; typedef struct SServerObj { @@ -99,6 +120,14 @@ typedef struct SServerObj { bool inited; } SServerObj; +SIpWhiteListTab* uvWhiteListCreate(); +void uvWhiteListDestroy(SIpWhiteListTab* pWhite); +void uvWhiteListAdd(SIpWhiteListTab* pWhite, char* user, SIpWhiteList* pList, int64_t ver); +void uvWhiteListUpdate(SIpWhiteListTab* pWhite, SHashObj* pTable); +bool uvWhiteListCheckConn(SIpWhiteListTab* pWhite, SSvrConn* pConn); +bool uvWhiteListFilte(SIpWhiteListTab* pWhite, char* user, uint32_t ip, int64_t ver); +void uvWhiteListSetConnVer(SIpWhiteListTab* pWhite, SSvrConn* pConn); + static void uvAllocConnBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); static void uvAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); static void uvOnRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf); @@ -141,8 +170,9 @@ static void uvHandleQuit(SSvrMsg* msg, SWorkThrd* thrd); static void uvHandleRelease(SSvrMsg* msg, SWorkThrd* thrd); static void uvHandleResp(SSvrMsg* msg, SWorkThrd* thrd); static void uvHandleRegister(SSvrMsg* msg, SWorkThrd* thrd); +static void uvHandleUpdate(SSvrMsg* pMsg, SWorkThrd* thrd); static void (*transAsyncHandle[])(SSvrMsg* msg, SWorkThrd* thrd) = {uvHandleResp, uvHandleQuit, uvHandleRelease, - uvHandleRegister, NULL}; + uvHandleRegister, uvHandleUpdate}; static void uvDestroyConn(uv_handle_t* handle); @@ -181,8 +211,134 @@ static void uvHandleActivityTimeout(uv_timer_t* handle) { tDebug("%p timeout since no activity", conn); } +static bool uvCheckIp(SIpV4Range* pRange, int32_t ip) { + // impl later + SubnetUtils subnet = {0}; + if (subnetInit(&subnet, pRange) != 0) { + return false; + } + return subnetCheckIp(&subnet, ip); +} +SIpWhiteListTab* uvWhiteListCreate() { + SIpWhiteListTab* pWhiteList = taosMemoryCalloc(1, sizeof(SIpWhiteListTab)); + + pWhiteList->pList = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), 0, HASH_NO_LOCK); + pWhiteList->ver = -1; + return pWhiteList; +} +void uvWhiteListDestroy(SIpWhiteListTab* pWhite) { + SHashObj* pWhiteList = pWhite->pList; + void* pIter = taosHashIterate(pWhiteList, NULL); + while (pIter) { + SWhiteUserList* pUserList = *(SWhiteUserList**)pIter; + taosMemoryFree(pUserList->pList); + taosMemoryFree(pUserList); + + pIter = taosHashIterate(pWhiteList, pIter); + } + taosHashCleanup(pWhiteList); + taosMemoryFree(pWhite); +} + +void uvWhiteListToStr(SWhiteUserList* plist, char* user, char** ppBuf) { + char* tmp = NULL; + int32_t tlen = transUtilSWhiteListToStr(plist->pList, &tmp); + + char* pBuf = taosMemoryCalloc(1, tlen + 64); + int32_t len = sprintf(pBuf, "user: %s, ver: %" PRId64 ", ip: {%s}", user, plist->ver, tmp); + taosMemoryFree(tmp); + + *ppBuf = pBuf; +} +void uvWhiteListDebug(SIpWhiteListTab* pWrite) { + SHashObj* pWhiteList = pWrite->pList; + void* pIter = taosHashIterate(pWhiteList, NULL); + while (pIter) { + size_t klen = 0; + char user[TSDB_USER_LEN + 1] = {0}; + char* pUser = taosHashGetKey(pIter, &klen); + memcpy(user, pUser, klen); + + SWhiteUserList* pUserList = *(SWhiteUserList**)pIter; + + char* buf = NULL; + uvWhiteListToStr(pUserList, user, &buf); + tDebug("ip-white-list %s", buf); + taosMemoryFree(buf); + pIter = taosHashIterate(pWhiteList, pIter); + } +} +void uvWhiteListAdd(SIpWhiteListTab* pWhite, char* user, SIpWhiteList* plist, int64_t ver) { + SHashObj* pWhiteList = pWhite->pList; + + SWhiteUserList** ppUserList = taosHashGet(pWhiteList, user, strlen(user)); + if (ppUserList == NULL || *ppUserList == NULL) { + SWhiteUserList* pUserList = taosMemoryCalloc(1, sizeof(SWhiteUserList)); + pUserList->ver = ver; + + pUserList->pList = plist; + + taosHashPut(pWhiteList, user, strlen(user), &pUserList, sizeof(void*)); + } else { + SWhiteUserList* pUserList = *ppUserList; + + taosMemoryFreeClear(pUserList->pList); + pUserList->ver = ver; + pUserList->pList = plist; + } + uvWhiteListDebug(pWhite); +} + +void uvWhiteListUpdate(SIpWhiteListTab* pWhite, SHashObj* pTable) { + pWhite->ver++; + // impl later +} + +static bool uvWhiteListIsDefaultAddr(uint32_t ip) { + // 127.0.0.1 + static SIpV4Range range = {.ip = 16777343, .mask = 32}; + return range.ip == ip; +} +bool uvWhiteListFilte(SIpWhiteListTab* pWhite, char* user, uint32_t ip, int64_t ver) { + // impl check + SHashObj* pWhiteList = pWhite->pList; + bool valid = false; + + if (uvWhiteListIsDefaultAddr(ip)) return true; + + SWhiteUserList** ppList = taosHashGet(pWhiteList, user, strlen(user)); + if (ppList == NULL || *ppList == NULL) { + return false; + } + SWhiteUserList* pUserList = *ppList; + if (pUserList->ver == ver) return true; + + SIpWhiteList* pIpWhiteList = pUserList->pList; + for (int i = 0; i < pIpWhiteList->num; i++) { + SIpV4Range* range = &pIpWhiteList->pIpRange[i]; + if (uvCheckIp(range, ip)) { + valid = true; + break; + } + } + return valid; +} +bool uvWhiteListCheckConn(SIpWhiteListTab* pWhite, SSvrConn* pConn) { + if (pConn->inType == TDMT_MND_STATUS || pConn->inType == TDMT_MND_RETRIEVE_IP_WHITE || + pConn->serverIp == pConn->clientIp || + pWhite->ver == pConn->whiteListVer /*|| strncmp(pConn->user, "_dnd", strlen("_dnd")) == 0*/) + return true; + + return uvWhiteListFilte(pWhite, pConn->user, pConn->clientIp, pConn->whiteListVer); +} +void uvWhiteListSetConnVer(SIpWhiteListTab* pWhite, SSvrConn* pConn) { + // if conn already check by current whiteLis + pConn->whiteListVer = pWhite->ver; +} + static bool uvHandleReq(SSvrConn* pConn) { - STrans* pTransInst = pConn->pTransInst; + STrans* pTransInst = pConn->pTransInst; + SWorkThrd* pThrd = pConn->hostThrd; STransMsgHead* pHead = NULL; @@ -199,8 +355,17 @@ static bool uvHandleReq(SSvrConn* pConn) { pHead->code = htonl(pHead->code); pHead->msgLen = htonl(pHead->msgLen); + pConn->inType = pHead->msgType; memcpy(pConn->user, pHead->user, strlen(pHead->user)); + int8_t forbiddenIp = 0; + if (pThrd->enableIpWhiteList) { + forbiddenIp = !uvWhiteListCheckConn(pThrd->pWhiteList, pConn) ? 1 : 0; + if (forbiddenIp == 0) { + uvWhiteListSetConnVer(pThrd->pWhiteList, pConn); + } + } + if (uvRecvReleaseReq(pConn, pHead)) { return true; } @@ -219,7 +384,6 @@ static bool uvHandleReq(SSvrConn* pConn) { transMsg.msgType = pHead->msgType; transMsg.code = pHead->code; - pConn->inType = pHead->msgType; if (pConn->status == ConnNormal) { if (pHead->persist == 1) { pConn->status = ConnAcquire; @@ -262,6 +426,7 @@ static bool uvHandleReq(SSvrConn* pConn) { transMsg.info.refId = pConn->refId; transMsg.info.traceId = pHead->traceId; transMsg.info.cliVer = htonl(pHead->compatibilityVer); + transMsg.info.forbiddenIp = forbiddenIp; tGTrace("%s handle %p conn:%p translated to app, refId:%" PRIu64, transLabel(pTransInst), transMsg.info.handle, pConn, pConn->refId); @@ -395,7 +560,7 @@ static void uvOnPipeWriteCb(uv_write_t* req, int status) { if (status == 0) { tTrace("success to dispatch conn to work thread"); } else { - tError("fail to dispatch conn to work thread, code:%s", uv_strerror(status)); + tError("fail to dispatch conn to work thread, reason:%s", uv_strerror(status)); } if (!uv_is_closing((uv_handle_t*)req->data)) { uv_close((uv_handle_t*)req->data, uvFreeCb); @@ -542,7 +707,7 @@ void uvWorkerAsyncCb(uv_async_t* handle) { } // release handle to rpc init - if (msg->type == Quit) { + if (msg->type == Quit || msg->type == Update) { (*transAsyncHandle[msg->type])(msg, pThrd); } else { STransMsg transMsg = msg->msg; @@ -638,7 +803,7 @@ static void uvPrepareCb(uv_prepare_t* handle) { continue; } // release handle to rpc init - if (msg->type == Quit) { + if (msg->type == Quit || msg->type == Update) { (*transAsyncHandle[msg->type])(msg, pThrd); continue; } else { @@ -796,7 +961,10 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { transSockInfo2Str(&sockname, pConn->src); struct sockaddr_in addr = *(struct sockaddr_in*)&peername; + struct sockaddr_in saddr = *(struct sockaddr_in*)&sockname; + pConn->clientIp = addr.sin_addr.s_addr; + pConn->serverIp = saddr.sin_addr.s_addr; pConn->port = ntohs(addr.sin_port); uv_read_start((uv_stream_t*)(pConn->pTcp), uvAllocRecvBufferCb, uvOnRecvCb); @@ -1066,9 +1234,10 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, SWorkThrd* thrd = (SWorkThrd*)taosMemoryCalloc(1, sizeof(SWorkThrd)); thrd->pTransInst = shandle; thrd->quit = false; - srv->pThreadObj[i] = thrd; thrd->pTransInst = shandle; + thrd->pWhiteList = uvWhiteListCreate(); + srv->pThreadObj[i] = thrd; srv->pipe[i] = (uv_pipe_t*)taosMemoryCalloc(2, sizeof(uv_pipe_t)); thrd->pipe = &(srv->pipe[i][1]); // init read @@ -1093,6 +1262,7 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, thrd->pTransInst = shandle; thrd->quit = false; thrd->pTransInst = shandle; + thrd->pWhiteList = uvWhiteListCreate(); srv->pipe[i] = (uv_pipe_t*)taosMemoryCalloc(2, sizeof(uv_pipe_t)); srv->pThreadObj[i] = thrd; @@ -1192,6 +1362,31 @@ void uvHandleRegister(SSvrMsg* msg, SWorkThrd* thrd) { taosMemoryFree(msg); } } +void uvHandleUpdate(SSvrMsg* msg, SWorkThrd* thrd) { + SUpdateIpWhite* req = msg->arg; + if (req != NULL) { + for (int i = 0; i < req->numOfUser; i++) { + SUpdateUserIpWhite* pUser = &req->pUserIpWhite[i]; + + int32_t sz = pUser->numOfRange * sizeof(SIpV4Range); + SIpWhiteList* pList = taosMemoryCalloc(1, sz + sizeof(SIpWhiteList)); + pList->num = pUser->numOfRange; + + memcpy(pList->pIpRange, pUser->pIpRanges, sz); + uvWhiteListAdd(thrd->pWhiteList, pUser->user, pList, pUser->ver); + } + + thrd->pWhiteList->ver = req->ver; + thrd->enableIpWhiteList = 1; + + tFreeSUpdateIpWhiteReq(req); + taosMemoryFree(req); + } else { + thrd->enableIpWhiteList = 0; + } + taosMemoryFree(msg); + return; +} void destroyWorkThrd(SWorkThrd* pThrd) { if (pThrd == NULL) { return; @@ -1200,6 +1395,9 @@ void destroyWorkThrd(SWorkThrd* pThrd) { SRV_RELEASE_UV(pThrd->loop); TRANS_DESTROY_ASYNC_POOL_MSG(pThrd->asyncPool, SSvrMsg, destroySmsg); transAsyncPoolDestroy(pThrd->asyncPool); + + uvWhiteListDestroy(pThrd->pWhiteList); + taosMemoryFree(pThrd->prepare); taosMemoryFree(pThrd->loop); taosMemoryFree(pThrd); @@ -1367,5 +1565,23 @@ _return2: rpcFreeCont(msg->pCont); return -1; } +void transSetIpWhiteList(void* thandle, void* arg, FilteFunc* func) { + STrans* pTransInst = (STrans*)transAcquireExHandle(transGetInstMgt(), (int64_t)thandle); + + tDebug("ip-white-list update on rpc"); + SServerObj* svrObj = pTransInst->tcphandle; + for (int i = 0; i < svrObj->numOfThreads; i++) { + SWorkThrd* pThrd = svrObj->pThreadObj[i]; + + SSvrMsg* msg = taosMemoryCalloc(1, sizeof(SSvrMsg)); + SUpdateIpWhite* pReq = (arg != NULL ? cloneSUpdateIpWhiteReq((SUpdateIpWhite*)arg) : NULL); + + msg->type = Update; + msg->arg = pReq; + + transAsyncSend(pThrd->asyncPool, &msg->q); + } + transReleaseExHandle(transGetInstMgt(), (int64_t)thandle); +} int transGetConnInfo(void* thandle, STransHandleInfo* pConnInfo) { return -1; } diff --git a/source/os/src/osRand.c b/source/os/src/osRand.c index 9cb6f6e52a..43abc75d4f 100644 --- a/source/os/src/osRand.c +++ b/source/os/src/osRand.c @@ -86,8 +86,9 @@ void taosRandStr(char* str, int32_t size) { } void taosRandStr2(char* str, int32_t size) { - const char* set = "abcdefghijklmnopqrstuvwxyz0123456789"; - int32_t len = 36; + + const char* set = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ@"; + int32_t len = strlen(set); for (int32_t i = 0; i < size; ++i) { str[i] = set[taosRand() % len]; diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index 2b2a0daf7b..faf1ecbfc7 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -746,9 +746,9 @@ bool taosValidIpAndPort(uint32_t ip, uint16_t port) { serverAdd.sin_port = (uint16_t)htons(port); fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); - if (fd < 0) { // exception + if (fd < 0) { // exception return false; - } else if (fd <= 2) { // in, out, err + } else if (fd <= 2) { // in, out, err taosCloseSocketNoCheck1(fd); return false; } @@ -895,32 +895,6 @@ int64_t taosCopyFds(TdSocketPtr pSrcSocket, TdSocketPtr pDestSocket, int64_t len } // Function converting an IP address string to an uint32_t. -uint32_t ip2uint(const char *const ip_addr) { - char ip_addr_cpy[20]; - char ip[5]; - - tstrncpy(ip_addr_cpy, ip_addr, sizeof(ip_addr_cpy)); - - char *s_start, *s_end; - s_start = ip_addr_cpy; - s_end = ip_addr_cpy; - - int32_t k; - - for (k = 0; *s_start != '\0'; s_start = s_end) { - for (s_end = s_start; *s_end != '.' && *s_end != '\0'; s_end++) { - } - if (*s_end == '.') { - *s_end = '\0'; - s_end++; - } - ip[k++] = (char)atoi(s_start); - } - - ip[k] = '\0'; - - return *((uint32_t *)ip); -} #endif // endif 0 diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 0fff280c59..9832720994 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -100,6 +100,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_APP_IS_STARTING, "Database is starting TAOS_DEFINE_ERROR(TSDB_CODE_APP_IS_STOPPING, "Database is closing down") TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_DATA_FMT, "Invalid data format") TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_CFG_VALUE, "Invalid configuration value") +TAOS_DEFINE_ERROR(TSDB_CODE_IP_NOT_IN_WHITE_LIST, "Not allowed to connect") //client TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_OPERATION, "Invalid operation") @@ -193,6 +194,10 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_USERS, "Too many users") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_ALTER_OPER, "Invalid alter operation") TAOS_DEFINE_ERROR(TSDB_CODE_MND_AUTH_FAILURE, "Authentication failure") TAOS_DEFINE_ERROR(TSDB_CODE_MND_PRIVILEDGE_EXIST, "User already have this priviledge") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_USER_HOST_EXIST, "Host already exist in ip white list") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_USER_HOST_NOT_EXIST, "Host not exist in ip white list") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_USER_HOST, "Too many host in ip white list") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_USER_LOCAL_HOST_NOT_DROP, "Host can not be dropped") //mnode-stable-part1 TAOS_DEFINE_ERROR(TSDB_CODE_MND_STB_ALREADY_EXIST, "STable already exists") @@ -216,7 +221,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_BUFSIZE, "Invalid func bufSize" TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_COMMENT, "Invalid func comment") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_RETRIEVE, "Invalid func retrieve msg") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST, "index already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST, "index already exists in db") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TAG_INDEX_NOT_EXIST, "index not exist") @@ -307,7 +312,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_STREAMS, "Too many streams") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TARGET_TABLE, "Cannot write the same stable as other stream") // mnode-sma -TAOS_DEFINE_ERROR(TSDB_CODE_MND_SMA_ALREADY_EXIST, "index already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_SMA_ALREADY_EXIST, "index already exists in db") TAOS_DEFINE_ERROR(TSDB_CODE_MND_SMA_NOT_EXIST, "index not exist") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_SMA_OPTION, "Invalid sma index option") diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 203b27d044..fc50b993bf 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -5,6 +5,54 @@ #unit-test ,,y,unit-test,bash test.sh +#system test +,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/scalar_function.py +,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/at_once_interval.py +,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/at_once_session.py +,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/at_once_state_window.py +,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/window_close_interval.py +,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/window_close_session.py +,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/window_close_state_window.py +,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/max_delay_interval.py +,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/max_delay_session.py +,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/at_once_interval_ext.py +,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/max_delay_interval_ext.py +,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/window_close_session_ext.py +,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/partition_interval.py +,,y,system-test,./pytest.sh python3 ./test.py -f 8-stream/pause_resume_test.py + +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stbJoin.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stbJoin.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stbJoin.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stbJoin.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/hint.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/hint.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/hint.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/hint.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_str.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_26.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_str.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_26.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/columnLenUpdated.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/columnLenUpdated.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/columnLenUpdated.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/columnLenUpdated.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_str.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_26.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/interval_limit_opt.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/partition_by_col.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/partition_by_col.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/partition_by_col.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/partition_by_col.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqShow.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropStb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeStb0.py @@ -22,6 +70,40 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqOffset.py ,,n,system-test,python3 ./test.py -f 7-tmq/tmqDropConsumer.py + +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/delete_stable.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/out_of_order.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/out_of_order.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_null_none.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_null_none.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_null_none.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_null_none.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_null_none.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/database_pre_suf.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/out_of_order.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/out_of_order.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_str.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_26.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/select_null.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/select_null.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/select_null.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/select_null.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/select_null.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/slimit.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/slimit.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/slimit.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/slimit.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/slimit.py -Q 4 + +,,y,system-test,./pytest.sh python3 ./test.py -f 3-enterprise/restore/restoreDnode.py -N 5 -M 3 -i False +,,y,system-test,./pytest.sh python3 ./test.py -f 3-enterprise/restore/restoreVnode.py -N 5 -M 3 -i False +,,y,system-test,./pytest.sh python3 ./test.py -f 3-enterprise/restore/restoreMnode.py -N 5 -M 3 -i False +,,y,system-test,./pytest.sh python3 ./test.py -f 3-enterprise/restore/restoreQnode.py -N 5 -M 3 -i False + ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/create_wrong_topic.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/dropDbR3ConflictTransaction.py -N 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/basic5.py @@ -79,5 +161,1110 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/raw_block_interface_test.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqSubscribeStb-r3.py -N 5 -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -n 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -i True +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -n 3 -i True +,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-19201.py +,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-21561.py +,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TS-3404.py +,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TS-3581.py +,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TS-3311.py +,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TS-3821.py + +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/balance_vgroups_r1.py -N 6 +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosShell.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosShellError.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosShellNetChk.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/telemetry.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/backquote_check.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosdMonitor.py +,,n,system-test,python3 ./test.py -f 0-others/taosdShell.py -N 5 -M 3 -Q 3 +,,n,system-test,python3 ./test.py -f 0-others/udfTest.py +,,n,system-test,python3 ./test.py -f 0-others/udf_create.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/udf_restart_taosd.py +,,n,system-test,python3 ./test.py -f 0-others/udf_cfg1.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/udf_cfg2.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/cachemodel.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/sysinfo.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/user_control.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/user_manage.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/user_privilege.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/user_privilege_show.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/user_privilege_all.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/fsync.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/multilevel.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/ttl.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/ttlChangeOnWrite.py +,,n,system-test,python3 ./test.py -f 0-others/compatibility.py +,,n,system-test,python3 ./test.py -f 0-others/tag_index_basic.py +,,n,system-test,python3 ./test.py -f 0-others/udfpy_main.py +,,n,system-test,python3 ./test.py -N 3 -f 0-others/walRetention.py +#,,n,system-test,python3 ./test.py -f 0-others/splitVGroup.py -N 5 +,,n,system-test,python3 ./test.py -f 0-others/timeRangeWise.py -N 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_database.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_replica.py -N 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/opentsdb_telnet_line_taosc_insert.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/opentsdb_json_taosc_insert.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/test_stmt_muti_insert_query.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/test_stmt_set_tbname_tag.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_stable.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_table.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/boundary.py +,,n,system-test,python3 ./test.py -f 1-insert/insertWithMoreVgroup.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/table_comment.py +#,,n,system-test,python3 ./test.py -f 1-insert/time_range_wise.py +#,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/block_wise.py +#,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/create_retentions.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/mutil_stage.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/table_param_ttl.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/table_param_ttl.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/update_data_muti_rows.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/db_tb_name_check.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/InsertFuturets.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/insert_wide_column.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_benchmark.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_1.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_1.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_1.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_1.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_1.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_2.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_2.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_2.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_2.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_2.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_3.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_3.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_3.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_3.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_3.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_4.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_4.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_4.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_4.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_4.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/precisionUS.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/precisionNS.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/show.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/show_tag_index.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/information_schema.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/abs.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/abs.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/and_or_for_byte.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/and_or_for_byte.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/apercentile.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/apercentile.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arccos.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arccos.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arcsin.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arcsin.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arctan.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arctan.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/avg.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/avg.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/between.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/between.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/bottom.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/bottom.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cast.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cast.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ceil.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ceil.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/char_length.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/char_length.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/check_tsdb.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/check_tsdb.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat_ws.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat_ws.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat_ws2.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat_ws2.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cos.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cos.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_partition.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_partition.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/countAlwaysReturnValue.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/countAlwaysReturnValue.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/db.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/db.py -N 3 -n 3 -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/diff.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/diff.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distinct.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distinct.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_apercentile.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_apercentile.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_avg.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_avg.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_count.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_count.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_max.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_max.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_min.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_min.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_spread.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_spread.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_stddev.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_stddev.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_sum.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_sum.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/explain.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/explain.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/first.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/first.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/floor.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/floor.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_null.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_null.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_stateduration.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_stateduration.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/histogram.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/histogram.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/hyperloglog.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/hyperloglog.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/interp.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/interp.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/irate.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/irate.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_row.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_row.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/leastsquares.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/leastsquares.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/length.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/length.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/limit.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/log.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/log.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/lower.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/lower.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ltrim.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ltrim.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mavg.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mavg.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_partition.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_partition.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_min_last_interval.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_row_interval.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/min.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/min.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mode.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mode.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Now.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Now.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/orderBy.py -N 5 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/percentile.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/percentile.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/pow.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/pow.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/query_cols_tags_and_or.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/query_cols_tags_and_or.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/round.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/round.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/rtrim.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/rtrim.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sample.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sample.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sin.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sin.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/smaBasic.py -N 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/smaTest.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/smaTest.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/sma_index.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml_TS-3724.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/varbinary.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/spread.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/spread.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sqrt.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sqrt.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/statecount.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/statecount.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stateduration.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stateduration.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/substr.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/substr.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sum.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sum.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tail.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tail.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tan.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tan.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Timediff.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Timediff.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/timetruncate.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/timetruncate.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/timezone.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/timezone.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/To_iso8601.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/To_iso8601.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/To_unixtimestamp.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/To_unixtimestamp.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Today.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Today.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/top.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/top.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tsbsQuery.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tsbsQuery.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ttl_comment.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ttl_comment.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/twa.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/twa.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/union.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/union.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/unique.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/unique.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/upper.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/upper.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/varchar.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/varchar.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/case_when.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/case_when.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/blockSMA.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/blockSMA.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/projectionDesc.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/projectionDesc.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/update_data.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/tb_100w_data_order.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/delete_childtable.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/delete_normaltable.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/delete_systable.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/keep_expired.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/stmt_error.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/drop.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/drop.py -N 3 -M 3 -i False -n 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join2.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/union1.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat2.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/json_tag.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQueryInterval.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/systable_func.py + +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity_1.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/elapsed.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/csum.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_diff.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tagFilter.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/projectionDesc.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ts_3405_3398_3423.py -N 3 -n 3 + +,,n,system-test,python3 ./test.py -f 2-query/queryQnode.py +,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode1mnode.py +,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode2mnode.py -N 5 +,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3 -i False +,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -N 5 -M 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -N 5 -M 3 -i False +,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeStopLoop.py -N 5 -M 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 6 -M 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 6 -M 3 -n 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -N 6 -M 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -N 6 -M 3 -n 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -N 6 -M 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -N 6 -M 3 -n 3 + +,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeModifyMeta.py -N 6 -M 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeModifyMeta.py -N 6 -M 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py -N 6 -M 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py -N 6 -M 3 -n 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py -N 6 -M 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py -N 6 -M 3 -n 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py -N 6 -M 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py -N 6 -M 3 -n 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertData.py -N 6 -M 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertData.py -N 6 -M 3 -n 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -N 6 -M 3 +#,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -N 6 -M 3 -n 3 +,,n,system-test,python3 ./test.py -f 6-cluster/manually-test/6dnode3mnodeInsertLessDataAlterRep3to1to3.py -N 6 -M 3 +#,,n,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeRoll.py -N 3 -C 1 +,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 7 -M 3 -C 6 +,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 7 -M 3 -C 6 -n 3 +#,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5 +,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeRecreateMnode.py -N 6 -M 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeStopFollowerLeader.py -N 5 -M 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -N 5 -M 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/vnode/4dnode1mnode_basic_createDb_replica1.py -N 4 -M 1 +,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas.py -N 4 -M 1 +,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas_querys.py -N 4 -M 1 +,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas.py -N 4 -M 1 +,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys.py -N 4 -M 1 +,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py -N 4 -M 1 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/between.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distinct.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/varchar.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ltrim.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/rtrim.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/length.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/char_length.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/upper.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/lower.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join2.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cast.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/substr.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/union.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/union1.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat2.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat_ws.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat_ws2.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/check_tsdb.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/spread.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/hyperloglog.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/explain.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/leastsquares.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/timezone.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Now.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Today.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/min.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mode.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/first.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/To_iso8601.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/To_unixtimestamp.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/timetruncate.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/diff.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Timediff.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/json_tag.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/top.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/bottom.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/percentile.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/apercentile.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/abs.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ceil.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/floor.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/round.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/log.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/pow.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sqrt.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sin.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cos.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tan.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arcsin.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arccos.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arctan.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/interp.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQueryInterval.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity_1.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/avg.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/elapsed.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/csum.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mavg.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sample.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_diff.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/unique.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stateduration.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_stateduration.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/statecount.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tail.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ttl_comment.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_count.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_max.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_min.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_sum.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_spread.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_apercentile.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_avg.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_stddev.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/twa.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/irate.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_null.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_partition.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_partition.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_min_last_interval.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_row_interval.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_row.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tsbsQuery.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/case_when.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/blockSMA.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/projectionDesc.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-21561.py -Q 2 + +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/between.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distinct.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/varchar.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ltrim.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/rtrim.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/length.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/char_length.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/upper.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/lower.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join2.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cast.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/substr.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/union.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/union1.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat2.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat_ws.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat_ws2.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/check_tsdb.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/spread.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/hyperloglog.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/explain.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/leastsquares.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/timezone.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Now.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Today.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/min.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mode.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/first.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/To_iso8601.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/To_unixtimestamp.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/timetruncate.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/diff.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Timediff.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/json_tag.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/top.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/bottom.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/percentile.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/apercentile.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/abs.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ceil.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/floor.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/round.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/log.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/pow.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sqrt.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sin.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cos.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tan.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arcsin.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arccos.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arctan.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQueryInterval.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity_1.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/avg.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/elapsed.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/csum.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mavg.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sample.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_diff.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/unique.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stateduration.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_stateduration.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/statecount.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tail.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ttl_comment.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_count.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_max.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_min.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_sum.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_spread.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_apercentile.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_avg.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_stddev.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/twa.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/irate.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_null.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_partition.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_partition.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_min_last_interval.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_row_interval.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_row.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tsbsQuery.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/interp.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/case_when.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/blockSMA.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/projectionDesc.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-21561.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/between.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distinct.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/varchar.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ltrim.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/rtrim.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/length.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/char_length.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/upper.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/lower.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join2.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/substr.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/union.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/union1.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat2.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat_ws.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat_ws2.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/check_tsdb.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/spread.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/hyperloglog.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/explain.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/leastsquares.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/timezone.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Now.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Today.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/min.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mode.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/first.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/To_iso8601.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/To_unixtimestamp.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/timetruncate.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/diff.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Timediff.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/json_tag.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/top.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/bottom.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/percentile.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/apercentile.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/abs.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ceil.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/floor.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/round.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/log.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/pow.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sqrt.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sin.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cos.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tan.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arcsin.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arccos.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arctan.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQueryInterval.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stablity_1.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/avg.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/elapsed.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/csum.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mavg.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sample.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cast.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_diff.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/unique.py -Q 4 +#,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stateduration.py -Q 4 +#,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_stateduration.py -Q 4 +#,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/statecount.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tail.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ttl_comment.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_count.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_max.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_min.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_sum.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_spread.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_apercentile.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_avg.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_stddev.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/twa.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/irate.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_null.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_partition.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_partition.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_min_last_interval.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_row_interval.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_row.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tsbsQuery.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/interp.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/case_when.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_select.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_select.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_select.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_select.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/insert_select.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/out_of_order.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/blockSMA.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/projectionDesc.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/odbc.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/fill_with_group.py +,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-21561.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-20582.py +,,n,system-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/insertMix.py -N 3 +,,n,system-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/stt.py -N 3 + +#tsim test +,,y,script,./test.sh -f tsim/tmq/basic2Of2ConsOverlap.sim +,,y,script,./test.sh -f tsim/parser/where.sim +,,y,script,./test.sh -f tsim/parser/join_manyblocks.sim +,,y,script,./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim +,,y,script,./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim +,,y,script,./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v2.sim +,,y,script,./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v3.sim +,,y,script,./test.sh -f tsim/parser/limit1.sim +,,y,script,./test.sh -f tsim/parser/union.sim +,,y,script,./test.sh -f tsim/parser/commit.sim +,,y,script,./test.sh -f tsim/parser/nestquery.sim +,,n,script,./test.sh -f tsim/valgrind/checkError7.sim +,,y,script,./test.sh -f tsim/parser/groupby.sim +,,y,script,./test.sh -f tsim/parser/sliding.sim +,,y,script,./test.sh -f tsim/dnode/balance2.sim +,,y,script,./test.sh -f tsim/vnode/replica3_repeat.sim +,,y,script,./test.sh -f tsim/parser/col_arithmetic_operation.sim +,,y,script,./test.sh -f tsim/trans/create_db.sim +,,y,script,./test.sh -f tsim/dnode/balance3.sim +,,y,script,./test.sh -f tsim/vnode/replica3_many.sim +,,y,script,./test.sh -f tsim/stable/metrics_idx.sim +# ,,y,script,./test.sh -f tsim/db/alter_replica_13.sim +,,y,script,./test.sh -f tsim/sync/3Replica1VgElect.sim +,,y,script,./test.sh -f tsim/sync/3Replica5VgElect.sim +,,n,script,./test.sh -f tsim/valgrind/checkError6.sim + +,,y,script,./test.sh -f tsim/user/basic.sim +,,y,script,./test.sh -f tsim/user/password.sim +,,y,script,./test.sh -f tsim/user/whitelist.sim +,,y,script,./test.sh -f tsim/user/privilege_db.sim +,,y,script,./test.sh -f tsim/user/privilege_sysinfo.sim +,,y,script,./test.sh -f tsim/user/privilege_topic.sim +,,y,script,./test.sh -f tsim/user/privilege_table.sim +,,y,script,./test.sh -f tsim/user/privilege_create_db.sim +,,y,script,./test.sh -f tsim/db/alter_option.sim +# ,,y,script,./test.sh -f tsim/db/alter_replica_31.sim +,,y,script,./test.sh -f tsim/db/basic1.sim +,,y,script,./test.sh -f tsim/db/basic2.sim +,,y,script,./test.sh -f tsim/db/basic3.sim +,,y,script,./test.sh -f tsim/db/basic4.sim +,,y,script,./test.sh -f tsim/db/basic5.sim +,,y,script,./test.sh -f tsim/db/basic6.sim +,,y,script,./test.sh -f tsim/db/commit.sim +,,y,script,./test.sh -f tsim/db/create_all_options.sim +,,y,script,./test.sh -f tsim/db/delete_reuse1.sim +,,y,script,./test.sh -f tsim/db/delete_reuse2.sim +,,y,script,./test.sh -f tsim/db/delete_reusevnode.sim +,,y,script,./test.sh -f tsim/db/delete_reusevnode2.sim +,,y,script,./test.sh -f tsim/db/delete_writing1.sim +,,y,script,./test.sh -f tsim/db/delete_writing2.sim +,,y,script,./test.sh -f tsim/db/error1.sim +,,y,script,./test.sh -f tsim/db/keep.sim +,,y,script,./test.sh -f tsim/db/len.sim +,,y,script,./test.sh -f tsim/db/repeat.sim +,,y,script,./test.sh -f tsim/db/show_create_db.sim +,,y,script,./test.sh -f tsim/db/show_create_table.sim +,,y,script,./test.sh -f tsim/db/tables.sim +,,y,script,./test.sh -f tsim/db/taosdlog.sim +,,y,script,./test.sh -f tsim/db/table_prefix_suffix.sim +,,y,script,./test.sh -f tsim/dnode/balance_replica1.sim +,,y,script,./test.sh -f tsim/dnode/balance_replica3.sim +,,y,script,./test.sh -f tsim/dnode/balance1.sim +,,y,script,./test.sh -f tsim/dnode/balancex.sim +,,y,script,./test.sh -f tsim/dnode/create_dnode.sim +,,y,script,./test.sh -f tsim/dnode/drop_dnode_has_mnode.sim +,,y,script,./test.sh -f tsim/dnode/drop_dnode_has_qnode_snode.sim +,,y,script,./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica1.sim +,,y,script,./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica3.sim +,,y,script,./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica1.sim +,,y,script,./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim +,,y,script,./test.sh -f tsim/dnode/drop_dnode_force.sim +,,y,script,./test.sh -f tsim/dnode/offline_reason.sim +,,y,script,./test.sh -f tsim/dnode/redistribute_vgroup_replica1.sim +,,y,script,./test.sh -f tsim/dnode/vnode_clean.sim +,,y,script,./test.sh -f tsim/dnode/use_dropped_dnode.sim +,,y,script,./test.sh -f tsim/dnode/split_vgroup_replica1.sim +,,y,script,./test.sh -f tsim/dnode/split_vgroup_replica3.sim +,,y,script,./test.sh -f tsim/import/basic.sim +,,y,script,./test.sh -f tsim/import/commit.sim +,,y,script,./test.sh -f tsim/import/large.sim +,,y,script,./test.sh -f tsim/import/replica1.sim +,,y,script,./test.sh -f tsim/insert/backquote.sim +,,y,script,./test.sh -f tsim/insert/basic.sim +,,y,script,./test.sh -f tsim/insert/basic0.sim +,,y,script,./test.sh -f tsim/insert/basic1.sim +,,y,script,./test.sh -f tsim/insert/basic2.sim +,,y,script,./test.sh -f tsim/insert/commit-merge0.sim +,,y,script,./test.sh -f tsim/insert/insert_drop.sim +,,y,script,./test.sh -f tsim/insert/insert_select.sim +,,y,script,./test.sh -f tsim/insert/null.sim +,,y,script,./test.sh -f tsim/insert/query_block1_file.sim +,,y,script,./test.sh -f tsim/insert/query_block1_memory.sim +,,y,script,./test.sh -f tsim/insert/query_block2_file.sim +,,y,script,./test.sh -f tsim/insert/query_block2_memory.sim +,,y,script,./test.sh -f tsim/insert/query_file_memory.sim +,,y,script,./test.sh -f tsim/insert/query_multi_file.sim +,,y,script,./test.sh -f tsim/insert/tcp.sim +,,y,script,./test.sh -f tsim/insert/update0.sim +,,y,script,./test.sh -f tsim/insert/delete0.sim +,,y,script,./test.sh -f tsim/insert/update1_sort_merge.sim +,,y,script,./test.sh -f tsim/insert/update2.sim +,,y,script,./test.sh -f tsim/parser/alter__for_community_version.sim +,,y,script,./test.sh -f tsim/parser/alter_column.sim +,,y,script,./test.sh -f tsim/parser/alter_stable.sim +,,y,script,./test.sh -f tsim/parser/alter.sim +,,y,script,./test.sh -f tsim/parser/alter1.sim +,,y,script,./test.sh -f tsim/parser/auto_create_tb_drop_tb.sim +,,y,script,./test.sh -f tsim/parser/auto_create_tb.sim +,,y,script,./test.sh -f tsim/parser/between_and.sim +,,y,script,./test.sh -f tsim/parser/binary_escapeCharacter.sim +,,y,script,./test.sh -f tsim/parser/columnValue_bigint.sim +,,y,script,./test.sh -f tsim/parser/columnValue_bool.sim +,,y,script,./test.sh -f tsim/parser/columnValue_double.sim +,,y,script,./test.sh -f tsim/parser/columnValue_float.sim +,,y,script,./test.sh -f tsim/parser/columnValue_int.sim +,,y,script,./test.sh -f tsim/parser/columnValue_smallint.sim +,,y,script,./test.sh -f tsim/parser/columnValue_tinyint.sim +,,y,script,./test.sh -f tsim/parser/columnValue_unsign.sim +,,y,script,./test.sh -f tsim/parser/condition.sim +,,y,script,./test.sh -f tsim/parser/condition_scl.sim +,,y,script,./test.sh -f tsim/parser/constCol.sim +,,y,script,./test.sh -f tsim/parser/create_db.sim +,,y,script,./test.sh -f tsim/parser/create_mt.sim +,,y,script,./test.sh -f tsim/parser/create_tb_with_tag_name.sim +,,y,script,./test.sh -f tsim/parser/create_tb.sim +,,y,script,./test.sh -f tsim/parser/dbtbnameValidate.sim +,,y,script,./test.sh -f tsim/parser/distinct.sim +,,y,script,./test.sh -f tsim/parser/fill_us.sim +,,y,script,./test.sh -f tsim/parser/fill.sim +,,y,script,./test.sh -f tsim/parser/first_last.sim +,,y,script,./test.sh -f tsim/parser/fill_stb.sim +,,y,script,./test.sh -f tsim/parser/interp.sim +,,y,script,./test.sh -f tsim/parser/fourArithmetic-basic.sim +,,y,script,./test.sh -f tsim/parser/function.sim +,,y,script,./test.sh -f tsim/parser/groupby-basic.sim +,,y,script,./test.sh -f tsim/parser/having_child.sim +,,y,script,./test.sh -f tsim/parser/having.sim +,,y,script,./test.sh -f tsim/parser/import_commit1.sim +,,y,script,./test.sh -f tsim/parser/import_commit2.sim +,,y,script,./test.sh -f tsim/parser/import_commit3.sim +,,y,script,./test.sh -f tsim/parser/import_file.sim +,,y,script,./test.sh -f tsim/parser/import.sim +,,y,script,./test.sh -f tsim/parser/insert_multiTbl.sim +,,y,script,./test.sh -f tsim/parser/insert_tb.sim +,,y,script,./test.sh -f tsim/parser/join_multitables.sim +,,y,script,./test.sh -f tsim/parser/join_multivnode.sim +,,y,script,./test.sh -f tsim/parser/join.sim +,,y,script,./test.sh -f tsim/parser/last_cache.sim +,,y,script,./test.sh -f tsim/parser/last_groupby.sim +,,y,script,./test.sh -f tsim/parser/lastrow.sim +,,y,script,./test.sh -f tsim/parser/lastrow2.sim +,,y,script,./test.sh -f tsim/parser/like.sim +,,y,script,./test.sh -f tsim/parser/limit.sim +,,y,script,./test.sh -f tsim/parser/mixed_blocks.sim +,,y,script,./test.sh -f tsim/parser/nchar.sim +,,y,script,./test.sh -f tsim/parser/null_char.sim +,,y,script,./test.sh -f tsim/parser/precision_ns.sim +,,y,script,./test.sh -f tsim/parser/projection_limit_offset.sim +,,y,script,./test.sh -f tsim/parser/regex.sim +,,y,script,./test.sh -f tsim/parser/regressiontest.sim +,,y,script,./test.sh -f tsim/parser/select_across_vnodes.sim +,,y,script,./test.sh -f tsim/parser/select_distinct_tag.sim +,,y,script,./test.sh -f tsim/parser/select_from_cache_disk.sim +,,y,script,./test.sh -f tsim/parser/select_with_tags.sim +,,y,script,./test.sh -f tsim/parser/selectResNum.sim +,,y,script,./test.sh -f tsim/parser/set_tag_vals.sim +,,y,script,./test.sh -f tsim/parser/single_row_in_tb.sim +,,y,script,./test.sh -f tsim/parser/slimit_alter_tags.sim +,,y,script,./test.sh -f tsim/parser/slimit.sim +,,y,script,./test.sh -f tsim/parser/slimit1.sim +,,y,script,./test.sh -f tsim/parser/stableOp.sim +,,y,script,./test.sh -f tsim/parser/tags_dynamically_specifiy.sim +,,y,script,./test.sh -f tsim/parser/tags_filter.sim +,,y,script,./test.sh -f tsim/parser/tbnameIn.sim +,,y,script,./test.sh -f tsim/parser/timestamp.sim +,,y,script,./test.sh -f tsim/parser/top_groupby.sim +,,y,script,./test.sh -f tsim/parser/topbot.sim +,,y,script,./test.sh -f tsim/parser/union_sysinfo.sim +,,y,script,./test.sh -f tsim/parser/slimit_limit.sim +,,y,script,./test.sh -f tsim/parser/table_merge_limit.sim +,,y,script,./test.sh -f tsim/query/tagLikeFilter.sim +,,y,script,./test.sh -f tsim/query/charScalarFunction.sim +,,y,script,./test.sh -f tsim/query/explain.sim +,,y,script,./test.sh -f tsim/query/interval-offset.sim +,,y,script,./test.sh -f tsim/query/interval.sim +,,y,script,./test.sh -f tsim/query/scalarFunction.sim +,,y,script,./test.sh -f tsim/query/scalarNull.sim +,,y,script,./test.sh -f tsim/query/session.sim +,,y,script,./test.sh -f tsim/query/udf.sim +,,n,script,./test.sh -f tsim/query/udfpy.sim +,,y,script,./test.sh -f tsim/query/udf_with_const.sim +,,y,script,./test.sh -f tsim/query/join_interval.sim +,,y,script,./test.sh -f tsim/query/join_pk.sim +,,y,script,./test.sh -f tsim/query/unionall_as_table.sim +,,y,script,./test.sh -f tsim/query/multi_order_by.sim +,,y,script,./test.sh -f tsim/query/sys_tbname.sim +,,y,script,./test.sh -f tsim/query/groupby.sim +,,y,script,./test.sh -f tsim/query/groupby_distinct.sim +,,y,script,./test.sh -f tsim/query/event.sim +,,y,script,./test.sh -f tsim/query/forceFill.sim +,,y,script,./test.sh -f tsim/query/emptyTsRange.sim +,,y,script,./test.sh -f tsim/query/emptyTsRange_scl.sim +,,y,script,./test.sh -f tsim/query/partitionby.sim +,,y,script,./test.sh -f tsim/query/tableCount.sim +,,y,script,./test.sh -f tsim/query/tag_scan.sim +,,y,script,./test.sh -f tsim/query/nullColSma.sim +,,y,script,./test.sh -f tsim/query/bug3398.sim +,,y,script,./test.sh -f tsim/query/explain_tsorder.sim +,,y,script,./test.sh -f tsim/qnode/basic1.sim +,,y,script,./test.sh -f tsim/snode/basic1.sim +,,y,script,./test.sh -f tsim/mnode/basic1.sim +,,y,script,./test.sh -f tsim/mnode/basic2.sim +,,y,script,./test.sh -f tsim/mnode/basic3.sim +,,y,script,./test.sh -f tsim/mnode/basic4.sim +,,y,script,./test.sh -f tsim/mnode/basic5.sim +,,y,script,./test.sh -f tsim/show/basic.sim +,,y,script,./test.sh -f tsim/table/autocreate.sim +,,y,script,./test.sh -f tsim/table/basic1.sim +,,y,script,./test.sh -f tsim/table/basic2.sim +,,y,script,./test.sh -f tsim/table/basic3.sim +,,y,script,./test.sh -f tsim/table/bigint.sim +,,y,script,./test.sh -f tsim/table/binary.sim +,,y,script,./test.sh -f tsim/table/bool.sim +,,y,script,./test.sh -f tsim/table/column_name.sim +,,y,script,./test.sh -f tsim/table/column_num.sim +,,y,script,./test.sh -f tsim/table/column_value.sim +,,y,script,./test.sh -f tsim/table/column2.sim +,,y,script,./test.sh -f tsim/table/createmulti.sim +,,y,script,./test.sh -f tsim/table/date.sim +,,y,script,./test.sh -f tsim/table/db.table.sim +,,y,script,./test.sh -f tsim/table/delete_reuse1.sim +,,y,script,./test.sh -f tsim/table/delete_reuse2.sim +,,y,script,./test.sh -f tsim/table/delete_writing.sim +,,y,script,./test.sh -f tsim/table/describe.sim +,,y,script,./test.sh -f tsim/table/double.sim +,,y,script,./test.sh -f tsim/table/float.sim +,,y,script,./test.sh -f tsim/table/hash.sim +,,y,script,./test.sh -f tsim/table/int.sim +,,y,script,./test.sh -f tsim/table/limit.sim +,,y,script,./test.sh -f tsim/table/smallint.sim +,,y,script,./test.sh -f tsim/table/table_len.sim +,,y,script,./test.sh -f tsim/table/table.sim +,,y,script,./test.sh -f tsim/table/tinyint.sim +,,y,script,./test.sh -f tsim/table/vgroup.sim +,,n,script,./test.sh -f tsim/stream/basic0.sim -g +,,y,script,./test.sh -f tsim/stream/basic1.sim +,,y,script,./test.sh -f tsim/stream/basic2.sim +,,y,script,./test.sh -f tsim/stream/basic3.sim +,,y,script,./test.sh -f tsim/stream/basic4.sim +,,y,script,./test.sh -f tsim/stream/checkpointInterval0.sim +,,y,script,./test.sh -f tsim/stream/checkStreamSTable1.sim +,,y,script,./test.sh -f tsim/stream/checkStreamSTable.sim +,,y,script,./test.sh -f tsim/stream/deleteInterval.sim +,,y,script,./test.sh -f tsim/stream/deleteSession.sim +,,y,script,./test.sh -f tsim/stream/deleteState.sim +,,y,script,./test.sh -f tsim/stream/distributeInterval0.sim +,,y,script,./test.sh -f tsim/stream/distributeIntervalRetrive0.sim +,,y,script,./test.sh -f tsim/stream/distributeSession0.sim +,,y,script,./test.sh -f tsim/stream/drop_stream.sim +,,y,script,./test.sh -f tsim/stream/fillHistoryBasic1.sim +,,y,script,./test.sh -f tsim/stream/fillHistoryBasic2.sim +,,y,script,./test.sh -f tsim/stream/fillHistoryBasic3.sim +,,y,script,./test.sh -f tsim/stream/fillIntervalDelete0.sim +,,y,script,./test.sh -f tsim/stream/fillIntervalDelete1.sim +,,y,script,./test.sh -f tsim/stream/fillIntervalLinear.sim +,,y,script,./test.sh -f tsim/stream/fillIntervalPartitionBy.sim +,,y,script,./test.sh -f tsim/stream/fillIntervalPrevNext1.sim +,,y,script,./test.sh -f tsim/stream/fillIntervalPrevNext.sim +,,y,script,./test.sh -f tsim/stream/fillIntervalRange.sim +,,y,script,./test.sh -f tsim/stream/fillIntervalValue.sim +,,y,script,./test.sh -f tsim/stream/ignoreCheckUpdate.sim +,,y,script,./test.sh -f tsim/stream/ignoreExpiredData.sim +,,y,script,./test.sh -f tsim/stream/partitionby1.sim +,,y,script,./test.sh -f tsim/stream/partitionbyColumnInterval.sim +,,y,script,./test.sh -f tsim/stream/partitionbyColumnSession.sim +,,y,script,./test.sh -f tsim/stream/partitionbyColumnState.sim +,,y,script,./test.sh -f tsim/stream/partitionby.sim +,,y,script,./test.sh -f tsim/stream/pauseAndResume.sim +,,y,script,./test.sh -f tsim/stream/schedSnode.sim +,,y,script,./test.sh -f tsim/stream/session0.sim +,,y,script,./test.sh -f tsim/stream/session1.sim +,,y,script,./test.sh -f tsim/stream/sliding.sim +,,y,script,./test.sh -f tsim/stream/state0.sim +,,y,script,./test.sh -f tsim/stream/state1.sim +,,y,script,./test.sh -f tsim/stream/triggerInterval0.sim +,,y,script,./test.sh -f tsim/stream/triggerSession0.sim +,,y,script,./test.sh -f tsim/stream/udTableAndTag0.sim +,,y,script,./test.sh -f tsim/stream/udTableAndTag1.sim +,,y,script,./test.sh -f tsim/stream/udTableAndTag2.sim +,,y,script,./test.sh -f tsim/stream/windowClose.sim +,,y,script,./test.sh -f tsim/trans/lossdata1.sim +,,y,script,./test.sh -f tsim/tmq/basic1.sim +,,y,script,./test.sh -f tsim/tmq/basic2.sim +,,y,script,./test.sh -f tsim/tmq/basic3.sim +,,y,script,./test.sh -f tsim/tmq/basic4.sim +,,y,script,./test.sh -f tsim/tmq/basic1Of2Cons.sim +,,y,script,./test.sh -f tsim/tmq/basic2Of2Cons.sim +,,y,script,./test.sh -f tsim/tmq/basic3Of2Cons.sim +,,y,script,./test.sh -f tsim/tmq/basic4Of2Cons.sim +,,y,script,./test.sh -f tsim/tmq/topic.sim +,,y,script,./test.sh -f tsim/tmq/snapshot.sim +,,y,script,./test.sh -f tsim/tmq/snapshot1.sim +,,y,script,./test.sh -f tsim/stable/alter_comment.sim +,,y,script,./test.sh -f tsim/stable/alter_count.sim +,,y,script,./test.sh -f tsim/stable/alter_import.sim +,,y,script,./test.sh -f tsim/stable/alter_insert1.sim +,,y,script,./test.sh -f tsim/stable/alter_insert2.sim +,,y,script,./test.sh -f tsim/stable/alter_metrics.sim +,,y,script,./test.sh -f tsim/stable/column_add.sim +,,y,script,./test.sh -f tsim/stable/column_drop.sim +,,y,script,./test.sh -f tsim/stable/column_modify.sim +,,y,script,./test.sh -f tsim/stable/disk.sim +,,y,script,./test.sh -f tsim/stable/dnode3.sim +,,y,script,./test.sh -f tsim/stable/metrics.sim +,,y,script,./test.sh -f tsim/stable/refcount.sim +,,y,script,./test.sh -f tsim/stable/tag_add.sim +,,y,script,./test.sh -f tsim/stable/tag_drop.sim +,,y,script,./test.sh -f tsim/stable/tag_filter.sim +,,y,script,./test.sh -f tsim/stable/tag_modify.sim +,,y,script,./test.sh -f tsim/stable/tag_rename.sim +,,y,script,./test.sh -f tsim/stable/values.sim +,,y,script,./test.sh -f tsim/stable/vnode3.sim +,,n,script,./test.sh -f tsim/sma/drop_sma.sim +,,y,script,./test.sh -f tsim/sma/sma_leak.sim +,,y,script,./test.sh -f tsim/sma/tsmaCreateInsertQuery.sim +,,y,script,./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim +,,y,script,./test.sh -f tsim/sma/rsmaPersistenceRecovery.sim +,,n,script,./test.sh -f tsim/valgrind/checkError1.sim +,,n,script,./test.sh -f tsim/valgrind/checkError2.sim +,,n,script,./test.sh -f tsim/valgrind/checkError3.sim +,,n,script,./test.sh -f tsim/valgrind/checkError4.sim +,,n,script,./test.sh -f tsim/valgrind/checkError5.sim +,,n,script,./test.sh -f tsim/valgrind/checkError8.sim +,,n,script,./test.sh -f tsim/valgrind/checkUdf.sim +,,y,script,./test.sh -f tsim/vnode/replica3_basic.sim +,,y,script,./test.sh -f tsim/vnode/replica3_vgroup.sim +,,y,script,./test.sh -f tsim/vnode/replica3_import.sim +,,y,script,./test.sh -f tsim/vnode/stable_balance_replica1.sim +,,y,script,./test.sh -f tsim/vnode/stable_dnode2_stop.sim +,,y,script,./test.sh -f tsim/vnode/stable_dnode2.sim +,,y,script,./test.sh -f tsim/vnode/stable_dnode3.sim +,,y,script,./test.sh -f tsim/vnode/stable_replica3_dnode6.sim +,,y,script,./test.sh -f tsim/vnode/stable_replica3_vnode3.sim +,,y,script,./test.sh -f tsim/sync/oneReplica1VgElect.sim +,,y,script,./test.sh -f tsim/sync/oneReplica5VgElect.sim +,,y,script,./test.sh -f tsim/catalog/alterInCurrent.sim +,,y,script,./test.sh -f tsim/scalar/in.sim +,,y,script,./test.sh -f tsim/scalar/scalar.sim +,,y,script,./test.sh -f tsim/scalar/filter.sim +,,y,script,./test.sh -f tsim/scalar/caseWhen.sim +,,y,script,./test.sh -f tsim/scalar/tsConvert.sim +,,y,script,./test.sh -f tsim/alter/cached_schema_after_alter.sim +,,y,script,./test.sh -f tsim/alter/dnode.sim +,,y,script,./test.sh -f tsim/alter/table.sim +,,y,script,./test.sh -f tsim/cache/new_metrics.sim +,,y,script,./test.sh -f tsim/cache/restart_table.sim +,,y,script,./test.sh -f tsim/cache/restart_metrics.sim +,,y,script,./test.sh -f tsim/column/commit.sim +,,y,script,./test.sh -f tsim/column/metrics.sim +,,y,script,./test.sh -f tsim/column/table.sim +,,y,script,./test.sh -f tsim/compress/commitlog.sim +,,y,script,./test.sh -f tsim/compress/compress2.sim +,,y,script,./test.sh -f tsim/compress/compress.sim +,,y,script,./test.sh -f tsim/compress/uncompress.sim +,,y,script,./test.sh -f tsim/compute/avg.sim +,,y,script,./test.sh -f tsim/compute/block_dist.sim +,,y,script,./test.sh -f tsim/compute/bottom.sim +,,y,script,./test.sh -f tsim/compute/count.sim +,,y,script,./test.sh -f tsim/compute/diff.sim +,,y,script,./test.sh -f tsim/compute/diff2.sim +,,y,script,./test.sh -f tsim/compute/first.sim +,,y,script,./test.sh -f tsim/compute/interval.sim +,,y,script,./test.sh -f tsim/compute/last_row.sim +,,y,script,./test.sh -f tsim/compute/last.sim +,,y,script,./test.sh -f tsim/compute/leastsquare.sim +,,y,script,./test.sh -f tsim/compute/max.sim +,,y,script,./test.sh -f tsim/compute/min.sim +,,y,script,./test.sh -f tsim/compute/null.sim +,,y,script,./test.sh -f tsim/compute/percentile.sim +,,y,script,./test.sh -f tsim/compute/stddev.sim +,,y,script,./test.sh -f tsim/compute/sum.sim +,,y,script,./test.sh -f tsim/compute/top.sim +,,y,script,./test.sh -f tsim/field/2.sim +,,y,script,./test.sh -f tsim/field/3.sim +,,y,script,./test.sh -f tsim/field/4.sim +,,y,script,./test.sh -f tsim/field/5.sim +,,y,script,./test.sh -f tsim/field/6.sim +,,y,script,./test.sh -f tsim/field/binary.sim +,,y,script,./test.sh -f tsim/field/bigint.sim +,,y,script,./test.sh -f tsim/field/bool.sim +,,y,script,./test.sh -f tsim/field/double.sim +,,y,script,./test.sh -f tsim/field/float.sim +,,y,script,./test.sh -f tsim/field/int.sim +,,y,script,./test.sh -f tsim/field/single.sim +,,y,script,./test.sh -f tsim/field/smallint.sim +,,y,script,./test.sh -f tsim/field/tinyint.sim +,,y,script,./test.sh -f tsim/field/unsigined_bigint.sim +,,y,script,./test.sh -f tsim/vector/metrics_field.sim +,,y,script,./test.sh -f tsim/vector/metrics_mix.sim +,,y,script,./test.sh -f tsim/vector/metrics_query.sim +,,y,script,./test.sh -f tsim/vector/metrics_tag.sim +,,y,script,./test.sh -f tsim/vector/metrics_time.sim +,,y,script,./test.sh -f tsim/vector/multi.sim +,,y,script,./test.sh -f tsim/vector/single.sim +,,y,script,./test.sh -f tsim/vector/table_field.sim +,,y,script,./test.sh -f tsim/vector/table_mix.sim +,,y,script,./test.sh -f tsim/vector/table_query.sim +,,y,script,./test.sh -f tsim/vector/table_time.sim +,,y,script,./test.sh -f tsim/wal/kill.sim +,,y,script,./test.sh -f tsim/tag/3.sim +,,y,script,./test.sh -f tsim/tag/4.sim +,,y,script,./test.sh -f tsim/tag/5.sim +,,y,script,./test.sh -f tsim/tag/6.sim +,,y,script,./test.sh -f tsim/tag/add.sim +,,y,script,./test.sh -f tsim/tag/bigint.sim +,,y,script,./test.sh -f tsim/tag/binary_binary.sim +,,y,script,./test.sh -f tsim/tag/binary.sim +,,y,script,./test.sh -f tsim/tag/bool_binary.sim +,,y,script,./test.sh -f tsim/tag/bool_int.sim +,,y,script,./test.sh -f tsim/tag/bool.sim +,,y,script,./test.sh -f tsim/tag/change.sim +,,y,script,./test.sh -f tsim/tag/column.sim +,,y,script,./test.sh -f tsim/tag/commit.sim +,,y,script,./test.sh -f tsim/tag/create.sim +,,y,script,./test.sh -f tsim/tag/delete.sim +,,y,script,./test.sh -f tsim/tag/double.sim +,,y,script,./test.sh -f tsim/tag/filter.sim +,,y,script,./test.sh -f tsim/tag/float.sim +,,y,script,./test.sh -f tsim/tag/int_binary.sim +,,y,script,./test.sh -f tsim/tag/int_float.sim +,,y,script,./test.sh -f tsim/tag/int.sim +,,y,script,./test.sh -f tsim/tag/set.sim +,,y,script,./test.sh -f tsim/tag/smallint.sim +,,y,script,./test.sh -f tsim/tag/tinyint.sim +,,y,script,./test.sh -f tsim/tag/drop_tag.sim +,,y,script,./test.sh -f tsim/tag/tbNameIn.sim +,,y,script,./test.sh -f tmp/monitor.sim +,,y,script,./test.sh -f tsim/tagindex/add_index.sim +,,n,script,./test.sh -f tsim/tagindex/sma_and_tag_index.sim + + +#develop test +,,n,develop-test,python3 ./test.py -f 2-query/table_count_scan.py +,,n,develop-test,python3 ./test.py -f 2-query/show_create_db.py +,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/auto_create_table_json.py +,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/custom_col_tag.py +,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/default_json.py +,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/demo.py +,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/insert_alltypes_json.py +,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/invalid_commandline.py +,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/json_tag.py +,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/query_json.py +,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/sample_csv_json.py +,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/sml_json_alltypes.py +,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/taosdemoTestQueryWithJson.py -R +,,n,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/telnet_tcp.py -R + +#docs-examples test +,,n,docs-examples-test,bash python.sh +,,n,docs-examples-test,bash node.sh +,,n,docs-examples-test,bash csharp.sh +,,n,docs-examples-test,bash jdbc.sh +,,n,docs-examples-test,bash go.sh +,,n,docs-examples-test,bash test_R.sh \ No newline at end of file diff --git a/tests/script/api/makefile b/tests/script/api/makefile index 6d55d8a75f..ac5c5c9138 100644 --- a/tests/script/api/makefile +++ b/tests/script/api/makefile @@ -16,6 +16,7 @@ exe: gcc $(CFLAGS) ./dbTableRoute.c -o $(ROOT)dbTableRoute $(LFLAGS) gcc $(CFLAGS) ./insertSameTs.c -o $(ROOT)insertSameTs $(LFLAGS) gcc $(CFLAGS) ./passwdTest.c -o $(ROOT)passwdTest $(LFLAGS) + gcc $(CFLAGS) ./whiteListTest.c -o $(ROOT)whiteListTest $(LFLAGS) clean: rm $(ROOT)batchprepare @@ -23,3 +24,4 @@ clean: rm $(ROOT)dbTableRoute rm $(ROOT)insertSameTs rm $(ROOT)passwdTest + rm $(ROOT)whiteListTest diff --git a/tests/script/api/whiteListTest.c b/tests/script/api/whiteListTest.c new file mode 100644 index 0000000000..1fe377f0f5 --- /dev/null +++ b/tests/script/api/whiteListTest.c @@ -0,0 +1,170 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +// TAOS standard API example. The same syntax as MySQL, but only a subset +// to compile: gcc -o demo demo.c -ltaos + +/** + * passwdTest.c + * - Run the test case in clear TDengine environment with default root passwd 'taosdata' + */ + +#include +#include +#include +#include +#include +#include "taos.h" // TAOS header file + +#define nDup 1 +#define nRoot 10 +#define nUser 10 +#define USER_LEN 24 +#define BUF_LEN 1024 + +void createUsers(TAOS *taos, const char *host); +void dropUsers(TAOS* taos); + +int nPassVerNotified = 0; +int nWhiteListVerNotified = 0; +TAOS *taosu[nRoot] = {0}; +char users[nUser][USER_LEN] = {0}; + +void __taos_notify_cb(void *param, void *ext, int type) { + switch (type) { + case TAOS_NOTIFY_PASSVER: { + ++nPassVerNotified; + printf("%s:%d type:%d user:%s ver:%d\n", __func__, __LINE__, type, param ? (char *)param : "NULL", *(int *)ext); + break; + } + case TAOS_NOTIFY_WHITELIST_VER: { + ++nWhiteListVerNotified; + printf("%s:%d type:%d user:%s ver:%"PRId64 "\n", __func__, __LINE__, type, param ? (char *)param : "NULL", *(int64_t *)ext); + break; + } + default: + printf("%s:%d unknown type:%d\n", __func__, __LINE__, type); + break; + } +} + +void __taos_async_whitelist_cb(void *param, int code, TAOS *taos, int numOfWhiteLists, uint64_t* pWhiteList) { + if (code == 0) { + printf("fetch whitelist cb. user: %s numofWhitelist: %d\n", param ? (char*)param : NULL, numOfWhiteLists); + for (int i = 0; i < numOfWhiteLists; ++i) { + printf(" %d: 0x%llx\n", i, pWhiteList[i]); + } + } else { + printf("fetch whitelist cb error %d\n", code); + } +} + +static void queryDB(TAOS *taos, char *command) { + int i; + TAOS_RES *pSql = NULL; + int32_t code = -1; + + for (i = 0; i < nDup; ++i) { + if (NULL != pSql) { + taos_free_result(pSql); + pSql = NULL; + } + + pSql = taos_query(taos, command); + code = taos_errno(pSql); + if (0 == code) { + break; + } + } + + if (code != 0) { + fprintf(stderr, "failed to run: %s, reason: %s\n", command, taos_errstr(pSql)); + taos_free_result(pSql); + taos_close(taos); + exit(EXIT_FAILURE); + } else { + fprintf(stderr, "success to run: %s\n", command); + } + + taos_free_result(pSql); +} + + +int main(int argc, char *argv[]) { + char qstr[1024]; + + // connect to server + if (argc < 2) { + printf("please input server-ip \n"); + return 0; + } + + TAOS *taos = taos_connect(argv[1], "root", "taosdata", NULL, 0); + if (taos == NULL) { + printf("failed to connect to server, reason:%s\n", "null taos" /*taos_errstr(taos)*/); + exit(1); + } + createUsers(taos, argv[1]); + + while (nWhiteListVerNotified < 10) { + printf("white list update notified %d times\n", nWhiteListVerNotified); + sleep(1); + } + printf("succeed in getting white list nofication. %d times\n", nWhiteListVerNotified); + dropUsers(taos); + taos_close(taos); + taos_cleanup(); +} + +void dropUsers(TAOS *taos) { + char qstr[1024]; + for (int i = 0; i < nUser; ++i) { + sprintf(users[i], "user%d", i); + sprintf(qstr, "DROP USER %s", users[i]); + queryDB(taos, qstr); + taos_close(taosu[i]); + } +} + +void createUsers(TAOS *taos, const char *host) { + char qstr[1024]; + // users + for (int i = 0; i < nUser; ++i) { + sprintf(users[i], "user%d", i); + sprintf(qstr, "CREATE USER %s PASS 'taosdata'", users[i]); + queryDB(taos, qstr); + + taosu[i] = taos_connect(host, users[i], "taosdata", NULL, 0); + if (taosu[i] == NULL) { + printf("failed to connect to server, user:%s, reason:%s\n", users[i], "null taos" /*taos_errstr(taos)*/); + exit(1); + } + + int code = taos_set_notify_cb(taosu[i], __taos_notify_cb, users[i], TAOS_NOTIFY_WHITELIST_VER); + + if (code != 0) { + fprintf(stderr, "failed to run: taos_set_notify_cb for user:%s since %d\n", users[i], code); + } else { + fprintf(stderr, "success to run: taos_set_notify_cb for user:%s\n", users[i]); + } + + // alter whitelist for users + sprintf(qstr, "alter user %s add host '%d.%d.%d.%d/24'", users[i], i, i, i, i); + queryDB(taos, qstr); + + taos_fetch_whitelist_a(taosu[i], __taos_async_whitelist_cb, users[i]); + } + +} diff --git a/tests/script/tsim/tagindex/add_index.sim b/tests/script/tsim/tagindex/add_index.sim index 70771fc63b..e73c7480ac 100644 --- a/tests/script/tsim/tagindex/add_index.sim +++ b/tests/script/tsim/tagindex/add_index.sim @@ -292,11 +292,12 @@ if $rows != 1 then return -1 endi -sql drop index $data[0][0] +#$drop_name=`$data[0][0]` +#sql drop index `$data[0][0]\` -if $rows != 0 then - return -1 -endi +#if $rows != 0 then +# return -1 +#endi sql_error drop index t2 @@ -304,7 +305,7 @@ sql_error drop index t3 -sql create index ti0 on $mtPrefix (t1) +#sql create index ti0 on $mtPrefix (t1) $i = $interval while $i < $limit diff --git a/tests/script/tsim/user/privilege_sysinfo.sim b/tests/script/tsim/user/privilege_sysinfo.sim index 815d0bd7b2..567d963c86 100644 --- a/tests/script/tsim/user/privilege_sysinfo.sim +++ b/tests/script/tsim/user/privilege_sysinfo.sim @@ -154,13 +154,14 @@ sql_error select * from information_schema.ins_mnodes sql_error select * from information_schema.ins_modules sql_error select * from information_schema.ins_qnodes sql_error select * from information_schema.ins_cluster +sql_error select * from information_schema.ins_users +sql_error select * from information_schema.ins_user_privilege sql select * from information_schema.ins_databases sql select * from information_schema.ins_functions sql select * from information_schema.ins_indexes sql select * from information_schema.ins_stables sql select * from information_schema.ins_tables sql select * from information_schema.ins_tags -sql select * from information_schema.ins_users sql select * from information_schema.ins_topics sql select * from information_schema.ins_subscriptions sql select * from information_schema.ins_streams diff --git a/tests/script/tsim/user/whitelist.sim b/tests/script/tsim/user/whitelist.sim index 53f16a90bb..4722c00efa 100644 --- a/tests/script/tsim/user/whitelist.sim +++ b/tests/script/tsim/user/whitelist.sim @@ -8,7 +8,7 @@ sql create user u_read pass 'taosdata1' host '127.0.0.1/24','192.168.1.0/24' sql create user u_write pass 'taosdata1' host '127.0.0.1','192.168.1.0' sql alter user u_read add host '3.3.3.4/24' -sql alter user u_write drop host '4.4.4.5/25' +sql_error alter user u_write drop host '4.4.4.5/25' sql show users if $rows != 3 then diff --git a/tests/script/wtest.bat b/tests/script/wtest.bat index 88ae703b7c..7321dbfd05 100644 --- a/tests/script/wtest.bat +++ b/tests/script/wtest.bat @@ -67,7 +67,7 @@ set "FILE_NAME=testSuite.sim" if "%1" == "-f" set "FILE_NAME=%2" set FILE_NAME=%FILE_NAME:/=\% -start cmd /k "timeout /t 600 /NOBREAK && taskkill /f /im tsim.exe & exit /b" +start cmd /k "timeout /t 800 /NOBREAK && taskkill /f /im tsim.exe & exit /b" rem echo FILE_NAME: %FILE_NAME% echo ExcuteCmd: %tsim% -c %CFG_DIR% -f %FILE_NAME% diff --git a/tests/system-test/0-others/information_schema.py b/tests/system-test/0-others/information_schema.py index baf0682fbb..245fc9ba21 100644 --- a/tests/system-test/0-others/information_schema.py +++ b/tests/system-test/0-others/information_schema.py @@ -217,7 +217,7 @@ class TDTestCase: tdSql.checkEqual(20470,len(tdSql.queryResult)) tdSql.query("select * from information_schema.ins_columns where db_name ='information_schema'") - tdSql.checkEqual(193, len(tdSql.queryResult)) + tdSql.checkEqual(194, len(tdSql.queryResult)) tdSql.query("select * from information_schema.ins_columns where db_name ='performance_schema'") tdSql.checkEqual(54, len(tdSql.queryResult)) diff --git a/tests/system-test/0-others/ttlChangeOnWrite.py b/tests/system-test/0-others/ttlChangeOnWrite.py index f9c63e4ec2..7bb10e25e8 100644 --- a/tests/system-test/0-others/ttlChangeOnWrite.py +++ b/tests/system-test/0-others/ttlChangeOnWrite.py @@ -24,7 +24,7 @@ class TDTestCase: for i in range(self.tables): tdSql.execute(f'create table t{i} using stb tags({i}) ttl {self.ttl}') - time.sleep(self.ttl * 2) + time.sleep(self.ttl + 3) tdSql.query('show tables') tdSql.checkRows(90) diff --git a/tests/system-test/2-query/partition_by_col.py b/tests/system-test/2-query/partition_by_col.py new file mode 100644 index 0000000000..1a394649d6 --- /dev/null +++ b/tests/system-test/2-query/partition_by_col.py @@ -0,0 +1,323 @@ +import taos +import sys +import time +import socket +import os +import threading +import math +from datetime import datetime + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +# from tmqCommon import * + +COMPARE_DATA = 0 +COMPARE_LEN = 1 + +class TDTestCase: + def __init__(self): + self.vgroups = 4 + self.ctbNum = 10 + self.rowsPerTbl = 10000 + self.duraion = '1h' + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def create_database(self,tsql, dbName,dropFlag=1,vgroups=2,replica=1, duration:str='1d'): + if dropFlag == 1: + tsql.execute("drop database if exists %s"%(dbName)) + + tsql.execute("create database if not exists %s vgroups %d replica %d duration %s"%(dbName, vgroups, replica, duration)) + tdLog.debug("complete to create database %s"%(dbName)) + return + + def create_stable(self,tsql, paraDict): + colString = tdCom.gen_column_type_str(colname_prefix=paraDict["colPrefix"], column_elm_list=paraDict["colSchema"]) + tagString = tdCom.gen_tag_type_str(tagname_prefix=paraDict["tagPrefix"], tag_elm_list=paraDict["tagSchema"]) + sqlString = f"create table if not exists %s.%s (%s) tags (%s)"%(paraDict["dbName"], paraDict["stbName"], colString, tagString) + tdLog.debug("%s"%(sqlString)) + tsql.execute(sqlString) + return + + def create_ctable(self,tsql=None, dbName='dbx',stbName='stb',ctbPrefix='ctb',ctbNum=1,ctbStartIdx=0): + for i in range(ctbNum): + sqlString = "create table %s.%s%d using %s.%s tags(%d, 'tb%d', 'tb%d', %d, %d, %d)" % \ + (dbName,ctbPrefix,i+ctbStartIdx,dbName,stbName,(i+ctbStartIdx) % 5,i+ctbStartIdx,i+ctbStartIdx,i+ctbStartIdx,i+ctbStartIdx,i+ctbStartIdx) + tsql.execute(sqlString) + + tdLog.debug("complete to create %d child tables by %s.%s" %(ctbNum, dbName, stbName)) + return + + def insert_data(self,tsql,dbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs,tsStep): + tdLog.debug("start to insert data ............") + tsql.execute("use %s" %dbName) + pre_insert = "insert into " + sql = pre_insert + + for i in range(ctbNum): + rowsBatched = 0 + sql += " %s%d values "%(ctbPrefix,i) + for j in range(rowsPerTbl): + if (i < ctbNum/2): + sql += "(%d, %d, %d, %d,%d,%d,%d,true,'binary%d', 'nchar%d') "%(startTs + j*tsStep, j%10, j%10, j%10, j%10, j%10, j%10, j%10, j%10) + else: + sql += "(%d, %d, NULL, %d,NULL,%d,%d,true,'binary%d', 'nchar%d') "%(startTs + j*tsStep, j%10, j%10, j%10, j%10, j%10, j%10) + rowsBatched += 1 + if ((rowsBatched == batchNum) or (j == rowsPerTbl - 1)): + tsql.execute(sql) + rowsBatched = 0 + if j < rowsPerTbl - 1: + sql = "insert into %s%d values " %(ctbPrefix,i) + else: + sql = "insert into " + if sql != pre_insert: + tsql.execute(sql) + tdLog.debug("insert data ............ [OK]") + return + + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'test', + 'dropFlag': 1, + 'vgroups': 2, + 'stbName': 'meters', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1}, + {'type': 'BIGINT', 'count':1}, + {'type': 'FLOAT', 'count':1}, + {'type': 'DOUBLE', 'count':1}, + {'type': 'smallint', 'count':1}, + {'type': 'tinyint', 'count':1}, + {'type': 'bool', 'count':1}, + {'type': 'binary', 'len':10, 'count':1}, + {'type': 'nchar', 'len':10, 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'nchar', 'len':20, 'count':1},{'type': 'binary', 'len':20, 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'smallint', 'count':1},{'type': 'DOUBLE', 'count':1}], + 'ctbPrefix': 't', + 'ctbStartIdx': 0, + 'ctbNum': 100, + 'rowsPerTbl': 10000, + 'batchNum': 3000, + 'startTs': 1537146000000, + 'tsStep': 600000} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tdLog.info("create database") + self.create_database(tsql=tdSql, dbName=paraDict["dbName"], dropFlag=paraDict["dropFlag"], vgroups=paraDict["vgroups"], replica=self.replicaVar, duration=self.duraion) + + tdLog.info("create stb") + self.create_stable(tsql=tdSql, paraDict=paraDict) + + tdLog.info("create child tables") + self.create_ctable(tsql=tdSql, dbName=paraDict["dbName"], \ + stbName=paraDict["stbName"],ctbPrefix=paraDict["ctbPrefix"],\ + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict["ctbStartIdx"]) + self.insert_data(tsql=tdSql, dbName=paraDict["dbName"],\ + ctbPrefix=paraDict["ctbPrefix"],ctbNum=paraDict["ctbNum"],\ + rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],\ + startTs=paraDict["startTs"],tsStep=paraDict["tsStep"]) + return + + def check_explain_res_has_row(self, plan_str_expect: str, rows): + plan_found = False + for row in rows: + if str(row).find(plan_str_expect) >= 0: + tdLog.debug("plan: [%s] found in: [%s]" % (plan_str_expect, str(row))) + plan_found = True + break + if not plan_found: + tdLog.exit("plan: %s not found in res: [%s]" % (plan_str_expect, str(rows))) + + + def test_sort_for_partition_hint(self): + sql = 'select count(*), c1 from meters partition by c1' + sql_hint = 'select /*+ sort_for_group() */count(*), c1 from meters partition by c1' + self.check_explain_res_has_row("Partition on", self.explain_sql(sql)) + self.check_explain_res_has_row("Sort", self.explain_sql(sql_hint)) + + sql = 'select count(*), c1, tbname from meters partition by tbname, c1' + sql_hint = 'select /*+ sort_for_group() */ count(*), c1, tbname from meters partition by tbname, c1' + self.check_explain_res_has_row("Partition on", self.explain_sql(sql)) + self.check_explain_res_has_row("Sort", self.explain_sql(sql_hint)) + + sql = 'select count(*), c1, tbname from meters partition by tbname, c1 interval(1s)' + sql_hint = 'select /*+ sort_for_group() */ count(*), c1, tbname from meters partition by tbname, c1 interval(1s)' + self.check_explain_res_has_row("Partition on", self.explain_sql(sql)) + self.check_explain_res_has_row("Sort", self.explain_sql(sql_hint)) + + sql = 'select count(*), c1, t1 from meters partition by t1, c1' + sql_hint = 'select /*+ sort_for_group() */ count(*), c1, t1 from meters partition by t1, c1' + self.check_explain_res_has_row("Partition on", self.explain_sql(sql)) + self.check_explain_res_has_row("Sort", self.explain_sql(sql_hint)) + + sql = 'select count(*), c1, t1 from meters partition by t1, c1 interval(1s)' + sql_hint = 'select /*+ sort_for_group() */ count(*), c1, t1 from meters partition by t1, c1 interval(1s)' + self.check_explain_res_has_row("Partition on", self.explain_sql(sql)) + self.check_explain_res_has_row("Sort", self.explain_sql(sql_hint)) + + sql = 'select count(*), c1 from meters partition by c1 interval(1s)' + sql_hint = 'select /*+ sort_for_group() */ count(*), c1 from meters partition by c1 interval(1s)' + self.check_explain_res_has_row("Partition on", self.explain_sql(sql)) + self.check_explain_res_has_row("Sort", self.explain_sql(sql_hint)) + + def add_order_by(self, sql: str, order_by: str, select_list: str = "*") -> str: + return "select %s from (%s)t order by %s" % (select_list, sql, order_by) + + def add_hint(self, sql: str) -> str: + return "select /*+ sort_for_group() */ %s" % sql[6:] + + def query_with_time(self, sql): + start = datetime.now() + tdSql.query(sql, queryTimes=1) + return (datetime.now().timestamp() - start.timestamp()) * 1000 + + def explain_sql(self, sql: str): + sql = "explain " + sql + tdSql.query(sql, queryTimes=1) + return tdSql.queryResult + + def query_and_compare_res(self, sql1, sql2, compare_what: int = 0): + dur = self.query_with_time(sql1) + tdLog.debug("sql1 query with time: [%f]" % dur) + res1 = tdSql.queryResult + dur = self.query_with_time(sql2) + tdLog.debug("sql2 query with time: [%f]" % dur) + res2 = tdSql.queryResult + if res1 is None or res2 is None: + tdLog.exit("res1 or res2 is None") + if compare_what <= COMPARE_LEN: + if len(res1) != len(res2): + tdLog.exit("query and copare failed cause different rows, sql1: [%s], rows: [%d], sql2: [%s], rows: [%d]" % (sql1, len(res1), sql2, len(res2))) + if compare_what == COMPARE_DATA: + for i in range(0, len(res1)): + if res1[i] != res2[i]: + tdLog.exit("compare failed for row: [%d], sqls: [%s] res1: [%s], sql2 : [%s] res2: [%s]" % (i, sql1, res1[i], sql2, res2[i])) + tdLog.debug("sql: [%s] and sql: [%s] have same results, rows: [%d]" % (sql1, sql2, len(res1))) + + def prepare_and_query_and_compare(self, sqls: [], order_by: str, select_list: str = "*", compare_what: int = 0): + for sql in sqls: + sql_hint = self.add_order_by(self.add_hint(sql), order_by, select_list) + sql = self.add_order_by(sql, order_by, select_list) + self.check_explain_res_has_row("Sort", self.explain_sql(sql_hint)) + self.check_explain_res_has_row("Partition", self.explain_sql(sql)) + self.query_and_compare_res(sql, sql_hint, compare_what=compare_what) + + def test_sort_for_partition_res(self): + sqls_par_c1_agg = [ + "select count(*), c1 from meters partition by c1", + "select count(*), min(c2), max(c3), c1 from meters partition by c1", + "select c1 from meters partition by c1", + ] + sqls_par_c1 = [ + "select * from meters partition by c1" + ] + sqls_par_c1_c2_agg = [ + "select count(*), c1, c2 from meters partition by c1, c2", + "select c1, c2 from meters partition by c1, c2", + "select count(*), c1, c2, min(c4), max(c5), sum(c6) from meters partition by c1, c2", + ] + sqls_par_c1_c2 = [ + "select * from meters partition by c1, c2" + ] + + sqls_par_tbname_c1 = [ + "select count(*), c1 , tbname as a from meters partition by tbname, c1" + ] + sqls_par_tag_c1 = [ + "select count(*), c1, t1 from meters partition by t1, c1" + ] + self.prepare_and_query_and_compare(sqls_par_c1_agg, "c1") + self.prepare_and_query_and_compare(sqls_par_c1, "c1, ts, c2", "c1, ts, c2") + self.prepare_and_query_and_compare(sqls_par_c1_c2_agg, "c1, c2") + self.prepare_and_query_and_compare(sqls_par_c1_c2, "c1, c2, ts, c3", "c1, c2, ts, c3") + self.prepare_and_query_and_compare(sqls_par_tbname_c1, "a, c1") + self.prepare_and_query_and_compare(sqls_par_tag_c1, "t1, c1") + + def get_interval_template_sqls(self, col_name): + sqls = [ + 'select _wstart as ts, count(*), tbname as a, %s from meters partition by tbname, %s interval(1s)' % (col_name, col_name), + #'select _wstart as ts, count(*), tbname as a, %s from meters partition by tbname, %s interval(30s)' % (col_name, col_name), + #'select _wstart as ts, count(*), tbname as a, %s from meters partition by tbname, %s interval(1m)' % (col_name, col_name), + #'select _wstart as ts, count(*), tbname as a, %s from meters partition by tbname, %s interval(30m)' % (col_name, col_name), + #'select _wstart as ts, count(*), tbname as a, %s from meters partition by tbname, %s interval(1h)' % (col_name, col_name), + + 'select _wstart as ts, count(*), t1 as a, %s from meters partition by t1, %s interval(1s)' % (col_name, col_name), + #'select _wstart as ts, count(*), t1 as a, %s from meters partition by t1, %s interval(30s)' % (col_name, col_name), + #'select _wstart as ts, count(*), t1 as a, %s from meters partition by t1, %s interval(1m)' % (col_name, col_name), + #'select _wstart as ts, count(*), t1 as a, %s from meters partition by t1, %s interval(30m)' % (col_name, col_name), + #'select _wstart as ts, count(*), t1 as a, %s from meters partition by t1, %s interval(1h)' % (col_name, col_name), + + 'select _wstart as ts, count(*), %s as a, %s from meters partition by %s interval(1s)' % (col_name, col_name, col_name), + #'select _wstart as ts, count(*), %s as a, %s from meters partition by %s interval(30s)' % (col_name, col_name, col_name), + #'select _wstart as ts, count(*), %s as a, %s from meters partition by %s interval(1m)' % (col_name, col_name, col_name), + #'select _wstart as ts, count(*), %s as a, %s from meters partition by %s interval(30m)' % (col_name, col_name, col_name), + #'select _wstart as ts, count(*), %s as a, %s from meters partition by %s interval(1h)' % (col_name, col_name, col_name), + + 'select _wstart as ts, count(*), tbname as a, %s from meters partition by %s, tbname interval(1s)' % (col_name, col_name), + 'select _wstart as ts, count(*), t1 as a, %s from meters partition by %s, t1 interval(1s)' % (col_name, col_name), + ] + order_list = 'a, %s, ts' % (col_name) + return (sqls, order_list) + + def test_sort_for_partition_interval(self): + sqls, order_list = self.get_interval_template_sqls('c1') + self.prepare_and_query_and_compare(sqls, order_list) + #sqls, order_list = self.get_interval_template_sqls('c2') + #self.prepare_and_query(sqls, order_list) + sqls, order_list = self.get_interval_template_sqls('c3') + self.prepare_and_query_and_compare(sqls, order_list) + #sqls, order_list = self.get_interval_template_sqls('c4') + #self.prepare_and_query(sqls, order_list) + #sqls, order_list = self.get_interval_template_sqls('c5') + #self.prepare_and_query(sqls, order_list) + sqls, order_list = self.get_interval_template_sqls('c6') + self.prepare_and_query_and_compare(sqls, order_list) + #sqls, order_list = self.get_interval_template_sqls('c7') + #self.prepare_and_query(sqls, order_list) + sqls, order_list = self.get_interval_template_sqls('c8') + self.prepare_and_query_and_compare(sqls, order_list) + sqls, order_list = self.get_interval_template_sqls('c9') + self.prepare_and_query_and_compare(sqls, order_list) + + def test_sort_for_partition_no_agg_limit(self): + sqls_template = [ + 'select * from meters partition by c1 slimit %d limit %d', + 'select * from meters partition by c2 slimit %d limit %d', + 'select * from meters partition by c8 slimit %d limit %d', + ] + sqls = [] + for sql in sqls_template: + sqls.append(sql % (1,1)) + sqls.append(sql % (1,10)) + sqls.append(sql % (10,10)) + sqls.append(sql % (100, 100)) + order_by_list = 'ts,c1,c2,c3,c4,c5,c6,c7,c8,c9,t1,t2,t3,t4,t5,t6' + + self.prepare_and_query_and_compare(sqls, order_by_list, compare_what=COMPARE_LEN) + + + def run(self): + self.prepareTestEnv() + #time.sleep(99999999) + self.test_sort_for_partition_hint() + self.test_sort_for_partition_res() + self.test_sort_for_partition_interval() + self.test_sort_for_partition_no_agg_limit() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/6-cluster/5dnode3mnodeRecreateMnode.py b/tests/system-test/6-cluster/5dnode3mnodeRecreateMnode.py index 650bc347aa..7af5982dec 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeRecreateMnode.py +++ b/tests/system-test/6-cluster/5dnode3mnodeRecreateMnode.py @@ -166,22 +166,23 @@ class TDTestCase: nodePort = 6030 + i*100 newTdSql=tdCom.newTdSql(port=nodePort) + + tdDnodes[1].stoptaosd() + dataPath = tdDnodes[1].dataDir os.system(f"rm -rf {dataPath}/*") os.system(f"rm -rf {dataPath}/.runing") - tdDnodes[1].stoptaosd() tdDnodes[1].starttaosd() sleep(5) for i in range(6): nodePort = 6030 + i*100 newTdSql=tdCom.newTdSql(port=nodePort) + tdDnodes[0].stoptaosd() dataPath = tdDnodes[0].dataDir os.system(f"rm -rf {dataPath}/*") os.system(f"rm -rf {dataPath}/.runing") - - tdDnodes[0].stoptaosd() tdDnodes[0].starttaosd() sleep(5) for i in range(6): diff --git a/tests/system-test/test.py b/tests/system-test/test.py index e0dc426004..6813530a5c 100644 --- a/tests/system-test/test.py +++ b/tests/system-test/test.py @@ -109,7 +109,7 @@ if __name__ == "__main__": websocket = False replicaVar = 1 asan = False - independentMnode = True + independentMnode = False previousCluster = False opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:N:M:Q:C:RWD:n:i:aP', [ 'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'restart', 'updateCfgDict', 'killv', 'execCmd','dnodeNums','mnodeNums','queryPolicy','createDnodeNums','restful','websocket','adaptercfgupdate','replicaVar','independentMnode','previous']) @@ -202,7 +202,7 @@ if __name__ == "__main__": createDnodeNums = value if key in ['-i', '--independentMnode']: - independentMnode = False + independentMnode = value if key in ['-R', '--restful']: restful = True diff --git a/tests/system-test/win-test-file b/tests/system-test/win-test-file index adea684ef0..443c27fd7e 100644 --- a/tests/system-test/win-test-file +++ b/tests/system-test/win-test-file @@ -18,6 +18,7 @@ python3 ./test.py -f 2-query/nestedQuery_math.py -Q 4 python3 ./test.py -f 2-query/nestedQuery_time.py -Q 4 python3 ./test.py -f 2-query/nestedQuery_26.py -Q 4 python3 ./test.py -f 2-query/interval_limit_opt.py -Q 4 +python3 ./test.py -f 2-query/partition_by_col.py -Q 4 python3 ./test.py -f 7-tmq/tmqShow.py python3 ./test.py -f 7-tmq/tmqDropStb.py python3 ./test.py -f 7-tmq/subscribeStb0.py