fix:conflicts from 3.0
This commit is contained in:
wangmm0220
commit
ad70dc178f
|
|
@ -2,7 +2,7 @@
|
|||
IF (DEFINED VERNUMBER)
|
||||
SET(TD_VER_NUMBER ${VERNUMBER})
|
||||
ELSE ()
|
||||
SET(TD_VER_NUMBER "3.1.2.0.alpha")
|
||||
SET(TD_VER_NUMBER "3.2.0.0.alpha")
|
||||
ENDIF ()
|
||||
|
||||
IF (DEFINED VERCOMPATIBLE)
|
||||
|
|
|
|||
|
|
@ -384,7 +384,7 @@ Shows information about all vgroups in the current database.
|
|||
## SHOW VNODES
|
||||
|
||||
```sql
|
||||
SHOW VNODES {dnode_id | dnode_endpoint};
|
||||
SHOW VNODES [ON DNODE dnode_id];
|
||||
```
|
||||
|
||||
Shows information about all vnodes in the system or about the vnodes for a specified dnode.
|
||||
|
|
|
|||
|
|
@ -327,7 +327,7 @@ SHOW [db_name.]VGROUPS;
|
|||
## SHOW VNODES
|
||||
|
||||
```sql
|
||||
SHOW VNODES {dnode_id | dnode_endpoint};
|
||||
SHOW VNODES [ON DNODE dnode_id];
|
||||
```
|
||||
|
||||
显示当前系统中所有 VNODE 或某个 DNODE 的 VNODE 的信息。
|
||||
|
|
|
|||
|
|
@ -125,6 +125,7 @@ typedef enum {
|
|||
|
||||
typedef enum {
|
||||
TAOS_NOTIFY_PASSVER = 0,
|
||||
TAOS_NOTIFY_WHITELIST_VER = 1
|
||||
} TAOS_NOTIFY_TYPE;
|
||||
|
||||
#define RET_MSG_LENGTH 1024
|
||||
|
|
@ -236,6 +237,9 @@ DLL_EXPORT void taos_set_hb_quit(int8_t quitByKill);
|
|||
|
||||
DLL_EXPORT int taos_set_notify_cb(TAOS *taos, __taos_notify_fn_t fp, void *param, int type);
|
||||
|
||||
typedef void (*__taos_async_whitelist_fn_t)(void *param, int code, TAOS *taos, int numOfWhiteLists, uint64_t* pWhiteLists);
|
||||
DLL_EXPORT void taos_fetch_whitelist_a(TAOS *taos, __taos_async_whitelist_fn_t fp, void *param);
|
||||
|
||||
/* --------------------------schemaless INTERFACE------------------------------- */
|
||||
|
||||
DLL_EXPORT TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision);
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ typedef struct SBlockOrderInfo {
|
|||
bool nullFirst;
|
||||
int32_t order;
|
||||
int32_t slotId;
|
||||
void* compFn;
|
||||
SColumnInfoData* pColData;
|
||||
} SBlockOrderInfo;
|
||||
|
||||
|
|
@ -82,6 +83,15 @@ static FORCE_INLINE bool colDataIsNull_s(const SColumnInfoData* pColumnInfoData,
|
|||
}
|
||||
}
|
||||
|
||||
static FORCE_INLINE bool colDataIsNull_t(const SColumnInfoData* pColumnInfoData, uint32_t row, bool isVarType) {
|
||||
if (!pColumnInfoData->hasNull) return false;
|
||||
if (isVarType) {
|
||||
return colDataIsNull_var(pColumnInfoData, row);
|
||||
} else {
|
||||
return pColumnInfoData->nullbitmap ? colDataIsNull_f(pColumnInfoData->nullbitmap, row) : false;
|
||||
}
|
||||
}
|
||||
|
||||
static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, uint32_t totalRows, uint32_t row,
|
||||
SColumnDataAgg* pColAgg) {
|
||||
if (!pColumnInfoData->hasNull) {
|
||||
|
|
@ -210,6 +220,10 @@ double blockDataGetSerialRowSize(const SSDataBlock* pBlock);
|
|||
size_t blockDataGetSerialMetaSize(uint32_t numOfCols);
|
||||
|
||||
int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo);
|
||||
/**
|
||||
* @brief find how many rows already in order start from first row
|
||||
*/
|
||||
int32_t blockDataGetSortedRows(SSDataBlock* pDataBlock, SArray* pOrderInfo);
|
||||
|
||||
int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows, bool clearPayload);
|
||||
int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows);
|
||||
|
|
|
|||
|
|
@ -849,6 +849,7 @@ typedef struct {
|
|||
int32_t authVer;
|
||||
char sVer[TSDB_VERSION_LEN];
|
||||
char sDetailVer[128];
|
||||
int64_t whiteListVer;
|
||||
} SConnectRsp;
|
||||
|
||||
int32_t tSerializeSConnectRsp(void* buf, int32_t bufLen, SConnectRsp* pRsp);
|
||||
|
|
@ -880,6 +881,12 @@ typedef struct SIpV4Range {
|
|||
uint32_t mask;
|
||||
} SIpV4Range;
|
||||
|
||||
typedef struct {
|
||||
int32_t num;
|
||||
SIpV4Range pIpRange[];
|
||||
} SIpWhiteList;
|
||||
|
||||
SIpWhiteList* cloneIpWhiteList(SIpWhiteList* pIpWhiteList);
|
||||
typedef struct {
|
||||
int8_t createType;
|
||||
int8_t superUser; // denote if it is a super user or not
|
||||
|
|
@ -895,6 +902,30 @@ int32_t tSerializeSCreateUserReq(void* buf, int32_t bufLen, SCreateUserReq* pReq
|
|||
int32_t tDeserializeSCreateUserReq(void* buf, int32_t bufLen, SCreateUserReq* pReq);
|
||||
void tFreeSCreateUserReq(SCreateUserReq* pReq);
|
||||
|
||||
typedef struct {
|
||||
int64_t ver;
|
||||
char user[TSDB_USER_LEN];
|
||||
int32_t numOfRange;
|
||||
SIpV4Range* pIpRanges;
|
||||
} SUpdateUserIpWhite;
|
||||
typedef struct {
|
||||
int64_t ver;
|
||||
int numOfUser;
|
||||
SUpdateUserIpWhite* pUserIpWhite;
|
||||
} SUpdateIpWhite;
|
||||
|
||||
int32_t tSerializeSUpdateIpWhite(void* buf, int32_t bufLen, SUpdateIpWhite* pReq);
|
||||
int32_t tDeserializeSUpdateIpWhite(void* buf, int32_t bufLen, SUpdateIpWhite* pReq);
|
||||
void tFreeSUpdateIpWhiteReq(SUpdateIpWhite* pReq);
|
||||
SUpdateIpWhite* cloneSUpdateIpWhiteReq(SUpdateIpWhite* pReq);
|
||||
|
||||
typedef struct {
|
||||
int64_t ipWhiteVer;
|
||||
} SRetrieveIpWhiteReq;
|
||||
|
||||
int32_t tSerializeRetrieveIpWhite(void* buf, int32_t bufLen, SRetrieveIpWhiteReq* pReq);
|
||||
int32_t tDeserializeRetrieveIpWhite(void* buf, int32_t bufLen, SRetrieveIpWhiteReq* pReq);
|
||||
|
||||
typedef struct {
|
||||
int8_t alterType;
|
||||
int8_t superUser;
|
||||
|
|
@ -935,12 +966,30 @@ typedef struct {
|
|||
SHashObj* readTbs;
|
||||
SHashObj* writeTbs;
|
||||
SHashObj* useDbs;
|
||||
int64_t whiteListVer;
|
||||
} SGetUserAuthRsp;
|
||||
|
||||
int32_t tSerializeSGetUserAuthRsp(void* buf, int32_t bufLen, SGetUserAuthRsp* pRsp);
|
||||
int32_t tDeserializeSGetUserAuthRsp(void* buf, int32_t bufLen, SGetUserAuthRsp* pRsp);
|
||||
void tFreeSGetUserAuthRsp(SGetUserAuthRsp* pRsp);
|
||||
|
||||
typedef struct {
|
||||
char user[TSDB_USER_LEN];
|
||||
} SGetUserWhiteListReq;
|
||||
|
||||
int32_t tSerializeSGetUserWhiteListReq(void* buf, int32_t bufLen, SGetUserWhiteListReq* pReq);
|
||||
int32_t tDeserializeSGetUserWhiteListReq(void* buf, int32_t bufLen, SGetUserWhiteListReq* pReq);
|
||||
|
||||
typedef struct {
|
||||
char user[TSDB_USER_LEN];
|
||||
int32_t numWhiteLists;
|
||||
SIpV4Range* pWhiteLists;
|
||||
} SGetUserWhiteListRsp;
|
||||
|
||||
int32_t tSerializeSGetUserWhiteListRsp(void* buf, int32_t bufLen, SGetUserWhiteListRsp* pRsp);
|
||||
int32_t tDeserializeSGetUserWhiteListRsp(void* buf, int32_t bufLen, SGetUserWhiteListRsp* pRsp);
|
||||
void tFreeSGetUserWhiteListRsp(SGetUserWhiteListRsp* pRsp);
|
||||
|
||||
/*
|
||||
* for client side struct, only column id, type, bytes are necessary
|
||||
* But for data in vnode side, we need all the following information.
|
||||
|
|
@ -1455,6 +1504,7 @@ typedef struct {
|
|||
SClusterCfg clusterCfg;
|
||||
SArray* pVloads; // array of SVnodeLoad
|
||||
int32_t statusSeq;
|
||||
int64_t ipWhiteVer;
|
||||
} SStatusReq;
|
||||
|
||||
int32_t tSerializeSStatusReq(void* buf, int32_t bufLen, SStatusReq* pReq);
|
||||
|
|
@ -1485,6 +1535,7 @@ typedef struct {
|
|||
SDnodeCfg dnodeCfg;
|
||||
SArray* pDnodeEps; // Array of SDnodeEp
|
||||
int32_t statusSeq;
|
||||
int64_t ipWhiteVer;
|
||||
} SStatusRsp;
|
||||
|
||||
int32_t tSerializeSStatusRsp(void* buf, int32_t bufLen, SStatusRsp* pRsp);
|
||||
|
|
|
|||
|
|
@ -52,6 +52,7 @@ typedef struct {
|
|||
void* data;
|
||||
void* mgmt;
|
||||
void* clientRpc;
|
||||
void* serverRpc;
|
||||
PutToQueueFp putToQueueFp;
|
||||
GetQueueSizeFp qsizeFp;
|
||||
SendReqFp sendReqFp;
|
||||
|
|
|
|||
|
|
@ -177,6 +177,8 @@ enum { // WARN: new msg should be appended to segment tail
|
|||
TD_DEF_MSG_TYPE(TDMT_MND_UPTIME_TIMER, "uptime-timer", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_MND_TMQ_LOST_CONSUMER_CLEAR, "lost-consumer-clear", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_MND_STREAM_HEARTBEAT, "stream-heartbeat", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_MND_RETRIEVE_IP_WHITE, "retrieve-ip-white", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_MND_GET_USER_WHITELIST, "get-user-whitelist", NULL, NULL)
|
||||
TD_DEF_MSG_TYPE(TDMT_MND_MAX_MSG, "mnd-max", NULL, NULL)
|
||||
|
||||
TD_DEF_MSG_TYPE(TDMT_MND_BALANCE_VGROUP_LEADER, "balance-vgroup-leader", NULL, NULL)
|
||||
|
|
|
|||
|
|
@ -366,6 +366,7 @@
|
|||
#define TK_NK_BIN 605 // bin format data 0b111
|
||||
#define TK_BATCH_SCAN 606
|
||||
#define TK_NO_BATCH_SCAN 607
|
||||
#define TK_SORT_FOR_GROUP 608
|
||||
|
||||
|
||||
#define TK_NK_NIL 65535
|
||||
|
|
|
|||
|
|
@ -136,6 +136,8 @@ typedef struct SAggLogicNode {
|
|||
bool hasTimeLineFunc;
|
||||
bool onlyHasKeepOrderFunc;
|
||||
bool hasGroupKeyOptimized;
|
||||
bool isGroupTb;
|
||||
bool isPartTb; // true if partition keys has tbname
|
||||
} SAggLogicNode;
|
||||
|
||||
typedef struct SProjectLogicNode {
|
||||
|
|
@ -221,6 +223,7 @@ typedef struct SMergeLogicNode {
|
|||
int32_t srcGroupId;
|
||||
bool groupSort;
|
||||
bool ignoreGroupId;
|
||||
bool inputWithGroupId;
|
||||
} SMergeLogicNode;
|
||||
|
||||
typedef enum EWindowType {
|
||||
|
|
@ -263,6 +266,7 @@ typedef struct SWindowLogicNode {
|
|||
int8_t igExpired;
|
||||
int8_t igCheckUpdate;
|
||||
EWindowAlgorithm windowAlgo;
|
||||
bool isPartTb;
|
||||
} SWindowLogicNode;
|
||||
|
||||
typedef struct SFillLogicNode {
|
||||
|
|
@ -279,8 +283,9 @@ typedef struct SSortLogicNode {
|
|||
SLogicNode node;
|
||||
SNodeList* pSortKeys;
|
||||
bool groupSort;
|
||||
int64_t maxRows;
|
||||
bool skipPKSortOpt;
|
||||
bool calcGroupId;
|
||||
bool excludePkCol; // exclude PK ts col when calc group id
|
||||
} SSortLogicNode;
|
||||
|
||||
typedef struct SPartitionLogicNode {
|
||||
|
|
@ -288,6 +293,10 @@ typedef struct SPartitionLogicNode {
|
|||
SNodeList* pPartitionKeys;
|
||||
SNodeList* pTags;
|
||||
SNode* pSubtable;
|
||||
|
||||
bool needBlockOutputTsOrder; // if true, partition output block will have ts order maintained
|
||||
int32_t pkTsColId;
|
||||
uint64_t pkTsColTbId;
|
||||
} SPartitionLogicNode;
|
||||
|
||||
typedef enum ESubplanType {
|
||||
|
|
@ -527,6 +536,7 @@ typedef struct SMergePhysiNode {
|
|||
int32_t srcGroupId;
|
||||
bool groupSort;
|
||||
bool ignoreGroupId;
|
||||
bool inputWithGroupId;
|
||||
} SMergePhysiNode;
|
||||
|
||||
typedef struct SWindowPhysiNode {
|
||||
|
|
@ -603,6 +613,8 @@ typedef struct SSortPhysiNode {
|
|||
SNodeList* pExprs; // these are expression list of order_by_clause and parameter expression of aggregate function
|
||||
SNodeList* pSortKeys; // element is SOrderByExprNode, and SOrderByExprNode::pExpr is SColumnNode
|
||||
SNodeList* pTargets;
|
||||
bool calcGroupId;
|
||||
bool excludePkCol;
|
||||
} SSortPhysiNode;
|
||||
|
||||
typedef SSortPhysiNode SGroupSortPhysiNode;
|
||||
|
|
@ -612,6 +624,9 @@ typedef struct SPartitionPhysiNode {
|
|||
SNodeList* pExprs; // these are expression list of partition_by_clause
|
||||
SNodeList* pPartitionKeys;
|
||||
SNodeList* pTargets;
|
||||
|
||||
bool needBlockOutputTsOrder;
|
||||
int32_t tsSlotId;
|
||||
} SPartitionPhysiNode;
|
||||
|
||||
typedef struct SStreamPartitionPhysiNode {
|
||||
|
|
|
|||
|
|
@ -119,6 +119,7 @@ typedef struct SLeftValueNode {
|
|||
typedef enum EHintOption {
|
||||
HINT_NO_BATCH_SCAN = 1,
|
||||
HINT_BATCH_SCAN,
|
||||
HINT_SORT_FOR_GROUP,
|
||||
} EHintOption;
|
||||
|
||||
typedef struct SHintNode {
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@ typedef struct {
|
|||
int64_t applyIndex;
|
||||
uint64_t applyTerm;
|
||||
char user[TSDB_USER_LEN];
|
||||
|
||||
} SRpcConnInfo;
|
||||
|
||||
typedef struct SRpcHandleInfo {
|
||||
|
|
@ -60,6 +61,8 @@ typedef struct SRpcHandleInfo {
|
|||
STraceId traceId;
|
||||
|
||||
SRpcConnInfo conn;
|
||||
int8_t forbiddenIp;
|
||||
|
||||
} SRpcHandleInfo;
|
||||
|
||||
typedef struct SRpcMsg {
|
||||
|
|
@ -162,6 +165,11 @@ int rpcSendRequestWithCtx(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg,
|
|||
int rpcSendRecv(void *shandle, SEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp);
|
||||
int rpcSetDefaultAddr(void *thandle, const char *ip, const char *fqdn);
|
||||
void *rpcAllocHandle();
|
||||
void rpcSetIpWhite(void *thandl, void *arg);
|
||||
|
||||
int32_t rpcUtilSIpRangeToStr(SIpV4Range *pRange, char *buf);
|
||||
|
||||
int32_t rpcUtilSWhiteListToStr(SIpWhiteList *pWhiteList, char **ppBuf);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
|
|
|
|||
|
|
@ -123,6 +123,8 @@ int32_t* taosGetErrno();
|
|||
#define TSDB_CODE_INVALID_DATA_FMT TAOS_DEF_ERROR_CODE(0, 0x0132)
|
||||
#define TSDB_CODE_INVALID_CFG_VALUE TAOS_DEF_ERROR_CODE(0, 0x0133)
|
||||
|
||||
#define TSDB_CODE_IP_NOT_IN_WHITE_LIST TAOS_DEF_ERROR_CODE(0, 0x0134)
|
||||
|
||||
//client
|
||||
#define TSDB_CODE_TSC_INVALID_OPERATION TAOS_DEF_ERROR_CODE(0, 0x0200)
|
||||
#define TSDB_CODE_TSC_INVALID_QHANDLE TAOS_DEF_ERROR_CODE(0, 0x0201)
|
||||
|
|
@ -245,6 +247,10 @@ int32_t* taosGetErrno();
|
|||
#define TSDB_CODE_MND_AUTH_FAILURE TAOS_DEF_ERROR_CODE(0, 0x0357)
|
||||
#define TSDB_CODE_MND_USER_NOT_AVAILABLE TAOS_DEF_ERROR_CODE(0, 0x0358)
|
||||
#define TSDB_CODE_MND_PRIVILEDGE_EXIST TAOS_DEF_ERROR_CODE(0, 0x0359)
|
||||
#define TSDB_CODE_MND_USER_HOST_EXIST TAOS_DEF_ERROR_CODE(0, 0x035A)
|
||||
#define TSDB_CODE_MND_USER_HOST_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x035B)
|
||||
#define TSDB_CODE_MND_TOO_MANY_USER_HOST TAOS_DEF_ERROR_CODE(0, 0x035C)
|
||||
#define TSDB_CODE_MND_USER_LOCAL_HOST_NOT_DROP TAOS_DEF_ERROR_CODE(0, 0x035D)
|
||||
|
||||
// mnode-stable-part1
|
||||
#define TSDB_CODE_MND_STB_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0360)
|
||||
|
|
|
|||
|
|
@ -202,6 +202,7 @@ typedef enum ELogicConditionType {
|
|||
#define TSDB_DB_NAME_LEN 65
|
||||
#define TSDB_DB_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN + TSDB_NAME_DELIMITER_LEN)
|
||||
#define TSDB_PRIVILEDGE_CONDITION_LEN 48 * 1024
|
||||
#define TSDB_PRIVILEDGE_HOST_LEN 48 * 1024
|
||||
|
||||
#define TSDB_FUNC_NAME_LEN 65
|
||||
#define TSDB_FUNC_COMMENT_LEN 1024 * 1024
|
||||
|
|
|
|||
|
|
@ -33,17 +33,14 @@ adapterName="taosadapter"
|
|||
benchmarkName="taosBenchmark"
|
||||
dumpName="taosdump"
|
||||
demoName="taosdemo"
|
||||
xname="taosx"
|
||||
|
||||
clientName2="taos"
|
||||
serverName2="${clientName2}d"
|
||||
configFile2="${clientName2}.cfg"
|
||||
productName2="TDengine"
|
||||
emailName2="taosdata.com"
|
||||
xname2="${clientName2}x"
|
||||
adapterName2="${clientName2}adapter"
|
||||
|
||||
explorerName="${clientName2}-explorer"
|
||||
benchmarkName2="${clientName2}Benchmark"
|
||||
demoName2="${clientName2}demo"
|
||||
dumpName2="${clientName2}dump"
|
||||
|
|
@ -218,8 +215,6 @@ function install_bin() {
|
|||
${csudo}rm -f ${bin_link_dir}/${demoName2} || :
|
||||
${csudo}rm -f ${bin_link_dir}/${benchmarkName2} || :
|
||||
${csudo}rm -f ${bin_link_dir}/${dumpName2} || :
|
||||
${csudo}rm -f ${bin_link_dir}/${xname2} || :
|
||||
${csudo}rm -f ${bin_link_dir}/${explorerName} || :
|
||||
${csudo}rm -f ${bin_link_dir}/set_core || :
|
||||
${csudo}rm -f ${bin_link_dir}/TDinsight.sh || :
|
||||
|
||||
|
|
@ -233,8 +228,6 @@ function install_bin() {
|
|||
[ -x ${install_main_dir}/bin/${benchmarkName2} ] && ${csudo}ln -sf ${install_main_dir}/bin/${benchmarkName2} ${bin_link_dir}/${demoName2} || :
|
||||
[ -x ${install_main_dir}/bin/${benchmarkName2} ] && ${csudo}ln -sf ${install_main_dir}/bin/${benchmarkName2} ${bin_link_dir}/${benchmarkName2} || :
|
||||
[ -x ${install_main_dir}/bin/${dumpName2} ] && ${csudo}ln -sf ${install_main_dir}/bin/${dumpName2} ${bin_link_dir}/${dumpName2} || :
|
||||
[ -x ${install_main_dir}/bin/${xname2} ] && ${csudo}ln -sf ${install_main_dir}/bin/${xname2} ${bin_link_dir}/${xname2} || :
|
||||
[ -x ${install_main_dir}/bin/${explorerName} ] && ${csudo}ln -sf ${install_main_dir}/bin/${explorerName} ${bin_link_dir}/${explorerName} || :
|
||||
[ -x ${install_main_dir}/bin/TDinsight.sh ] && ${csudo}ln -sf ${install_main_dir}/bin/TDinsight.sh ${bin_link_dir}/TDinsight.sh || :
|
||||
if [ "$clientName2" == "${clientName}" ]; then
|
||||
[ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || :
|
||||
|
|
@ -703,26 +696,6 @@ function clean_service_on_systemd() {
|
|||
# if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then
|
||||
# ${csudo}rm -f ${service_config_dir}/${serverName2}.service
|
||||
# fi
|
||||
x_service_config="${service_config_dir}/${xName2}.service"
|
||||
if [ -e "$x_service_config" ]; then
|
||||
if systemctl is-active --quiet ${xName2}; then
|
||||
echo "${productName2} ${xName2} is running, stopping it..."
|
||||
${csudo}systemctl stop ${xName2} &>/dev/null || echo &>/dev/null
|
||||
fi
|
||||
${csudo}systemctl disable ${xName2} &>/dev/null || echo &>/dev/null
|
||||
${csudo}rm -f ${x_service_config}
|
||||
fi
|
||||
|
||||
explorer_service_config="${service_config_dir}/${explorerName2}.service"
|
||||
if [ -e "$explorer_service_config" ]; then
|
||||
if systemctl is-active --quiet ${explorerName2}; then
|
||||
echo "${productName2} ${explorerName2} is running, stopping it..."
|
||||
${csudo}systemctl stop ${explorerName2} &>/dev/null || echo &>/dev/null
|
||||
fi
|
||||
${csudo}systemctl disable ${explorerName2} &>/dev/null || echo &>/dev/null
|
||||
${csudo}rm -f ${explorer_service_config}
|
||||
${csudo}rm -f /etc/${clientName2}/explorer.toml
|
||||
fi
|
||||
}
|
||||
|
||||
function install_service_on_systemd() {
|
||||
|
|
|
|||
|
|
@ -135,6 +135,12 @@ typedef struct {
|
|||
__taos_notify_fn_t fp;
|
||||
} SPassInfo;
|
||||
|
||||
typedef struct {
|
||||
int64_t ver;
|
||||
void* param;
|
||||
__taos_notify_fn_t fp;
|
||||
} SWhiteListInfo;
|
||||
|
||||
typedef struct STscObj {
|
||||
char user[TSDB_USER_LEN];
|
||||
char pass[TSDB_PASSWORD_LEN];
|
||||
|
|
@ -152,6 +158,7 @@ typedef struct STscObj {
|
|||
SAppInstInfo* pAppInfo;
|
||||
SHashObj* pRequests;
|
||||
SPassInfo passInfo;
|
||||
SWhiteListInfo whiteListInfo;
|
||||
} STscObj;
|
||||
|
||||
typedef struct STscDbg {
|
||||
|
|
|
|||
|
|
@ -116,6 +116,19 @@ static int32_t hbUpdateUserAuthInfo(SAppHbMgr *pAppHbMgr, SUserAuthBatchRsp *bat
|
|||
atomic_load_32(&passInfo->ver), pTscObj->id);
|
||||
}
|
||||
}
|
||||
|
||||
if (pTscObj->whiteListInfo.fp) {
|
||||
SWhiteListInfo *whiteListInfo = &pTscObj->whiteListInfo;
|
||||
int64_t oldVer = atomic_load_64(&whiteListInfo->ver);
|
||||
if (oldVer < pRsp->whiteListVer) {
|
||||
atomic_store_64(&whiteListInfo->ver, pRsp->whiteListVer);
|
||||
if (whiteListInfo->fp) {
|
||||
(*whiteListInfo->fp)(whiteListInfo->param, &pRsp->whiteListVer, TAOS_NOTIFY_WHITELIST_VER);
|
||||
}
|
||||
tscDebug("update whitelist version of user %s from %"PRId64" to %"PRId64", tscRid:%" PRIi64, pRsp->user, oldVer,
|
||||
atomic_load_64(&whiteListInfo->ver), pTscObj->id);
|
||||
}
|
||||
}
|
||||
releaseTscObj(pReq->connKey.tscRid);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -140,6 +140,13 @@ int taos_set_notify_cb(TAOS *taos, __taos_notify_fn_t fp, void *param, int type)
|
|||
taosThreadMutexUnlock(&pObj->mutex);
|
||||
break;
|
||||
}
|
||||
case TAOS_NOTIFY_WHITELIST_VER: {
|
||||
taosThreadMutexLock(&pObj->mutex);
|
||||
pObj->whiteListInfo.fp = fp;
|
||||
pObj->whiteListInfo.param = param;
|
||||
taosThreadMutexUnlock(&pObj->mutex);
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
terrno = TSDB_CODE_INVALID_PARA;
|
||||
releaseTscObj(*(int64_t *)taos);
|
||||
|
|
@ -151,6 +158,113 @@ int taos_set_notify_cb(TAOS *taos, __taos_notify_fn_t fp, void *param, int type)
|
|||
return 0;
|
||||
}
|
||||
|
||||
typedef struct SFetchWhiteListInfo{
|
||||
int64_t connId;
|
||||
__taos_async_whitelist_fn_t userCbFn;
|
||||
void* userParam;
|
||||
} SFetchWhiteListInfo;
|
||||
|
||||
int32_t fetchWhiteListCallbackFn(void* param, SDataBuf* pMsg, int32_t code) {
|
||||
SFetchWhiteListInfo* pInfo = (SFetchWhiteListInfo*)param;
|
||||
TAOS* taos = &pInfo->connId;
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
pInfo->userCbFn(pInfo->userParam, code, taos, 0, NULL);
|
||||
taosMemoryFree(pMsg->pData);
|
||||
taosMemoryFree(pMsg->pEpSet);
|
||||
taosMemoryFree(pInfo);
|
||||
return code;
|
||||
}
|
||||
|
||||
SGetUserWhiteListRsp wlRsp;
|
||||
tDeserializeSGetUserWhiteListRsp(pMsg->pData, pMsg->len, &wlRsp);
|
||||
|
||||
uint64_t* pWhiteLists = taosMemoryMalloc(wlRsp.numWhiteLists * sizeof(uint64_t));
|
||||
if (pWhiteLists == NULL) {
|
||||
taosMemoryFree(pMsg->pData);
|
||||
taosMemoryFree(pMsg->pEpSet);
|
||||
taosMemoryFree(pInfo);
|
||||
tFreeSGetUserWhiteListRsp(&wlRsp);
|
||||
}
|
||||
|
||||
for (int i = 0; i < wlRsp.numWhiteLists; ++i) {
|
||||
pWhiteLists[i] = ((uint64_t)wlRsp.pWhiteLists[i].mask << 32) | wlRsp.pWhiteLists[i].ip;
|
||||
}
|
||||
|
||||
pInfo->userCbFn(pInfo->userParam, code, taos, wlRsp.numWhiteLists, pWhiteLists);
|
||||
|
||||
taosMemoryFree(pWhiteLists);
|
||||
taosMemoryFree(pMsg->pData);
|
||||
taosMemoryFree(pMsg->pEpSet);
|
||||
taosMemoryFree(pInfo);
|
||||
tFreeSGetUserWhiteListRsp(&wlRsp);
|
||||
return code;
|
||||
}
|
||||
|
||||
void taos_fetch_whitelist_a(TAOS *taos, __taos_async_whitelist_fn_t fp, void *param) {
|
||||
if (NULL == taos) {
|
||||
fp(param, TSDB_CODE_INVALID_PARA, taos, 0, NULL);
|
||||
return;
|
||||
}
|
||||
|
||||
int64_t connId = *(int64_t*)taos;
|
||||
|
||||
STscObj *pTsc = acquireTscObj(connId);
|
||||
if (NULL == pTsc) {
|
||||
fp(param, TSDB_CODE_TSC_DISCONNECTED, taos, 0, NULL);
|
||||
return;
|
||||
}
|
||||
|
||||
SGetUserWhiteListReq req;
|
||||
memcpy(req.user, pTsc->user, TSDB_USER_LEN);
|
||||
int32_t msgLen = tSerializeSGetUserWhiteListReq(NULL, 0, &req);
|
||||
void* pReq = taosMemoryMalloc(msgLen);
|
||||
if (pReq == NULL) {
|
||||
fp(param, TSDB_CODE_OUT_OF_MEMORY, taos, 0, NULL);
|
||||
releaseTscObj(connId);
|
||||
return;
|
||||
}
|
||||
|
||||
if (tSerializeSGetUserWhiteListReq(pReq, msgLen, &req) < 0) {
|
||||
fp(param, TSDB_CODE_INVALID_PARA, taos, 0, NULL);
|
||||
taosMemoryFree(pReq);
|
||||
releaseTscObj(connId);
|
||||
return;
|
||||
}
|
||||
|
||||
SFetchWhiteListInfo* pParam = taosMemoryMalloc(sizeof(SFetchWhiteListInfo));
|
||||
if (pParam == NULL) {
|
||||
fp(param, TSDB_CODE_OUT_OF_MEMORY, taos, 0, NULL);
|
||||
taosMemoryFree(pReq);
|
||||
releaseTscObj(connId);
|
||||
return;
|
||||
}
|
||||
|
||||
pParam->connId = connId;
|
||||
pParam->userCbFn = fp;
|
||||
pParam->userParam = param;
|
||||
SMsgSendInfo* pSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo));
|
||||
if (pSendInfo == NULL) {
|
||||
fp(param, TSDB_CODE_OUT_OF_MEMORY, taos, 0, NULL);
|
||||
taosMemoryFree(pParam);
|
||||
taosMemoryFree(pReq);
|
||||
releaseTscObj(connId);
|
||||
return;
|
||||
}
|
||||
|
||||
pSendInfo->msgInfo = (SDataBuf){.pData = pReq, .len = msgLen, .handle = NULL};
|
||||
pSendInfo->requestId = generateRequestId();
|
||||
pSendInfo->requestObjRefId = 0;
|
||||
pSendInfo->param = pParam;
|
||||
pSendInfo->fp = fetchWhiteListCallbackFn;
|
||||
pSendInfo->msgType = TDMT_MND_GET_USER_WHITELIST;
|
||||
|
||||
int64_t transportId = 0;
|
||||
SEpSet epSet = getEpSet_s(&pTsc->pAppInfo->mgmtEp);
|
||||
asyncSendMsgToServer(pTsc->pAppInfo->pTransporter, &epSet, &transportId, pSendInfo);
|
||||
releaseTscObj(connId);
|
||||
return;
|
||||
}
|
||||
|
||||
void taos_close_internal(void *taos) {
|
||||
if (taos == NULL) {
|
||||
return;
|
||||
|
|
|
|||
|
|
@ -139,6 +139,7 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) {
|
|||
pTscObj->connType = connectRsp.connType;
|
||||
pTscObj->passInfo.ver = connectRsp.passVer;
|
||||
pTscObj->authVer = connectRsp.authVer;
|
||||
pTscObj->whiteListInfo.ver = connectRsp.whiteListVer;
|
||||
|
||||
hbRegisterConn(pTscObj->pAppInfo->pAppHbMgr, pTscObj->id, connectRsp.clusterId, connectRsp.connType);
|
||||
|
||||
|
|
|
|||
|
|
@ -221,6 +221,7 @@ static const SSysDbTableSchema userUsersSchema[] = {
|
|||
{.name = "enable", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = false},
|
||||
{.name = "sysinfo", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = false},
|
||||
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false},
|
||||
{.name = "allowed_host", .bytes = TSDB_PRIVILEDGE_HOST_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
|
||||
};
|
||||
|
||||
GRANTS_SCHEMA;
|
||||
|
|
@ -326,7 +327,7 @@ static const SSysTableMeta infosMeta[] = {
|
|||
{TSDB_INS_TABLE_TAGS, userTagsSchema, tListLen(userTagsSchema), false},
|
||||
{TSDB_INS_TABLE_COLS, userColsSchema, tListLen(userColsSchema), false},
|
||||
// {TSDB_INS_TABLE_TABLE_DISTRIBUTED, userTblDistSchema, tListLen(userTblDistSchema)},
|
||||
{TSDB_INS_TABLE_USERS, userUsersSchema, tListLen(userUsersSchema), false},
|
||||
{TSDB_INS_TABLE_USERS, userUsersSchema, tListLen(userUsersSchema), true},
|
||||
{TSDB_INS_TABLE_LICENCES, grantsSchema, tListLen(grantsSchema), true},
|
||||
{TSDB_INS_TABLE_VGROUPS, vgroupsSchema, tListLen(vgroupsSchema), true},
|
||||
{TSDB_INS_TABLE_CONFIGS, configSchema, tListLen(configSchema), false},
|
||||
|
|
@ -336,7 +337,7 @@ static const SSysTableMeta infosMeta[] = {
|
|||
{TSDB_INS_TABLE_STREAMS, streamSchema, tListLen(streamSchema), false},
|
||||
{TSDB_INS_TABLE_STREAM_TASKS, streamTaskSchema, tListLen(streamTaskSchema), false},
|
||||
{TSDB_INS_TABLE_VNODES, vnodesSchema, tListLen(vnodesSchema), true},
|
||||
{TSDB_INS_TABLE_USER_PRIVILEGES, userUserPrivilegesSchema, tListLen(userUserPrivilegesSchema), false},
|
||||
{TSDB_INS_TABLE_USER_PRIVILEGES, userUserPrivilegesSchema, tListLen(userUserPrivilegesSchema), true},
|
||||
};
|
||||
|
||||
static const SSysDbTableSchema connectionsSchema[] = {
|
||||
|
|
|
|||
|
|
@ -933,7 +933,13 @@ int32_t dataBlockCompar(const void* p1, const void* p2, const void* param) {
|
|||
return 0;
|
||||
}
|
||||
}
|
||||
__compar_fn_t fn = getKeyComparFunc(pColInfoData->info.type, pOrder->order);
|
||||
|
||||
__compar_fn_t fn;
|
||||
if (pOrder->compFn) {
|
||||
fn = pOrder->compFn;
|
||||
} else {
|
||||
fn = getKeyComparFunc(pColInfoData->info.type, pOrder->order);
|
||||
}
|
||||
|
||||
int ret = fn(left1, right1);
|
||||
if (ret == 0) {
|
||||
|
|
@ -1099,6 +1105,7 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) {
|
|||
for (int32_t i = 0; i < taosArrayGetSize(helper.orderInfo); ++i) {
|
||||
struct SBlockOrderInfo* pInfo = taosArrayGet(helper.orderInfo, i);
|
||||
pInfo->pColData = taosArrayGet(pDataBlock->pDataBlock, pInfo->slotId);
|
||||
pInfo->compFn = getKeyComparFunc(pInfo->pColData->info.type, pInfo->order);
|
||||
}
|
||||
|
||||
terrno = 0;
|
||||
|
|
@ -2509,3 +2516,20 @@ void trimDataBlock(SSDataBlock* pBlock, int32_t totalRows, const bool* pBoolList
|
|||
int32_t blockGetEncodeSize(const SSDataBlock* pBlock) {
|
||||
return blockDataGetSerialMetaSize(taosArrayGetSize(pBlock->pDataBlock)) + blockDataGetSize(pBlock);
|
||||
}
|
||||
|
||||
int32_t blockDataGetSortedRows(SSDataBlock* pDataBlock, SArray* pOrderInfo) {
|
||||
if (!pDataBlock || !pOrderInfo) return 0;
|
||||
for (int32_t i = 0; i < taosArrayGetSize(pOrderInfo); ++i) {
|
||||
SBlockOrderInfo* pOrder = taosArrayGet(pOrderInfo, i);
|
||||
pOrder->pColData = taosArrayGet(pDataBlock->pDataBlock, pOrder->slotId);
|
||||
pOrder->compFn = getKeyComparFunc(pOrder->pColData->info.type, pOrder->order);
|
||||
}
|
||||
SSDataBlockSortHelper sortHelper = {.orderInfo = pOrderInfo, .pDataBlock = pDataBlock};
|
||||
int32_t rowIdx = 0, nextRowIdx = 1;
|
||||
for (; rowIdx < pDataBlock->info.rows && nextRowIdx < pDataBlock->info.rows; ++rowIdx, ++nextRowIdx) {
|
||||
if (dataBlockCompar(&nextRowIdx, &rowIdx, &sortHelper) < 0) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return nextRowIdx;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1116,6 +1116,8 @@ int32_t tSerializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) {
|
|||
if (tEncodeI64(&encoder, reserved) < 0) return -1;
|
||||
if (tEncodeI64(&encoder, reserved) < 0) return -1;
|
||||
}
|
||||
|
||||
if (tEncodeI64(&encoder, pReq->ipWhiteVer) < 0) return -1;
|
||||
tEndEncode(&encoder);
|
||||
|
||||
int32_t tlen = encoder.pos;
|
||||
|
|
@ -1226,6 +1228,10 @@ int32_t tDeserializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) {
|
|||
if (tDecodeI64(&decoder, &reserved) < 0) return -1;
|
||||
}
|
||||
}
|
||||
if (!tDecodeIsEnd(&decoder)) {
|
||||
if (tDecodeI64(&decoder, &pReq->ipWhiteVer) < 0) return -1;
|
||||
}
|
||||
|
||||
tEndDecode(&decoder);
|
||||
tDecoderClear(&decoder);
|
||||
return 0;
|
||||
|
|
@ -1258,6 +1264,8 @@ int32_t tSerializeSStatusRsp(void *buf, int32_t bufLen, SStatusRsp *pRsp) {
|
|||
}
|
||||
|
||||
if (tEncodeI32(&encoder, pRsp->statusSeq) < 0) return -1;
|
||||
|
||||
if (tEncodeI64(&encoder, pRsp->ipWhiteVer) < 0) return -1;
|
||||
tEndEncode(&encoder);
|
||||
|
||||
int32_t tlen = encoder.pos;
|
||||
|
|
@ -1300,6 +1308,10 @@ int32_t tDeserializeSStatusRsp(void *buf, int32_t bufLen, SStatusRsp *pRsp) {
|
|||
}
|
||||
|
||||
if (tDecodeI32(&decoder, &pRsp->statusSeq) < 0) return -1;
|
||||
|
||||
if (!tDecodeIsEnd(&decoder)) {
|
||||
if (tDecodeI64(&decoder, &pRsp->ipWhiteVer) < 0) return -1;
|
||||
}
|
||||
tEndDecode(&decoder);
|
||||
tDecoderClear(&decoder);
|
||||
return 0;
|
||||
|
|
@ -1371,6 +1383,15 @@ int32_t tDeserializeSDropUserReq(void *buf, int32_t bufLen, SDropUserReq *pReq)
|
|||
return 0;
|
||||
}
|
||||
|
||||
SIpWhiteList *cloneIpWhiteList(SIpWhiteList *pIpWhiteList) {
|
||||
if (pIpWhiteList == NULL) return NULL;
|
||||
|
||||
int32_t sz = sizeof(SIpWhiteList) + pIpWhiteList->num * sizeof(SIpV4Range);
|
||||
SIpWhiteList *pNew = taosMemoryCalloc(1, sz);
|
||||
memcpy(pNew, pIpWhiteList, sz);
|
||||
return pNew;
|
||||
}
|
||||
|
||||
int32_t tSerializeSCreateUserReq(void *buf, int32_t bufLen, SCreateUserReq *pReq) {
|
||||
SEncoder encoder = {0};
|
||||
tEncoderInit(&encoder, buf, bufLen);
|
||||
|
|
@ -1414,12 +1435,123 @@ int32_t tDeserializeSCreateUserReq(void *buf, int32_t bufLen, SCreateUserReq *pR
|
|||
}
|
||||
tEndDecode(&decoder);
|
||||
|
||||
|
||||
tDecoderClear(&decoder);
|
||||
return 0;
|
||||
}
|
||||
|
||||
void tFreeSCreateUserReq(SCreateUserReq *pReq) { taosMemoryFree(pReq->pIpRanges); }
|
||||
|
||||
int32_t tSerializeSUpdateIpWhite(void *buf, int32_t bufLen, SUpdateIpWhite *pReq) {
|
||||
// impl later
|
||||
SEncoder encoder = {0};
|
||||
tEncoderInit(&encoder, buf, bufLen);
|
||||
if (tStartEncode(&encoder) < 0) return -1;
|
||||
if (tEncodeI64(&encoder, pReq->ver) < 0) return -1;
|
||||
if (tEncodeI32(&encoder, pReq->numOfUser) < 0) return -1;
|
||||
for (int i = 0; i < pReq->numOfUser; i++) {
|
||||
SUpdateUserIpWhite *pUser = &(pReq->pUserIpWhite[i]);
|
||||
|
||||
if (tEncodeI64(&encoder, pUser->ver) < 0) return -1;
|
||||
if (tEncodeCStr(&encoder, pUser->user) < 0) return -1;
|
||||
if (tEncodeI32(&encoder, pUser->numOfRange) < 0) return -1;
|
||||
for (int j = 0; j < pUser->numOfRange; j++) {
|
||||
SIpV4Range *pRange = &pUser->pIpRanges[j];
|
||||
if (tEncodeU32(&encoder, pRange->ip) < 0) return -1;
|
||||
if (tEncodeU32(&encoder, pRange->mask) < 0) return -1;
|
||||
}
|
||||
}
|
||||
|
||||
tEndEncode(&encoder);
|
||||
|
||||
int32_t tlen = encoder.pos;
|
||||
tEncoderClear(&encoder);
|
||||
return tlen;
|
||||
}
|
||||
int32_t tDeserializeSUpdateIpWhite(void *buf, int32_t bufLen, SUpdateIpWhite *pReq) {
|
||||
SDecoder decoder = {0};
|
||||
tDecoderInit(&decoder, buf, bufLen);
|
||||
|
||||
if (tStartDecode(&decoder) < 0) return -1;
|
||||
// impl later
|
||||
if (tDecodeI64(&decoder, &pReq->ver) < 0) return -1;
|
||||
if (tDecodeI32(&decoder, &pReq->numOfUser) < 0) return -1;
|
||||
|
||||
pReq->pUserIpWhite = taosMemoryCalloc(1, sizeof(SUpdateUserIpWhite) * pReq->numOfUser);
|
||||
for (int i = 0; i < pReq->numOfUser; i++) {
|
||||
SUpdateUserIpWhite *pUserWhite = &pReq->pUserIpWhite[i];
|
||||
if (tDecodeI64(&decoder, &pUserWhite->ver) < 0) return -1;
|
||||
if (tDecodeCStrTo(&decoder, pUserWhite->user) < 0) return -1;
|
||||
if (tDecodeI32(&decoder, &pUserWhite->numOfRange) < 0) return -1;
|
||||
|
||||
pUserWhite->pIpRanges = taosMemoryCalloc(1, pUserWhite->numOfRange * sizeof(SIpV4Range));
|
||||
for (int j = 0; j < pUserWhite->numOfRange; j++) {
|
||||
SIpV4Range *pRange = &pUserWhite->pIpRanges[j];
|
||||
if (tDecodeU32(&decoder, &pRange->ip) < 0) return -1;
|
||||
if (tDecodeU32(&decoder, &pRange->mask) < 0) return -1;
|
||||
}
|
||||
}
|
||||
|
||||
tEndDecode(&decoder);
|
||||
tDecoderClear(&decoder);
|
||||
return 0;
|
||||
}
|
||||
void tFreeSUpdateIpWhiteReq(SUpdateIpWhite *pReq) {
|
||||
for (int i = 0; i < pReq->numOfUser; i++) {
|
||||
SUpdateUserIpWhite *pUserWhite = &pReq->pUserIpWhite[i];
|
||||
taosMemoryFree(pUserWhite->pIpRanges);
|
||||
}
|
||||
taosMemoryFree(pReq->pUserIpWhite);
|
||||
// impl later
|
||||
return;
|
||||
}
|
||||
SUpdateIpWhite *cloneSUpdateIpWhiteReq(SUpdateIpWhite *pReq) {
|
||||
SUpdateIpWhite *pClone = taosMemoryCalloc(1, sizeof(SUpdateIpWhite));
|
||||
|
||||
pClone->numOfUser = pReq->numOfUser;
|
||||
pClone->ver = pReq->ver;
|
||||
pClone->pUserIpWhite = taosMemoryCalloc(1, sizeof(SUpdateUserIpWhite) * pReq->numOfUser);
|
||||
|
||||
for (int i = 0; i < pReq->numOfUser; i++) {
|
||||
SUpdateUserIpWhite *pNew = &pClone->pUserIpWhite[i];
|
||||
SUpdateUserIpWhite *pOld = &pReq->pUserIpWhite[i];
|
||||
|
||||
pNew->ver = pOld->ver;
|
||||
memcpy(pNew->user, pOld->user, strlen(pOld->user));
|
||||
pNew->numOfRange = pOld->numOfRange;
|
||||
|
||||
int32_t sz = pOld->numOfRange * sizeof(SIpV4Range);
|
||||
pNew->pIpRanges = taosMemoryCalloc(1, sz);
|
||||
memcpy(pNew->pIpRanges, pOld->pIpRanges, sz);
|
||||
}
|
||||
return pClone;
|
||||
}
|
||||
int32_t tSerializeRetrieveIpWhite(void *buf, int32_t bufLen, SRetrieveIpWhiteReq *pReq) {
|
||||
SEncoder encoder = {0};
|
||||
tEncoderInit(&encoder, buf, bufLen);
|
||||
if (tStartEncode(&encoder) < 0) return -1;
|
||||
|
||||
if (tEncodeI64(&encoder, pReq->ipWhiteVer) < 0) {
|
||||
return -1;
|
||||
}
|
||||
tEndEncode(&encoder);
|
||||
|
||||
int32_t tlen = encoder.pos;
|
||||
tEncoderClear(&encoder);
|
||||
return tlen;
|
||||
}
|
||||
int32_t tDeserializeRetrieveIpWhite(void *buf, int32_t bufLen, SRetrieveIpWhiteReq *pReq) {
|
||||
SDecoder decoder = {0};
|
||||
tDecoderInit(&decoder, buf, bufLen);
|
||||
|
||||
if (tStartDecode(&decoder) < 0) return -1;
|
||||
// impl later
|
||||
if (tDecodeI64(&decoder, &pReq->ipWhiteVer) < 0) return -1;
|
||||
tEndDecode(&decoder);
|
||||
tDecoderClear(&decoder);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t tSerializeSAlterUserReq(void *buf, int32_t bufLen, SAlterUserReq *pReq) {
|
||||
SEncoder encoder = {0};
|
||||
tEncoderInit(&encoder, buf, bufLen);
|
||||
|
|
@ -1599,7 +1731,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp)
|
|||
|
||||
// since 3.0.7.0
|
||||
if (tEncodeI32(pEncoder, pRsp->passVer) < 0) return -1;
|
||||
|
||||
if (tEncodeI64(pEncoder, pRsp->whiteListVer) < 0) return -1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -1731,6 +1863,11 @@ int32_t tDeserializeSGetUserAuthRspImpl(SDecoder *pDecoder, SGetUserAuthRsp *pRs
|
|||
} else {
|
||||
pRsp->passVer = 0;
|
||||
}
|
||||
if (!tDecodeIsEnd(pDecoder)) {
|
||||
if (tDecodeI64(pDecoder, &pRsp->whiteListVer) < 0) goto _err;
|
||||
} else {
|
||||
pRsp->whiteListVer = 0;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
_err:
|
||||
|
|
@ -1769,6 +1906,73 @@ void tFreeSGetUserAuthRsp(SGetUserAuthRsp *pRsp) {
|
|||
taosHashCleanup(pRsp->useDbs);
|
||||
}
|
||||
|
||||
int32_t tSerializeSGetUserWhiteListReq(void *buf, int32_t bufLen, SGetUserWhiteListReq *pReq) {
|
||||
SEncoder encoder = {0};
|
||||
tEncoderInit(&encoder, buf, bufLen);
|
||||
|
||||
if (tStartEncode(&encoder) < 0) return -1;
|
||||
if (tEncodeCStr(&encoder, pReq->user) < 0) return -1;
|
||||
tEndEncode(&encoder);
|
||||
|
||||
int32_t tlen = encoder.pos;
|
||||
tEncoderClear(&encoder);
|
||||
return tlen;
|
||||
}
|
||||
|
||||
int32_t tDeserializeSGetUserWhiteListReq(void *buf, int32_t bufLen, SGetUserWhiteListReq *pReq) {
|
||||
SDecoder decoder = {0};
|
||||
tDecoderInit(&decoder, buf, bufLen);
|
||||
|
||||
if (tStartDecode(&decoder) < 0) return -1;
|
||||
if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1;
|
||||
tEndDecode(&decoder);
|
||||
|
||||
tDecoderClear(&decoder);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t tSerializeSGetUserWhiteListRsp(void* buf, int32_t bufLen, SGetUserWhiteListRsp* pRsp) {
|
||||
SEncoder encoder = {0};
|
||||
tEncoderInit(&encoder, buf, bufLen);
|
||||
|
||||
if (tStartEncode(&encoder) < 0) return -1;
|
||||
if (tEncodeCStr(&encoder, pRsp->user) < 0) return -1;
|
||||
if (tEncodeI32(&encoder, pRsp->numWhiteLists) < 0) return -1;
|
||||
for (int i = 0; i < pRsp->numWhiteLists; ++i) {
|
||||
if (tEncodeU32(&encoder, pRsp->pWhiteLists[i].ip) < 0) return -1;
|
||||
if (tEncodeU32(&encoder, pRsp->pWhiteLists[i].mask) < 0) return -1;
|
||||
}
|
||||
tEndEncode(&encoder);
|
||||
|
||||
int32_t tlen = encoder.pos;
|
||||
tEncoderClear(&encoder);
|
||||
return tlen;
|
||||
}
|
||||
|
||||
int32_t tDeserializeSGetUserWhiteListRsp(void* buf, int32_t bufLen, SGetUserWhiteListRsp* pRsp) {
|
||||
SDecoder decoder = {0};
|
||||
tDecoderInit(&decoder, buf, bufLen);
|
||||
|
||||
if (tStartDecode(&decoder) < 0) return -1;
|
||||
if (tDecodeCStrTo(&decoder, pRsp->user) < 0) return -1;
|
||||
|
||||
if (tDecodeI32(&decoder, &pRsp->numWhiteLists) < 0) return -1;
|
||||
pRsp->pWhiteLists = taosMemoryMalloc(pRsp->numWhiteLists * sizeof(SIpV4Range));
|
||||
if (pRsp->pWhiteLists == NULL) return -1;
|
||||
for (int32_t i = 0; i < pRsp->numWhiteLists; ++i) {
|
||||
if (tDecodeU32(&decoder, &(pRsp->pWhiteLists[i].ip)) < 0) return -1;
|
||||
if (tDecodeU32(&decoder, &(pRsp->pWhiteLists[i].mask)) < 0) return -1;
|
||||
}
|
||||
|
||||
tEndDecode(&decoder);
|
||||
tDecoderClear(&decoder);
|
||||
return 0;
|
||||
}
|
||||
|
||||
void tFreeSGetUserWhiteListRsp(SGetUserWhiteListRsp* pRsp) {
|
||||
taosMemoryFree(pRsp->pWhiteLists);
|
||||
}
|
||||
|
||||
int32_t tSerializeSCreateDropMQSNodeReq(void *buf, int32_t bufLen, SMCreateQnodeReq *pReq) {
|
||||
SEncoder encoder = {0};
|
||||
tEncoderInit(&encoder, buf, bufLen);
|
||||
|
|
@ -4235,6 +4439,7 @@ int32_t tSerializeSConnectRsp(void *buf, int32_t bufLen, SConnectRsp *pRsp) {
|
|||
if (tEncodeCStr(&encoder, pRsp->sDetailVer) < 0) return -1;
|
||||
if (tEncodeI32(&encoder, pRsp->passVer) < 0) return -1;
|
||||
if (tEncodeI32(&encoder, pRsp->authVer) < 0) return -1;
|
||||
if (tEncodeI64(&encoder, pRsp->whiteListVer) < 0) return -1;
|
||||
tEndEncode(&encoder);
|
||||
|
||||
int32_t tlen = encoder.pos;
|
||||
|
|
@ -4271,6 +4476,11 @@ int32_t tDeserializeSConnectRsp(void *buf, int32_t bufLen, SConnectRsp *pRsp) {
|
|||
pRsp->authVer = 0;
|
||||
}
|
||||
|
||||
if (!tDecodeIsEnd(&decoder)) {
|
||||
if (tDecodeI64(&decoder, &pRsp->whiteListVer) < 0) return -1;
|
||||
} else {
|
||||
pRsp->whiteListVer = 0;
|
||||
}
|
||||
tEndDecode(&decoder);
|
||||
|
||||
tDecoderClear(&decoder);
|
||||
|
|
|
|||
|
|
@ -30,7 +30,36 @@ static void dmUpdateDnodeCfg(SDnodeMgmt *pMgmt, SDnodeCfg *pCfg) {
|
|||
taosThreadRwlockUnlock(&pMgmt->pData->lock);
|
||||
}
|
||||
}
|
||||
static void dmMayShouldUpdateIpWhiteList(SDnodeMgmt *pMgmt, int64_t ver) {
|
||||
dDebug("ip-white-list on dnode ver: %" PRId64 ", status ver: %" PRId64 "", pMgmt->pData->ipWhiteVer, ver);
|
||||
if (pMgmt->pData->ipWhiteVer == ver) {
|
||||
if (ver == 0) {
|
||||
dDebug("disable ip-white-list on dnode ver: %" PRId64 ", status ver: %" PRId64 "", pMgmt->pData->ipWhiteVer, ver);
|
||||
rpcSetIpWhite(pMgmt->msgCb.serverRpc, NULL);
|
||||
// pMgmt->ipWhiteVer = ver;
|
||||
}
|
||||
return;
|
||||
}
|
||||
int64_t oldVer = pMgmt->pData->ipWhiteVer;
|
||||
// pMgmt->ipWhiteVer = ver;
|
||||
|
||||
SRetrieveIpWhiteReq req = {.ipWhiteVer = oldVer};
|
||||
int32_t contLen = tSerializeRetrieveIpWhite(NULL, 0, &req);
|
||||
void *pHead = rpcMallocCont(contLen);
|
||||
tSerializeRetrieveIpWhite(pHead, contLen, &req);
|
||||
|
||||
SRpcMsg rpcMsg = {.pCont = pHead,
|
||||
.contLen = contLen,
|
||||
.msgType = TDMT_MND_RETRIEVE_IP_WHITE,
|
||||
.info.ahandle = (void *)0x9527,
|
||||
.info.refId = 0,
|
||||
.info.noResp = 0};
|
||||
SEpSet epset = {0};
|
||||
|
||||
dmGetMnodeEpSet(pMgmt->pData, &epset);
|
||||
|
||||
rpcSendRequest(pMgmt->msgCb.clientRpc, &epset, &rpcMsg, NULL);
|
||||
}
|
||||
static void dmProcessStatusRsp(SDnodeMgmt *pMgmt, SRpcMsg *pRsp) {
|
||||
const STraceId *trace = &pRsp->info.traceId;
|
||||
dGTrace("status rsp received from mnode, statusSeq:%d code:0x%x", pMgmt->statusSeq, pRsp->code);
|
||||
|
|
@ -55,6 +84,7 @@ static void dmProcessStatusRsp(SDnodeMgmt *pMgmt, SRpcMsg *pRsp) {
|
|||
dmUpdateDnodeCfg(pMgmt, &statusRsp.dnodeCfg);
|
||||
dmUpdateEps(pMgmt->pData, statusRsp.pDnodeEps);
|
||||
}
|
||||
dmMayShouldUpdateIpWhiteList(pMgmt, statusRsp.ipWhiteVer);
|
||||
}
|
||||
tFreeSStatusRsp(&statusRsp);
|
||||
}
|
||||
|
|
@ -111,6 +141,7 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) {
|
|||
|
||||
pMgmt->statusSeq++;
|
||||
req.statusSeq = pMgmt->statusSeq;
|
||||
req.ipWhiteVer = pMgmt->pData->ipWhiteVer;
|
||||
|
||||
int32_t contLen = tSerializeSStatusReq(NULL, 0, &req);
|
||||
void *pHead = rpcMallocCont(contLen);
|
||||
|
|
|
|||
|
|
@ -55,6 +55,7 @@ static int32_t dmOpenMgmt(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) {
|
|||
pMgmt->getMnodeLoadsFp = pInput->getMnodeLoadsFp;
|
||||
pMgmt->getQnodeLoadsFp = pInput->getQnodeLoadsFp;
|
||||
|
||||
// pMgmt->pData->ipWhiteVer = 0;
|
||||
if (dmStartWorker(pMgmt) != 0) {
|
||||
return -1;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -161,6 +161,9 @@ SArray *mmGetMsgHandles() {
|
|||
if (dmSetMgmtHandle(pArray, TDMT_MND_PAUSE_STREAM, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_MND_RESUME_STREAM, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
|
||||
|
||||
|
||||
if (dmSetMgmtHandle(pArray, TDMT_MND_RETRIEVE_IP_WHITE, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_MND_GET_USER_WHITELIST, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_MND_GET_INDEX, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_MND_GET_TABLE_INDEX, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
|
||||
if (dmSetMgmtHandle(pArray, TDMT_MND_TMQ_CREATE_TOPIC, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
|
||||
|
|
|
|||
|
|
@ -63,6 +63,29 @@ static void dmConvertErrCode(tmsg_t msgType) {
|
|||
terrno = TSDB_CODE_VND_STOPPED;
|
||||
}
|
||||
}
|
||||
static void dmUpdateRpcIpWhite(SDnodeData *pData, void *pTrans, SRpcMsg *pRpc) {
|
||||
SUpdateIpWhite ipWhite = {0}; // aosMemoryCalloc(1, sizeof(SUpdateIpWhite));
|
||||
tDeserializeSUpdateIpWhite(pRpc->pCont, pRpc->contLen, &ipWhite);
|
||||
|
||||
rpcSetIpWhite(pTrans, &ipWhite);
|
||||
pData->ipWhiteVer = ipWhite.ver;
|
||||
|
||||
tFreeSUpdateIpWhiteReq(&ipWhite);
|
||||
|
||||
rpcFreeCont(pRpc->pCont);
|
||||
}
|
||||
static bool dmIsForbiddenIp(int8_t forbidden, char *user, uint32_t clientIp) {
|
||||
if (forbidden) {
|
||||
SIpV4Range range = {.ip = clientIp, .mask = 32};
|
||||
char buf[36] = {0};
|
||||
|
||||
rpcUtilSIpRangeToStr(&range, buf);
|
||||
dError("User:%s host:%s not in ip white list", user, buf);
|
||||
return true;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) {
|
||||
SDnodeTrans *pTrans = &pDnode->trans;
|
||||
int32_t code = -1;
|
||||
|
|
@ -81,6 +104,12 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) {
|
|||
goto _OVER;
|
||||
}
|
||||
|
||||
bool isForbidden = dmIsForbiddenIp(pRpc->info.forbiddenIp, pRpc->info.conn.user, pRpc->info.conn.clientIp);
|
||||
if (isForbidden) {
|
||||
terrno = TSDB_CODE_IP_NOT_IN_WHITE_LIST;
|
||||
goto _OVER;
|
||||
}
|
||||
|
||||
switch (pRpc->msgType) {
|
||||
case TDMT_DND_NET_TEST:
|
||||
dmProcessNetTestReq(pDnode, pRpc);
|
||||
|
|
@ -97,6 +126,10 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) {
|
|||
dmSetMnodeEpSet(&pDnode->data, pEpSet);
|
||||
}
|
||||
break;
|
||||
case TDMT_MND_RETRIEVE_IP_WHITE_RSP: {
|
||||
dmUpdateRpcIpWhite(&pDnode->data, pTrans->serverRpc, pRpc);
|
||||
return;
|
||||
} break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
|
@ -372,6 +405,7 @@ void dmCleanupServer(SDnode *pDnode) {
|
|||
SMsgCb dmGetMsgcb(SDnode *pDnode) {
|
||||
SMsgCb msgCb = {
|
||||
.clientRpc = pDnode->trans.clientRpc,
|
||||
.serverRpc = pDnode->trans.serverRpc,
|
||||
.sendReqFp = dmSendReq,
|
||||
.sendRspFp = dmSendRsp,
|
||||
.registerBrokenLinkArgFp = dmRegisterBrokenLinkArg,
|
||||
|
|
|
|||
|
|
@ -107,6 +107,7 @@ typedef struct {
|
|||
TdThreadRwlock lock;
|
||||
SMsgCb msgCb;
|
||||
bool validMnodeEps;
|
||||
int64_t ipWhiteVer;
|
||||
} SDnodeData;
|
||||
|
||||
typedef struct {
|
||||
|
|
|
|||
|
|
@ -289,6 +289,9 @@ typedef struct {
|
|||
int32_t acctId;
|
||||
int32_t authVersion;
|
||||
int32_t passVersion;
|
||||
int64_t ipWhiteListVer;
|
||||
SIpWhiteList* pIpWhiteList;
|
||||
|
||||
SHashObj* readDbs;
|
||||
SHashObj* writeDbs;
|
||||
SHashObj* topics;
|
||||
|
|
|
|||
|
|
@ -130,6 +130,7 @@ typedef struct SMnode {
|
|||
SGrantInfo grant;
|
||||
MndMsgFp msgFp[TDMT_MAX];
|
||||
SMsgCb msgCb;
|
||||
int64_t ipWhiteVer;
|
||||
} SMnode;
|
||||
|
||||
void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp);
|
||||
|
|
@ -140,6 +141,8 @@ bool mndGetRestored(SMnode *pMnode);
|
|||
void mndSetStop(SMnode *pMnode);
|
||||
bool mndGetStop(SMnode *pMnode);
|
||||
|
||||
SArray *mndGetAllDnodeFqdns(SMnode *pMnode);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -33,6 +33,9 @@ int32_t mndCheckTopicPrivilegeByName(SMnode *pMnode, const char *user, EOperType
|
|||
int32_t mndCheckShowPrivilege(SMnode *pMnode, const char *user, EShowType showType, const char *dbname);
|
||||
int32_t mndCheckAlterUserPrivilege(SUserObj *pOperUser, SUserObj *pUser, SAlterUserReq *pAlter);
|
||||
int32_t mndSetUserAuthRsp(SMnode *pMnode, SUserObj *pUser, SGetUserAuthRsp *pRsp);
|
||||
int32_t mndSetUserWhiteListRsp(SMnode* pMnode, SUserObj* pUser, SGetUserWhiteListRsp* pWhiteListRsp);
|
||||
int32_t mndEnableIpWhiteList(SMnode *pMnode);
|
||||
int32_t mndFetchIpWhiteList(SIpWhiteList *ipList, char **buf);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
|
|
|
|||
|
|
@ -23,6 +23,10 @@
|
|||
extern "C" {
|
||||
#endif
|
||||
|
||||
enum {
|
||||
IP_WHITE_ADD,
|
||||
IP_WHITE_DROP,
|
||||
};
|
||||
int32_t mndInitUser(SMnode *pMnode);
|
||||
void mndCleanupUser(SMnode *pMnode);
|
||||
SUserObj *mndAcquireUser(SMnode *pMnode, const char *userName);
|
||||
|
|
@ -40,6 +44,13 @@ int32_t mndUserRemoveTopic(SMnode *pMnode, STrans *pTrans, char *topic);
|
|||
|
||||
int32_t mndUserDupObj(SUserObj *pUser, SUserObj *pNew);
|
||||
void mndUserFreeObj(SUserObj *pUser);
|
||||
|
||||
int64_t mndGetIpWhiteVer(SMnode *pMnode);
|
||||
|
||||
void mndUpdateIpWhite(SMnode *pMnode, char *user, char *fqdn, int8_t type, int8_t lock);
|
||||
|
||||
int32_t mndRefreshUserIpWhiteList(SMnode *pMnode);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -51,6 +51,12 @@ enum {
|
|||
DND_CONN_ACTIVE_CODE,
|
||||
};
|
||||
|
||||
enum {
|
||||
DND_CREATE,
|
||||
DND_ADD,
|
||||
DND_DROP,
|
||||
};
|
||||
|
||||
static int32_t mndCreateDefaultDnode(SMnode *pMnode);
|
||||
static SSdbRaw *mndDnodeActionEncode(SDnodeObj *pDnode);
|
||||
static SSdbRow *mndDnodeActionDecode(SSdbRaw *pRaw);
|
||||
|
|
@ -103,6 +109,9 @@ int32_t mndInitDnode(SMnode *pMnode) {
|
|||
return sdbSetTable(pMnode->pSdb, table);
|
||||
}
|
||||
|
||||
SIpWhiteList *mndCreateIpWhiteOfDnode(SMnode *pMnode);
|
||||
SIpWhiteList *mndAddIpWhiteOfDnode(SIpWhiteList *pIpWhiteList, char *fqdn);
|
||||
SIpWhiteList *mndRmIpWhiteOfDnode(SIpWhiteList *pIpWhiteList, char *fqdn);
|
||||
void mndCleanupDnode(SMnode *pMnode) {}
|
||||
|
||||
static int32_t mndCreateDefaultDnode(SMnode *pMnode) {
|
||||
|
|
@ -130,6 +139,7 @@ static int32_t mndCreateDefaultDnode(SMnode *pMnode) {
|
|||
|
||||
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
|
||||
code = 0;
|
||||
mndUpdateIpWhite(pMnode, TSDB_DEFAULT_USER, dnodeObj.fqdn, IP_WHITE_ADD, 1);
|
||||
|
||||
_OVER:
|
||||
mndTransDrop(pTrans);
|
||||
|
|
@ -518,13 +528,16 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
|
|||
}
|
||||
}
|
||||
|
||||
pMnode->ipWhiteVer = mndGetIpWhiteVer(pMnode);
|
||||
|
||||
int64_t dnodeVer = sdbGetTableVer(pMnode->pSdb, SDB_DNODE) + sdbGetTableVer(pMnode->pSdb, SDB_MNODE);
|
||||
int64_t curMs = taosGetTimestampMs();
|
||||
bool online = mndIsDnodeOnline(pDnode, curMs);
|
||||
bool dnodeChanged = (statusReq.dnodeVer == 0) || (statusReq.dnodeVer != dnodeVer);
|
||||
bool reboot = (pDnode->rebootTime != statusReq.rebootTime);
|
||||
bool supportVnodesChanged = pDnode->numOfSupportVnodes != statusReq.numOfSupportVnodes;
|
||||
bool needCheck = !online || dnodeChanged || reboot || supportVnodesChanged;
|
||||
bool needCheck =
|
||||
!online || dnodeChanged || reboot || supportVnodesChanged || pMnode->ipWhiteVer != statusReq.ipWhiteVer;
|
||||
|
||||
const STraceId *trace = &pReq->info.traceId;
|
||||
mGTrace("dnode:%d, status received, accessTimes:%d check:%d online:%d reboot:%d changed:%d statusSeq:%d", pDnode->id,
|
||||
|
|
@ -645,6 +658,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) {
|
|||
}
|
||||
|
||||
mndGetDnodeEps(pMnode, statusRsp.pDnodeEps);
|
||||
statusRsp.ipWhiteVer = pMnode->ipWhiteVer;
|
||||
|
||||
int32_t contLen = tSerializeSStatusRsp(NULL, 0, &statusRsp);
|
||||
void *pHead = rpcMallocCont(contLen);
|
||||
|
|
@ -691,6 +705,7 @@ static int32_t mndCreateDnode(SMnode *pMnode, SRpcMsg *pReq, SCreateDnodeReq *pC
|
|||
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
|
||||
code = 0;
|
||||
|
||||
mndUpdateIpWhite(pMnode, TSDB_DEFAULT_USER, dnodeObj.fqdn, IP_WHITE_ADD, 1);
|
||||
_OVER:
|
||||
mndTransDrop(pTrans);
|
||||
sdbFreeRaw(pRaw);
|
||||
|
|
@ -983,6 +998,7 @@ static int32_t mndDropDnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SM
|
|||
|
||||
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
|
||||
|
||||
mndUpdateIpWhite(pMnode, TSDB_DEFAULT_USER, pDnode->fqdn, IP_WHITE_DROP, 1);
|
||||
code = 0;
|
||||
|
||||
_OVER:
|
||||
|
|
@ -1455,3 +1471,19 @@ _err:
|
|||
terrno = TSDB_CODE_INVALID_CFG;
|
||||
return -1;
|
||||
}
|
||||
|
||||
SArray *mndGetAllDnodeFqdns(SMnode *pMnode) {
|
||||
SDnodeObj *pObj = NULL;
|
||||
void *pIter = NULL;
|
||||
SSdb *pSdb = pMnode->pSdb;
|
||||
SArray *fqdns = taosArrayInit(4, sizeof(void *));
|
||||
while (1) {
|
||||
pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pObj);
|
||||
if (pIter == NULL) break;
|
||||
|
||||
char *fqdn = taosStrdup(pObj->fqdn);
|
||||
taosArrayPush(fqdns, &fqdn);
|
||||
sdbRelease(pSdb, pObj);
|
||||
}
|
||||
return fqdns;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -76,12 +76,19 @@ int32_t mndBuildInsTableSchema(SMnode *pMnode, const char *dbFName, const char *
|
|||
}
|
||||
|
||||
STableMetaRsp *pMeta = taosHashGet(pMnode->infosMeta, tbName, strlen(tbName));
|
||||
if (NULL == pMeta || (!sysinfo && pMeta->sysInfo)) {
|
||||
if (NULL == pMeta) {
|
||||
mError("invalid information schema table name:%s", tbName);
|
||||
terrno = TSDB_CODE_MND_INVALID_SYS_TABLENAME;
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (!sysinfo && pMeta->sysInfo) {
|
||||
mError("no permission to get schema of table name:%s", tbName);
|
||||
terrno = TSDB_CODE_PAR_PERMISSION_DENIED;
|
||||
return -1;
|
||||
}
|
||||
|
||||
|
||||
*pRsp = *pMeta;
|
||||
|
||||
pRsp->pSchemas = taosMemoryCalloc(pMeta->numOfColumns, sizeof(SSchema));
|
||||
|
|
|
|||
|
|
@ -28,10 +28,33 @@ int32_t mndCheckDbPrivilege(SMnode *pMnode, const char *user, EOperType operType
|
|||
int32_t mndCheckDbPrivilegeByName(SMnode *pMnode, const char *user, EOperType operType, const char *dbname) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t mndCheckTopicPrivilege(SMnode *pMnode, const char *user, EOperType operType, SMqTopicObj *pTopic) { return 0; }
|
||||
int32_t mndCheckTopicPrivilegeByName(SMnode *pMnode, const char *user, EOperType operType, const char *topicName) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
// TODO: for community version use the commented version
|
||||
int32_t mndSetUserWhiteListRsp(SMnode *pMnode, SUserObj *pUser, SGetUserWhiteListRsp *pWhiteListRsp) {
|
||||
memcpy(pWhiteListRsp->user, pUser->user, TSDB_USER_LEN);
|
||||
pWhiteListRsp->numWhiteLists = 1;
|
||||
pWhiteListRsp->pWhiteLists = taosMemoryMalloc(pWhiteListRsp->numWhiteLists * sizeof(SIpV4Range));
|
||||
if (pWhiteListRsp->pWhiteLists == NULL) {
|
||||
return TSDB_CODE_OUT_OF_MEMORY;
|
||||
}
|
||||
memset(pWhiteListRsp->pWhiteLists, 0, pWhiteListRsp->numWhiteLists * sizeof(SIpV4Range));
|
||||
// pWhiteListRsp->numWhiteLists = pUser->pIpWhiteList->num;
|
||||
// pWhiteListRsp->pWhiteLists = taosMemoryMalloc(pWhiteListRsp->numWhiteLists * sizeof(SIpV4Range));
|
||||
// if (pWhiteListRsp->pWhiteLists == NULL) {
|
||||
// return TSDB_CODE_OUT_OF_MEMORY;
|
||||
// }
|
||||
// memcpy(pWhiteListRsp->pWhiteLists, pUser->pIpWhiteList->pIpRange,
|
||||
// pWhiteListRsp->numWhiteLists * sizeof(SIpV4Range));
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t mndSetUserAuthRsp(SMnode *pMnode, SUserObj *pUser, SGetUserAuthRsp *pRsp) {
|
||||
memcpy(pRsp->user, pUser->user, TSDB_USER_LEN);
|
||||
pRsp->superAuth = 1;
|
||||
|
|
@ -39,6 +62,14 @@ int32_t mndSetUserAuthRsp(SMnode *pMnode, SUserObj *pUser, SGetUserAuthRsp *pRsp
|
|||
pRsp->sysInfo = pUser->sysInfo;
|
||||
pRsp->version = pUser->authVersion;
|
||||
pRsp->passVer = pUser->passVersion;
|
||||
pRsp->whiteListVer = pUser->ipWhiteListVer;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t mndEnableIpWhiteList(SMnode *pMnode) { return 1; }
|
||||
|
||||
int32_t mndFetchIpWhiteList(SIpWhiteList *ipList, char **buf) {
|
||||
*buf = NULL;
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
|
@ -290,6 +290,7 @@ _CONNECT:
|
|||
connectRsp.svrTimestamp = taosGetTimestampSec();
|
||||
connectRsp.passVer = pUser->passVersion;
|
||||
connectRsp.authVer = pUser->authVersion;
|
||||
connectRsp.whiteListVer = pUser->ipWhiteListVer;
|
||||
|
||||
strcpy(connectRsp.sVer, version);
|
||||
snprintf(connectRsp.sDetailVer, sizeof(connectRsp.sDetailVer), "ver:%s\nbuild:%s\ngitinfo:%s", version, buildinfo,
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@
|
|||
|
||||
#define _DEFAULT_SOURCE
|
||||
#include "mndStb.h"
|
||||
#include "audit.h"
|
||||
#include "mndDb.h"
|
||||
#include "mndDnode.h"
|
||||
#include "mndIndex.h"
|
||||
|
|
@ -31,7 +32,6 @@
|
|||
#include "mndUser.h"
|
||||
#include "mndVgroup.h"
|
||||
#include "tname.h"
|
||||
#include "audit.h"
|
||||
|
||||
#define STB_VER_NUMBER 1
|
||||
#define STB_RESERVE_SIZE 64
|
||||
|
|
@ -858,7 +858,19 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat
|
|||
}
|
||||
return 0;
|
||||
}
|
||||
static int32_t mndGenIdxNameForFirstTag(char *fullname, char *dbname, char *tagname) {
|
||||
char randStr[24] = {0};
|
||||
|
||||
int8_t start = 8;
|
||||
int8_t end = sizeof(randStr) - 1;
|
||||
// gen rand str len [base:end]
|
||||
// note: ignore rand performance issues
|
||||
int64_t len = taosRand() % (end - start + 1) + start;
|
||||
|
||||
taosRandStr2(randStr, len);
|
||||
sprintf(fullname, "%s.%s_%s", dbname, tagname, randStr);
|
||||
return 0;
|
||||
}
|
||||
static int32_t mndCreateStb(SMnode *pMnode, SRpcMsg *pReq, SMCreateStbReq *pCreate, SDbObj *pDb) {
|
||||
SStbObj stbObj = {0};
|
||||
int32_t code = -1;
|
||||
|
|
@ -871,11 +883,8 @@ static int32_t mndCreateStb(SMnode *pMnode, SRpcMsg *pReq, SMCreateStbReq *pCrea
|
|||
mInfo("trans:%d, used to create stb:%s", pTrans->id, pCreate->name);
|
||||
if (mndBuildStbFromReq(pMnode, &stbObj, pCreate, pDb) != 0) goto _OVER;
|
||||
|
||||
char randStr[24] = {0};
|
||||
taosRandStr2(randStr, tListLen(randStr) - 1);
|
||||
SSchema *pSchema = &(stbObj.pTags[0]);
|
||||
sprintf(fullIdxName, "%s.%s_%s", pDb->name, pSchema->name, randStr);
|
||||
|
||||
mndGenIdxNameForFirstTag(fullIdxName, pDb->name, pSchema->name);
|
||||
SSIdx idx = {0};
|
||||
if (mndAcquireGlobalIdx(pMnode, fullIdxName, SDB_IDX, &idx) == 0 && idx.pIdx != NULL) {
|
||||
terrno = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST;
|
||||
|
|
@ -1067,8 +1076,7 @@ static int32_t mndBuildStbFromAlter(SStbObj *pStb, SStbObj *pDst, SMCreateStbReq
|
|||
}
|
||||
|
||||
static char *mndAuditFieldTypeStr(int32_t type) {
|
||||
switch (type)
|
||||
{
|
||||
switch (type) {
|
||||
case TSDB_DATA_TYPE_NULL:
|
||||
return "null";
|
||||
case TSDB_DATA_TYPE_BOOL:
|
||||
|
|
@ -1127,8 +1135,7 @@ static void mndAuditFieldStr(char* detail, SArray *arr, int32_t len, int32_t max
|
|||
if (detialLen > 0 && detialLen < max - fieldLen - 1) {
|
||||
strcat(detail, ", ");
|
||||
detialLen += fieldLen;
|
||||
}
|
||||
else{
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
sprintf(field, "%s:%s", pField->name, mndAuditFieldTypeStr(pField->type));
|
||||
|
|
@ -1136,8 +1143,7 @@ static void mndAuditFieldStr(char* detail, SArray *arr, int32_t len, int32_t max
|
|||
if (detialLen < max - fieldLen - 1) {
|
||||
strcat(detail, field);
|
||||
detialLen += fieldLen;
|
||||
}
|
||||
else{
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
|
@ -1252,14 +1258,17 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) {
|
|||
if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
|
||||
|
||||
char detail[AUDIT_DETAIL_MAX] = {0};
|
||||
sprintf(detail, "colVer:%d, delay1:%" PRId64 ", delay2:%" PRId64 ", deleteMark1:%" PRId64 ", "
|
||||
"deleteMark2:%" PRId64 ", igExists:%d, numOfColumns:%d, numOfFuncs:%d, numOfTags:%d, "
|
||||
"source:%d, suid:%" PRId64 ", tagVer:%d, ttl:%d, "
|
||||
sprintf(detail,
|
||||
"colVer:%d, delay1:%" PRId64 ", delay2:%" PRId64 ", deleteMark1:%" PRId64
|
||||
", "
|
||||
"deleteMark2:%" PRId64
|
||||
", igExists:%d, numOfColumns:%d, numOfFuncs:%d, numOfTags:%d, "
|
||||
"source:%d, suid:%" PRId64
|
||||
", tagVer:%d, ttl:%d, "
|
||||
"watermark1:%" PRId64 ", watermark2:%" PRId64,
|
||||
createReq.colVer, createReq.delay1, createReq.delay2, createReq.deleteMark1,
|
||||
createReq.deleteMark2, createReq.igExists, createReq.numOfColumns, createReq.numOfFuncs, createReq.numOfTags,
|
||||
createReq.source, createReq.suid, createReq.tagVer, createReq.ttl,
|
||||
createReq.watermark1, createReq.watermark2);
|
||||
createReq.colVer, createReq.delay1, createReq.delay2, createReq.deleteMark1, createReq.deleteMark2,
|
||||
createReq.igExists, createReq.numOfColumns, createReq.numOfFuncs, createReq.numOfTags, createReq.source,
|
||||
createReq.suid, createReq.tagVer, createReq.ttl, createReq.watermark1, createReq.watermark2);
|
||||
|
||||
mndAuditFieldStr(detail, createReq.pColumns, createReq.numOfColumns, AUDIT_DETAIL_MAX);
|
||||
mndAuditFieldStr(detail, createReq.pTags, createReq.numOfTags, AUDIT_DETAIL_MAX);
|
||||
|
|
@ -2338,8 +2347,7 @@ static int32_t mndProcessAlterStbReq(SRpcMsg *pReq) {
|
|||
if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
|
||||
|
||||
char detail[2000] = {0};
|
||||
sprintf(detail, "alterType:%d, numOfFields:%d, ttl:%d" ,
|
||||
alterReq.alterType, alterReq.numOfFields, alterReq.ttl);
|
||||
sprintf(detail, "alterType:%d, numOfFields:%d, ttl:%d", alterReq.alterType, alterReq.numOfFields, alterReq.ttl);
|
||||
|
||||
SName name = {0};
|
||||
tNameFromString(&name, alterReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
|
||||
|
|
@ -2608,8 +2616,7 @@ static int32_t mndProcessDropStbReq(SRpcMsg *pReq) {
|
|||
if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS;
|
||||
|
||||
char detail[2000] = {0};
|
||||
sprintf(detail, "igNotExists:%d, source:%d" ,
|
||||
dropReq.igNotExists, dropReq.source);
|
||||
sprintf(detail, "igNotExists:%d, source:%d", dropReq.igNotExists, dropReq.source);
|
||||
|
||||
SName name = {0};
|
||||
tNameFromString(&name, dropReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@
|
|||
#include "mndSync.h"
|
||||
#include "mndCluster.h"
|
||||
#include "mndTrans.h"
|
||||
#include "mndUser.h"
|
||||
|
||||
static int32_t mndSyncEqCtrlMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) {
|
||||
if (pMsg == NULL || pMsg->pCont == NULL) {
|
||||
|
|
@ -304,6 +305,7 @@ void mndRestoreFinish(const SSyncFSM *pFsm, const SyncIndex commitIdx) {
|
|||
} else {
|
||||
mInfo("vgId:1, sync restore finished");
|
||||
}
|
||||
mndRefreshUserIpWhiteList(pMnode);
|
||||
|
||||
ASSERT(commitIdx == mndSyncAppliedIndex(pFsm));
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -1948,6 +1948,10 @@ static int32_t vnodeProcessDropIndexReq(SVnode *pVnode, int64_t ver, void *pReq,
|
|||
extern int32_t vnodeProcessCompactVnodeReqImpl(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
|
||||
|
||||
static int32_t vnodeProcessCompactVnodeReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
|
||||
if (!pVnode->restored) {
|
||||
vInfo("vgId:%d, ignore compact req during restoring. ver:%" PRId64, TD_VID(pVnode), ver);
|
||||
return 0;
|
||||
}
|
||||
return vnodeProcessCompactVnodeReqImpl(pVnode, ver, pReq, len, pRsp);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -196,4 +196,22 @@ void updateTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pWin, int64_t
|
|||
|
||||
SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTables, SArray* pUidTagList, void* pVnode,
|
||||
SStorageAPI* pStorageAPI);
|
||||
|
||||
/**
|
||||
* @brief build a tuple into keyBuf
|
||||
* @param [out] keyBuf the output buf
|
||||
* @param [in] pSortGroupCols the cols to build
|
||||
* @param [in] pBlock block the tuple in
|
||||
*/
|
||||
int32_t buildKeys(char* keyBuf, const SArray* pSortGroupCols, const SSDataBlock* pBlock, int32_t rowIndex);
|
||||
|
||||
int32_t compKeys(const SArray* pSortGroupCols, const char* oldkeyBuf, int32_t oldKeysLen, const SSDataBlock* pDataBlock,
|
||||
int32_t rowIndex);
|
||||
|
||||
uint64_t calcGroupId(char *pData, int32_t len);
|
||||
|
||||
SNodeList* makeColsNodeArrFromSortKeys(SNodeList* pSortKeys);
|
||||
|
||||
int32_t extractKeysLen(const SArray* keys);
|
||||
|
||||
#endif // TDENGINE_EXECUTIL_H
|
||||
|
|
|
|||
|
|
@ -194,6 +194,16 @@ void tsortSetClosed(SSortHandle* pHandle);
|
|||
void tsortSetSingleTableMerge(SSortHandle* pHandle);
|
||||
void tsortSetAbortCheckFn(SSortHandle* pHandle, bool (*checkFn)(void* param), void* param);
|
||||
|
||||
/**
|
||||
* @brief comp the tuple with keyBuf, if not equal, new keys will be built in keyBuf, newLen will be stored in keyLen
|
||||
* @param [in] pSortCols cols to comp and build
|
||||
* @param [in, out] pass in the old keys, if comp not equal, new keys will be built in it.
|
||||
* @param [in, out] keyLen the old keysLen, if comp not equal, new keysLen will be stored in it.
|
||||
* @param [in] the tuple to comp with
|
||||
* @retval 0 if comp equal, 1 if not
|
||||
*/
|
||||
int32_t tsortCompAndBuildKeys(const SArray* pSortCols, char* keyBuf, int32_t* keyLen, const STupleHandle* pTuple);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -303,6 +303,7 @@ static int32_t createDataBlockForEmptyInput(SOperatorInfo* pOperator, SSDataBloc
|
|||
|
||||
SOperatorInfo* downstream = pOperator->pDownstream[0];
|
||||
if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_PARTITION ||
|
||||
downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_SORT ||
|
||||
(downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN &&
|
||||
((STableScanInfo*)downstream->info)->hasGroupByTag == true)) {
|
||||
return TSDB_CODE_SUCCESS;
|
||||
|
|
|
|||
|
|
@ -2261,3 +2261,104 @@ void updateTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pWin, int64_t
|
|||
ts[3] = pWin->skey; // window start key
|
||||
ts[4] = pWin->ekey + delta; // window end key
|
||||
}
|
||||
|
||||
int32_t compKeys(const SArray* pSortGroupCols, const char* oldkeyBuf, int32_t oldKeysLen, const SSDataBlock* pBlock, int32_t rowIndex) {
|
||||
SColumnDataAgg* pColAgg = NULL;
|
||||
const char* isNull = oldkeyBuf;
|
||||
const char* p = oldkeyBuf + sizeof(int8_t) * pSortGroupCols->size;
|
||||
|
||||
for (int32_t i = 0; i < pSortGroupCols->size; ++i) {
|
||||
const SColumn* pCol = (SColumn*)TARRAY_GET_ELEM(pSortGroupCols, i);
|
||||
const SColumnInfoData* pColInfoData = TARRAY_GET_ELEM(pBlock->pDataBlock, pCol->slotId);
|
||||
if (pBlock->pBlockAgg) pColAgg = pBlock->pBlockAgg[pCol->slotId];
|
||||
|
||||
if (colDataIsNull(pColInfoData, pBlock->info.rows, rowIndex, pColAgg)) {
|
||||
if (isNull[i] != 1) return 1;
|
||||
} else {
|
||||
if (isNull[i] != 0) return 1;
|
||||
const char* val = colDataGetData(pColInfoData, rowIndex);
|
||||
if (pCol->type == TSDB_DATA_TYPE_JSON) {
|
||||
int32_t len = getJsonValueLen(val);
|
||||
if (memcmp(p, val, len) != 0) return 1;
|
||||
p += len;
|
||||
} else if (IS_VAR_DATA_TYPE(pCol->type)) {
|
||||
if (memcmp(p, val, varDataTLen(val)) != 0) return 1;
|
||||
p += varDataTLen(val);
|
||||
} else {
|
||||
if (0 != memcmp(p, val, pCol->bytes)) return 1;
|
||||
p += pCol->bytes;
|
||||
}
|
||||
}
|
||||
}
|
||||
if ((int32_t)(p - oldkeyBuf) != oldKeysLen) return 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t buildKeys(char* keyBuf, const SArray* pSortGroupCols, const SSDataBlock* pBlock,
|
||||
int32_t rowIndex) {
|
||||
uint32_t colNum = pSortGroupCols->size;
|
||||
SColumnDataAgg* pColAgg = NULL;
|
||||
char* isNull = keyBuf;
|
||||
char* p = keyBuf + sizeof(int8_t) * colNum;
|
||||
|
||||
for (int32_t i = 0; i < colNum; ++i) {
|
||||
const SColumn* pCol = (SColumn*)TARRAY_GET_ELEM(pSortGroupCols, i);
|
||||
const SColumnInfoData* pColInfoData = TARRAY_GET_ELEM(pBlock->pDataBlock, pCol->slotId);
|
||||
if (pCol->slotId > pBlock->pDataBlock->size) continue;
|
||||
|
||||
if (pBlock->pBlockAgg) pColAgg = pBlock->pBlockAgg[pCol->slotId];
|
||||
|
||||
if (colDataIsNull(pColInfoData, pBlock->info.rows, rowIndex, pColAgg)) {
|
||||
isNull[i] = 1;
|
||||
} else {
|
||||
isNull[i] = 0;
|
||||
const char* val = colDataGetData(pColInfoData, rowIndex);
|
||||
if (pCol->type == TSDB_DATA_TYPE_JSON) {
|
||||
int32_t len = getJsonValueLen(val);
|
||||
memcpy(p, val, len);
|
||||
p += len;
|
||||
} else if (IS_VAR_DATA_TYPE(pCol->type)) {
|
||||
varDataCopy(p, val);
|
||||
p += varDataTLen(val);
|
||||
} else {
|
||||
memcpy(p, val, pCol->bytes);
|
||||
p += pCol->bytes;
|
||||
}
|
||||
}
|
||||
}
|
||||
return (int32_t)(p - keyBuf);
|
||||
}
|
||||
|
||||
uint64_t calcGroupId(char* pData, int32_t len) {
|
||||
T_MD5_CTX context;
|
||||
tMD5Init(&context);
|
||||
tMD5Update(&context, (uint8_t*)pData, len);
|
||||
tMD5Final(&context);
|
||||
|
||||
// NOTE: only extract the initial 8 bytes of the final MD5 digest
|
||||
uint64_t id = 0;
|
||||
memcpy(&id, context.digest, sizeof(uint64_t));
|
||||
if (0 == id) memcpy(&id, context.digest + 8, sizeof(uint64_t));
|
||||
return id;
|
||||
}
|
||||
|
||||
SNodeList* makeColsNodeArrFromSortKeys(SNodeList* pSortKeys) {
|
||||
SNode* node;
|
||||
SNodeList* ret = NULL;
|
||||
FOREACH(node, pSortKeys) {
|
||||
SOrderByExprNode* pSortKey = (SOrderByExprNode*)node;
|
||||
nodesListMakeAppend(&ret, pSortKey->pExpr);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
int32_t extractKeysLen(const SArray* keys) {
|
||||
int32_t len = 0;
|
||||
int32_t keyNum = taosArrayGetSize(keys);
|
||||
for (int32_t i = 0; i < keyNum; ++i) {
|
||||
SColumn* pCol = (SColumn*)taosArrayGet(keys, i);
|
||||
len += pCol->bytes;
|
||||
}
|
||||
len += sizeof(int8_t) * keyNum; //null flag
|
||||
return len;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -56,6 +56,10 @@ typedef struct SPartitionOperatorInfo {
|
|||
int32_t groupIndex; // group index
|
||||
int32_t pageIndex; // page index of current group
|
||||
SExprSupp scalarSup;
|
||||
|
||||
int32_t remainRows;
|
||||
int32_t orderedRows;
|
||||
SArray* pOrderInfoArr;
|
||||
} SPartitionOperatorInfo;
|
||||
|
||||
static void* getCurrentDataGroupInfo(const SPartitionOperatorInfo* pInfo, SDataGroupInfo** pGroupInfo, int32_t len);
|
||||
|
|
@ -635,20 +639,6 @@ void* getCurrentDataGroupInfo(const SPartitionOperatorInfo* pInfo, SDataGroupInf
|
|||
return pPage;
|
||||
}
|
||||
|
||||
uint64_t calcGroupId(char* pData, int32_t len) {
|
||||
T_MD5_CTX context;
|
||||
tMD5Init(&context);
|
||||
tMD5Update(&context, (uint8_t*)pData, len);
|
||||
tMD5Final(&context);
|
||||
|
||||
// NOTE: only extract the initial 8 bytes of the final MD5 digest
|
||||
uint64_t id = 0;
|
||||
memcpy(&id, context.digest, sizeof(uint64_t));
|
||||
if (0 == id)
|
||||
memcpy(&id, context.digest + 8, sizeof(uint64_t));
|
||||
return id;
|
||||
}
|
||||
|
||||
int32_t* setupColumnOffset(const SSDataBlock* pBlock, int32_t rowCapacity) {
|
||||
size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
|
||||
int32_t* offset = taosMemoryCalloc(numOfCols, sizeof(int32_t));
|
||||
|
|
@ -699,6 +689,8 @@ static SSDataBlock* buildPartitionResult(SOperatorInfo* pOperator) {
|
|||
SPartitionOperatorInfo* pInfo = pOperator->info;
|
||||
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
|
||||
|
||||
if (pInfo->remainRows == 0) {
|
||||
blockDataCleanup(pInfo->binfo.pRes);
|
||||
SDataGroupInfo* pGroupInfo =
|
||||
(pInfo->groupIndex != -1) ? taosArrayGet(pInfo->sortedGroupArray, pInfo->groupIndex) : NULL;
|
||||
if (pInfo->groupIndex == -1 || pInfo->pageIndex >= taosArrayGetSize(pGroupInfo->pPageList)) {
|
||||
|
|
@ -726,10 +718,20 @@ static SSDataBlock* buildPartitionResult(SOperatorInfo* pOperator) {
|
|||
|
||||
pInfo->pageIndex += 1;
|
||||
releaseBufPage(pInfo->pBuf, page);
|
||||
|
||||
pInfo->binfo.pRes->info.dataLoad = 1;
|
||||
blockDataUpdateTsWindow(pInfo->binfo.pRes, 0);
|
||||
pInfo->binfo.pRes->info.id.groupId = pGroupInfo->groupId;
|
||||
pInfo->binfo.pRes->info.dataLoad = 1;
|
||||
pInfo->orderedRows = 0;
|
||||
}
|
||||
|
||||
if (pInfo->pOrderInfoArr) {
|
||||
pInfo->binfo.pRes->info.rows += pInfo->remainRows;
|
||||
blockDataTrimFirstRows(pInfo->binfo.pRes, pInfo->orderedRows);
|
||||
pInfo->orderedRows = blockDataGetSortedRows(pInfo->binfo.pRes, pInfo->pOrderInfoArr);
|
||||
pInfo->remainRows = pInfo->binfo.pRes->info.rows - pInfo->orderedRows;
|
||||
pInfo->binfo.pRes->info.rows = pInfo->orderedRows;
|
||||
}
|
||||
|
||||
blockDataUpdateTsWindow(pInfo->binfo.pRes, 0);
|
||||
|
||||
pOperator->resultInfo.totalRows += pInfo->binfo.pRes->info.rows;
|
||||
return pInfo->binfo.pRes;
|
||||
|
|
@ -746,7 +748,6 @@ static SSDataBlock* hashPartition(SOperatorInfo* pOperator) {
|
|||
SSDataBlock* pRes = pInfo->binfo.pRes;
|
||||
|
||||
if (pOperator->status == OP_RES_TO_RETURN) {
|
||||
blockDataCleanup(pRes);
|
||||
return buildPartitionResult(pOperator);
|
||||
}
|
||||
|
||||
|
|
@ -829,6 +830,7 @@ static void destroyPartitionOperatorInfo(void* param) {
|
|||
|
||||
cleanupExprSupp(&pInfo->scalarSup);
|
||||
destroyDiskbasedBuf(pInfo->pBuf);
|
||||
taosArrayDestroy(pInfo->pOrderInfoArr);
|
||||
taosMemoryFreeClear(param);
|
||||
}
|
||||
|
||||
|
|
@ -846,6 +848,17 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartition
|
|||
SExprInfo* pExprInfo = createExprInfo(pPartNode->pTargets, NULL, &numOfCols);
|
||||
pInfo->pGroupCols = makeColumnArrayFromList(pPartNode->pPartitionKeys);
|
||||
|
||||
if (pPartNode->needBlockOutputTsOrder) {
|
||||
SBlockOrderInfo order = {.order = ORDER_ASC, .pColData = NULL, .nullFirst = false, .slotId = pPartNode->tsSlotId};
|
||||
pInfo->pOrderInfoArr = taosArrayInit(1, sizeof(SBlockOrderInfo));
|
||||
if (!pInfo->pOrderInfoArr) {
|
||||
terrno = TSDB_CODE_OUT_OF_MEMORY;
|
||||
pTaskInfo->code = terrno;
|
||||
goto _error;
|
||||
}
|
||||
taosArrayPush(pInfo->pOrderInfoArr, &order);
|
||||
}
|
||||
|
||||
if (pPartNode->pExprs != NULL) {
|
||||
int32_t num = 0;
|
||||
SExprInfo* pExprInfo1 = createExprInfo(pPartNode->pExprs, NULL, &num);
|
||||
|
|
|
|||
|
|
@ -19,6 +19,15 @@
|
|||
#include "querytask.h"
|
||||
#include "tdatablock.h"
|
||||
|
||||
typedef struct SSortOpGroupIdCalc {
|
||||
STupleHandle* pSavedTuple;
|
||||
SArray* pSortColsArr;
|
||||
char* keyBuf;
|
||||
int32_t lastKeysLen; // default to be 0
|
||||
uint64_t lastGroupId;
|
||||
bool excludePKCol;
|
||||
} SSortOpGroupIdCalc;
|
||||
|
||||
typedef struct SSortOperatorInfo {
|
||||
SOptrBasicInfo binfo;
|
||||
uint32_t sortBufSize; // max buffer size for in-memory sort
|
||||
|
|
@ -31,6 +40,7 @@ typedef struct SSortOperatorInfo {
|
|||
SLimitInfo limitInfo;
|
||||
uint64_t maxTupleLength;
|
||||
int64_t maxRows;
|
||||
SSortOpGroupIdCalc* pGroupIdCalc;
|
||||
} SSortOperatorInfo;
|
||||
|
||||
static SSDataBlock* doSort(SOperatorInfo* pOperator);
|
||||
|
|
@ -40,6 +50,8 @@ static int32_t getExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain
|
|||
static void destroySortOperatorInfo(void* param);
|
||||
static int32_t calcSortOperMaxTupleLength(SSortOperatorInfo* pSortOperInfo, SNodeList* pSortKeys);
|
||||
|
||||
static void destroySortOpGroupIdCalc(SSortOpGroupIdCalc* pCalc);
|
||||
|
||||
// todo add limit/offset impl
|
||||
SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* pSortNode, SExecTaskInfo* pTaskInfo) {
|
||||
SSortOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSortOperatorInfo));
|
||||
|
|
@ -78,6 +90,34 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode*
|
|||
|
||||
pInfo->binfo.pRes = createDataBlockFromDescNode(pDescNode);
|
||||
pInfo->pSortInfo = createSortInfo(pSortNode->pSortKeys);
|
||||
|
||||
if (pSortNode->calcGroupId) {
|
||||
int32_t keyLen;
|
||||
SSortOpGroupIdCalc* pGroupIdCalc = pInfo->pGroupIdCalc = taosMemoryCalloc(1, sizeof(SSortOpGroupIdCalc));
|
||||
if (!pGroupIdCalc) {
|
||||
code = TSDB_CODE_OUT_OF_MEMORY;
|
||||
goto _error;
|
||||
}
|
||||
SNodeList* pSortColsNodeArr = makeColsNodeArrFromSortKeys(pSortNode->pSortKeys);
|
||||
if (!pSortColsNodeArr) code = TSDB_CODE_OUT_OF_MEMORY;
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
pGroupIdCalc->pSortColsArr = makeColumnArrayFromList(pSortColsNodeArr);
|
||||
if (!pGroupIdCalc->pSortColsArr) code = TSDB_CODE_OUT_OF_MEMORY;
|
||||
nodesClearList(pSortColsNodeArr);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
// PK ts col should always at last, see partColOptCreateSort
|
||||
if (pSortNode->excludePkCol) taosArrayPop(pGroupIdCalc->pSortColsArr);
|
||||
keyLen = extractKeysLen(pGroupIdCalc->pSortColsArr);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
pGroupIdCalc->lastKeysLen = 0;
|
||||
pGroupIdCalc->keyBuf = taosMemoryCalloc(1, keyLen);
|
||||
if (!pGroupIdCalc->keyBuf) code = TSDB_CODE_OUT_OF_MEMORY;
|
||||
}
|
||||
}
|
||||
if (code != TSDB_CODE_SUCCESS) goto _error;
|
||||
|
||||
pInfo->binfo.inputTsOrder = pSortNode->node.inputTsOrder;
|
||||
pInfo->binfo.outputTsOrder = pSortNode->node.outputTsOrder;
|
||||
initLimitInfo(pSortNode->node.pLimit, pSortNode->node.pSlimit, &pInfo->limitInfo);
|
||||
|
|
@ -129,6 +169,52 @@ void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle) {
|
|||
pBlock->info.rows += 1;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief get next tuple with group id attached, here assume that all tuples are sorted by group keys
|
||||
* @param [in, out] pBlock the output block, the group id will be saved in it
|
||||
* @retval NULL if next group tuple arrived and this new group tuple will be saved in pInfo.pSavedTuple
|
||||
* @retval NULL if no more tuples
|
||||
*/
|
||||
static STupleHandle* nextTupleWithGroupId(SSortHandle* pHandle, SSortOperatorInfo* pInfo, SSDataBlock* pBlock) {
|
||||
STupleHandle* retTuple = pInfo->pGroupIdCalc->pSavedTuple;
|
||||
if (!retTuple) {
|
||||
retTuple = tsortNextTuple(pHandle);
|
||||
}
|
||||
|
||||
if (retTuple) {
|
||||
int32_t newGroup;
|
||||
if (pInfo->pGroupIdCalc->pSavedTuple) {
|
||||
newGroup = true;
|
||||
pInfo->pGroupIdCalc->pSavedTuple = NULL;
|
||||
} else {
|
||||
newGroup = tsortCompAndBuildKeys(pInfo->pGroupIdCalc->pSortColsArr, pInfo->pGroupIdCalc->keyBuf,
|
||||
&pInfo->pGroupIdCalc->lastKeysLen, retTuple);
|
||||
}
|
||||
bool emptyBlock = pBlock->info.rows == 0;
|
||||
if (newGroup) {
|
||||
if (!emptyBlock) {
|
||||
// new group arrived, and we have already copied some tuples for cur group, save the new group tuple, return
|
||||
// NULL. Note that the keyBuf and lastKeysLen has been updated to new value
|
||||
pInfo->pGroupIdCalc->pSavedTuple = retTuple;
|
||||
retTuple = NULL;
|
||||
} else {
|
||||
// new group with empty block
|
||||
pInfo->pGroupIdCalc->lastGroupId = pBlock->info.id.groupId =
|
||||
calcGroupId(pInfo->pGroupIdCalc->keyBuf, pInfo->pGroupIdCalc->lastKeysLen);
|
||||
}
|
||||
} else {
|
||||
if (emptyBlock) {
|
||||
// new block but not new group, assign last group id to it
|
||||
pBlock->info.id.groupId = pInfo->pGroupIdCalc->lastGroupId;
|
||||
} else {
|
||||
// not new group and not empty block and ret NOT NULL, just return the tuple
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return retTuple;
|
||||
}
|
||||
|
||||
SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity, SArray* pColMatchInfo,
|
||||
SSortOperatorInfo* pInfo) {
|
||||
blockDataCleanup(pDataBlock);
|
||||
|
|
@ -140,8 +226,13 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i
|
|||
|
||||
blockDataEnsureCapacity(p, capacity);
|
||||
|
||||
STupleHandle* pTupleHandle;
|
||||
while (1) {
|
||||
STupleHandle* pTupleHandle = tsortNextTuple(pHandle);
|
||||
if (pInfo->pGroupIdCalc) {
|
||||
pTupleHandle = nextTupleWithGroupId(pHandle, pInfo, p);
|
||||
} else {
|
||||
pTupleHandle = tsortNextTuple(pHandle);
|
||||
}
|
||||
if (pTupleHandle == NULL) {
|
||||
break;
|
||||
}
|
||||
|
|
@ -168,6 +259,7 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i
|
|||
pDataBlock->info.dataLoad = 1;
|
||||
pDataBlock->info.rows = p->info.rows;
|
||||
pDataBlock->info.scanFlag = p->info.scanFlag;
|
||||
pDataBlock->info.id.groupId = p->info.id.groupId;
|
||||
}
|
||||
|
||||
blockDataDestroy(p);
|
||||
|
|
@ -281,6 +373,7 @@ void destroySortOperatorInfo(void* param) {
|
|||
tsortDestroySortHandle(pInfo->pSortHandle);
|
||||
taosArrayDestroy(pInfo->pSortInfo);
|
||||
taosArrayDestroy(pInfo->matchInfo.pList);
|
||||
destroySortOpGroupIdCalc(pInfo->pGroupIdCalc);
|
||||
taosMemoryFreeClear(param);
|
||||
}
|
||||
|
||||
|
|
@ -309,6 +402,14 @@ static int32_t calcSortOperMaxTupleLength(SSortOperatorInfo* pSortOperInfo, SNod
|
|||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
static void destroySortOpGroupIdCalc(SSortOpGroupIdCalc* pCalc) {
|
||||
if (pCalc) {
|
||||
taosArrayDestroy(pCalc->pSortColsArr);
|
||||
taosMemoryFree(pCalc->keyBuf);
|
||||
taosMemoryFree(pCalc);
|
||||
}
|
||||
}
|
||||
|
||||
//=====================================================================================
|
||||
// Group Sort Operator
|
||||
typedef enum EChildOperatorStatus { CHILD_OP_NEW_GROUP, CHILD_OP_SAME_GROUP, CHILD_OP_FINISHED } EChildOperatorStatus;
|
||||
|
|
@ -591,6 +692,7 @@ typedef struct SMultiwayMergeOperatorInfo {
|
|||
bool ignoreGroupId;
|
||||
uint64_t groupId;
|
||||
STupleHandle* prefetchedTuple;
|
||||
bool inputWithGroupId;
|
||||
} SMultiwayMergeOperatorInfo;
|
||||
|
||||
int32_t openMultiwayMergeOperator(SOperatorInfo* pOperator) {
|
||||
|
|
@ -641,7 +743,7 @@ static void doGetSortedBlockData(SMultiwayMergeOperatorInfo* pInfo, SSortHandle*
|
|||
|
||||
while (1) {
|
||||
STupleHandle* pTupleHandle = NULL;
|
||||
if (pInfo->groupSort) {
|
||||
if (pInfo->groupSort || pInfo->inputWithGroupId) {
|
||||
if (pInfo->prefetchedTuple == NULL) {
|
||||
pTupleHandle = tsortNextTuple(pHandle);
|
||||
} else {
|
||||
|
|
@ -662,7 +764,7 @@ static void doGetSortedBlockData(SMultiwayMergeOperatorInfo* pInfo, SSortHandle*
|
|||
break;
|
||||
}
|
||||
|
||||
if (pInfo->groupSort) {
|
||||
if (pInfo->groupSort || pInfo->inputWithGroupId) {
|
||||
uint64_t tupleGroupId = tsortGetGroupId(pTupleHandle);
|
||||
if (pInfo->groupId == 0 || pInfo->groupId == tupleGroupId) {
|
||||
appendOneRowToDataBlock(p, pTupleHandle);
|
||||
|
|
@ -842,6 +944,7 @@ SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** downStreams, size
|
|||
pInfo->sortBufSize = pInfo->bufPageSize * (numStreams + 1); // one additional is reserved for merged result.
|
||||
pInfo->binfo.inputTsOrder = pMergePhyNode->node.inputTsOrder;
|
||||
pInfo->binfo.outputTsOrder = pMergePhyNode->node.outputTsOrder;
|
||||
pInfo->inputWithGroupId = pMergePhyNode->inputWithGroupId;
|
||||
|
||||
setOperatorInfo(pOperator, "MultiwayMergeOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE, false, OP_NOT_OPENED, pInfo, pTaskInfo);
|
||||
pOperator->fpSet = createOperatorFpSet(openMultiwayMergeOperator, doMultiwayMerge, NULL,
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@
|
|||
#include "tsort.h"
|
||||
#include "tutil.h"
|
||||
#include "tsimplehash.h"
|
||||
#include "executil.h"
|
||||
|
||||
struct STupleHandle {
|
||||
SSDataBlock* pBlock;
|
||||
|
|
@ -615,15 +616,18 @@ int32_t msortComparFn(const void* pLeft, const void* pRight, void* param) {
|
|||
int ret = pParam->cmpFn(left1, right1);
|
||||
return ret;
|
||||
} else {
|
||||
bool isVarType;
|
||||
for (int32_t i = 0; i < pInfo->size; ++i) {
|
||||
SBlockOrderInfo* pOrder = TARRAY_GET_ELEM(pInfo, i);
|
||||
SColumnInfoData* pLeftColInfoData = TARRAY_GET_ELEM(pLeftBlock->pDataBlock, pOrder->slotId);
|
||||
SColumnInfoData* pRightColInfoData = TARRAY_GET_ELEM(pRightBlock->pDataBlock, pOrder->slotId);
|
||||
isVarType = IS_VAR_DATA_TYPE(pLeftColInfoData->info.type);
|
||||
|
||||
if (pLeftColInfoData->hasNull || pRightColInfoData->hasNull) {
|
||||
bool leftNull = false;
|
||||
if (pLeftColInfoData->hasNull) {
|
||||
if (pLeftBlock->pBlockAgg == NULL) {
|
||||
leftNull = colDataIsNull_s(pLeftColInfoData, pLeftSource->src.rowIndex);
|
||||
leftNull = colDataIsNull_t(pLeftColInfoData, pLeftSource->src.rowIndex, isVarType);
|
||||
} else {
|
||||
leftNull = colDataIsNull(pLeftColInfoData, pLeftBlock->info.rows, pLeftSource->src.rowIndex,
|
||||
pLeftBlock->pBlockAgg[i]);
|
||||
|
|
@ -633,7 +637,7 @@ int32_t msortComparFn(const void* pLeft, const void* pRight, void* param) {
|
|||
bool rightNull = false;
|
||||
if (pRightColInfoData->hasNull) {
|
||||
if (pRightBlock->pBlockAgg == NULL) {
|
||||
rightNull = colDataIsNull_s(pRightColInfoData, pRightSource->src.rowIndex);
|
||||
rightNull = colDataIsNull_t(pRightColInfoData, pRightSource->src.rowIndex, isVarType);
|
||||
} else {
|
||||
rightNull = colDataIsNull(pRightColInfoData, pRightBlock->info.rows, pRightSource->src.rowIndex,
|
||||
pRightBlock->pBlockAgg[i]);
|
||||
|
|
@ -651,11 +655,22 @@ int32_t msortComparFn(const void* pLeft, const void* pRight, void* param) {
|
|||
if (leftNull) {
|
||||
return pOrder->nullFirst ? -1 : 1;
|
||||
}
|
||||
}
|
||||
|
||||
void* left1 = colDataGetData(pLeftColInfoData, pLeftSource->src.rowIndex);
|
||||
void* right1 = colDataGetData(pRightColInfoData, pRightSource->src.rowIndex);
|
||||
void* left1, *right1;
|
||||
if (isVarType) {
|
||||
left1 = colDataGetVarData(pLeftColInfoData, pLeftSource->src.rowIndex);
|
||||
right1 = colDataGetVarData(pRightColInfoData, pRightSource->src.rowIndex);
|
||||
} else {
|
||||
left1 = colDataGetNumData(pLeftColInfoData, pLeftSource->src.rowIndex);
|
||||
right1 = colDataGetNumData(pRightColInfoData, pRightSource->src.rowIndex);
|
||||
}
|
||||
|
||||
__compar_fn_t fn = getKeyComparFunc(pLeftColInfoData->info.type, pOrder->order);
|
||||
__compar_fn_t fn = pOrder->compFn;
|
||||
if (!fn) {
|
||||
fn = getKeyComparFunc(pLeftColInfoData->info.type, pOrder->order);
|
||||
pOrder->compFn = fn;
|
||||
}
|
||||
|
||||
int ret = fn(left1, right1);
|
||||
if (ret == 0) {
|
||||
|
|
@ -1567,3 +1582,15 @@ SSortExecInfo tsortGetSortExecInfo(SSortHandle* pHandle) {
|
|||
|
||||
return info;
|
||||
}
|
||||
|
||||
int32_t tsortCompAndBuildKeys(const SArray* pSortCols, char* keyBuf, int32_t* keyLen,
|
||||
const STupleHandle* pTuple) {
|
||||
int32_t ret;
|
||||
if (0 == compKeys(pSortCols, keyBuf, *keyLen, pTuple->pBlock, pTuple->rowIndex)) {
|
||||
ret = 0;
|
||||
} else {
|
||||
*keyLen = buildKeys(keyBuf, pSortCols, pTuple->pBlock, pTuple->rowIndex);
|
||||
ret = 1;
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -444,6 +444,7 @@ static int32_t logicAggCopy(const SAggLogicNode* pSrc, SAggLogicNode* pDst) {
|
|||
CLONE_NODE_LIST_FIELD(pGroupKeys);
|
||||
CLONE_NODE_LIST_FIELD(pAggFuncs);
|
||||
COPY_SCALAR_FIELD(hasGroupKeyOptimized);
|
||||
COPY_SCALAR_FIELD(isPartTb);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
|
|
@ -489,6 +490,7 @@ static int32_t logicMergeCopy(const SMergeLogicNode* pSrc, SMergeLogicNode* pDst
|
|||
COPY_SCALAR_FIELD(srcGroupId);
|
||||
COPY_SCALAR_FIELD(groupSort);
|
||||
COPY_SCALAR_FIELD(ignoreGroupId);
|
||||
COPY_SCALAR_FIELD(inputWithGroupId);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
|
|
@ -531,6 +533,8 @@ static int32_t logicSortCopy(const SSortLogicNode* pSrc, SSortLogicNode* pDst) {
|
|||
COPY_BASE_OBJECT_FIELD(node, logicNodeCopy);
|
||||
CLONE_NODE_LIST_FIELD(pSortKeys);
|
||||
COPY_SCALAR_FIELD(groupSort);
|
||||
COPY_SCALAR_FIELD(calcGroupId);
|
||||
COPY_SCALAR_FIELD(excludePkCol);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
|
|
@ -539,6 +543,9 @@ static int32_t logicPartitionCopy(const SPartitionLogicNode* pSrc, SPartitionLog
|
|||
CLONE_NODE_LIST_FIELD(pPartitionKeys);
|
||||
CLONE_NODE_LIST_FIELD(pTags);
|
||||
CLONE_NODE_FIELD(pSubtable);
|
||||
COPY_SCALAR_FIELD(needBlockOutputTsOrder);
|
||||
COPY_SCALAR_FIELD(pkTsColId);
|
||||
COPY_SCALAR_FIELD(pkTsColTbId);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
|
|
@ -678,6 +685,8 @@ static int32_t physiPartitionCopy(const SPartitionPhysiNode* pSrc, SPartitionPhy
|
|||
CLONE_NODE_LIST_FIELD(pExprs);
|
||||
CLONE_NODE_LIST_FIELD(pPartitionKeys);
|
||||
CLONE_NODE_LIST_FIELD(pTargets);
|
||||
COPY_SCALAR_FIELD(needBlockOutputTsOrder);
|
||||
COPY_SCALAR_FIELD(tsSlotId);
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -2271,6 +2271,7 @@ static const char* jkMergePhysiPlanNumOfChannels = "NumOfChannels";
|
|||
static const char* jkMergePhysiPlanSrcGroupId = "SrcGroupId";
|
||||
static const char* jkMergePhysiPlanGroupSort = "GroupSort";
|
||||
static const char* jkMergePhysiPlanIgnoreGroupID = "IgnoreGroupID";
|
||||
static const char* jkMergePhysiPlanInputWithGroupId = "InputWithGroupId";
|
||||
|
||||
static int32_t physiMergeNodeToJson(const void* pObj, SJson* pJson) {
|
||||
const SMergePhysiNode* pNode = (const SMergePhysiNode*)pObj;
|
||||
|
|
@ -2294,6 +2295,9 @@ static int32_t physiMergeNodeToJson(const void* pObj, SJson* pJson) {
|
|||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tjsonAddBoolToObject(pJson, jkMergePhysiPlanIgnoreGroupID, pNode->ignoreGroupId);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tjsonAddBoolToObject(pJson, jkMergePhysiPlanInputWithGroupId, pNode->inputWithGroupId);
|
||||
}
|
||||
|
||||
return code;
|
||||
}
|
||||
|
|
@ -2327,7 +2331,8 @@ static int32_t jsonToPhysiMergeNode(const SJson* pJson, void* pObj) {
|
|||
static const char* jkSortPhysiPlanExprs = "Exprs";
|
||||
static const char* jkSortPhysiPlanSortKeys = "SortKeys";
|
||||
static const char* jkSortPhysiPlanTargets = "Targets";
|
||||
static const char* jkSortPhysiPlanMaxRows = "MaxRows";
|
||||
static const char* jkSortPhysiPlanCalcGroupIds = "CalcGroupIds";
|
||||
static const char* jkSortPhysiPlanExcludePKCol = "ExcludePKCol";
|
||||
|
||||
static int32_t physiSortNodeToJson(const void* pObj, SJson* pJson) {
|
||||
const SSortPhysiNode* pNode = (const SSortPhysiNode*)pObj;
|
||||
|
|
@ -2342,6 +2347,12 @@ static int32_t physiSortNodeToJson(const void* pObj, SJson* pJson) {
|
|||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = nodeListToJson(pJson, jkSortPhysiPlanTargets, pNode->pTargets);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tjsonAddBoolToObject(pJson, jkSortPhysiPlanCalcGroupIds, pNode->calcGroupId);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tjsonAddBoolToObject(pJson, jkSortPhysiPlanExcludePKCol, pNode->excludePkCol);
|
||||
}
|
||||
|
||||
return code;
|
||||
}
|
||||
|
|
@ -2359,6 +2370,12 @@ static int32_t jsonToPhysiSortNode(const SJson* pJson, void* pObj) {
|
|||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = jsonToNodeList(pJson, jkSortPhysiPlanTargets, &pNode->pTargets);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tjsonGetBoolValue(pJson, jkSortPhysiPlanCalcGroupIds, &pNode->calcGroupId);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code= tjsonGetBoolValue(pJson, jkSortPhysiPlanExcludePKCol, &pNode->excludePkCol);
|
||||
}
|
||||
|
||||
return code;
|
||||
}
|
||||
|
|
@ -2644,6 +2661,8 @@ static int32_t jsonToPhysiEventWindowNode(const SJson* pJson, void* pObj) {
|
|||
static const char* jkPartitionPhysiPlanExprs = "Exprs";
|
||||
static const char* jkPartitionPhysiPlanPartitionKeys = "PartitionKeys";
|
||||
static const char* jkPartitionPhysiPlanTargets = "Targets";
|
||||
static const char* jkPartitionPhysiPlanNeedBlockOutputTsOrder = "NeedBlockOutputTsOrder";
|
||||
static const char* jkPartitionPhysiPlanTsSlotId = "tsSlotId";
|
||||
|
||||
static int32_t physiPartitionNodeToJson(const void* pObj, SJson* pJson) {
|
||||
const SPartitionPhysiNode* pNode = (const SPartitionPhysiNode*)pObj;
|
||||
|
|
@ -2658,6 +2677,12 @@ static int32_t physiPartitionNodeToJson(const void* pObj, SJson* pJson) {
|
|||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = nodeListToJson(pJson, jkPartitionPhysiPlanTargets, pNode->pTargets);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
tjsonAddBoolToObject(pJson, jkPartitionPhysiPlanNeedBlockOutputTsOrder, pNode->needBlockOutputTsOrder);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
tjsonAddIntegerToObject(pJson, jkPartitionPhysiPlanTsSlotId, pNode->tsSlotId);
|
||||
}
|
||||
|
||||
return code;
|
||||
}
|
||||
|
|
@ -2675,6 +2700,12 @@ static int32_t jsonToPhysiPartitionNode(const SJson* pJson, void* pObj) {
|
|||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = jsonToNodeList(pJson, jkPartitionPhysiPlanTargets, &pNode->pTargets);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tjsonGetBoolValue(pJson, jkPartitionPhysiPlanNeedBlockOutputTsOrder, &pNode->needBlockOutputTsOrder);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tjsonGetIntValue(pJson, jkPartitionPhysiPlanTsSlotId, &pNode->tsSlotId);
|
||||
}
|
||||
|
||||
return code;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2682,6 +2682,7 @@ enum {
|
|||
PHY_MERGE_CODE_SRC_GROUP_ID,
|
||||
PHY_MERGE_CODE_GROUP_SORT,
|
||||
PHY_MERGE_CODE_IGNORE_GROUP_ID,
|
||||
PHY_MERGE_CODE_INPUT_WITH_GROUP_ID,
|
||||
};
|
||||
|
||||
static int32_t physiMergeNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
|
||||
|
|
@ -2706,6 +2707,9 @@ static int32_t physiMergeNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
|
|||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tlvEncodeBool(pEncoder, PHY_MERGE_CODE_IGNORE_GROUP_ID, pNode->ignoreGroupId);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tlvEncodeBool(pEncoder, PHY_MERGE_CODE_INPUT_WITH_GROUP_ID, pNode->inputWithGroupId);
|
||||
}
|
||||
|
||||
return code;
|
||||
}
|
||||
|
|
@ -2738,6 +2742,9 @@ static int32_t msgToPhysiMergeNode(STlvDecoder* pDecoder, void* pObj) {
|
|||
case PHY_MERGE_CODE_IGNORE_GROUP_ID:
|
||||
code = tlvDecodeBool(pTlv, &pNode->ignoreGroupId);
|
||||
break;
|
||||
case PHY_MERGE_CODE_INPUT_WITH_GROUP_ID:
|
||||
code = tlvDecodeBool(pTlv, &pNode->inputWithGroupId);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
|
@ -2746,7 +2753,14 @@ static int32_t msgToPhysiMergeNode(STlvDecoder* pDecoder, void* pObj) {
|
|||
return code;
|
||||
}
|
||||
|
||||
enum { PHY_SORT_CODE_BASE_NODE = 1, PHY_SORT_CODE_EXPR, PHY_SORT_CODE_SORT_KEYS, PHY_SORT_CODE_TARGETS };
|
||||
enum {
|
||||
PHY_SORT_CODE_BASE_NODE = 1,
|
||||
PHY_SORT_CODE_EXPR,
|
||||
PHY_SORT_CODE_SORT_KEYS,
|
||||
PHY_SORT_CODE_TARGETS,
|
||||
PHY_SORT_CODE_CALC_GROUPID,
|
||||
PHY_SORT_CODE_EXCLUDE_PK_COL
|
||||
};
|
||||
|
||||
static int32_t physiSortNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
|
||||
const SSortPhysiNode* pNode = (const SSortPhysiNode*)pObj;
|
||||
|
|
@ -2761,6 +2775,12 @@ static int32_t physiSortNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
|
|||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tlvEncodeObj(pEncoder, PHY_SORT_CODE_TARGETS, nodeListToMsg, pNode->pTargets);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tlvEncodeBool(pEncoder, PHY_SORT_CODE_CALC_GROUPID, pNode->calcGroupId);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tlvEncodeBool(pEncoder, PHY_SORT_CODE_EXCLUDE_PK_COL, pNode->excludePkCol);
|
||||
}
|
||||
|
||||
return code;
|
||||
}
|
||||
|
|
@ -2784,6 +2804,11 @@ static int32_t msgToPhysiSortNode(STlvDecoder* pDecoder, void* pObj) {
|
|||
case PHY_SORT_CODE_TARGETS:
|
||||
code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pTargets);
|
||||
break;
|
||||
case PHY_SORT_CODE_CALC_GROUPID:
|
||||
code = tlvDecodeBool(pTlv, &pNode->calcGroupId);
|
||||
break;
|
||||
case PHY_SORT_CODE_EXCLUDE_PK_COL:
|
||||
code = tlvDecodeBool(pTlv, &pNode->excludePkCol);
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
|
@ -3142,7 +3167,14 @@ static int32_t msgToPhysiEventWindowNode(STlvDecoder* pDecoder, void* pObj) {
|
|||
return code;
|
||||
}
|
||||
|
||||
enum { PHY_PARTITION_CODE_BASE_NODE = 1, PHY_PARTITION_CODE_EXPR, PHY_PARTITION_CODE_KEYS, PHY_PARTITION_CODE_TARGETS };
|
||||
enum {
|
||||
PHY_PARTITION_CODE_BASE_NODE = 1,
|
||||
PHY_PARTITION_CODE_EXPR,
|
||||
PHY_PARTITION_CODE_KEYS,
|
||||
PHY_PARTITION_CODE_TARGETS,
|
||||
PHY_PARTITION_CODE_HAS_OUTPUT_TS_ORDER,
|
||||
PHY_PARTITION_CODE_TS_SLOTID
|
||||
};
|
||||
|
||||
static int32_t physiPartitionNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
|
||||
const SPartitionPhysiNode* pNode = (const SPartitionPhysiNode*)pObj;
|
||||
|
|
@ -3157,6 +3189,12 @@ static int32_t physiPartitionNodeToMsg(const void* pObj, STlvEncoder* pEncoder)
|
|||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tlvEncodeObj(pEncoder, PHY_PARTITION_CODE_TARGETS, nodeListToMsg, pNode->pTargets);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tlvEncodeBool(pEncoder, PHY_PARTITION_CODE_HAS_OUTPUT_TS_ORDER, pNode->needBlockOutputTsOrder);
|
||||
}
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = tlvEncodeI32(pEncoder, PHY_PARTITION_CODE_TS_SLOTID, pNode->tsSlotId);
|
||||
}
|
||||
|
||||
return code;
|
||||
}
|
||||
|
|
@ -3180,6 +3218,12 @@ static int32_t msgToPhysiPartitionNode(STlvDecoder* pDecoder, void* pObj) {
|
|||
case PHY_PARTITION_CODE_TARGETS:
|
||||
code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pTargets);
|
||||
break;
|
||||
case PHY_PARTITION_CODE_HAS_OUTPUT_TS_ORDER:
|
||||
code = tlvDecodeBool(pTlv, &pNode->needBlockOutputTsOrder);
|
||||
break;
|
||||
case PHY_PARTITION_CODE_TS_SLOTID:
|
||||
code = tlvDecodeI32(pTlv, &pNode->tsSlotId);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,8 +13,8 @@
|
|||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
#include <uv.h>
|
||||
#include <regex.h>
|
||||
#include <uv.h>
|
||||
|
||||
#include "parAst.h"
|
||||
#include "parUtil.h"
|
||||
|
|
@ -359,6 +359,9 @@ bool addHintNodeToList(SAstCreateContext* pCxt, SNodeList** ppHintList, EHintOpt
|
|||
}
|
||||
break;
|
||||
}
|
||||
case HINT_SORT_FOR_GROUP:
|
||||
if (paramNum > 0) return true;
|
||||
break;
|
||||
default:
|
||||
return true;
|
||||
}
|
||||
|
|
@ -421,6 +424,14 @@ SNodeList* createHintNodeList(SAstCreateContext* pCxt, const SToken* pLiteral) {
|
|||
}
|
||||
opt = HINT_NO_BATCH_SCAN;
|
||||
break;
|
||||
case TK_SORT_FOR_GROUP:
|
||||
lastComma = false;
|
||||
if (0 != opt || inParamList) {
|
||||
quit = true;
|
||||
break;
|
||||
}
|
||||
opt = HINT_SORT_FOR_GROUP;
|
||||
break;
|
||||
case TK_NK_LP:
|
||||
lastComma = false;
|
||||
if (0 == opt || inParamList) {
|
||||
|
|
@ -1666,9 +1677,7 @@ static int32_t getIpV4RangeFromWhitelistItem(char* ipRange, SIpV4Range* pIpRange
|
|||
code = TSDB_CODE_PAR_INVALID_IP_RANGE;
|
||||
} else {
|
||||
pIpRange->ip = addr.s_addr;
|
||||
uint32_t mask = (1 << (32 - prefix)) - 1;
|
||||
mask = htonl(~mask);
|
||||
pIpRange->mask = mask;
|
||||
pIpRange->mask = prefix;
|
||||
code = TSDB_CODE_SUCCESS;
|
||||
}
|
||||
} else {
|
||||
|
|
@ -1678,7 +1687,7 @@ static int32_t getIpV4RangeFromWhitelistItem(char* ipRange, SIpV4Range* pIpRange
|
|||
struct in_addr addr;
|
||||
if (uv_inet_pton(AF_INET, ipCopy, &addr) == 0) {
|
||||
pIpRange->ip = addr.s_addr;
|
||||
pIpRange->mask = 0xFFFFFFFF;
|
||||
pIpRange->mask = 32;
|
||||
code = TSDB_CODE_SUCCESS;
|
||||
} else {
|
||||
code = TSDB_CODE_PAR_INVALID_IP_RANGE;
|
||||
|
|
|
|||
|
|
@ -207,6 +207,7 @@ static SKeyword keywordTable[] = {
|
|||
{"SMALLINT", TK_SMALLINT},
|
||||
{"SNODE", TK_SNODE},
|
||||
{"SNODES", TK_SNODES},
|
||||
{"SORT_FOR_GROUP", TK_SORT_FOR_GROUP},
|
||||
{"SOFFSET", TK_SOFFSET},
|
||||
{"SPLIT", TK_SPLIT},
|
||||
{"STABLE", TK_STABLE},
|
||||
|
|
|
|||
|
|
@ -45,11 +45,14 @@ int32_t scaleOutLogicPlan(SPlanContext* pCxt, SLogicSubplan* pLogicSubplan, SQue
|
|||
int32_t createPhysiPlan(SPlanContext* pCxt, SQueryLogicPlan* pLogicPlan, SQueryPlan** pPlan, SArray* pExecNodeList);
|
||||
|
||||
bool getBatchScanOptionFromHint(SNodeList* pList);
|
||||
bool getSortForGroupOptHint(SNodeList* pList);
|
||||
SLogicNode* getLogicNodeRootNode(SLogicNode* pCurr);
|
||||
int32_t collectTableAliasFromNodes(SNode* pNode, SSHashObj** ppRes);
|
||||
bool isPartTableAgg(SAggLogicNode* pAgg);
|
||||
bool isPartTagAgg(SAggLogicNode* pAgg);
|
||||
bool isPartTableWinodw(SWindowLogicNode* pWindow);
|
||||
bool keysHasCol(SNodeList* pKeys);
|
||||
bool keysHasTbname(SNodeList* pKeys);
|
||||
|
||||
#define CLONE_LIMIT 1
|
||||
#define CLONE_SLIMIT 1 << 1
|
||||
|
|
|
|||
|
|
@ -741,6 +741,9 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect,
|
|||
}
|
||||
nodesDestroyList(pOutputGroupKeys);
|
||||
|
||||
pAgg->isGroupTb = pAgg->pGroupKeys ? keysHasTbname(pAgg->pGroupKeys) : 0;
|
||||
pAgg->isPartTb = pSelect->pPartitionByList ? keysHasTbname(pSelect->pPartitionByList) : 0;
|
||||
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
*pLogicNode = (SLogicNode*)pAgg;
|
||||
} else {
|
||||
|
|
@ -962,6 +965,7 @@ static int32_t createWindowLogicNodeByInterval(SLogicPlanContext* pCxt, SInterva
|
|||
nodesDestroyNode((SNode*)pWindow);
|
||||
return TSDB_CODE_OUT_OF_MEMORY;
|
||||
}
|
||||
pWindow->isPartTb = pSelect->pPartitionByList ? keysHasTbname(pSelect->pPartitionByList) : 0;
|
||||
|
||||
return createWindowLogicNodeFinalize(pCxt, pSelect, pWindow, pLogicNode);
|
||||
}
|
||||
|
|
@ -993,7 +997,6 @@ static int32_t createWindowLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSele
|
|||
if (NULL == pSelect->pWindow) {
|
||||
return TSDB_CODE_SUCCESS;
|
||||
}
|
||||
|
||||
switch (nodeType(pSelect->pWindow)) {
|
||||
case QUERY_NODE_STATE_WINDOW:
|
||||
return createWindowLogicNodeByState(pCxt, (SStateWindowNode*)pSelect->pWindow, pSelect, pLogicNode);
|
||||
|
|
@ -1260,6 +1263,15 @@ static int32_t createPartitionLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pS
|
|||
}
|
||||
}
|
||||
|
||||
if (keysHasCol(pPartition->pPartitionKeys) && pSelect->pWindow &&
|
||||
nodeType(pSelect->pWindow) == QUERY_NODE_INTERVAL_WINDOW) {
|
||||
pPartition->needBlockOutputTsOrder = true;
|
||||
SIntervalWindowNode* pInterval = (SIntervalWindowNode*)pSelect->pWindow;
|
||||
SColumnNode* pTsCol = (SColumnNode*)pInterval->pCol;
|
||||
pPartition->pkTsColId = pTsCol->colId;
|
||||
pPartition->pkTsColTbId = pTsCol->tableId;
|
||||
}
|
||||
|
||||
if (TSDB_CODE_SUCCESS == code && NULL != pSelect->pTags) {
|
||||
pPartition->pTags = nodesCloneList(pSelect->pTags);
|
||||
if (NULL == pPartition->pTags) {
|
||||
|
|
|
|||
|
|
@ -1661,22 +1661,6 @@ static int32_t smaIndexOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSub
|
|||
return smaIndexOptimizeImpl(pCxt, pLogicSubplan, pScan);
|
||||
}
|
||||
|
||||
static EDealRes partTagsOptHasColImpl(SNode* pNode, void* pContext) {
|
||||
if (QUERY_NODE_COLUMN == nodeType(pNode)) {
|
||||
if (COLUMN_TYPE_TAG != ((SColumnNode*)pNode)->colType && COLUMN_TYPE_TBNAME != ((SColumnNode*)pNode)->colType) {
|
||||
*(bool*)pContext = true;
|
||||
return DEAL_RES_END;
|
||||
}
|
||||
}
|
||||
return DEAL_RES_CONTINUE;
|
||||
}
|
||||
|
||||
static bool planOptNodeListHasCol(SNodeList* pKeys) {
|
||||
bool hasCol = false;
|
||||
nodesWalkExprs(pKeys, partTagsOptHasColImpl, &hasCol);
|
||||
return hasCol;
|
||||
}
|
||||
|
||||
static EDealRes partTagsOptHasTbname(SNode* pNode, void* pContext) {
|
||||
if (QUERY_NODE_COLUMN == nodeType(pNode)) {
|
||||
if (COLUMN_TYPE_TBNAME == ((SColumnNode*)pNode)->colType) {
|
||||
|
|
@ -1755,7 +1739,7 @@ static bool partTagsOptMayBeOptimized(SLogicNode* pNode) {
|
|||
return false;
|
||||
}
|
||||
|
||||
return !planOptNodeListHasCol(partTagsGetPartKeys(pNode)) && partTagsOptAreSupportedFuncs(partTagsGetFuncs(pNode));
|
||||
return !keysHasCol(partTagsGetPartKeys(pNode)) && partTagsOptAreSupportedFuncs(partTagsGetFuncs(pNode));
|
||||
}
|
||||
|
||||
static int32_t partTagsOptRebuildTbanme(SNodeList* pPartKeys) {
|
||||
|
|
@ -2042,6 +2026,7 @@ static int32_t eliminateProjOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan*
|
|||
}
|
||||
|
||||
int32_t code = replaceLogicNode(pLogicSubplan, (SLogicNode*)pProjectNode, pChild);
|
||||
if (pProjectNode->node.pHint && !pChild->pHint) TSWAP(pProjectNode->node.pHint, pChild->pHint);
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
NODES_CLEAR_LIST(pProjectNode->node.pChildren);
|
||||
nodesDestroyNode((SNode*)pProjectNode);
|
||||
|
|
@ -2734,7 +2719,7 @@ static bool tagScanOptShouldBeOptimized(SLogicNode* pNode) {
|
|||
}
|
||||
|
||||
SAggLogicNode* pAgg = (SAggLogicNode*)(pNode->pParent);
|
||||
if (NULL == pAgg->pGroupKeys || NULL != pAgg->pAggFuncs || planOptNodeListHasCol(pAgg->pGroupKeys) ||
|
||||
if (NULL == pAgg->pGroupKeys || NULL != pAgg->pAggFuncs || keysHasCol(pAgg->pGroupKeys) ||
|
||||
!planOptNodeListHasTbname(pAgg->pGroupKeys)) {
|
||||
return false;
|
||||
}
|
||||
|
|
@ -2853,13 +2838,14 @@ static bool pushDownLimitTo(SLogicNode* pNodeWithLimit, SLogicNode* pNodeLimitPu
|
|||
cloneLimit(pNodeWithLimit, pNodeLimitPushTo, CLONE_LIMIT_SLIMIT);
|
||||
return true;
|
||||
}
|
||||
case QUERY_NODE_LOGIC_PLAN_SORT:
|
||||
if (((SSortLogicNode*)pNodeLimitPushTo)->calcGroupId) break;
|
||||
// fall through
|
||||
case QUERY_NODE_LOGIC_PLAN_FILL:
|
||||
case QUERY_NODE_LOGIC_PLAN_SORT: {
|
||||
cloneLimit(pNodeWithLimit, pNodeLimitPushTo, CLONE_LIMIT_SLIMIT);
|
||||
SNode* pChild = NULL;
|
||||
FOREACH(pChild, pNodeLimitPushTo->pChildren) { pushDownLimitHow(pNodeLimitPushTo, (SLogicNode*)pChild); }
|
||||
return true;
|
||||
}
|
||||
case QUERY_NODE_LOGIC_PLAN_AGG: {
|
||||
if (nodeType(pNodeWithLimit) == QUERY_NODE_LOGIC_PLAN_PROJECT &&
|
||||
(isPartTagAgg((SAggLogicNode*)pNodeLimitPushTo) || isPartTableAgg((SAggLogicNode*)pNodeLimitPushTo))) {
|
||||
|
|
@ -3587,6 +3573,96 @@ static int32_t stableJoinOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicS
|
|||
return stbJoinOptRewriteStableJoin(pCxt, pNode, pLogicSubplan);
|
||||
}
|
||||
|
||||
static bool partColOptShouldBeOptimized(SLogicNode* pNode) {
|
||||
if (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode)) {
|
||||
SPartitionLogicNode* pPartition = (SPartitionLogicNode*)pNode;
|
||||
if (keysHasCol(pPartition->pPartitionKeys)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static SSortLogicNode* partColOptCreateSort(SPartitionLogicNode* pPartition) {
|
||||
SNode* node;
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
SSortLogicNode* pSort = (SSortLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_SORT);
|
||||
if (pSort) {
|
||||
bool alreadyPartByPKTs = false;
|
||||
pSort->groupSort = false;
|
||||
FOREACH(node, pPartition->pPartitionKeys) {
|
||||
SOrderByExprNode* pOrder = (SOrderByExprNode*)nodesMakeNode(QUERY_NODE_ORDER_BY_EXPR);
|
||||
if (QUERY_NODE_COLUMN == nodeType(node) && ((SColumnNode*)node)->colId == pPartition->pkTsColId &&
|
||||
((SColumnNode*)node)->tableId == pPartition->pkTsColTbId)
|
||||
alreadyPartByPKTs = true;
|
||||
if (!pOrder) {
|
||||
code = TSDB_CODE_OUT_OF_MEMORY;
|
||||
} else {
|
||||
nodesListMakeAppend(&pSort->pSortKeys, (SNode*)pOrder);
|
||||
pOrder->order = ORDER_ASC;
|
||||
pOrder->pExpr = nodesCloneNode(node);
|
||||
if (!pOrder->pExpr) code = TSDB_CODE_OUT_OF_MEMORY;
|
||||
}
|
||||
}
|
||||
|
||||
if (pPartition->needBlockOutputTsOrder && !alreadyPartByPKTs) {
|
||||
SOrderByExprNode* pOrder = (SOrderByExprNode*)nodesMakeNode(QUERY_NODE_ORDER_BY_EXPR);
|
||||
if (!pOrder) {
|
||||
code = TSDB_CODE_OUT_OF_MEMORY;
|
||||
} else {
|
||||
pSort->excludePkCol = true;
|
||||
nodesListMakeAppend(&pSort->pSortKeys, (SNode*)pOrder);
|
||||
pOrder->order = ORDER_ASC;
|
||||
pOrder->pExpr = 0;
|
||||
FOREACH(node, pPartition->node.pTargets) {
|
||||
if (nodeType(node) == QUERY_NODE_COLUMN) {
|
||||
SColumnNode* pCol = (SColumnNode*)node;
|
||||
if (pCol->colId == pPartition->pkTsColId && pCol->tableId == pPartition->pkTsColTbId) {
|
||||
pOrder->pExpr = nodesCloneNode((SNode*)pCol);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!pOrder->pExpr) {
|
||||
code = TSDB_CODE_PAR_INTERNAL_ERROR;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (code != TSDB_CODE_SUCCESS) {
|
||||
nodesDestroyNode((SNode*)pSort);
|
||||
pSort = NULL;
|
||||
}
|
||||
return pSort;
|
||||
}
|
||||
|
||||
static int32_t partitionColsOpt(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
SPartitionLogicNode* pNode = (SPartitionLogicNode*)optFindPossibleNode(pLogicSubplan->pNode, partColOptShouldBeOptimized);
|
||||
if (NULL == pNode) return TSDB_CODE_SUCCESS;
|
||||
|
||||
SLogicNode* pRootNode = getLogicNodeRootNode((SLogicNode*)pNode);
|
||||
if (!pRootNode->pHint || !getSortForGroupOptHint(pRootNode->pHint)) {
|
||||
return code;
|
||||
}
|
||||
|
||||
// replace with sort node
|
||||
SSortLogicNode* pSort = partColOptCreateSort(pNode);
|
||||
if (!pSort) {
|
||||
// if sort create failed, we eat the error, skip the optimization
|
||||
code = TSDB_CODE_SUCCESS;
|
||||
} else {
|
||||
TSWAP(pSort->node.pChildren, pNode->node.pChildren);
|
||||
TSWAP(pSort->node.pTargets, pNode->node.pTargets);
|
||||
optResetParent((SLogicNode*)pSort);
|
||||
pSort->calcGroupId = true;
|
||||
code = replaceLogicNode(pLogicSubplan, (SLogicNode*)pNode, (SLogicNode*)pSort);
|
||||
if (code == TSDB_CODE_SUCCESS) {
|
||||
pCxt->optimized = true;
|
||||
} else {
|
||||
nodesDestroyNode((SNode*)pSort);
|
||||
}
|
||||
}
|
||||
return code;
|
||||
}
|
||||
|
||||
// clang-format off
|
||||
static const SOptimizeRule optimizeRuleSet[] = {
|
||||
|
|
@ -3606,6 +3682,7 @@ static const SOptimizeRule optimizeRuleSet[] = {
|
|||
{.pName = "TableCountScan", .optimizeFunc = tableCountScanOptimize},
|
||||
{.pName = "EliminateProject", .optimizeFunc = eliminateProjOptimize},
|
||||
{.pName = "EliminateSetOperator", .optimizeFunc = eliminateSetOpOptimize},
|
||||
{.pName = "PartitionCols", .optimizeFunc = partitionColsOpt},
|
||||
};
|
||||
// clang-format on
|
||||
|
||||
|
|
|
|||
|
|
@ -1749,6 +1749,8 @@ static int32_t createSortPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren
|
|||
SNodeList* pPrecalcExprs = NULL;
|
||||
SNodeList* pSortKeys = NULL;
|
||||
int32_t code = rewritePrecalcExprs(pCxt, pSortLogicNode->pSortKeys, &pPrecalcExprs, &pSortKeys);
|
||||
pSort->calcGroupId = pSortLogicNode->calcGroupId;
|
||||
pSort->excludePkCol = pSortLogicNode->excludePkCol;
|
||||
|
||||
SDataBlockDescNode* pChildTupe = (((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc);
|
||||
// push down expression to pOutputDataBlockDesc of child node
|
||||
|
|
@ -1797,6 +1799,7 @@ static int32_t createPartitionPhysiNodeImpl(SPhysiPlanContext* pCxt, SNodeList*
|
|||
SNodeList* pPrecalcExprs = NULL;
|
||||
SNodeList* pPartitionKeys = NULL;
|
||||
int32_t code = rewritePrecalcExprs(pCxt, pPartLogicNode->pPartitionKeys, &pPrecalcExprs, &pPartitionKeys);
|
||||
pPart->needBlockOutputTsOrder = pPartLogicNode->needBlockOutputTsOrder;
|
||||
|
||||
SDataBlockDescNode* pChildTupe = (((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc);
|
||||
// push down expression to pOutputDataBlockDesc of child node
|
||||
|
|
@ -1818,6 +1821,22 @@ static int32_t createPartitionPhysiNodeImpl(SPhysiPlanContext* pCxt, SNodeList*
|
|||
}
|
||||
}
|
||||
|
||||
if (pPart->needBlockOutputTsOrder) {
|
||||
SNode* node;
|
||||
bool found = false;
|
||||
FOREACH(node, pPartLogicNode->node.pTargets) {
|
||||
if (nodeType(node) == QUERY_NODE_COLUMN) {
|
||||
SColumnNode* pCol = (SColumnNode*)node;
|
||||
if (pCol->tableId == pPartLogicNode->pkTsColTbId && pCol->colId == pPartLogicNode->pkTsColId) {
|
||||
pPart->tsSlotId = pCol->slotId;
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!found) code = TSDB_CODE_PLAN_INTERNAL_ERROR;
|
||||
}
|
||||
|
||||
if (TSDB_CODE_SUCCESS == code) {
|
||||
code = setConditionsSlotId(pCxt, (const SLogicNode*)pPartLogicNode, (SPhysiNode*)pPart);
|
||||
}
|
||||
|
|
@ -1944,6 +1963,7 @@ static int32_t createMergePhysiNode(SPhysiPlanContext* pCxt, SMergeLogicNode* pM
|
|||
pMerge->srcGroupId = pMergeLogicNode->srcGroupId;
|
||||
pMerge->groupSort = pMergeLogicNode->groupSort;
|
||||
pMerge->ignoreGroupId = pMergeLogicNode->ignoreGroupId;
|
||||
pMerge->inputWithGroupId = pMergeLogicNode->inputWithGroupId;
|
||||
|
||||
int32_t code = addDataBlockSlots(pCxt, pMergeLogicNode->pInputs, pMerge->node.pOutputDataBlockDesc);
|
||||
|
||||
|
|
|
|||
|
|
@ -244,7 +244,12 @@ static bool stbSplHasMultiTbScan(bool streamQuery, SLogicNode* pNode) {
|
|||
}
|
||||
pChild = nodesListGetNode(((SLogicNode*)pChild)->pChildren, 0);
|
||||
}
|
||||
return (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pChild) && stbSplIsMultiTbScan(streamQuery, (SScanLogicNode*)pChild));
|
||||
if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pChild) && stbSplIsMultiTbScan(streamQuery, (SScanLogicNode*)pChild)) {
|
||||
return true;
|
||||
} else if (QUERY_NODE_LOGIC_PLAN_SORT == nodeType(pChild)) {
|
||||
return stbSplHasMultiTbScan(streamQuery, (SLogicNode*)pChild);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static bool stbSplIsMultiTbScanChild(bool streamQuery, SLogicNode* pNode) {
|
||||
|
|
@ -519,6 +524,11 @@ static int32_t stbSplRewriteFromMergeNode(SMergeLogicNode* pMerge, SLogicNode* p
|
|||
}
|
||||
break;
|
||||
}
|
||||
case QUERY_NODE_LOGIC_PLAN_SORT: {
|
||||
SSortLogicNode* pSort = (SSortLogicNode*)pNode;
|
||||
if (pSort->calcGroupId) pMerge->inputWithGroupId = true;
|
||||
break;
|
||||
}
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -358,12 +358,12 @@ static bool stbNotSystemScan(SLogicNode* pNode) {
|
|||
}
|
||||
}
|
||||
|
||||
static bool stbHasPartTbname(SNodeList* pPartKeys) {
|
||||
if (NULL == pPartKeys) {
|
||||
bool keysHasTbname(SNodeList* pKeys) {
|
||||
if (NULL == pKeys) {
|
||||
return false;
|
||||
}
|
||||
SNode* pPartKey = NULL;
|
||||
FOREACH(pPartKey, pPartKeys) {
|
||||
FOREACH(pPartKey, pKeys) {
|
||||
if (QUERY_NODE_GROUPING_SET == nodeType(pPartKey)) {
|
||||
pPartKey = nodesListGetNode(((SGroupingSetNode*)pPartKey)->pParameterList, 0);
|
||||
}
|
||||
|
|
@ -390,10 +390,10 @@ bool isPartTableAgg(SAggLogicNode* pAgg) {
|
|||
return false;
|
||||
}
|
||||
if (NULL != pAgg->pGroupKeys) {
|
||||
return stbHasPartTbname(pAgg->pGroupKeys) &&
|
||||
return (pAgg->isGroupTb || keysHasTbname(pAgg->pGroupKeys)) &&
|
||||
stbNotSystemScan((SLogicNode*)nodesListGetNode(pAgg->node.pChildren, 0));
|
||||
}
|
||||
return stbHasPartTbname(stbGetPartKeys((SLogicNode*)nodesListGetNode(pAgg->node.pChildren, 0)));
|
||||
return pAgg->isPartTb || keysHasTbname(stbGetPartKeys((SLogicNode*)nodesListGetNode(pAgg->node.pChildren, 0)));
|
||||
}
|
||||
|
||||
static bool stbHasPartTag(SNodeList* pPartKeys) {
|
||||
|
|
@ -430,6 +430,17 @@ bool getBatchScanOptionFromHint(SNodeList* pList) {
|
|||
return batchScan;
|
||||
}
|
||||
|
||||
bool getSortForGroupOptHint(SNodeList* pList) {
|
||||
SNode* pNode;
|
||||
FOREACH(pNode, pList) {
|
||||
SHintNode* pHint = (SHintNode*)pNode;
|
||||
if (pHint->option == HINT_SORT_FOR_GROUP) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
int32_t collectTableAliasFromNodes(SNode* pNode, SSHashObj** ppRes) {
|
||||
int32_t code = TSDB_CODE_SUCCESS;
|
||||
SLogicNode* pCurr = (SLogicNode*)pNode;
|
||||
|
|
@ -467,7 +478,7 @@ bool isPartTagAgg(SAggLogicNode* pAgg) {
|
|||
}
|
||||
|
||||
bool isPartTableWinodw(SWindowLogicNode* pWindow) {
|
||||
return stbHasPartTbname(stbGetPartKeys((SLogicNode*)nodesListGetNode(pWindow->node.pChildren, 0)));
|
||||
return pWindow->isPartTb || keysHasTbname(stbGetPartKeys((SLogicNode*)nodesListGetNode(pWindow->node.pChildren, 0)));
|
||||
}
|
||||
|
||||
bool cloneLimit(SLogicNode* pParent, SLogicNode* pChild, uint8_t cloneWhat) {
|
||||
|
|
@ -490,3 +501,19 @@ bool cloneLimit(SLogicNode* pParent, SLogicNode* pChild, uint8_t cloneWhat) {
|
|||
}
|
||||
return cloned;
|
||||
}
|
||||
|
||||
static EDealRes partTagsOptHasColImpl(SNode* pNode, void* pContext) {
|
||||
if (QUERY_NODE_COLUMN == nodeType(pNode)) {
|
||||
if (COLUMN_TYPE_TAG != ((SColumnNode*)pNode)->colType && COLUMN_TYPE_TBNAME != ((SColumnNode*)pNode)->colType) {
|
||||
*(bool*)pContext = true;
|
||||
return DEAL_RES_END;
|
||||
}
|
||||
}
|
||||
return DEAL_RES_CONTINUE;
|
||||
}
|
||||
|
||||
bool keysHasCol(SNodeList* pKeys) {
|
||||
bool hasCol = false;
|
||||
nodesWalkExprs(pKeys, partTagsOptHasColImpl, &hasCol);
|
||||
return hasCol;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1712,25 +1712,24 @@ int streamStateGetCfIdx(SStreamState* pState, const char* funcName) {
|
|||
if (pState != NULL && idx != -1) {
|
||||
SBackendCfWrapper* wrapper = pState->pTdbState->pBackendCfWrapper;
|
||||
rocksdb_column_family_handle_t* cf = NULL;
|
||||
taosThreadRwlockRdlock(&wrapper->rwLock);
|
||||
taosThreadRwlockWrlock(&wrapper->rwLock);
|
||||
cf = wrapper->pHandle[idx];
|
||||
taosThreadRwlockUnlock(&wrapper->rwLock);
|
||||
if (cf == NULL) {
|
||||
char buf[128] = {0};
|
||||
GEN_COLUMN_FAMILY_NAME(buf, wrapper->idstr, ginitDict[idx].key);
|
||||
char* err = NULL;
|
||||
|
||||
taosThreadRwlockWrlock(&wrapper->rwLock);
|
||||
cf = rocksdb_create_column_family(wrapper->rocksdb, wrapper->cfOpts[idx], buf, &err);
|
||||
if (err != NULL) {
|
||||
idx = -1;
|
||||
qError("failed to to open cf, %p %s_%s, reason:%s", pState, wrapper->idstr, funcName, err);
|
||||
taosMemoryFree(err);
|
||||
} else {
|
||||
qDebug("succ to to open cf, %p %s_%s", pState, wrapper->idstr, funcName);
|
||||
wrapper->pHandle[idx] = cf;
|
||||
}
|
||||
taosThreadRwlockUnlock(&wrapper->rwLock);
|
||||
}
|
||||
taosThreadRwlockUnlock(&wrapper->rwLock);
|
||||
}
|
||||
|
||||
return idx;
|
||||
|
|
|
|||
|
|
@ -29,6 +29,8 @@ extern "C" {
|
|||
#include "ttrace.h"
|
||||
#include "tutil.h"
|
||||
|
||||
typedef bool (*FilteFunc)(void* arg);
|
||||
|
||||
typedef void* queue[2];
|
||||
/* Private macros. */
|
||||
#define QUEUE_NEXT(q) (*(queue**)&((*(q))[0]))
|
||||
|
|
@ -308,6 +310,7 @@ int transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pMsg, STransMs
|
|||
int transSendResponse(const STransMsg* msg);
|
||||
int transRegisterMsg(const STransMsg* msg);
|
||||
int transSetDefaultAddr(void* shandle, const char* ip, const char* fqdn);
|
||||
void transSetIpWhiteList(void* shandle, void* arg, FilteFunc* func);
|
||||
|
||||
int transSockInfo2Str(struct sockaddr* sockname, char* dst);
|
||||
|
||||
|
|
@ -434,6 +437,23 @@ int32_t transGetRefMgt();
|
|||
int32_t transGetInstMgt();
|
||||
|
||||
void transHttpEnvDestroy();
|
||||
|
||||
typedef struct {
|
||||
uint32_t netmask;
|
||||
uint32_t address;
|
||||
uint32_t network;
|
||||
uint32_t broadcast;
|
||||
char info[32];
|
||||
int8_t type;
|
||||
} SubnetUtils;
|
||||
|
||||
int32_t subnetInit(SubnetUtils* pUtils, SIpV4Range* pRange);
|
||||
int32_t subnetCheckIp(SubnetUtils* pUtils, uint32_t ip);
|
||||
int32_t subnetDebugInfoToBuf(SubnetUtils* pUtils, char* buf);
|
||||
|
||||
int32_t transUtilSIpRangeToStr(SIpV4Range* pRange, char* buf);
|
||||
int32_t transUtilSWhiteListToStr(SIpWhiteList* pWhiteList, char** ppBuf);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -178,13 +178,21 @@ void rpcUnrefHandle(void* handle, int8_t type) { (*taosUnRefHandle[type])(handle
|
|||
int rpcRegisterBrokenLinkArg(SRpcMsg* msg) { return transRegisterMsg(msg); }
|
||||
int rpcReleaseHandle(void* handle, int8_t type) { return (*transReleaseHandle[type])(handle); }
|
||||
|
||||
// client only
|
||||
int rpcSetDefaultAddr(void* thandle, const char* ip, const char* fqdn) {
|
||||
// later
|
||||
return transSetDefaultAddr(thandle, ip, fqdn);
|
||||
}
|
||||
// server only
|
||||
void rpcSetIpWhite(void* thandle, void* arg) { transSetIpWhiteList(thandle, arg, NULL); }
|
||||
|
||||
void* rpcAllocHandle() { return (void*)transAllocHandle(); }
|
||||
|
||||
int32_t rpcUtilSIpRangeToStr(SIpV4Range* pRange, char* buf) { return transUtilSIpRangeToStr(pRange, buf); }
|
||||
int32_t rpcUtilSWhiteListToStr(SIpWhiteList* pWhiteList, char** ppBuf) {
|
||||
return transUtilSWhiteListToStr(pWhiteList, ppBuf);
|
||||
}
|
||||
|
||||
int32_t rpcInit() {
|
||||
transInit();
|
||||
return 0;
|
||||
|
|
|
|||
|
|
@ -654,3 +654,106 @@ void transDestoryExHandle(void* handle) {
|
|||
}
|
||||
taosMemoryFree(handle);
|
||||
}
|
||||
|
||||
// void subnetIp2int(const char* const ip_addr, uint8_t* dst) {
|
||||
// char ip_addr_cpy[20];
|
||||
// char ip[5];
|
||||
|
||||
// tstrncpy(ip_addr_cpy, ip_addr, sizeof(ip_addr_cpy));
|
||||
|
||||
// char *s_start, *s_end;
|
||||
// s_start = ip_addr_cpy;
|
||||
// s_end = ip_addr_cpy;
|
||||
|
||||
// int32_t k = 0;
|
||||
|
||||
// for (k = 0; *s_start != '\0'; s_start = s_end) {
|
||||
// for (s_end = s_start; *s_end != '.' && *s_end != '\0'; s_end++) {
|
||||
// }
|
||||
// if (*s_end == '.') {
|
||||
// *s_end = '\0';
|
||||
// s_end++;
|
||||
// }
|
||||
// dst[k++] = (char)atoi(s_start);
|
||||
// }
|
||||
// }
|
||||
|
||||
uint32_t subnetIpRang2Int(SIpV4Range* pRange) {
|
||||
uint32_t ip = pRange->ip;
|
||||
return ((ip & 0xFF) << 24) | ((ip & 0xFF00) << 8) | ((ip & 0xFF0000) >> 8) | ((ip >> 24) & 0xFF);
|
||||
}
|
||||
int32_t subnetInit(SubnetUtils* pUtils, SIpV4Range* pRange) {
|
||||
if (pRange->mask == 32) {
|
||||
pUtils->type = 0;
|
||||
pUtils->address = pRange->ip;
|
||||
return 0;
|
||||
}
|
||||
pUtils->address = subnetIpRang2Int(pRange);
|
||||
|
||||
for (int i = 0; i < pRange->mask; i++) {
|
||||
pUtils->netmask |= (1 << (31 - i));
|
||||
}
|
||||
|
||||
pUtils->network = pUtils->address & pUtils->netmask;
|
||||
pUtils->broadcast = (pUtils->network) | (pUtils->netmask ^ 0xFFFFFFFF);
|
||||
pUtils->type = (pRange->mask == 32 ? 0 : 1);
|
||||
|
||||
return 0;
|
||||
}
|
||||
int32_t subnetDebugInfoToBuf(SubnetUtils* pUtils, char* buf) {
|
||||
sprintf(buf, "raw: %s, address: %d, netmask:%d, network:%d, broadcast:%d", pUtils->info, pUtils->address,
|
||||
pUtils->netmask, pUtils->network, pUtils->broadcast);
|
||||
return 0;
|
||||
}
|
||||
int32_t subnetCheckIp(SubnetUtils* pUtils, uint32_t ip) {
|
||||
// impl later
|
||||
if (pUtils == NULL) return false;
|
||||
if (pUtils->type == 0) {
|
||||
return pUtils->address == ip;
|
||||
} else {
|
||||
SIpV4Range range = {.ip = ip, .mask = 32};
|
||||
|
||||
uint32_t t = subnetIpRang2Int(&range);
|
||||
return t >= pUtils->network && t <= pUtils->broadcast;
|
||||
}
|
||||
}
|
||||
|
||||
int32_t transUtilSIpRangeToStr(SIpV4Range* pRange, char* buf) {
|
||||
int32_t len = 0;
|
||||
|
||||
struct in_addr addr;
|
||||
addr.s_addr = pRange->ip;
|
||||
|
||||
uv_inet_ntop(AF_INET, &addr, buf, 32);
|
||||
|
||||
len = strlen(buf);
|
||||
|
||||
if (pRange->mask != 32) {
|
||||
len += sprintf(buf + len, "/%d", pRange->mask);
|
||||
}
|
||||
buf[len] = 0;
|
||||
return len;
|
||||
}
|
||||
|
||||
int32_t transUtilSWhiteListToStr(SIpWhiteList* pList, char** ppBuf) {
|
||||
if (pList->num == 0) {
|
||||
*ppBuf = NULL;
|
||||
return 0;
|
||||
}
|
||||
int32_t len = 0;
|
||||
char* pBuf = taosMemoryCalloc(1, pList->num * 36);
|
||||
|
||||
for (int i = 0; i < pList->num; i++) {
|
||||
SIpV4Range* pRange = &pList->pIpRange[i];
|
||||
|
||||
char tbuf[32] = {0};
|
||||
int tlen = transUtilSIpRangeToStr(pRange, tbuf);
|
||||
len += sprintf(pBuf + len, "%s,", tbuf);
|
||||
}
|
||||
if (len > 0) {
|
||||
pBuf[len - 1] = 0;
|
||||
}
|
||||
|
||||
*ppBuf = pBuf;
|
||||
return len;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -43,6 +43,7 @@ typedef struct SSvrConn {
|
|||
|
||||
ConnStatus status;
|
||||
|
||||
uint32_t serverIp;
|
||||
uint32_t clientIp;
|
||||
uint16_t port;
|
||||
|
||||
|
|
@ -55,6 +56,8 @@ typedef struct SSvrConn {
|
|||
char user[TSDB_UNI_LEN]; // user ID for the link
|
||||
char secret[TSDB_PASSWORD_LEN];
|
||||
char ckey[TSDB_PASSWORD_LEN]; // ciphering key
|
||||
|
||||
int64_t whiteListVer;
|
||||
} SSvrConn;
|
||||
|
||||
typedef struct SSvrMsg {
|
||||
|
|
@ -62,8 +65,22 @@ typedef struct SSvrMsg {
|
|||
STransMsg msg;
|
||||
queue q;
|
||||
STransMsgType type;
|
||||
|
||||
void* arg;
|
||||
FilteFunc func;
|
||||
|
||||
} SSvrMsg;
|
||||
|
||||
typedef struct {
|
||||
int64_t ver;
|
||||
SIpWhiteList* pList;
|
||||
// SArray* list;
|
||||
|
||||
} SWhiteUserList;
|
||||
typedef struct {
|
||||
SHashObj* pList;
|
||||
int64_t ver;
|
||||
} SIpWhiteListTab;
|
||||
typedef struct SWorkThrd {
|
||||
TdThread thread;
|
||||
uv_connect_t connect_req;
|
||||
|
|
@ -77,6 +94,10 @@ typedef struct SWorkThrd {
|
|||
queue conn;
|
||||
void* pTransInst;
|
||||
bool quit;
|
||||
|
||||
SIpWhiteListTab* pWhiteList;
|
||||
int64_t whiteListVer;
|
||||
int8_t enableIpWhiteList;
|
||||
} SWorkThrd;
|
||||
|
||||
typedef struct SServerObj {
|
||||
|
|
@ -99,6 +120,14 @@ typedef struct SServerObj {
|
|||
bool inited;
|
||||
} SServerObj;
|
||||
|
||||
SIpWhiteListTab* uvWhiteListCreate();
|
||||
void uvWhiteListDestroy(SIpWhiteListTab* pWhite);
|
||||
void uvWhiteListAdd(SIpWhiteListTab* pWhite, char* user, SIpWhiteList* pList, int64_t ver);
|
||||
void uvWhiteListUpdate(SIpWhiteListTab* pWhite, SHashObj* pTable);
|
||||
bool uvWhiteListCheckConn(SIpWhiteListTab* pWhite, SSvrConn* pConn);
|
||||
bool uvWhiteListFilte(SIpWhiteListTab* pWhite, char* user, uint32_t ip, int64_t ver);
|
||||
void uvWhiteListSetConnVer(SIpWhiteListTab* pWhite, SSvrConn* pConn);
|
||||
|
||||
static void uvAllocConnBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf);
|
||||
static void uvAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf);
|
||||
static void uvOnRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf);
|
||||
|
|
@ -141,8 +170,9 @@ static void uvHandleQuit(SSvrMsg* msg, SWorkThrd* thrd);
|
|||
static void uvHandleRelease(SSvrMsg* msg, SWorkThrd* thrd);
|
||||
static void uvHandleResp(SSvrMsg* msg, SWorkThrd* thrd);
|
||||
static void uvHandleRegister(SSvrMsg* msg, SWorkThrd* thrd);
|
||||
static void uvHandleUpdate(SSvrMsg* pMsg, SWorkThrd* thrd);
|
||||
static void (*transAsyncHandle[])(SSvrMsg* msg, SWorkThrd* thrd) = {uvHandleResp, uvHandleQuit, uvHandleRelease,
|
||||
uvHandleRegister, NULL};
|
||||
uvHandleRegister, uvHandleUpdate};
|
||||
|
||||
static void uvDestroyConn(uv_handle_t* handle);
|
||||
|
||||
|
|
@ -181,8 +211,134 @@ static void uvHandleActivityTimeout(uv_timer_t* handle) {
|
|||
tDebug("%p timeout since no activity", conn);
|
||||
}
|
||||
|
||||
static bool uvCheckIp(SIpV4Range* pRange, int32_t ip) {
|
||||
// impl later
|
||||
SubnetUtils subnet = {0};
|
||||
if (subnetInit(&subnet, pRange) != 0) {
|
||||
return false;
|
||||
}
|
||||
return subnetCheckIp(&subnet, ip);
|
||||
}
|
||||
SIpWhiteListTab* uvWhiteListCreate() {
|
||||
SIpWhiteListTab* pWhiteList = taosMemoryCalloc(1, sizeof(SIpWhiteListTab));
|
||||
|
||||
pWhiteList->pList = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), 0, HASH_NO_LOCK);
|
||||
pWhiteList->ver = -1;
|
||||
return pWhiteList;
|
||||
}
|
||||
void uvWhiteListDestroy(SIpWhiteListTab* pWhite) {
|
||||
SHashObj* pWhiteList = pWhite->pList;
|
||||
void* pIter = taosHashIterate(pWhiteList, NULL);
|
||||
while (pIter) {
|
||||
SWhiteUserList* pUserList = *(SWhiteUserList**)pIter;
|
||||
taosMemoryFree(pUserList->pList);
|
||||
taosMemoryFree(pUserList);
|
||||
|
||||
pIter = taosHashIterate(pWhiteList, pIter);
|
||||
}
|
||||
taosHashCleanup(pWhiteList);
|
||||
taosMemoryFree(pWhite);
|
||||
}
|
||||
|
||||
void uvWhiteListToStr(SWhiteUserList* plist, char* user, char** ppBuf) {
|
||||
char* tmp = NULL;
|
||||
int32_t tlen = transUtilSWhiteListToStr(plist->pList, &tmp);
|
||||
|
||||
char* pBuf = taosMemoryCalloc(1, tlen + 64);
|
||||
int32_t len = sprintf(pBuf, "user: %s, ver: %" PRId64 ", ip: {%s}", user, plist->ver, tmp);
|
||||
taosMemoryFree(tmp);
|
||||
|
||||
*ppBuf = pBuf;
|
||||
}
|
||||
void uvWhiteListDebug(SIpWhiteListTab* pWrite) {
|
||||
SHashObj* pWhiteList = pWrite->pList;
|
||||
void* pIter = taosHashIterate(pWhiteList, NULL);
|
||||
while (pIter) {
|
||||
size_t klen = 0;
|
||||
char user[TSDB_USER_LEN + 1] = {0};
|
||||
char* pUser = taosHashGetKey(pIter, &klen);
|
||||
memcpy(user, pUser, klen);
|
||||
|
||||
SWhiteUserList* pUserList = *(SWhiteUserList**)pIter;
|
||||
|
||||
char* buf = NULL;
|
||||
uvWhiteListToStr(pUserList, user, &buf);
|
||||
tDebug("ip-white-list %s", buf);
|
||||
taosMemoryFree(buf);
|
||||
pIter = taosHashIterate(pWhiteList, pIter);
|
||||
}
|
||||
}
|
||||
void uvWhiteListAdd(SIpWhiteListTab* pWhite, char* user, SIpWhiteList* plist, int64_t ver) {
|
||||
SHashObj* pWhiteList = pWhite->pList;
|
||||
|
||||
SWhiteUserList** ppUserList = taosHashGet(pWhiteList, user, strlen(user));
|
||||
if (ppUserList == NULL || *ppUserList == NULL) {
|
||||
SWhiteUserList* pUserList = taosMemoryCalloc(1, sizeof(SWhiteUserList));
|
||||
pUserList->ver = ver;
|
||||
|
||||
pUserList->pList = plist;
|
||||
|
||||
taosHashPut(pWhiteList, user, strlen(user), &pUserList, sizeof(void*));
|
||||
} else {
|
||||
SWhiteUserList* pUserList = *ppUserList;
|
||||
|
||||
taosMemoryFreeClear(pUserList->pList);
|
||||
pUserList->ver = ver;
|
||||
pUserList->pList = plist;
|
||||
}
|
||||
uvWhiteListDebug(pWhite);
|
||||
}
|
||||
|
||||
void uvWhiteListUpdate(SIpWhiteListTab* pWhite, SHashObj* pTable) {
|
||||
pWhite->ver++;
|
||||
// impl later
|
||||
}
|
||||
|
||||
static bool uvWhiteListIsDefaultAddr(uint32_t ip) {
|
||||
// 127.0.0.1
|
||||
static SIpV4Range range = {.ip = 16777343, .mask = 32};
|
||||
return range.ip == ip;
|
||||
}
|
||||
bool uvWhiteListFilte(SIpWhiteListTab* pWhite, char* user, uint32_t ip, int64_t ver) {
|
||||
// impl check
|
||||
SHashObj* pWhiteList = pWhite->pList;
|
||||
bool valid = false;
|
||||
|
||||
if (uvWhiteListIsDefaultAddr(ip)) return true;
|
||||
|
||||
SWhiteUserList** ppList = taosHashGet(pWhiteList, user, strlen(user));
|
||||
if (ppList == NULL || *ppList == NULL) {
|
||||
return false;
|
||||
}
|
||||
SWhiteUserList* pUserList = *ppList;
|
||||
if (pUserList->ver == ver) return true;
|
||||
|
||||
SIpWhiteList* pIpWhiteList = pUserList->pList;
|
||||
for (int i = 0; i < pIpWhiteList->num; i++) {
|
||||
SIpV4Range* range = &pIpWhiteList->pIpRange[i];
|
||||
if (uvCheckIp(range, ip)) {
|
||||
valid = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
return valid;
|
||||
}
|
||||
bool uvWhiteListCheckConn(SIpWhiteListTab* pWhite, SSvrConn* pConn) {
|
||||
if (pConn->inType == TDMT_MND_STATUS || pConn->inType == TDMT_MND_RETRIEVE_IP_WHITE ||
|
||||
pConn->serverIp == pConn->clientIp ||
|
||||
pWhite->ver == pConn->whiteListVer /*|| strncmp(pConn->user, "_dnd", strlen("_dnd")) == 0*/)
|
||||
return true;
|
||||
|
||||
return uvWhiteListFilte(pWhite, pConn->user, pConn->clientIp, pConn->whiteListVer);
|
||||
}
|
||||
void uvWhiteListSetConnVer(SIpWhiteListTab* pWhite, SSvrConn* pConn) {
|
||||
// if conn already check by current whiteLis
|
||||
pConn->whiteListVer = pWhite->ver;
|
||||
}
|
||||
|
||||
static bool uvHandleReq(SSvrConn* pConn) {
|
||||
STrans* pTransInst = pConn->pTransInst;
|
||||
SWorkThrd* pThrd = pConn->hostThrd;
|
||||
|
||||
STransMsgHead* pHead = NULL;
|
||||
|
||||
|
|
@ -199,8 +355,17 @@ static bool uvHandleReq(SSvrConn* pConn) {
|
|||
pHead->code = htonl(pHead->code);
|
||||
pHead->msgLen = htonl(pHead->msgLen);
|
||||
|
||||
pConn->inType = pHead->msgType;
|
||||
memcpy(pConn->user, pHead->user, strlen(pHead->user));
|
||||
|
||||
int8_t forbiddenIp = 0;
|
||||
if (pThrd->enableIpWhiteList) {
|
||||
forbiddenIp = !uvWhiteListCheckConn(pThrd->pWhiteList, pConn) ? 1 : 0;
|
||||
if (forbiddenIp == 0) {
|
||||
uvWhiteListSetConnVer(pThrd->pWhiteList, pConn);
|
||||
}
|
||||
}
|
||||
|
||||
if (uvRecvReleaseReq(pConn, pHead)) {
|
||||
return true;
|
||||
}
|
||||
|
|
@ -219,7 +384,6 @@ static bool uvHandleReq(SSvrConn* pConn) {
|
|||
transMsg.msgType = pHead->msgType;
|
||||
transMsg.code = pHead->code;
|
||||
|
||||
pConn->inType = pHead->msgType;
|
||||
if (pConn->status == ConnNormal) {
|
||||
if (pHead->persist == 1) {
|
||||
pConn->status = ConnAcquire;
|
||||
|
|
@ -262,6 +426,7 @@ static bool uvHandleReq(SSvrConn* pConn) {
|
|||
transMsg.info.refId = pConn->refId;
|
||||
transMsg.info.traceId = pHead->traceId;
|
||||
transMsg.info.cliVer = htonl(pHead->compatibilityVer);
|
||||
transMsg.info.forbiddenIp = forbiddenIp;
|
||||
|
||||
tGTrace("%s handle %p conn:%p translated to app, refId:%" PRIu64, transLabel(pTransInst), transMsg.info.handle, pConn,
|
||||
pConn->refId);
|
||||
|
|
@ -395,7 +560,7 @@ static void uvOnPipeWriteCb(uv_write_t* req, int status) {
|
|||
if (status == 0) {
|
||||
tTrace("success to dispatch conn to work thread");
|
||||
} else {
|
||||
tError("fail to dispatch conn to work thread, code:%s", uv_strerror(status));
|
||||
tError("fail to dispatch conn to work thread, reason:%s", uv_strerror(status));
|
||||
}
|
||||
if (!uv_is_closing((uv_handle_t*)req->data)) {
|
||||
uv_close((uv_handle_t*)req->data, uvFreeCb);
|
||||
|
|
@ -542,7 +707,7 @@ void uvWorkerAsyncCb(uv_async_t* handle) {
|
|||
}
|
||||
|
||||
// release handle to rpc init
|
||||
if (msg->type == Quit) {
|
||||
if (msg->type == Quit || msg->type == Update) {
|
||||
(*transAsyncHandle[msg->type])(msg, pThrd);
|
||||
} else {
|
||||
STransMsg transMsg = msg->msg;
|
||||
|
|
@ -638,7 +803,7 @@ static void uvPrepareCb(uv_prepare_t* handle) {
|
|||
continue;
|
||||
}
|
||||
// release handle to rpc init
|
||||
if (msg->type == Quit) {
|
||||
if (msg->type == Quit || msg->type == Update) {
|
||||
(*transAsyncHandle[msg->type])(msg, pThrd);
|
||||
continue;
|
||||
} else {
|
||||
|
|
@ -796,7 +961,10 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) {
|
|||
transSockInfo2Str(&sockname, pConn->src);
|
||||
|
||||
struct sockaddr_in addr = *(struct sockaddr_in*)&peername;
|
||||
struct sockaddr_in saddr = *(struct sockaddr_in*)&sockname;
|
||||
|
||||
pConn->clientIp = addr.sin_addr.s_addr;
|
||||
pConn->serverIp = saddr.sin_addr.s_addr;
|
||||
pConn->port = ntohs(addr.sin_port);
|
||||
|
||||
uv_read_start((uv_stream_t*)(pConn->pTcp), uvAllocRecvBufferCb, uvOnRecvCb);
|
||||
|
|
@ -1066,9 +1234,10 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads,
|
|||
SWorkThrd* thrd = (SWorkThrd*)taosMemoryCalloc(1, sizeof(SWorkThrd));
|
||||
thrd->pTransInst = shandle;
|
||||
thrd->quit = false;
|
||||
srv->pThreadObj[i] = thrd;
|
||||
thrd->pTransInst = shandle;
|
||||
thrd->pWhiteList = uvWhiteListCreate();
|
||||
|
||||
srv->pThreadObj[i] = thrd;
|
||||
srv->pipe[i] = (uv_pipe_t*)taosMemoryCalloc(2, sizeof(uv_pipe_t));
|
||||
thrd->pipe = &(srv->pipe[i][1]); // init read
|
||||
|
||||
|
|
@ -1093,6 +1262,7 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads,
|
|||
thrd->pTransInst = shandle;
|
||||
thrd->quit = false;
|
||||
thrd->pTransInst = shandle;
|
||||
thrd->pWhiteList = uvWhiteListCreate();
|
||||
|
||||
srv->pipe[i] = (uv_pipe_t*)taosMemoryCalloc(2, sizeof(uv_pipe_t));
|
||||
srv->pThreadObj[i] = thrd;
|
||||
|
|
@ -1192,6 +1362,31 @@ void uvHandleRegister(SSvrMsg* msg, SWorkThrd* thrd) {
|
|||
taosMemoryFree(msg);
|
||||
}
|
||||
}
|
||||
void uvHandleUpdate(SSvrMsg* msg, SWorkThrd* thrd) {
|
||||
SUpdateIpWhite* req = msg->arg;
|
||||
if (req != NULL) {
|
||||
for (int i = 0; i < req->numOfUser; i++) {
|
||||
SUpdateUserIpWhite* pUser = &req->pUserIpWhite[i];
|
||||
|
||||
int32_t sz = pUser->numOfRange * sizeof(SIpV4Range);
|
||||
SIpWhiteList* pList = taosMemoryCalloc(1, sz + sizeof(SIpWhiteList));
|
||||
pList->num = pUser->numOfRange;
|
||||
|
||||
memcpy(pList->pIpRange, pUser->pIpRanges, sz);
|
||||
uvWhiteListAdd(thrd->pWhiteList, pUser->user, pList, pUser->ver);
|
||||
}
|
||||
|
||||
thrd->pWhiteList->ver = req->ver;
|
||||
thrd->enableIpWhiteList = 1;
|
||||
|
||||
tFreeSUpdateIpWhiteReq(req);
|
||||
taosMemoryFree(req);
|
||||
} else {
|
||||
thrd->enableIpWhiteList = 0;
|
||||
}
|
||||
taosMemoryFree(msg);
|
||||
return;
|
||||
}
|
||||
void destroyWorkThrd(SWorkThrd* pThrd) {
|
||||
if (pThrd == NULL) {
|
||||
return;
|
||||
|
|
@ -1200,6 +1395,9 @@ void destroyWorkThrd(SWorkThrd* pThrd) {
|
|||
SRV_RELEASE_UV(pThrd->loop);
|
||||
TRANS_DESTROY_ASYNC_POOL_MSG(pThrd->asyncPool, SSvrMsg, destroySmsg);
|
||||
transAsyncPoolDestroy(pThrd->asyncPool);
|
||||
|
||||
uvWhiteListDestroy(pThrd->pWhiteList);
|
||||
|
||||
taosMemoryFree(pThrd->prepare);
|
||||
taosMemoryFree(pThrd->loop);
|
||||
taosMemoryFree(pThrd);
|
||||
|
|
@ -1367,5 +1565,23 @@ _return2:
|
|||
rpcFreeCont(msg->pCont);
|
||||
return -1;
|
||||
}
|
||||
void transSetIpWhiteList(void* thandle, void* arg, FilteFunc* func) {
|
||||
STrans* pTransInst = (STrans*)transAcquireExHandle(transGetInstMgt(), (int64_t)thandle);
|
||||
|
||||
tDebug("ip-white-list update on rpc");
|
||||
SServerObj* svrObj = pTransInst->tcphandle;
|
||||
for (int i = 0; i < svrObj->numOfThreads; i++) {
|
||||
SWorkThrd* pThrd = svrObj->pThreadObj[i];
|
||||
|
||||
SSvrMsg* msg = taosMemoryCalloc(1, sizeof(SSvrMsg));
|
||||
SUpdateIpWhite* pReq = (arg != NULL ? cloneSUpdateIpWhiteReq((SUpdateIpWhite*)arg) : NULL);
|
||||
|
||||
msg->type = Update;
|
||||
msg->arg = pReq;
|
||||
|
||||
transAsyncSend(pThrd->asyncPool, &msg->q);
|
||||
}
|
||||
transReleaseExHandle(transGetInstMgt(), (int64_t)thandle);
|
||||
}
|
||||
|
||||
int transGetConnInfo(void* thandle, STransHandleInfo* pConnInfo) { return -1; }
|
||||
|
|
|
|||
|
|
@ -86,8 +86,9 @@ void taosRandStr(char* str, int32_t size) {
|
|||
}
|
||||
|
||||
void taosRandStr2(char* str, int32_t size) {
|
||||
const char* set = "abcdefghijklmnopqrstuvwxyz0123456789";
|
||||
int32_t len = 36;
|
||||
|
||||
const char* set = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ@";
|
||||
int32_t len = strlen(set);
|
||||
|
||||
for (int32_t i = 0; i < size; ++i) {
|
||||
str[i] = set[taosRand() % len];
|
||||
|
|
|
|||
|
|
@ -895,32 +895,6 @@ int64_t taosCopyFds(TdSocketPtr pSrcSocket, TdSocketPtr pDestSocket, int64_t len
|
|||
}
|
||||
|
||||
// Function converting an IP address string to an uint32_t.
|
||||
uint32_t ip2uint(const char *const ip_addr) {
|
||||
char ip_addr_cpy[20];
|
||||
char ip[5];
|
||||
|
||||
tstrncpy(ip_addr_cpy, ip_addr, sizeof(ip_addr_cpy));
|
||||
|
||||
char *s_start, *s_end;
|
||||
s_start = ip_addr_cpy;
|
||||
s_end = ip_addr_cpy;
|
||||
|
||||
int32_t k;
|
||||
|
||||
for (k = 0; *s_start != '\0'; s_start = s_end) {
|
||||
for (s_end = s_start; *s_end != '.' && *s_end != '\0'; s_end++) {
|
||||
}
|
||||
if (*s_end == '.') {
|
||||
*s_end = '\0';
|
||||
s_end++;
|
||||
}
|
||||
ip[k++] = (char)atoi(s_start);
|
||||
}
|
||||
|
||||
ip[k] = '\0';
|
||||
|
||||
return *((uint32_t *)ip);
|
||||
}
|
||||
|
||||
#endif // endif 0
|
||||
|
||||
|
|
|
|||
|
|
@ -100,6 +100,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_APP_IS_STARTING, "Database is starting
|
|||
TAOS_DEFINE_ERROR(TSDB_CODE_APP_IS_STOPPING, "Database is closing down")
|
||||
TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_DATA_FMT, "Invalid data format")
|
||||
TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_CFG_VALUE, "Invalid configuration value")
|
||||
TAOS_DEFINE_ERROR(TSDB_CODE_IP_NOT_IN_WHITE_LIST, "Not allowed to connect")
|
||||
|
||||
//client
|
||||
TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_OPERATION, "Invalid operation")
|
||||
|
|
@ -193,6 +194,10 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_USERS, "Too many users")
|
|||
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_ALTER_OPER, "Invalid alter operation")
|
||||
TAOS_DEFINE_ERROR(TSDB_CODE_MND_AUTH_FAILURE, "Authentication failure")
|
||||
TAOS_DEFINE_ERROR(TSDB_CODE_MND_PRIVILEDGE_EXIST, "User already have this priviledge")
|
||||
TAOS_DEFINE_ERROR(TSDB_CODE_MND_USER_HOST_EXIST, "Host already exist in ip white list")
|
||||
TAOS_DEFINE_ERROR(TSDB_CODE_MND_USER_HOST_NOT_EXIST, "Host not exist in ip white list")
|
||||
TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_USER_HOST, "Too many host in ip white list")
|
||||
TAOS_DEFINE_ERROR(TSDB_CODE_MND_USER_LOCAL_HOST_NOT_DROP, "Host can not be dropped")
|
||||
|
||||
//mnode-stable-part1
|
||||
TAOS_DEFINE_ERROR(TSDB_CODE_MND_STB_ALREADY_EXIST, "STable already exists")
|
||||
|
|
@ -216,7 +221,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_BUFSIZE, "Invalid func bufSize"
|
|||
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_COMMENT, "Invalid func comment")
|
||||
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_RETRIEVE, "Invalid func retrieve msg")
|
||||
|
||||
TAOS_DEFINE_ERROR(TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST, "index already exists")
|
||||
TAOS_DEFINE_ERROR(TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST, "index already exists in db")
|
||||
TAOS_DEFINE_ERROR(TSDB_CODE_MND_TAG_INDEX_NOT_EXIST, "index not exist")
|
||||
|
||||
|
||||
|
|
@ -307,7 +312,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_STREAMS, "Too many streams")
|
|||
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TARGET_TABLE, "Cannot write the same stable as other stream")
|
||||
|
||||
// mnode-sma
|
||||
TAOS_DEFINE_ERROR(TSDB_CODE_MND_SMA_ALREADY_EXIST, "index already exists")
|
||||
TAOS_DEFINE_ERROR(TSDB_CODE_MND_SMA_ALREADY_EXIST, "index already exists in db")
|
||||
TAOS_DEFINE_ERROR(TSDB_CODE_MND_SMA_NOT_EXIST, "index not exist")
|
||||
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_SMA_OPTION, "Invalid sma index option")
|
||||
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -16,6 +16,7 @@ exe:
|
|||
gcc $(CFLAGS) ./dbTableRoute.c -o $(ROOT)dbTableRoute $(LFLAGS)
|
||||
gcc $(CFLAGS) ./insertSameTs.c -o $(ROOT)insertSameTs $(LFLAGS)
|
||||
gcc $(CFLAGS) ./passwdTest.c -o $(ROOT)passwdTest $(LFLAGS)
|
||||
gcc $(CFLAGS) ./whiteListTest.c -o $(ROOT)whiteListTest $(LFLAGS)
|
||||
|
||||
clean:
|
||||
rm $(ROOT)batchprepare
|
||||
|
|
@ -23,3 +24,4 @@ clean:
|
|||
rm $(ROOT)dbTableRoute
|
||||
rm $(ROOT)insertSameTs
|
||||
rm $(ROOT)passwdTest
|
||||
rm $(ROOT)whiteListTest
|
||||
|
|
|
|||
|
|
@ -0,0 +1,170 @@
|
|||
/*
|
||||
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
|
||||
*
|
||||
* This program is free software: you can use, redistribute, and/or modify
|
||||
* it under the terms of the GNU Affero General Public License, version 3
|
||||
* or later ("AGPL"), as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||
* FITNESS FOR A PARTICULAR PURPOSE.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
// TAOS standard API example. The same syntax as MySQL, but only a subset
|
||||
// to compile: gcc -o demo demo.c -ltaos
|
||||
|
||||
/**
|
||||
* passwdTest.c
|
||||
* - Run the test case in clear TDengine environment with default root passwd 'taosdata'
|
||||
*/
|
||||
|
||||
#include <inttypes.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <unistd.h>
|
||||
#include "taos.h" // TAOS header file
|
||||
|
||||
#define nDup 1
|
||||
#define nRoot 10
|
||||
#define nUser 10
|
||||
#define USER_LEN 24
|
||||
#define BUF_LEN 1024
|
||||
|
||||
void createUsers(TAOS *taos, const char *host);
|
||||
void dropUsers(TAOS* taos);
|
||||
|
||||
int nPassVerNotified = 0;
|
||||
int nWhiteListVerNotified = 0;
|
||||
TAOS *taosu[nRoot] = {0};
|
||||
char users[nUser][USER_LEN] = {0};
|
||||
|
||||
void __taos_notify_cb(void *param, void *ext, int type) {
|
||||
switch (type) {
|
||||
case TAOS_NOTIFY_PASSVER: {
|
||||
++nPassVerNotified;
|
||||
printf("%s:%d type:%d user:%s ver:%d\n", __func__, __LINE__, type, param ? (char *)param : "NULL", *(int *)ext);
|
||||
break;
|
||||
}
|
||||
case TAOS_NOTIFY_WHITELIST_VER: {
|
||||
++nWhiteListVerNotified;
|
||||
printf("%s:%d type:%d user:%s ver:%"PRId64 "\n", __func__, __LINE__, type, param ? (char *)param : "NULL", *(int64_t *)ext);
|
||||
break;
|
||||
}
|
||||
default:
|
||||
printf("%s:%d unknown type:%d\n", __func__, __LINE__, type);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
void __taos_async_whitelist_cb(void *param, int code, TAOS *taos, int numOfWhiteLists, uint64_t* pWhiteList) {
|
||||
if (code == 0) {
|
||||
printf("fetch whitelist cb. user: %s numofWhitelist: %d\n", param ? (char*)param : NULL, numOfWhiteLists);
|
||||
for (int i = 0; i < numOfWhiteLists; ++i) {
|
||||
printf(" %d: 0x%llx\n", i, pWhiteList[i]);
|
||||
}
|
||||
} else {
|
||||
printf("fetch whitelist cb error %d\n", code);
|
||||
}
|
||||
}
|
||||
|
||||
static void queryDB(TAOS *taos, char *command) {
|
||||
int i;
|
||||
TAOS_RES *pSql = NULL;
|
||||
int32_t code = -1;
|
||||
|
||||
for (i = 0; i < nDup; ++i) {
|
||||
if (NULL != pSql) {
|
||||
taos_free_result(pSql);
|
||||
pSql = NULL;
|
||||
}
|
||||
|
||||
pSql = taos_query(taos, command);
|
||||
code = taos_errno(pSql);
|
||||
if (0 == code) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (code != 0) {
|
||||
fprintf(stderr, "failed to run: %s, reason: %s\n", command, taos_errstr(pSql));
|
||||
taos_free_result(pSql);
|
||||
taos_close(taos);
|
||||
exit(EXIT_FAILURE);
|
||||
} else {
|
||||
fprintf(stderr, "success to run: %s\n", command);
|
||||
}
|
||||
|
||||
taos_free_result(pSql);
|
||||
}
|
||||
|
||||
|
||||
int main(int argc, char *argv[]) {
|
||||
char qstr[1024];
|
||||
|
||||
// connect to server
|
||||
if (argc < 2) {
|
||||
printf("please input server-ip \n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
TAOS *taos = taos_connect(argv[1], "root", "taosdata", NULL, 0);
|
||||
if (taos == NULL) {
|
||||
printf("failed to connect to server, reason:%s\n", "null taos" /*taos_errstr(taos)*/);
|
||||
exit(1);
|
||||
}
|
||||
createUsers(taos, argv[1]);
|
||||
|
||||
while (nWhiteListVerNotified < 10) {
|
||||
printf("white list update notified %d times\n", nWhiteListVerNotified);
|
||||
sleep(1);
|
||||
}
|
||||
printf("succeed in getting white list nofication. %d times\n", nWhiteListVerNotified);
|
||||
dropUsers(taos);
|
||||
taos_close(taos);
|
||||
taos_cleanup();
|
||||
}
|
||||
|
||||
void dropUsers(TAOS *taos) {
|
||||
char qstr[1024];
|
||||
for (int i = 0; i < nUser; ++i) {
|
||||
sprintf(users[i], "user%d", i);
|
||||
sprintf(qstr, "DROP USER %s", users[i]);
|
||||
queryDB(taos, qstr);
|
||||
taos_close(taosu[i]);
|
||||
}
|
||||
}
|
||||
|
||||
void createUsers(TAOS *taos, const char *host) {
|
||||
char qstr[1024];
|
||||
// users
|
||||
for (int i = 0; i < nUser; ++i) {
|
||||
sprintf(users[i], "user%d", i);
|
||||
sprintf(qstr, "CREATE USER %s PASS 'taosdata'", users[i]);
|
||||
queryDB(taos, qstr);
|
||||
|
||||
taosu[i] = taos_connect(host, users[i], "taosdata", NULL, 0);
|
||||
if (taosu[i] == NULL) {
|
||||
printf("failed to connect to server, user:%s, reason:%s\n", users[i], "null taos" /*taos_errstr(taos)*/);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
int code = taos_set_notify_cb(taosu[i], __taos_notify_cb, users[i], TAOS_NOTIFY_WHITELIST_VER);
|
||||
|
||||
if (code != 0) {
|
||||
fprintf(stderr, "failed to run: taos_set_notify_cb for user:%s since %d\n", users[i], code);
|
||||
} else {
|
||||
fprintf(stderr, "success to run: taos_set_notify_cb for user:%s\n", users[i]);
|
||||
}
|
||||
|
||||
// alter whitelist for users
|
||||
sprintf(qstr, "alter user %s add host '%d.%d.%d.%d/24'", users[i], i, i, i, i);
|
||||
queryDB(taos, qstr);
|
||||
|
||||
taos_fetch_whitelist_a(taosu[i], __taos_async_whitelist_cb, users[i]);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -292,11 +292,12 @@ if $rows != 1 then
|
|||
return -1
|
||||
endi
|
||||
|
||||
sql drop index $data[0][0]
|
||||
#$drop_name=`$data[0][0]`
|
||||
#sql drop index `$data[0][0]\`
|
||||
|
||||
if $rows != 0 then
|
||||
return -1
|
||||
endi
|
||||
#if $rows != 0 then
|
||||
# return -1
|
||||
#endi
|
||||
|
||||
|
||||
sql_error drop index t2
|
||||
|
|
@ -304,7 +305,7 @@ sql_error drop index t3
|
|||
|
||||
|
||||
|
||||
sql create index ti0 on $mtPrefix (t1)
|
||||
#sql create index ti0 on $mtPrefix (t1)
|
||||
|
||||
$i = $interval
|
||||
while $i < $limit
|
||||
|
|
|
|||
|
|
@ -154,13 +154,14 @@ sql_error select * from information_schema.ins_mnodes
|
|||
sql_error select * from information_schema.ins_modules
|
||||
sql_error select * from information_schema.ins_qnodes
|
||||
sql_error select * from information_schema.ins_cluster
|
||||
sql_error select * from information_schema.ins_users
|
||||
sql_error select * from information_schema.ins_user_privilege
|
||||
sql select * from information_schema.ins_databases
|
||||
sql select * from information_schema.ins_functions
|
||||
sql select * from information_schema.ins_indexes
|
||||
sql select * from information_schema.ins_stables
|
||||
sql select * from information_schema.ins_tables
|
||||
sql select * from information_schema.ins_tags
|
||||
sql select * from information_schema.ins_users
|
||||
sql select * from information_schema.ins_topics
|
||||
sql select * from information_schema.ins_subscriptions
|
||||
sql select * from information_schema.ins_streams
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ sql create user u_read pass 'taosdata1' host '127.0.0.1/24','192.168.1.0/24'
|
|||
sql create user u_write pass 'taosdata1' host '127.0.0.1','192.168.1.0'
|
||||
|
||||
sql alter user u_read add host '3.3.3.4/24'
|
||||
sql alter user u_write drop host '4.4.4.5/25'
|
||||
sql_error alter user u_write drop host '4.4.4.5/25'
|
||||
|
||||
sql show users
|
||||
if $rows != 3 then
|
||||
|
|
|
|||
|
|
@ -67,7 +67,7 @@ set "FILE_NAME=testSuite.sim"
|
|||
if "%1" == "-f" set "FILE_NAME=%2"
|
||||
set FILE_NAME=%FILE_NAME:/=\%
|
||||
|
||||
start cmd /k "timeout /t 600 /NOBREAK && taskkill /f /im tsim.exe & exit /b"
|
||||
start cmd /k "timeout /t 800 /NOBREAK && taskkill /f /im tsim.exe & exit /b"
|
||||
|
||||
rem echo FILE_NAME: %FILE_NAME%
|
||||
echo ExcuteCmd: %tsim% -c %CFG_DIR% -f %FILE_NAME%
|
||||
|
|
|
|||
|
|
@ -217,7 +217,7 @@ class TDTestCase:
|
|||
tdSql.checkEqual(20470,len(tdSql.queryResult))
|
||||
|
||||
tdSql.query("select * from information_schema.ins_columns where db_name ='information_schema'")
|
||||
tdSql.checkEqual(193, len(tdSql.queryResult))
|
||||
tdSql.checkEqual(194, len(tdSql.queryResult))
|
||||
|
||||
tdSql.query("select * from information_schema.ins_columns where db_name ='performance_schema'")
|
||||
tdSql.checkEqual(54, len(tdSql.queryResult))
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ class TDTestCase:
|
|||
for i in range(self.tables):
|
||||
tdSql.execute(f'create table t{i} using stb tags({i}) ttl {self.ttl}')
|
||||
|
||||
time.sleep(self.ttl * 2)
|
||||
time.sleep(self.ttl + 3)
|
||||
tdSql.query('show tables')
|
||||
tdSql.checkRows(90)
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,323 @@
|
|||
import taos
|
||||
import sys
|
||||
import time
|
||||
import socket
|
||||
import os
|
||||
import threading
|
||||
import math
|
||||
from datetime import datetime
|
||||
|
||||
from util.log import *
|
||||
from util.sql import *
|
||||
from util.cases import *
|
||||
from util.dnodes import *
|
||||
from util.common import *
|
||||
# from tmqCommon import *
|
||||
|
||||
COMPARE_DATA = 0
|
||||
COMPARE_LEN = 1
|
||||
|
||||
class TDTestCase:
|
||||
def __init__(self):
|
||||
self.vgroups = 4
|
||||
self.ctbNum = 10
|
||||
self.rowsPerTbl = 10000
|
||||
self.duraion = '1h'
|
||||
|
||||
def init(self, conn, logSql, replicaVar=1):
|
||||
self.replicaVar = int(replicaVar)
|
||||
tdLog.debug(f"start to excute {__file__}")
|
||||
tdSql.init(conn.cursor(), False)
|
||||
|
||||
def create_database(self,tsql, dbName,dropFlag=1,vgroups=2,replica=1, duration:str='1d'):
|
||||
if dropFlag == 1:
|
||||
tsql.execute("drop database if exists %s"%(dbName))
|
||||
|
||||
tsql.execute("create database if not exists %s vgroups %d replica %d duration %s"%(dbName, vgroups, replica, duration))
|
||||
tdLog.debug("complete to create database %s"%(dbName))
|
||||
return
|
||||
|
||||
def create_stable(self,tsql, paraDict):
|
||||
colString = tdCom.gen_column_type_str(colname_prefix=paraDict["colPrefix"], column_elm_list=paraDict["colSchema"])
|
||||
tagString = tdCom.gen_tag_type_str(tagname_prefix=paraDict["tagPrefix"], tag_elm_list=paraDict["tagSchema"])
|
||||
sqlString = f"create table if not exists %s.%s (%s) tags (%s)"%(paraDict["dbName"], paraDict["stbName"], colString, tagString)
|
||||
tdLog.debug("%s"%(sqlString))
|
||||
tsql.execute(sqlString)
|
||||
return
|
||||
|
||||
def create_ctable(self,tsql=None, dbName='dbx',stbName='stb',ctbPrefix='ctb',ctbNum=1,ctbStartIdx=0):
|
||||
for i in range(ctbNum):
|
||||
sqlString = "create table %s.%s%d using %s.%s tags(%d, 'tb%d', 'tb%d', %d, %d, %d)" % \
|
||||
(dbName,ctbPrefix,i+ctbStartIdx,dbName,stbName,(i+ctbStartIdx) % 5,i+ctbStartIdx,i+ctbStartIdx,i+ctbStartIdx,i+ctbStartIdx,i+ctbStartIdx)
|
||||
tsql.execute(sqlString)
|
||||
|
||||
tdLog.debug("complete to create %d child tables by %s.%s" %(ctbNum, dbName, stbName))
|
||||
return
|
||||
|
||||
def insert_data(self,tsql,dbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs,tsStep):
|
||||
tdLog.debug("start to insert data ............")
|
||||
tsql.execute("use %s" %dbName)
|
||||
pre_insert = "insert into "
|
||||
sql = pre_insert
|
||||
|
||||
for i in range(ctbNum):
|
||||
rowsBatched = 0
|
||||
sql += " %s%d values "%(ctbPrefix,i)
|
||||
for j in range(rowsPerTbl):
|
||||
if (i < ctbNum/2):
|
||||
sql += "(%d, %d, %d, %d,%d,%d,%d,true,'binary%d', 'nchar%d') "%(startTs + j*tsStep, j%10, j%10, j%10, j%10, j%10, j%10, j%10, j%10)
|
||||
else:
|
||||
sql += "(%d, %d, NULL, %d,NULL,%d,%d,true,'binary%d', 'nchar%d') "%(startTs + j*tsStep, j%10, j%10, j%10, j%10, j%10, j%10)
|
||||
rowsBatched += 1
|
||||
if ((rowsBatched == batchNum) or (j == rowsPerTbl - 1)):
|
||||
tsql.execute(sql)
|
||||
rowsBatched = 0
|
||||
if j < rowsPerTbl - 1:
|
||||
sql = "insert into %s%d values " %(ctbPrefix,i)
|
||||
else:
|
||||
sql = "insert into "
|
||||
if sql != pre_insert:
|
||||
tsql.execute(sql)
|
||||
tdLog.debug("insert data ............ [OK]")
|
||||
return
|
||||
|
||||
def prepareTestEnv(self):
|
||||
tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ")
|
||||
paraDict = {'dbName': 'test',
|
||||
'dropFlag': 1,
|
||||
'vgroups': 2,
|
||||
'stbName': 'meters',
|
||||
'colPrefix': 'c',
|
||||
'tagPrefix': 't',
|
||||
'colSchema': [{'type': 'INT', 'count':1},
|
||||
{'type': 'BIGINT', 'count':1},
|
||||
{'type': 'FLOAT', 'count':1},
|
||||
{'type': 'DOUBLE', 'count':1},
|
||||
{'type': 'smallint', 'count':1},
|
||||
{'type': 'tinyint', 'count':1},
|
||||
{'type': 'bool', 'count':1},
|
||||
{'type': 'binary', 'len':10, 'count':1},
|
||||
{'type': 'nchar', 'len':10, 'count':1}],
|
||||
'tagSchema': [{'type': 'INT', 'count':1},{'type': 'nchar', 'len':20, 'count':1},{'type': 'binary', 'len':20, 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'smallint', 'count':1},{'type': 'DOUBLE', 'count':1}],
|
||||
'ctbPrefix': 't',
|
||||
'ctbStartIdx': 0,
|
||||
'ctbNum': 100,
|
||||
'rowsPerTbl': 10000,
|
||||
'batchNum': 3000,
|
||||
'startTs': 1537146000000,
|
||||
'tsStep': 600000}
|
||||
|
||||
paraDict['vgroups'] = self.vgroups
|
||||
paraDict['ctbNum'] = self.ctbNum
|
||||
paraDict['rowsPerTbl'] = self.rowsPerTbl
|
||||
|
||||
tdLog.info("create database")
|
||||
self.create_database(tsql=tdSql, dbName=paraDict["dbName"], dropFlag=paraDict["dropFlag"], vgroups=paraDict["vgroups"], replica=self.replicaVar, duration=self.duraion)
|
||||
|
||||
tdLog.info("create stb")
|
||||
self.create_stable(tsql=tdSql, paraDict=paraDict)
|
||||
|
||||
tdLog.info("create child tables")
|
||||
self.create_ctable(tsql=tdSql, dbName=paraDict["dbName"], \
|
||||
stbName=paraDict["stbName"],ctbPrefix=paraDict["ctbPrefix"],\
|
||||
ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict["ctbStartIdx"])
|
||||
self.insert_data(tsql=tdSql, dbName=paraDict["dbName"],\
|
||||
ctbPrefix=paraDict["ctbPrefix"],ctbNum=paraDict["ctbNum"],\
|
||||
rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],\
|
||||
startTs=paraDict["startTs"],tsStep=paraDict["tsStep"])
|
||||
return
|
||||
|
||||
def check_explain_res_has_row(self, plan_str_expect: str, rows):
|
||||
plan_found = False
|
||||
for row in rows:
|
||||
if str(row).find(plan_str_expect) >= 0:
|
||||
tdLog.debug("plan: [%s] found in: [%s]" % (plan_str_expect, str(row)))
|
||||
plan_found = True
|
||||
break
|
||||
if not plan_found:
|
||||
tdLog.exit("plan: %s not found in res: [%s]" % (plan_str_expect, str(rows)))
|
||||
|
||||
|
||||
def test_sort_for_partition_hint(self):
|
||||
sql = 'select count(*), c1 from meters partition by c1'
|
||||
sql_hint = 'select /*+ sort_for_group() */count(*), c1 from meters partition by c1'
|
||||
self.check_explain_res_has_row("Partition on", self.explain_sql(sql))
|
||||
self.check_explain_res_has_row("Sort", self.explain_sql(sql_hint))
|
||||
|
||||
sql = 'select count(*), c1, tbname from meters partition by tbname, c1'
|
||||
sql_hint = 'select /*+ sort_for_group() */ count(*), c1, tbname from meters partition by tbname, c1'
|
||||
self.check_explain_res_has_row("Partition on", self.explain_sql(sql))
|
||||
self.check_explain_res_has_row("Sort", self.explain_sql(sql_hint))
|
||||
|
||||
sql = 'select count(*), c1, tbname from meters partition by tbname, c1 interval(1s)'
|
||||
sql_hint = 'select /*+ sort_for_group() */ count(*), c1, tbname from meters partition by tbname, c1 interval(1s)'
|
||||
self.check_explain_res_has_row("Partition on", self.explain_sql(sql))
|
||||
self.check_explain_res_has_row("Sort", self.explain_sql(sql_hint))
|
||||
|
||||
sql = 'select count(*), c1, t1 from meters partition by t1, c1'
|
||||
sql_hint = 'select /*+ sort_for_group() */ count(*), c1, t1 from meters partition by t1, c1'
|
||||
self.check_explain_res_has_row("Partition on", self.explain_sql(sql))
|
||||
self.check_explain_res_has_row("Sort", self.explain_sql(sql_hint))
|
||||
|
||||
sql = 'select count(*), c1, t1 from meters partition by t1, c1 interval(1s)'
|
||||
sql_hint = 'select /*+ sort_for_group() */ count(*), c1, t1 from meters partition by t1, c1 interval(1s)'
|
||||
self.check_explain_res_has_row("Partition on", self.explain_sql(sql))
|
||||
self.check_explain_res_has_row("Sort", self.explain_sql(sql_hint))
|
||||
|
||||
sql = 'select count(*), c1 from meters partition by c1 interval(1s)'
|
||||
sql_hint = 'select /*+ sort_for_group() */ count(*), c1 from meters partition by c1 interval(1s)'
|
||||
self.check_explain_res_has_row("Partition on", self.explain_sql(sql))
|
||||
self.check_explain_res_has_row("Sort", self.explain_sql(sql_hint))
|
||||
|
||||
def add_order_by(self, sql: str, order_by: str, select_list: str = "*") -> str:
|
||||
return "select %s from (%s)t order by %s" % (select_list, sql, order_by)
|
||||
|
||||
def add_hint(self, sql: str) -> str:
|
||||
return "select /*+ sort_for_group() */ %s" % sql[6:]
|
||||
|
||||
def query_with_time(self, sql):
|
||||
start = datetime.now()
|
||||
tdSql.query(sql, queryTimes=1)
|
||||
return (datetime.now().timestamp() - start.timestamp()) * 1000
|
||||
|
||||
def explain_sql(self, sql: str):
|
||||
sql = "explain " + sql
|
||||
tdSql.query(sql, queryTimes=1)
|
||||
return tdSql.queryResult
|
||||
|
||||
def query_and_compare_res(self, sql1, sql2, compare_what: int = 0):
|
||||
dur = self.query_with_time(sql1)
|
||||
tdLog.debug("sql1 query with time: [%f]" % dur)
|
||||
res1 = tdSql.queryResult
|
||||
dur = self.query_with_time(sql2)
|
||||
tdLog.debug("sql2 query with time: [%f]" % dur)
|
||||
res2 = tdSql.queryResult
|
||||
if res1 is None or res2 is None:
|
||||
tdLog.exit("res1 or res2 is None")
|
||||
if compare_what <= COMPARE_LEN:
|
||||
if len(res1) != len(res2):
|
||||
tdLog.exit("query and copare failed cause different rows, sql1: [%s], rows: [%d], sql2: [%s], rows: [%d]" % (sql1, len(res1), sql2, len(res2)))
|
||||
if compare_what == COMPARE_DATA:
|
||||
for i in range(0, len(res1)):
|
||||
if res1[i] != res2[i]:
|
||||
tdLog.exit("compare failed for row: [%d], sqls: [%s] res1: [%s], sql2 : [%s] res2: [%s]" % (i, sql1, res1[i], sql2, res2[i]))
|
||||
tdLog.debug("sql: [%s] and sql: [%s] have same results, rows: [%d]" % (sql1, sql2, len(res1)))
|
||||
|
||||
def prepare_and_query_and_compare(self, sqls: [], order_by: str, select_list: str = "*", compare_what: int = 0):
|
||||
for sql in sqls:
|
||||
sql_hint = self.add_order_by(self.add_hint(sql), order_by, select_list)
|
||||
sql = self.add_order_by(sql, order_by, select_list)
|
||||
self.check_explain_res_has_row("Sort", self.explain_sql(sql_hint))
|
||||
self.check_explain_res_has_row("Partition", self.explain_sql(sql))
|
||||
self.query_and_compare_res(sql, sql_hint, compare_what=compare_what)
|
||||
|
||||
def test_sort_for_partition_res(self):
|
||||
sqls_par_c1_agg = [
|
||||
"select count(*), c1 from meters partition by c1",
|
||||
"select count(*), min(c2), max(c3), c1 from meters partition by c1",
|
||||
"select c1 from meters partition by c1",
|
||||
]
|
||||
sqls_par_c1 = [
|
||||
"select * from meters partition by c1"
|
||||
]
|
||||
sqls_par_c1_c2_agg = [
|
||||
"select count(*), c1, c2 from meters partition by c1, c2",
|
||||
"select c1, c2 from meters partition by c1, c2",
|
||||
"select count(*), c1, c2, min(c4), max(c5), sum(c6) from meters partition by c1, c2",
|
||||
]
|
||||
sqls_par_c1_c2 = [
|
||||
"select * from meters partition by c1, c2"
|
||||
]
|
||||
|
||||
sqls_par_tbname_c1 = [
|
||||
"select count(*), c1 , tbname as a from meters partition by tbname, c1"
|
||||
]
|
||||
sqls_par_tag_c1 = [
|
||||
"select count(*), c1, t1 from meters partition by t1, c1"
|
||||
]
|
||||
self.prepare_and_query_and_compare(sqls_par_c1_agg, "c1")
|
||||
self.prepare_and_query_and_compare(sqls_par_c1, "c1, ts, c2", "c1, ts, c2")
|
||||
self.prepare_and_query_and_compare(sqls_par_c1_c2_agg, "c1, c2")
|
||||
self.prepare_and_query_and_compare(sqls_par_c1_c2, "c1, c2, ts, c3", "c1, c2, ts, c3")
|
||||
self.prepare_and_query_and_compare(sqls_par_tbname_c1, "a, c1")
|
||||
self.prepare_and_query_and_compare(sqls_par_tag_c1, "t1, c1")
|
||||
|
||||
def get_interval_template_sqls(self, col_name):
|
||||
sqls = [
|
||||
'select _wstart as ts, count(*), tbname as a, %s from meters partition by tbname, %s interval(1s)' % (col_name, col_name),
|
||||
#'select _wstart as ts, count(*), tbname as a, %s from meters partition by tbname, %s interval(30s)' % (col_name, col_name),
|
||||
#'select _wstart as ts, count(*), tbname as a, %s from meters partition by tbname, %s interval(1m)' % (col_name, col_name),
|
||||
#'select _wstart as ts, count(*), tbname as a, %s from meters partition by tbname, %s interval(30m)' % (col_name, col_name),
|
||||
#'select _wstart as ts, count(*), tbname as a, %s from meters partition by tbname, %s interval(1h)' % (col_name, col_name),
|
||||
|
||||
'select _wstart as ts, count(*), t1 as a, %s from meters partition by t1, %s interval(1s)' % (col_name, col_name),
|
||||
#'select _wstart as ts, count(*), t1 as a, %s from meters partition by t1, %s interval(30s)' % (col_name, col_name),
|
||||
#'select _wstart as ts, count(*), t1 as a, %s from meters partition by t1, %s interval(1m)' % (col_name, col_name),
|
||||
#'select _wstart as ts, count(*), t1 as a, %s from meters partition by t1, %s interval(30m)' % (col_name, col_name),
|
||||
#'select _wstart as ts, count(*), t1 as a, %s from meters partition by t1, %s interval(1h)' % (col_name, col_name),
|
||||
|
||||
'select _wstart as ts, count(*), %s as a, %s from meters partition by %s interval(1s)' % (col_name, col_name, col_name),
|
||||
#'select _wstart as ts, count(*), %s as a, %s from meters partition by %s interval(30s)' % (col_name, col_name, col_name),
|
||||
#'select _wstart as ts, count(*), %s as a, %s from meters partition by %s interval(1m)' % (col_name, col_name, col_name),
|
||||
#'select _wstart as ts, count(*), %s as a, %s from meters partition by %s interval(30m)' % (col_name, col_name, col_name),
|
||||
#'select _wstart as ts, count(*), %s as a, %s from meters partition by %s interval(1h)' % (col_name, col_name, col_name),
|
||||
|
||||
'select _wstart as ts, count(*), tbname as a, %s from meters partition by %s, tbname interval(1s)' % (col_name, col_name),
|
||||
'select _wstart as ts, count(*), t1 as a, %s from meters partition by %s, t1 interval(1s)' % (col_name, col_name),
|
||||
]
|
||||
order_list = 'a, %s, ts' % (col_name)
|
||||
return (sqls, order_list)
|
||||
|
||||
def test_sort_for_partition_interval(self):
|
||||
sqls, order_list = self.get_interval_template_sqls('c1')
|
||||
self.prepare_and_query_and_compare(sqls, order_list)
|
||||
#sqls, order_list = self.get_interval_template_sqls('c2')
|
||||
#self.prepare_and_query(sqls, order_list)
|
||||
sqls, order_list = self.get_interval_template_sqls('c3')
|
||||
self.prepare_and_query_and_compare(sqls, order_list)
|
||||
#sqls, order_list = self.get_interval_template_sqls('c4')
|
||||
#self.prepare_and_query(sqls, order_list)
|
||||
#sqls, order_list = self.get_interval_template_sqls('c5')
|
||||
#self.prepare_and_query(sqls, order_list)
|
||||
sqls, order_list = self.get_interval_template_sqls('c6')
|
||||
self.prepare_and_query_and_compare(sqls, order_list)
|
||||
#sqls, order_list = self.get_interval_template_sqls('c7')
|
||||
#self.prepare_and_query(sqls, order_list)
|
||||
sqls, order_list = self.get_interval_template_sqls('c8')
|
||||
self.prepare_and_query_and_compare(sqls, order_list)
|
||||
sqls, order_list = self.get_interval_template_sqls('c9')
|
||||
self.prepare_and_query_and_compare(sqls, order_list)
|
||||
|
||||
def test_sort_for_partition_no_agg_limit(self):
|
||||
sqls_template = [
|
||||
'select * from meters partition by c1 slimit %d limit %d',
|
||||
'select * from meters partition by c2 slimit %d limit %d',
|
||||
'select * from meters partition by c8 slimit %d limit %d',
|
||||
]
|
||||
sqls = []
|
||||
for sql in sqls_template:
|
||||
sqls.append(sql % (1,1))
|
||||
sqls.append(sql % (1,10))
|
||||
sqls.append(sql % (10,10))
|
||||
sqls.append(sql % (100, 100))
|
||||
order_by_list = 'ts,c1,c2,c3,c4,c5,c6,c7,c8,c9,t1,t2,t3,t4,t5,t6'
|
||||
|
||||
self.prepare_and_query_and_compare(sqls, order_by_list, compare_what=COMPARE_LEN)
|
||||
|
||||
|
||||
def run(self):
|
||||
self.prepareTestEnv()
|
||||
#time.sleep(99999999)
|
||||
self.test_sort_for_partition_hint()
|
||||
self.test_sort_for_partition_res()
|
||||
self.test_sort_for_partition_interval()
|
||||
self.test_sort_for_partition_no_agg_limit()
|
||||
|
||||
def stop(self):
|
||||
tdSql.close()
|
||||
tdLog.success(f"{__file__} successfully executed")
|
||||
|
||||
event = threading.Event()
|
||||
|
||||
tdCases.addLinux(__file__, TDTestCase())
|
||||
tdCases.addWindows(__file__, TDTestCase())
|
||||
|
|
@ -166,22 +166,23 @@ class TDTestCase:
|
|||
nodePort = 6030 + i*100
|
||||
newTdSql=tdCom.newTdSql(port=nodePort)
|
||||
|
||||
|
||||
tdDnodes[1].stoptaosd()
|
||||
|
||||
dataPath = tdDnodes[1].dataDir
|
||||
os.system(f"rm -rf {dataPath}/*")
|
||||
os.system(f"rm -rf {dataPath}/.runing")
|
||||
|
||||
tdDnodes[1].stoptaosd()
|
||||
tdDnodes[1].starttaosd()
|
||||
sleep(5)
|
||||
for i in range(6):
|
||||
nodePort = 6030 + i*100
|
||||
newTdSql=tdCom.newTdSql(port=nodePort)
|
||||
|
||||
tdDnodes[0].stoptaosd()
|
||||
dataPath = tdDnodes[0].dataDir
|
||||
os.system(f"rm -rf {dataPath}/*")
|
||||
os.system(f"rm -rf {dataPath}/.runing")
|
||||
|
||||
tdDnodes[0].stoptaosd()
|
||||
tdDnodes[0].starttaosd()
|
||||
sleep(5)
|
||||
for i in range(6):
|
||||
|
|
|
|||
|
|
@ -109,7 +109,7 @@ if __name__ == "__main__":
|
|||
websocket = False
|
||||
replicaVar = 1
|
||||
asan = False
|
||||
independentMnode = True
|
||||
independentMnode = False
|
||||
previousCluster = False
|
||||
opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:N:M:Q:C:RWD:n:i:aP', [
|
||||
'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'restart', 'updateCfgDict', 'killv', 'execCmd','dnodeNums','mnodeNums','queryPolicy','createDnodeNums','restful','websocket','adaptercfgupdate','replicaVar','independentMnode','previous'])
|
||||
|
|
@ -202,7 +202,7 @@ if __name__ == "__main__":
|
|||
createDnodeNums = value
|
||||
|
||||
if key in ['-i', '--independentMnode']:
|
||||
independentMnode = False
|
||||
independentMnode = value
|
||||
|
||||
if key in ['-R', '--restful']:
|
||||
restful = True
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ python3 ./test.py -f 2-query/nestedQuery_math.py -Q 4
|
|||
python3 ./test.py -f 2-query/nestedQuery_time.py -Q 4
|
||||
python3 ./test.py -f 2-query/nestedQuery_26.py -Q 4
|
||||
python3 ./test.py -f 2-query/interval_limit_opt.py -Q 4
|
||||
python3 ./test.py -f 2-query/partition_by_col.py -Q 4
|
||||
python3 ./test.py -f 7-tmq/tmqShow.py
|
||||
python3 ./test.py -f 7-tmq/tmqDropStb.py
|
||||
python3 ./test.py -f 7-tmq/subscribeStb0.py
|
||||
|
|
|
|||
Loading…
Reference in New Issue