From 916a7f292fa4370bfd64ff057085e187004354f2 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Mon, 18 May 2020 03:21:44 +0000 Subject: [PATCH 1/4] [TD-319] put protection on host name length --- src/client/inc/tscUtil.h | 1 + src/client/src/tscServer.c | 30 +++++------------------------- src/client/src/tscSql.c | 28 +++++++--------------------- src/client/src/tscSystem.c | 13 +++++-------- src/client/src/tscUtil.c | 21 +++++++++++++++++++++ src/common/inc/tglobal.h | 2 +- src/common/src/tglobal.c | 4 ++-- src/inc/taoserror.h | 1 + 8 files changed, 43 insertions(+), 57 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index dbbec01329..715d76e072 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -264,6 +264,7 @@ bool hasMoreVnodesToTry(SSqlObj *pSql); void tscTryQueryNextVnode(SSqlObj *pSql, __async_cb_func_t fp); void tscAsyncQuerySingleRowForNextVnode(void *param, TAOS_RES *tres, int numOfRows); void tscTryQueryNextClause(SSqlObj* pSql, void (*queryFp)()); +int tscSetMgmtIpListFromCfg(const char *first, const char *second); void* malloc_throw(size_t size); void* calloc_throw(size_t nmemb, size_t size); diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 110377d112..91a54eb27e 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -67,7 +67,7 @@ void tscPrintMgmtIp() { } } -void tscSetMgmtIpListFromCluster(SRpcIpSet *pIpList) { +void tscSetMgmtIpList(SRpcIpSet *pIpList) { tscMgmtIpSet.numOfIps = pIpList->numOfIps; tscMgmtIpSet.inUse = pIpList->inUse; for (int32_t i = 0; i < tscMgmtIpSet.numOfIps; ++i) { @@ -75,16 +75,6 @@ void tscSetMgmtIpListFromCluster(SRpcIpSet *pIpList) { } } -void tscSetMgmtIpListFromEdge() { - if (tscMgmtIpSet.numOfIps != 1) { - tscMgmtIpSet.numOfIps = 1; - tscMgmtIpSet.inUse = 0; - taosGetFqdnPortFromEp(tsFirst, tscMgmtIpSet.fqdn[0], &tscMgmtIpSet.port[0]); - tscTrace("edge mgmt IP list:"); - tscPrintMgmtIp(); - } -} - void tscUpdateIpSet(void *ahandle, SRpcIpSet *pIpSet) { tscMgmtIpSet = *pIpSet; tscTrace("mgmt IP list is changed for ufp is called, numOfIps:%d inUse:%d", tscMgmtIpSet.numOfIps, tscMgmtIpSet.inUse); @@ -93,18 +83,6 @@ void tscUpdateIpSet(void *ahandle, SRpcIpSet *pIpSet) { } } -void tscSetMgmtIpList(SRpcIpSet *pIpList) { - /* - * The iplist returned by the cluster edition is the current management nodes - * and the iplist returned by the edge edition is empty - */ - if (pIpList->numOfIps != 0) { - tscSetMgmtIpListFromCluster(pIpList); - } else { - tscSetMgmtIpListFromEdge(); - } -} - /* * For each management node, try twice at least in case of poor network situation. * If the client start to connect to a non-management node from the client, and the first retry may fail due to @@ -132,7 +110,8 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { if (code == 0) { SCMHeartBeatRsp *pRsp = (SCMHeartBeatRsp *)pRes->pRsp; SRpcIpSet * pIpList = &pRsp->ipList; - tscSetMgmtIpList(pIpList); + if (pIpList->numOfIps > 0) + tscSetMgmtIpList(pIpList); if (pRsp->killConnection) { tscKillConnection(pObj); @@ -2224,7 +2203,8 @@ int tscProcessConnectRsp(SSqlObj *pSql) { assert(len <= tListLen(pObj->db)); strncpy(pObj->db, temp, tListLen(pObj->db)); - tscSetMgmtIpList(&pConnect->ipList); + if (pConnect->ipList.numOfIps > 0) + tscSetMgmtIpList(&pConnect->ipList); strcpy(pObj->sversion, pConnect->serverVersion); pObj->writeAuth = pConnect->writeAuth; diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 87292f4fe6..d8ec104a50 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -65,32 +65,18 @@ STscObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con terrno = TSDB_CODE_INVALID_PASS; return NULL; } - + + if (ip) { + if (tscSetMgmtIpListFromCfg(ip, NULL) < 0) return NULL; + if (port) tscMgmtIpSet.port[0] = port; + } + void *pDnodeConn = NULL; if (tscInitRpc(user, pass, &pDnodeConn) != 0) { terrno = TSDB_CODE_NETWORK_UNAVAIL; return NULL; } - - tscMgmtIpSet.numOfIps = 0; - - if (ip && ip[0]) { - tscMgmtIpSet.inUse = 0; - tscMgmtIpSet.numOfIps = 1; - strcpy(tscMgmtIpSet.fqdn[0], ip); - tscMgmtIpSet.port[0] = port? port: tsDnodeShellPort; - } else { - if (tsFirst[0] != 0) { - taosGetFqdnPortFromEp(tsFirst, tscMgmtIpSet.fqdn[tscMgmtIpSet.numOfIps], &tscMgmtIpSet.port[tscMgmtIpSet.numOfIps]); - tscMgmtIpSet.numOfIps++; - } - - if (tsSecond[0] != 0) { - taosGetFqdnPortFromEp(tsSecond, tscMgmtIpSet.fqdn[tscMgmtIpSet.numOfIps], &tscMgmtIpSet.port[tscMgmtIpSet.numOfIps]); - tscMgmtIpSet.numOfIps++; - } - } - + STscObj *pObj = (STscObj *)calloc(1, sizeof(STscObj)); if (NULL == pObj) { terrno = TSDB_CODE_CLI_OUT_OF_MEMORY; diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index 5d8652a631..5d56fef1e9 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -23,6 +23,7 @@ #include "tutil.h" #include "tsched.h" #include "tscLog.h" +#include "tscUtil.h" #include "tsclient.h" #include "tglobal.h" #include "tconfig.h" @@ -114,14 +115,10 @@ void taos_init_imp() { taosInitNote(tsNumOfLogLines / 10, 1, (char*)"tsc_note"); } - tscMgmtIpSet.inUse = 0; - tscMgmtIpSet.numOfIps = 1; - taosGetFqdnPortFromEp(tsFirst, tscMgmtIpSet.fqdn[0], &tscMgmtIpSet.port[0]); - - if (tsSecond[0] && strcmp(tsSecond, tsFirst) != 0) { - tscMgmtIpSet.numOfIps = 2; - taosGetFqdnPortFromEp(tsSecond, tscMgmtIpSet.fqdn[1], &tscMgmtIpSet.port[1]); - } + if (tscSetMgmtIpListFromCfg(tsFirst, tsSecond) < 0) { + tscError("failed to init mgmt IP list"); + return; + } tscInitMsgsFp(); int queueSize = tsMaxVnodeConnections + tsMaxMeterConnections + tsMaxMgmtConnections + tsMaxMgmtConnections; diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 1d1e06d3a9..40d058d189 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -2163,3 +2163,24 @@ char* strdup_throw(const char* str) { } return p; } + +int tscSetMgmtIpListFromCfg(const char *first, const char *second) { + tscMgmtIpSet.numOfIps = 0; + tscMgmtIpSet.inUse = 0; + + if (first && first[0] != 0) { + if (strlen(first) >= TSDB_FQDN_LEN) return TSDB_CODE_INVALID_FQDN; + taosGetFqdnPortFromEp(first, tscMgmtIpSet.fqdn[tscMgmtIpSet.numOfIps], &tscMgmtIpSet.port[tscMgmtIpSet.numOfIps]); + tscMgmtIpSet.numOfIps++; + } + + if (second && second[0] != 0) { + if (strlen(second) >= TSDB_FQDN_LEN) return TSDB_CODE_INVALID_FQDN; + taosGetFqdnPortFromEp(second, tscMgmtIpSet.fqdn[tscMgmtIpSet.numOfIps], &tscMgmtIpSet.port[tscMgmtIpSet.numOfIps]); + tscMgmtIpSet.numOfIps++; + } + + if ( tscMgmtIpSet.numOfIps == 0) return TSDB_CODE_INVALID_FQDN; + + return 0; +} diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index b077f40945..da8f3cd1e1 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -175,7 +175,7 @@ void taosInitGlobalCfg(); bool taosCheckGlobalCfg(); void taosSetAllDebugFlag(); bool taosCfgDynamicOptions(char *msg); -int taosGetFqdnPortFromEp(char *ep, char *fqdn, uint16_t *port); +int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port); #ifdef __cplusplus } diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 90637265b3..18d8c9ebe2 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -61,7 +61,7 @@ int32_t tscEmbedded = 0; */ int64_t tsMsPerDay[] = {86400000L, 86400000000L}; -char tsFirst[TSDB_FQDN_LEN] = {0}; +char tsFirst[TSDB_FQDN_LEN] = {0}; char tsSecond[TSDB_FQDN_LEN] = {0}; char tsArbitrator[TSDB_FQDN_LEN] = {0}; char tsLocalEp[TSDB_FQDN_LEN] = {0}; // Local End Point, hostname:port @@ -1252,7 +1252,7 @@ bool taosCheckGlobalCfg() { return true; } -int taosGetFqdnPortFromEp(char *ep, char *fqdn, uint16_t *port) { +int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port) { *port = 0; strcpy(fqdn, ep); diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index f9bc1404fe..1390d66113 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -170,6 +170,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_QHANDLE, 0, 459, "invalid handle" TAOS_DEFINE_ERROR(TSDB_CODE_QUERY_CANCELLED, 0, 460, "query cancelled") TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_IE, 0, 461, "invalid ie") TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_VALUE, 0, 462, "invalid value") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_FQDN, 0, 463, "invalid FQDN") // others TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_FILE_FORMAT, 0, 500, "invalid file format") From 3c09bc1e088457c583b37d67c597ce2a0a72be8c Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Mon, 18 May 2020 03:32:25 +0000 Subject: [PATCH 2/4] add the error code --- src/client/src/tscUtil.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 40d058d189..4667606aa8 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -2169,18 +2169,27 @@ int tscSetMgmtIpListFromCfg(const char *first, const char *second) { tscMgmtIpSet.inUse = 0; if (first && first[0] != 0) { - if (strlen(first) >= TSDB_FQDN_LEN) return TSDB_CODE_INVALID_FQDN; + if (strlen(first) >= TSDB_FQDN_LEN) { + terrno = TSDB_CODE_INVALID_FQDN; + return -1; + } taosGetFqdnPortFromEp(first, tscMgmtIpSet.fqdn[tscMgmtIpSet.numOfIps], &tscMgmtIpSet.port[tscMgmtIpSet.numOfIps]); tscMgmtIpSet.numOfIps++; } if (second && second[0] != 0) { - if (strlen(second) >= TSDB_FQDN_LEN) return TSDB_CODE_INVALID_FQDN; + if (strlen(second) >= TSDB_FQDN_LEN) { + terrno = TSDB_CODE_INVALID_FQDN; + return -1; + } taosGetFqdnPortFromEp(second, tscMgmtIpSet.fqdn[tscMgmtIpSet.numOfIps], &tscMgmtIpSet.port[tscMgmtIpSet.numOfIps]); tscMgmtIpSet.numOfIps++; } - if ( tscMgmtIpSet.numOfIps == 0) return TSDB_CODE_INVALID_FQDN; + if ( tscMgmtIpSet.numOfIps == 0) { + terrno = TSDB_CODE_INVALID_FQDN; + return -1; + } return 0; } From d18d4df151cea74ece01ecf571e9da3fd9d3623d Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 18 May 2020 13:15:21 +0800 Subject: [PATCH 3/4] [TD-319] fixed crash while fqdn length changed --- src/inc/taosdef.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 61e82720b1..ba015d7bbf 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -218,7 +218,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_LOCALE_LEN 64 #define TSDB_TIMEZONE_LEN 64 -#define TSDB_FQDN_LEN 72 +#define TSDB_FQDN_LEN 256 #define TSDB_IPv4ADDR_LEN 16 #define TSDB_FILENAME_LEN 128 #define TSDB_METER_VNODE_BITS 20 @@ -234,7 +234,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_DEFAULT_PKT_SIZE 65480 //same as RPC_MAX_UDP_SIZE #define TSDB_PAYLOAD_SIZE (TSDB_DEFAULT_PKT_SIZE - 100) -#define TSDB_DEFAULT_PAYLOAD_SIZE 1024 // default payload size +#define TSDB_DEFAULT_PAYLOAD_SIZE 2048 // default payload size #define TSDB_EXTRA_PAYLOAD_SIZE 128 // extra bytes for auth #define TSDB_CQ_SQL_SIZE 1024 #define TSDB_MAX_VNODES 256 From e9f50a0ade23bdb8f6723997d03e9ad0cdb7869a Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 18 May 2020 13:43:37 +0800 Subject: [PATCH 4/4] [TD-359] change table uid from int64_t to uint64_t --- src/inc/taosmsg.h | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index e58bcf5237..728e733f84 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -187,13 +187,13 @@ typedef struct SMsgHead { // Submit message for one table typedef struct SSubmitBlk { - int64_t uid; // table unique id - int32_t tid; // table id - int32_t padding; // TODO just for padding here - int32_t sversion; // data schema version - int32_t len; // data part length, not including the SSubmitBlk head - int16_t numOfRows; // total number of rows in current submit block - char data[]; + uint64_t uid; // table unique id + int32_t tid; // table id + int32_t padding; // TODO just for padding here + int32_t sversion; // data schema version + int32_t len; // data part length, not including the SSubmitBlk head + int16_t numOfRows; // total number of rows in current submit block + char data[]; } SSubmitBlk; // Submit message for this TSDB @@ -327,9 +327,9 @@ typedef struct { } SMDDropTableMsg; typedef struct { - int32_t contLen; - int32_t vgId; - int64_t uid; + int32_t contLen; + int32_t vgId; + uint64_t uid; char tableId[TSDB_TABLE_ID_LEN + 1]; } SMDDropSTableMsg; @@ -404,9 +404,9 @@ typedef struct SColumnInfo { } SColumnInfo; typedef struct STableIdInfo { - int64_t uid; - int32_t tid; - TSKEY key; // last accessed ts, for subscription + uint64_t uid; + int32_t tid; + TSKEY key; // last accessed ts, for subscription } STableIdInfo; typedef struct STimeWindow {