From e8877f5f2165a57c51e312e79fda6c043788173e Mon Sep 17 00:00:00 2001 From: liu0x54 Date: Sat, 20 Jun 2020 08:42:49 +0000 Subject: [PATCH 01/27] [TD-711] fix the token define conflicts --- src/inc/ttokendef.h | 10 ++++++++++ src/util/inc/tstoken.h | 9 +-------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/inc/ttokendef.h b/src/inc/ttokendef.h index d388bc9dbe..7648aadc60 100644 --- a/src/inc/ttokendef.h +++ b/src/inc/ttokendef.h @@ -221,6 +221,16 @@ #define TK_INTO 203 #define TK_VALUES 204 + +#define TK_SPACE 300 +#define TK_COMMENT 301 +#define TK_ILLEGAL 302 +#define TK_HEX 303 // hex number 0x123 +#define TK_OCT 304 // oct number +#define TK_BIN 305 // bin format data 0b111 +#define TK_FILE 306 +#define TK_QUESTION 307 // denoting the placeholder of "?",when invoking statement bind query + #endif diff --git a/src/util/inc/tstoken.h b/src/util/inc/tstoken.h index 74687e9c18..7b0e498d76 100644 --- a/src/util/inc/tstoken.h +++ b/src/util/inc/tstoken.h @@ -24,14 +24,7 @@ extern "C" { #include "tutil.h" #include "ttokendef.h" -#define TK_SPACE 200 -#define TK_COMMENT 201 -#define TK_ILLEGAL 202 -#define TK_HEX 203 // hex number 0x123 -#define TK_OCT 204 // oct number -#define TK_BIN 205 // bin format data 0b111 -#define TK_FILE 206 -#define TK_QUESTION 207 // denoting the placeholder of "?",when invoking statement bind query + #define TSQL_TBNAME "TBNAME" #define TSQL_TBNAME_L "tbname" From dba77aa8e3cc600847ccf34ebce39f34203209c7 Mon Sep 17 00:00:00 2001 From: liu0x54 Date: Sat, 20 Jun 2020 08:51:06 +0000 Subject: [PATCH 02/27] [TD-711] fix the token define conflicts --- src/query/src/qtokenizer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/src/qtokenizer.c b/src/query/src/qtokenizer.c index aa9f6fddca..80d59a384e 100644 --- a/src/query/src/qtokenizer.c +++ b/src/query/src/qtokenizer.c @@ -25,7 +25,7 @@ // All the keywords of the SQL language are stored in a hash table typedef struct SKeyword { const char* name; // The keyword name - uint8_t type; // type + uint16_t type; // type uint8_t len; // length } SKeyword; From c9d13a431688b49138439ee77f9ac9f00c6bdd46 Mon Sep 17 00:00:00 2001 From: liu0x54 Date: Sat, 20 Jun 2020 08:55:30 +0000 Subject: [PATCH 03/27] [TD-711] fix the token define conflicts --- src/inc/ttokendef.h | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/inc/ttokendef.h b/src/inc/ttokendef.h index 7648aadc60..b313d6ebb1 100644 --- a/src/inc/ttokendef.h +++ b/src/inc/ttokendef.h @@ -222,14 +222,14 @@ #define TK_VALUES 204 -#define TK_SPACE 300 -#define TK_COMMENT 301 -#define TK_ILLEGAL 302 -#define TK_HEX 303 // hex number 0x123 -#define TK_OCT 304 // oct number -#define TK_BIN 305 // bin format data 0b111 -#define TK_FILE 306 -#define TK_QUESTION 307 // denoting the placeholder of "?",when invoking statement bind query +#define TK_SPACE 300 +#define TK_COMMENT 301 +#define TK_ILLEGAL 302 +#define TK_HEX 303 // hex number 0x123 +#define TK_OCT 304 // oct number +#define TK_BIN 305 // bin format data 0b111 +#define TK_FILE 306 +#define TK_QUESTION 307 // denoting the placeholder of "?",when invoking statement bind query #endif From c0ae4e75767e3dd9d2068044063f181b775abcc3 Mon Sep 17 00:00:00 2001 From: Hui Li Date: Sat, 20 Jun 2020 17:08:23 +0800 Subject: [PATCH 04/27] [TD-598] --- src/client/src/tscParseInsert.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index e65acc2483..69bc4a3e20 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -986,14 +986,16 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { return code; } -int validateTableName(char *tblName, int len) { +int validateTableName(char *tblName, int len, SSQLToken* psTblToken) { char buf[TSDB_TABLE_ID_LEN] = {0}; tstrncpy(buf, tblName, sizeof(buf)); - SSQLToken token = {.n = len, .type = TK_ID, .z = buf}; - tSQLGetToken(buf, &token.type); + psTblToken->n = len; + psTblToken->type = TK_ID; + psTblToken->z = buf; + tSQLGetToken(buf, &psTblToken->type); - return tscValidateName(&token); + return tscValidateName(psTblToken); } static int32_t validateDataSource(SSqlCmd *pCmd, int8_t type, const char *sql) { @@ -1077,14 +1079,14 @@ int tsParseInsertSql(SSqlObj *pSql) { } pCmd->curSql = sToken.z; - + SSQLToken sTblToken; // Check if the table name available or not - if (validateTableName(sToken.z, sToken.n) != TSDB_CODE_SUCCESS) { + if (validateTableName(sToken.z, sToken.n, &sTblToken) != TSDB_CODE_SUCCESS) { code = tscInvalidSQLErrMsg(pCmd->payload, "table name invalid", sToken.z); goto _error; } - if ((code = tscSetTableFullName(pTableMetaInfo, &sToken, pSql)) != TSDB_CODE_SUCCESS) { + if ((code = tscSetTableFullName(pTableMetaInfo, &sTblToken, pSql)) != TSDB_CODE_SUCCESS) { goto _error; } From e90d241eeb682102dbdeb7da7cbb1ace62b6a5f2 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sat, 20 Jun 2020 11:41:12 +0000 Subject: [PATCH 05/27] stop UDP/TCP connection first, then close all connections, then clean up --- src/rpc/inc/rpcTcp.h | 1 + src/rpc/src/rpcMain.c | 30 ++++++++++++++++++++++++++---- src/rpc/src/rpcTcp.c | 13 +++++++++---- src/rpc/src/rpcUdp.c | 8 ++------ 4 files changed, 38 insertions(+), 14 deletions(-) diff --git a/src/rpc/inc/rpcTcp.h b/src/rpc/inc/rpcTcp.h index 40fab00056..0b0e5bd0fb 100644 --- a/src/rpc/inc/rpcTcp.h +++ b/src/rpc/inc/rpcTcp.h @@ -21,6 +21,7 @@ extern "C" { #endif void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle); +void taosStopTcpServer(void *param); void taosCleanUpTcpServer(void *param); void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int num, void *fp, void *shandle); diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index 007a511adf..780ec5ccba 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -147,12 +147,19 @@ void *(*taosInitConn[])(uint32_t ip, uint16_t port, char *label, int threads, vo }; void (*taosCleanUpConn[])(void *thandle) = { - taosCleanUpUdpConnection, - taosCleanUpUdpConnection, + NULL, + NULL, taosCleanUpTcpServer, taosCleanUpTcpClient }; +void (*taosStopConn[])(void *thandle) = { + taosCleanUpUdpConnection, + taosCleanUpUdpConnection, + taosStopTcpServer, + NULL +}; + int (*taosSendData[])(uint32_t ip, uint16_t port, void *data, int len, void *chandle) = { taosSendUdpData, taosSendUdpData, @@ -289,14 +296,26 @@ void *rpcOpen(const SRpcInit *pInit) { void rpcClose(void *param) { SRpcInfo *pRpc = (SRpcInfo *)param; + // stop connection to outside first + if (taosStopConn[pRpc->connType | RPC_CONN_TCP]) + (*taosStopConn[pRpc->connType | RPC_CONN_TCP])(pRpc->tcphandle); + + if (taosStopConn[pRpc->connType]) + (*taosStopConn[pRpc->connType])(pRpc->udphandle); + + // close all connections for (int i = 0; i < pRpc->sessions; ++i) { if (pRpc->connList && pRpc->connList[i].user[0]) { rpcCloseConn((void *)(pRpc->connList + i)); } } - (*taosCleanUpConn[pRpc->connType | RPC_CONN_TCP])(pRpc->tcphandle); - (*taosCleanUpConn[pRpc->connType])(pRpc->udphandle); + // clean up + if (taosCleanUpConn[pRpc->connType | RPC_CONN_TCP]) + (*taosCleanUpConn[pRpc->connType | RPC_CONN_TCP])(pRpc->tcphandle); + + if (taosCleanUpConn[pRpc->connType]) + (*taosCleanUpConn[pRpc->connType])(pRpc->udphandle); tTrace("%s rpc is closed", pRpc->label); rpcDecRef(pRpc); @@ -588,6 +607,7 @@ static void rpcReleaseConn(SRpcConn *pConn) { pConn->inTranId = 0; pConn->outTranId = 0; pConn->secured = 0; + pConn->peerId = 0; pConn->peerIp = 0; pConn->peerPort = 0; pConn->pReqMsg = NULL; @@ -627,6 +647,7 @@ static SRpcConn *rpcAllocateClientConn(SRpcInfo *pRpc) { pConn->spi = pRpc->spi; pConn->encrypt = pRpc->encrypt; if (pConn->spi) memcpy(pConn->secret, pRpc->secret, TSDB_KEY_LEN); + tTrace("%s %p client connection is allocated", pRpc->label, pConn); } return pConn; @@ -681,6 +702,7 @@ static SRpcConn *rpcAllocateServerConn(SRpcInfo *pRpc, SRecvInfo *pRecv) { } taosHashPut(pRpc->hash, hashstr, size, (char *)&pConn, POINTER_BYTES); + tTrace("%s %p server connection is allocated", pRpc->label, pConn); } return pConn; diff --git a/src/rpc/src/rpcTcp.c b/src/rpc/src/rpcTcp.c index 5d156492c7..11d2c57dda 100644 --- a/src/rpc/src/rpcTcp.c +++ b/src/rpc/src/rpcTcp.c @@ -190,14 +190,19 @@ static void taosStopTcpThread(SThreadObj* pThreadObj) { } } +void taosStopTcpServer(void *handle) { + SServerObj *pServerObj = handle; + + tTrace("TCP:%s, stop accept new connections", pServerObj->label); + if (pServerObj == NULL) return; + if(pServerObj->fd >=0) shutdown(pServerObj->fd, SHUT_RD); + if(pServerObj->thread) pthread_join(pServerObj->thread, NULL); +} void taosCleanUpTcpServer(void *handle) { SServerObj *pServerObj = handle; SThreadObj *pThreadObj; - if (pServerObj == NULL) return; - if(pServerObj->fd >=0) shutdown(pServerObj->fd, SHUT_RD); - if(pServerObj->thread) pthread_join(pServerObj->thread, NULL); for (int i = 0; i < pServerObj->numOfThreads; ++i) { pThreadObj = pServerObj->pThreadObj + i; @@ -226,7 +231,7 @@ static void *taosAcceptTcpConnection(void *arg) { connFd = accept(pServerObj->fd, (struct sockaddr *)&caddr, &addrlen); if (connFd == -1) { if (errno == EINVAL) { - tTrace("%s TCP server socket was shutdown, exiting...", pServerObj->label); + tTrace("%s TCP server stop accepting new connections, exiting", pServerObj->label); break; } diff --git a/src/rpc/src/rpcUdp.c b/src/rpc/src/rpcUdp.c index 41446f87fb..35e06e633b 100644 --- a/src/rpc/src/rpcUdp.c +++ b/src/rpc/src/rpcUdp.c @@ -30,7 +30,6 @@ #define RPC_MAX_UDP_SIZE 65480 typedef struct { - void *signature; int index; int fd; uint16_t port; // peer port @@ -111,7 +110,6 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads pConn->processData = fp; pConn->index = i; pConn->pSet = pSet; - pConn->signature = pConn; int code = pthread_create(&pConn->thread, &thAttr, taosRecvUdpData, pConn); if (code != 0) { @@ -140,8 +138,6 @@ void taosCleanUpUdpConnection(void *handle) { for (int i = 0; i < pSet->threads; ++i) { pConn = pSet->udpConn + i; - pConn->signature = NULL; - if (pConn->fd >=0) shutdown(pConn->fd, SHUT_RDWR); if (pConn->fd >=0) taosCloseSocket(pConn->fd); } @@ -185,7 +181,7 @@ static void *taosRecvUdpData(void *param) { while (1) { dataLen = recvfrom(pConn->fd, pConn->buffer, RPC_MAX_UDP_SIZE, 0, (struct sockaddr *)&sourceAdd, &addLen); if(dataLen <= 0) { - tTrace("%s UDP socket was closed, exiting", pConn->label); + tTrace("%s UDP socket was closed, exiting(%s)", pConn->label, strerror(errno)); break; } @@ -221,7 +217,7 @@ static void *taosRecvUdpData(void *param) { int taosSendUdpData(uint32_t ip, uint16_t port, void *data, int dataLen, void *chandle) { SUdpConn *pConn = (SUdpConn *)chandle; - if (pConn == NULL || pConn->signature != pConn) return -1; + if (pConn == NULL) return -1; struct sockaddr_in destAdd; memset(&destAdd, 0, sizeof(destAdd)); From a3bb1d5f2b7eae6128379349dcf185dcb8b5869b Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sat, 20 Jun 2020 12:51:14 +0000 Subject: [PATCH 06/27] tune the code --- src/rpc/inc/rpcTcp.h | 1 + src/rpc/inc/rpcUdp.h | 1 + src/rpc/src/rpcMain.c | 24 +++++++++--------------- src/rpc/src/rpcTcp.c | 14 +++++++++++--- src/rpc/src/rpcUdp.c | 19 +++++++++++++++++-- 5 files changed, 39 insertions(+), 20 deletions(-) diff --git a/src/rpc/inc/rpcTcp.h b/src/rpc/inc/rpcTcp.h index 0b0e5bd0fb..6ef8fc2d92 100644 --- a/src/rpc/inc/rpcTcp.h +++ b/src/rpc/inc/rpcTcp.h @@ -25,6 +25,7 @@ void taosStopTcpServer(void *param); void taosCleanUpTcpServer(void *param); void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int num, void *fp, void *shandle); +void taosStopTcpClient(void *chandle); void taosCleanUpTcpClient(void *chandle); void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port); diff --git a/src/rpc/inc/rpcUdp.h b/src/rpc/inc/rpcUdp.h index fd60f4a089..c1da6a9240 100644 --- a/src/rpc/inc/rpcUdp.h +++ b/src/rpc/inc/rpcUdp.h @@ -23,6 +23,7 @@ extern "C" { #include "taosdef.h" void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int, void *fp, void *shandle); +void taosStopUdpConnection(void *handle); void taosCleanUpUdpConnection(void *handle); int taosSendUdpData(uint32_t ip, uint16_t port, void *data, int dataLen, void *chandle); void *taosOpenUdpConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port); diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index 780ec5ccba..f48d207d7b 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -147,17 +147,17 @@ void *(*taosInitConn[])(uint32_t ip, uint16_t port, char *label, int threads, vo }; void (*taosCleanUpConn[])(void *thandle) = { - NULL, - NULL, + taosCleanUpUdpConnection, + taosCleanUpUdpConnection, taosCleanUpTcpServer, taosCleanUpTcpClient }; void (*taosStopConn[])(void *thandle) = { - taosCleanUpUdpConnection, - taosCleanUpUdpConnection, + taosStopUdpConnection, + taosStopUdpConnection, taosStopTcpServer, - NULL + taosStopTcpClient, }; int (*taosSendData[])(uint32_t ip, uint16_t port, void *data, int len, void *chandle) = { @@ -297,11 +297,8 @@ void rpcClose(void *param) { SRpcInfo *pRpc = (SRpcInfo *)param; // stop connection to outside first - if (taosStopConn[pRpc->connType | RPC_CONN_TCP]) - (*taosStopConn[pRpc->connType | RPC_CONN_TCP])(pRpc->tcphandle); - - if (taosStopConn[pRpc->connType]) - (*taosStopConn[pRpc->connType])(pRpc->udphandle); + (*taosStopConn[pRpc->connType | RPC_CONN_TCP])(pRpc->tcphandle); + (*taosStopConn[pRpc->connType])(pRpc->udphandle); // close all connections for (int i = 0; i < pRpc->sessions; ++i) { @@ -311,11 +308,8 @@ void rpcClose(void *param) { } // clean up - if (taosCleanUpConn[pRpc->connType | RPC_CONN_TCP]) - (*taosCleanUpConn[pRpc->connType | RPC_CONN_TCP])(pRpc->tcphandle); - - if (taosCleanUpConn[pRpc->connType]) - (*taosCleanUpConn[pRpc->connType])(pRpc->udphandle); + (*taosCleanUpConn[pRpc->connType | RPC_CONN_TCP])(pRpc->tcphandle); + (*taosCleanUpConn[pRpc->connType])(pRpc->udphandle); tTrace("%s rpc is closed", pRpc->label); rpcDecRef(pRpc); diff --git a/src/rpc/src/rpcTcp.c b/src/rpc/src/rpcTcp.c index 11d2c57dda..aa94accceb 100644 --- a/src/rpc/src/rpcTcp.c +++ b/src/rpc/src/rpcTcp.c @@ -193,10 +193,11 @@ static void taosStopTcpThread(SThreadObj* pThreadObj) { void taosStopTcpServer(void *handle) { SServerObj *pServerObj = handle; - tTrace("TCP:%s, stop accept new connections", pServerObj->label); if (pServerObj == NULL) return; if(pServerObj->fd >=0) shutdown(pServerObj->fd, SHUT_RD); if(pServerObj->thread) pthread_join(pServerObj->thread, NULL); + + tTrace("%s TCP server is stopped", pServerObj->label); } void taosCleanUpTcpServer(void *handle) { @@ -210,7 +211,7 @@ void taosCleanUpTcpServer(void *handle) { pthread_mutex_destroy(&(pThreadObj->mutex)); } - tTrace("TCP:%s, TCP server is cleaned up", pServerObj->label); + tTrace("%s TCP server is cleaned up", pServerObj->label); tfree(pServerObj->pThreadObj); tfree(pServerObj); @@ -309,12 +310,19 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int num, void * return pThreadObj; } +void taosStopTcpClient(void *chandle) { + SThreadObj *pThreadObj = chandle; + if (pThreadObj == NULL) return; + + tTrace ("%s TCP client is stopped", pThreadObj->label); +} + void taosCleanUpTcpClient(void *chandle) { SThreadObj *pThreadObj = chandle; if (pThreadObj == NULL) return; taosStopTcpThread(pThreadObj); - tTrace ("%s, all connections are cleaned up", pThreadObj->label); + tTrace ("%s TCP client is cleaned up", pThreadObj->label); tfree(pThreadObj); } diff --git a/src/rpc/src/rpcUdp.c b/src/rpc/src/rpcUdp.c index 35e06e633b..f88db3a226 100644 --- a/src/rpc/src/rpcUdp.c +++ b/src/rpc/src/rpcUdp.c @@ -130,7 +130,7 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads return pSet; } -void taosCleanUpUdpConnection(void *handle) { +void taosStopUdpConnection(void *handle) { SUdpConnSet *pSet = (SUdpConnSet *)handle; SUdpConn *pConn; @@ -146,9 +146,24 @@ void taosCleanUpUdpConnection(void *handle) { pConn = pSet->udpConn + i; if (pConn->thread) pthread_join(pConn->thread, NULL); tfree(pConn->buffer); - tTrace("%s UDP thread is closed, inedx:%d", pConn->label, i); + // tTrace("%s UDP thread is closed, index:%d", pConn->label, i); } + tTrace("%s UDP is stopped", pSet->label); +} + +void taosCleanUpUdpConnection(void *handle) { + SUdpConnSet *pSet = (SUdpConnSet *)handle; + SUdpConn *pConn; + + if (pSet == NULL) return; + + for (int i = 0; i < pSet->threads; ++i) { + pConn = pSet->udpConn + i; + if (pConn->fd >=0) taosCloseSocket(pConn->fd); + } + + tTrace("%s UDP is cleaned up", pSet->label); tfree(pSet); } From f95d577615f78c4a57b34d1699c766f0c58911e5 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 21 Jun 2020 02:10:51 +0000 Subject: [PATCH 07/27] add loop.sh for test --- tests/script/loop.sh | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100755 tests/script/loop.sh diff --git a/tests/script/loop.sh b/tests/script/loop.sh new file mode 100755 index 0000000000..d41fe8d4aa --- /dev/null +++ b/tests/script/loop.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +################################################## +# +# Do simulation test +# +################################################## + +set +e +#set -x + +CMD_NAME= +LOOP_TIMES=5 + +while getopts "f:t:" arg +do + case $arg in + f) + CMD_NAME=$OPTARG + ;; + t) + LOOP_TIMES=$OPTARG + ;; + ?) + echo "unknow argument" + ;; + esac +done + +echo LOOP_TIMES ${LOOP_TIMES} +echo CMD_NAME ${CMD_NAME} + +for (( i=0; i<$LOOP_TIMES; i++ )) +do + echo loop $i + echo cmd $CMD_NAME + $CMD_NAME +done From 06cc1ec244f1ae6f5a8e23065399626d40605c12 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 21 Jun 2020 02:12:31 +0000 Subject: [PATCH 08/27] loop.sh --- tests/script/loop.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/script/loop.sh b/tests/script/loop.sh index d41fe8d4aa..adafe2ed60 100755 --- a/tests/script/loop.sh +++ b/tests/script/loop.sh @@ -6,7 +6,7 @@ # ################################################## -set +e +set -e #set -x CMD_NAME= @@ -30,9 +30,10 @@ done echo LOOP_TIMES ${LOOP_TIMES} echo CMD_NAME ${CMD_NAME} -for (( i=0; i<$LOOP_TIMES; i++ )) +for ((i=0; i<$LOOP_TIMES; i++ )) do echo loop $i echo cmd $CMD_NAME $CMD_NAME + sleep 2 done From 811bd5d2f2a266831c45a472f0b90dde9da9cadb Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 21 Jun 2020 02:44:15 +0000 Subject: [PATCH 09/27] fix bug while mnode replica --- src/mnode/src/mnodeDnode.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index 0f2bbb8235..e16195499b 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -348,7 +348,6 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) { pRsp->dnodeCfg.dnodeId = htonl(pDnode->dnodeId); pRsp->dnodeCfg.moduleStatus = htonl((int32_t)pDnode->isMgmt); pRsp->dnodeCfg.numOfVnodes = htonl(openVnodes); - mnodeGetMnodeInfos(&pRsp->mnodes); SDMVgroupAccess *pAccess = (SDMVgroupAccess *)((char *)pRsp + sizeof(SDMStatusRsp)); for (int32_t j = 0; j < openVnodes; ++j) { @@ -392,6 +391,10 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) { } pDnode->lastAccess = tsAccessSquence; + + //this func should be called after sdb replica changed + mnodeGetMnodeInfos(&pRsp->mnodes); + mnodeDecDnodeRef(pDnode); pMsg->rpcRsp.len = contLen; From 61faa484113fee64740ad317c3b81f703343100f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 21 Jun 2020 10:48:24 +0800 Subject: [PATCH 10/27] scripts --- tests/script/jenkins/unique.txt | 57 +++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/tests/script/jenkins/unique.txt b/tests/script/jenkins/unique.txt index 4f194cde93..da0aee3d34 100644 --- a/tests/script/jenkins/unique.txt +++ b/tests/script/jenkins/unique.txt @@ -35,6 +35,7 @@ cd ../../../debug; make ./test.sh -f unique/db/replica_reduce31.sim ./test.sh -f unique/db/replica_part.sim +./test.sh -f unique/dnode/alternativeRole.sim ./test.sh -f unique/dnode/balance1.sim ./test.sh -f unique/dnode/balance2.sim ./test.sh -f unique/dnode/balance3.sim @@ -75,3 +76,59 @@ cd ../../../debug; make ./test.sh -f unique/vnode/replica3_basic.sim ./test.sh -f unique/vnode/replica3_repeat.sim ./test.sh -f unique/vnode/replica3_vgroup.sim + +./test.sh -f general/stream/metrics_1.sim +./test.sh -f general/stream/metrics_del.sim +./test.sh -f general/stream/metrics_n.sim +./test.sh -f general/stream/metrics_replica1_vnoden.sim +#./test.sh -f general/stream/new_stream.sim +./test.sh -f general/stream/restart_stream.sim +./test.sh -f general/stream/stream_1.sim +./test.sh -f general/stream/stream_2.sim +./test.sh -f general/stream/stream_3.sim +./test.sh -f general/stream/stream_restart.sim +./test.sh -f general/stream/table_1.sim +./test.sh -f general/stream/table_del.sim +./test.sh -f general/stream/table_n.sim +./test.sh -f general/stream/table_replica1_vnoden.sim + +./test.sh -f unique/arbitrator/check_cluster_cfg_para.sim +./test.sh -f unique/arbitrator/dn2_mn1_cache_file_sync.sim +./test.sh -f unique/arbitrator/dn3_mn1_full_createTableFail.sim +./test.sh -f unique/arbitrator/dn3_mn1_full_dropDnodeFail.sim +./test.sh -f unique/arbitrator/dn3_mn1_multiCreateDropTable.sim +./test.sh -f unique/arbitrator/dn3_mn1_nw_disable_timeout_autoDropDnode.sim +./test.sh -f unique/arbitrator/dn3_mn1_replica2_wal1_AddDelDnode.sim +./test.sh -f unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim +./test.sh -f unique/arbitrator/dn3_mn1_replica_change.sim +./test.sh -f unique/arbitrator/dn3_mn1_stopDnode_timeout.sim +./test.sh -f unique/arbitrator/dn3_mn1_vnode_change.sim +./test.sh -f unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim +./test.sh -f unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim +./test.sh -f unique/arbitrator/dn3_mn1_vnode_noCorruptFile_offline.sim +./test.sh -f unique/arbitrator/dn3_mn1_vnode_delDir.sim +./test.sh -f unique/arbitrator/dn3_mn1_r2_vnode_delDir.sim +./test.sh -f unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim +./test.sh -f unique/arbitrator/dn3_mn1_vnode_nomaster.sim +./test.sh -f unique/arbitrator/dn3_mn2_killDnode.sim +./test.sh -f unique/arbitrator/insert_duplicationTs.sim +./test.sh -f unique/arbitrator/offline_replica2_alterTable_online.sim +./test.sh -f unique/arbitrator/offline_replica2_alterTag_online.sim +./test.sh -f unique/arbitrator/offline_replica2_createTable_online.sim +./test.sh -f unique/arbitrator/offline_replica2_dropDb_online.sim +./test.sh -f unique/arbitrator/offline_replica2_dropTable_online.sim +./test.sh -f unique/arbitrator/offline_replica3_alterTable_online.sim +./test.sh -f unique/arbitrator/offline_replica3_alterTag_online.sim +./test.sh -f unique/arbitrator/offline_replica3_createTable_online.sim +./test.sh -f unique/arbitrator/offline_replica3_dropDb_online.sim +./test.sh -f unique/arbitrator/offline_replica3_dropTable_online.sim +./test.sh -f unique/arbitrator/replica_changeWithArbitrator.sim +./test.sh -f unique/arbitrator/sync_replica2_alterTable_add.sim +./test.sh -f unique/arbitrator/sync_replica2_alterTable_drop.sim +./test.sh -f unique/arbitrator/sync_replica2_dropDb.sim +./test.sh -f unique/arbitrator/sync_replica2_dropTable.sim +./test.sh -f unique/arbitrator/sync_replica3_alterTable_add.sim +./test.sh -f unique/arbitrator/sync_replica3_alterTable_drop.sim +./test.sh -f unique/arbitrator/sync_replica3_dropDb.sim +./test.sh -f unique/arbitrator/sync_replica3_dropTable.sim + From 1721714f56f2618c47ca0df00860afa32339baac Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sun, 21 Jun 2020 02:57:07 +0000 Subject: [PATCH 11/27] add debug info --- src/rpc/src/rpcMain.c | 18 +++++++----------- src/rpc/src/rpcUdp.c | 2 +- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index f48d207d7b..f62df56771 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -964,11 +964,9 @@ static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv) { terrno = 0; pConn = rpcProcessMsgHead(pRpc, pRecv); - if (pHead->msgType < TSDB_MSG_TYPE_CM_HEARTBEAT || (rpcDebugFlag & 16)) { - tTrace("%s %p %p, %s received from 0x%x:%hu, parse code:0x%x len:%d sig:0x%08x:0x%08x:%d code:0x%x", + tTrace("%s %p %p, %s received from 0x%x:%hu, parse code:0x%x len:%d sig:0x%08x:0x%08x:%d code:0x%x", pRpc->label, pConn, (void *)pHead->ahandle, taosMsg[pHead->msgType], pRecv->ip, pRecv->port, terrno, pRecv->msgLen, pHead->sourceId, pHead->destId, pHead->tranId, pHead->code); - } int32_t code = terrno; if (code != TSDB_CODE_RPC_ALREADY_PROCESSED) { @@ -1196,16 +1194,14 @@ static void rpcSendMsgToPeer(SRpcConn *pConn, void *msg, int msgLen) { msgLen = rpcAddAuthPart(pConn, msg, msgLen); if ( rpcIsReq(pHead->msgType)) { - if (pHead->msgType < TSDB_MSG_TYPE_CM_HEARTBEAT || (rpcDebugFlag & 16)) - tTrace("%s, %s is sent to %s:%hu, len:%d sig:0x%08x:0x%08x:%d", - pConn->info, taosMsg[pHead->msgType], pConn->peerFqdn, pConn->peerPort, - msgLen, pHead->sourceId, pHead->destId, pHead->tranId); + tTrace("%s, %s is sent to %s:%hu, len:%d sig:0x%08x:0x%08x:%d", + pConn->info, taosMsg[pHead->msgType], pConn->peerFqdn, pConn->peerPort, + msgLen, pHead->sourceId, pHead->destId, pHead->tranId); } else { if (pHead->code == 0) pConn->secured = 1; // for success response, set link as secured - if (pHead->msgType < TSDB_MSG_TYPE_CM_HEARTBEAT || (rpcDebugFlag & 16)) - tTrace("%s, %s is sent to 0x%x:%hu, code:0x%x len:%d sig:0x%08x:0x%08x:%d", - pConn->info, taosMsg[pHead->msgType], pConn->peerIp, pConn->peerPort, - htonl(pHead->code), msgLen, pHead->sourceId, pHead->destId, pHead->tranId); + tTrace("%s, %s is sent to 0x%x:%hu, code:0x%x len:%d sig:0x%08x:0x%08x:%d", + pConn->info, taosMsg[pHead->msgType], pConn->peerIp, pConn->peerPort, + htonl(pHead->code), msgLen, pHead->sourceId, pHead->destId, pHead->tranId); } //tTrace("connection type is: %d", pConn->connType); diff --git a/src/rpc/src/rpcUdp.c b/src/rpc/src/rpcUdp.c index f88db3a226..a4c7d6c145 100644 --- a/src/rpc/src/rpcUdp.c +++ b/src/rpc/src/rpcUdp.c @@ -175,7 +175,7 @@ void *taosOpenUdpConnection(void *shandle, void *thandle, uint32_t ip, uint16_t SUdpConn *pConn = pSet->udpConn + pSet->index; pConn->port = port; - tTrace("%s UDP connection is setup, ip:%x:%hu", pConn->label, ip, port); + tTrace("%s UDP connection is setup, ip:%x:%hu localPort:%hu", pConn->label, ip, port, pConn->localPort); return pConn; } From 8336bbcf27a42f10718e37b595f537729dff2ab8 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 21 Jun 2020 04:11:08 +0000 Subject: [PATCH 12/27] fix commit assert error --- src/tsdb/src/tsdbMemTable.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index e928cdc385..2989daad49 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -222,11 +222,12 @@ int tsdbAsyncCommit(STsdbRepo *pRepo) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; } + pRepo->commit = 0; } ASSERT(pRepo->commit == 0); - if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_START); if (pRepo->mem != NULL) { + if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_START); if (tsdbLockRepo(pRepo) < 0) return -1; pRepo->imem = pRepo->mem; pRepo->mem = NULL; @@ -468,9 +469,6 @@ _err: static void tsdbEndCommit(STsdbRepo *pRepo) { ASSERT(pRepo->commit == 1); - tsdbLockRepo(pRepo); - pRepo->commit = 0; - tsdbUnlockRepo(pRepo); if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_OVER); } From 40dace79a325e4dc51a625358d2596e19dae84d9 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sun, 21 Jun 2020 09:16:26 +0000 Subject: [PATCH 13/27] add more debug info for TCP --- src/rpc/src/rpcTcp.c | 43 +++++++++++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/src/rpc/src/rpcTcp.c b/src/rpc/src/rpcTcp.c index aa94accceb..82168f0b0e 100644 --- a/src/rpc/src/rpcTcp.c +++ b/src/rpc/src/rpcTcp.c @@ -248,13 +248,13 @@ static void *taosAcceptTcpConnection(void *arg) { SFdObj *pFdObj = taosMallocFdObj(pThreadObj, connFd); if (pFdObj) { pFdObj->ip = caddr.sin_addr.s_addr; - pFdObj->port = caddr.sin_port; - tTrace("%s new connection from %s:%hu, FD:%p, numOfFds:%d", pServerObj->label, - inet_ntoa(caddr.sin_addr), pFdObj->port, pFdObj, pThreadObj->numOfFds); + pFdObj->port = htons(caddr.sin_port); + tTrace("%s new TCP connection from %s:%hu, fd:%d FD:%p numOfFds:%d", pServerObj->label, + inet_ntoa(caddr.sin_addr), pFdObj->port, connFd, pFdObj, pThreadObj->numOfFds); } else { close(connFd); tError("%s failed to malloc FdObj(%s) for connection from:%s:%hu", pServerObj->label, strerror(errno), - inet_ntoa(caddr.sin_addr), caddr.sin_port); + inet_ntoa(caddr.sin_addr), htons(caddr.sin_port)); } // pick up next thread for next connection @@ -333,14 +333,22 @@ void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uin int fd = taosOpenTcpClientSocket(ip, port, pThreadObj->ip); if (fd < 0) return NULL; + struct sockaddr_in sin; + uint16_t localPort = 0; + unsigned int addrlen = sizeof(sin); + if (getsockname(fd, (struct sockaddr *)&sin, &addrlen) == 0 && + sin.sin_family == AF_INET && addrlen == sizeof(sin)) { + localPort = (uint16_t)ntohs(sin.sin_port); + } + SFdObj *pFdObj = taosMallocFdObj(pThreadObj, fd); if (pFdObj) { pFdObj->thandle = thandle; pFdObj->port = port; pFdObj->ip = ip; - tTrace("%s %p, TCP connection to 0x%x:%hu is created, FD:%p numOfFds:%d", - pThreadObj->label, thandle, ip, port, pFdObj, pThreadObj->numOfFds); + tTrace("%s %p TCP connection to 0x%x:%hu is created, localPort:%hu FD:%p numOfFds:%d", + pThreadObj->label, thandle, ip, port, localPort, pFdObj, pThreadObj->numOfFds); } else { close(fd); tError("%s failed to malloc client FdObj(%s)", pThreadObj->label, strerror(errno)); @@ -353,7 +361,10 @@ void taosCloseTcpConnection(void *chandle) { SFdObj *pFdObj = chandle; if (pFdObj == NULL) return; - pFdObj->thandle = NULL; + SThreadObj *pThreadObj = pFdObj->pThreadObj; + tTrace("%s %p TCP connection will be closed, FD:%p", pThreadObj->label, pFdObj->thandle, pFdObj); + + // pFdObj->thandle = NULL; pFdObj->closedByApp = 1; shutdown(pFdObj->fd, SHUT_WR); } @@ -398,14 +409,14 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) { headLen = taosReadMsg(pFdObj->fd, &rpcHead, sizeof(SRpcHead)); if (headLen != sizeof(SRpcHead)) { - tTrace("%s %p, read error, headLen:%d", pThreadObj->label, pFdObj->thandle, headLen); + tTrace("%s %p read error, headLen:%d", pThreadObj->label, pFdObj->thandle, headLen); return -1; } msgLen = (int32_t)htonl((uint32_t)rpcHead.msgLen); buffer = malloc(msgLen + tsRpcOverhead); if ( NULL == buffer) { - tError("%s %p, TCP malloc(size:%d) fail", pThreadObj->label, pFdObj->thandle, msgLen); + tError("%s %p TCP malloc(size:%d) fail", pThreadObj->label, pFdObj->thandle, msgLen); return -1; } @@ -414,8 +425,8 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) { retLen = taosReadMsg(pFdObj->fd, msg + headLen, leftLen); if (leftLen != retLen) { - tError("%s %p, read error, leftLen:%d retLen:%d", - pThreadObj->label, pFdObj->thandle, leftLen, retLen); + tError("%s %p read error, leftLen:%d retLen:%d FD:%p", + pThreadObj->label, pFdObj->thandle, leftLen, retLen, pFdObj); free(buffer); return -1; } @@ -459,19 +470,19 @@ static void *taosProcessTcpData(void *param) { pFdObj = events[i].data.ptr; if (events[i].events & EPOLLERR) { - tTrace("%s %p, error happened on FD", pThreadObj->label, pFdObj->thandle); + tTrace("%s %p FD:%p epoll errors", pThreadObj->label, pFdObj->thandle, pFdObj); taosReportBrokenLink(pFdObj); continue; } if (events[i].events & EPOLLRDHUP) { - tTrace("%s %p, FD RD hang up", pThreadObj->label, pFdObj->thandle); + tTrace("%s %p FD:%p RD hang up", pThreadObj->label, pFdObj->thandle, pFdObj); taosReportBrokenLink(pFdObj); continue; } if (events[i].events & EPOLLHUP) { - tTrace("%s %p, FD hang up", pThreadObj->label, pFdObj->thandle); + tTrace("%s %p FD:%p hang up", pThreadObj->label, pFdObj->thandle, pFdObj); taosReportBrokenLink(pFdObj); continue; } @@ -540,7 +551,7 @@ static void taosFreeFdObj(SFdObj *pFdObj) { pThreadObj->numOfFds--; if (pThreadObj->numOfFds < 0) - tError("%s %p, TCP thread:%d, number of FDs is negative!!!", + tError("%s %p TCP thread:%d, number of FDs is negative!!!", pThreadObj->label, pFdObj->thandle, pThreadObj->threadId); if (pFdObj->prev) { @@ -555,7 +566,7 @@ static void taosFreeFdObj(SFdObj *pFdObj) { pthread_mutex_unlock(&pThreadObj->mutex); - tTrace("%s %p, FD:%p is cleaned, numOfFds:%d", + tTrace("%s %p TCP connection is closed, FD:%p numOfFds:%d", pThreadObj->label, pFdObj->thandle, pFdObj, pThreadObj->numOfFds); tfree(pFdObj); From 0e1c6aafb39607247b7f618670fc48b064e54c4c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 21 Jun 2020 10:41:12 +0000 Subject: [PATCH 14/27] fix coverity scan problems --- src/tsdb/src/tsdbBuffer.c | 1 + src/tsdb/src/tsdbFile.c | 4 +++- src/tsdb/src/tsdbMain.c | 7 ++++--- src/tsdb/src/tsdbMemTable.c | 4 +--- src/tsdb/src/tsdbMeta.c | 3 ++- 5 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/tsdb/src/tsdbBuffer.c b/src/tsdb/src/tsdbBuffer.c index 686fc4bbec..16f06c82bb 100644 --- a/src/tsdb/src/tsdbBuffer.c +++ b/src/tsdb/src/tsdbBuffer.c @@ -124,6 +124,7 @@ SListNode *tsdbAllocBufBlockFromPool(STsdbRepo *pRepo) { } SListNode * pNode = tdListPopHead(pBufPool->bufBlockList); + ASSERT(pNode != NULL); STsdbBufBlock *pBufBlock = NULL; tdListNodeGetData(pBufPool->bufBlockList, pNode, (void *)(&pBufBlock)); diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index ffac8da944..8ebb62f341 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -358,7 +358,9 @@ void tsdbRemoveFileGroup(STsdbRepo *pRepo, SFileGroup *pFGroup) { ASSERT(pFileH->nFGroups >= 0); for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) { - remove(fileGroup.files[type].fname); + if (remove(fileGroup.files[type].fname) < 0) { + tsdbError("vgId:%d failed to remove file %s", REPO_ID(pRepo), fileGroup.files[type].fname); + } tsdbDestroyFile(&fileGroup.files[type]); } } diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 80e5e8ea0d..ac334e8ed1 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -237,12 +237,13 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_ } SFile *pFile = &pFGroup->files[(*index) % 3]; - strcpy(fname, pFile->fname); + fname = strdup(pFile->fname); } } if (stat(fname, &fState) < 0) { tfree(sdup); + tfree(fname); return 0; } @@ -566,7 +567,7 @@ static int32_t tsdbSaveConfig(char *rootDir, STsdbCfg *pCfg) { _err: tfree(fname); - if (fd > 0) close(fd); + if (fd >= 0) close(fd); return -1; } @@ -609,7 +610,7 @@ static int tsdbLoadConfig(char *rootDir, STsdbCfg *pCfg) { _err: tfree(fname); - if (fd > 0) close(fd); + if (fd >= 0) close(fd); return -1; } diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 2989daad49..b1de90bc29 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -524,8 +524,6 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe goto _err; } - free(dataDir); - // Open files for write/read if (tsdbSetAndOpenHelperFile(pHelper, pGroup) < 0) { tsdbError("vgId:%d failed to set helper file since %s", REPO_ID(pRepo), tstrerror(terrno)); @@ -599,7 +597,7 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe return 0; _err: - // ASSERT(false); + tfree(dataDir); tsdbCloseHelperFile(pHelper, 1); return -1; } diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index beb8f33052..83392462ab 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -147,6 +147,7 @@ int tsdbDropTable(TSDB_REPO_T *repo, STableId tableId) { tsdbInsertTableAct(pRepo, TSDB_DROP_META, buf, tTable); tsdbRemoveTableFromMeta(pRepo, tTable, false, true); } + tSkipListDestroyIter(pIter); } tsdbRemoveTableFromMeta(pRepo, pTable, true, true); @@ -270,7 +271,6 @@ STableCfg *tsdbCreateTableCfgFromMsg(SMDCreateTableMsg *pMsg) { _err: tdDestroyTSchemaBuilder(&schemaBuilder); tsdbClearTableCfg(pCfg); - tfree(pCfg); return NULL; } @@ -309,6 +309,7 @@ int tsdbUpdateTagValue(TSDB_REPO_T *repo, SUpdateTableTagValMsg *pMsg) { int32_t code = tsdbUpdateTable(pRepo, super, pTableCfg); if (code != TSDB_CODE_SUCCESS) { + tsdbClearTableCfg(pTableCfg); return code; } tsdbClearTableCfg(pTableCfg); From 469cf6d6078c74313fc0cd01163b5a8ce3dc3b46 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Sun, 21 Jun 2020 14:38:54 +0000 Subject: [PATCH 15/27] remove re-useaddr for UDP socket --- src/util/src/tsocket.c | 29 +---------------------------- 1 file changed, 1 insertion(+), 28 deletions(-) diff --git a/src/util/src/tsocket.c b/src/util/src/tsocket.c index 98ed8d1ed7..e19d91837f 100644 --- a/src/util/src/tsocket.c +++ b/src/util/src/tsocket.c @@ -222,9 +222,7 @@ int taosReadn(int fd, char *ptr, int nbytes) { int taosOpenUdpSocket(uint32_t ip, uint16_t port) { struct sockaddr_in localAddr; int sockFd; - int ttl = 128; - int reuse, nocheck; - int bufSize = 8192000; + int bufSize = 1024000; uTrace("open udp socket:0x%x:%hu", ip, port); @@ -238,31 +236,6 @@ int taosOpenUdpSocket(uint32_t ip, uint16_t port) { return -1; } - reuse = 1; - if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) { - uError("setsockopt SO_REUSEADDR failed): %d (%s)", errno, strerror(errno)); - close(sockFd); - return -1; - }; - - nocheck = 1; - if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_NO_CHECK, (void *)&nocheck, sizeof(nocheck)) < 0) { - if (!taosSkipSocketCheck()) { - uError("setsockopt SO_NO_CHECK failed: %d (%s)", errno, strerror(errno)); - close(sockFd); - return -1; - } else { - uPrint("Skipping setsockopt SO_NO_CHECK error: %d (%s)", errno, strerror(errno)); - } - } - - ttl = 128; - if (taosSetSockOpt(sockFd, IPPROTO_IP, IP_TTL, (void *)&ttl, sizeof(ttl)) < 0) { - uError("setsockopt IP_TTL failed: %d (%s)", errno, strerror(errno)); - close(sockFd); - return -1; - } - if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_SNDBUF, (void *)&bufSize, sizeof(bufSize)) != 0) { uError("failed to set the send buffer size for UDP socket\n"); close(sockFd); From f829bebf68a98dab0a3e9581006f48366e69a3d7 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 22 Jun 2020 01:40:23 +0000 Subject: [PATCH 16/27] update table lastkey while insert data --- src/tsdb/src/tsdbMemTable.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index b1de90bc29..7a360c25f9 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -99,6 +99,7 @@ int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) { if (tSkipListPut(pTableData->pData, pNode) == NULL) { tsdbFreeBytes(pRepo, (void *)pNode, bytes); } else { + if (TABLE_LASTKEY(pTable) < key) TABLE_LASTKEY(pTable) = key; if (pMemTable->keyFirst > key) pMemTable->keyFirst = key; if (pMemTable->keyLast < key) pMemTable->keyLast = key; pMemTable->numOfRows++; From 540852b1f9f8a9f9a366a03b3b56ad28db7a254a Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Mon, 22 Jun 2020 09:46:52 +0800 Subject: [PATCH 17/27] fix td-452 --- src/client/src/tscParseInsert.c | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 69bc4a3e20..4020d201ce 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -46,18 +46,20 @@ static int32_t tscToInteger(SSQLToken *pToken, int64_t *value, char **endPtr) { return TK_ILLEGAL; } - int32_t radix = 10; - - int32_t radixList[3] = {16, 8, 2}; // the integer number with different radix: hex, oct, bin - if (pToken->type == TK_HEX || pToken->type == TK_OCT || pToken->type == TK_BIN) { - radix = radixList[pToken->type - TK_HEX]; - } - errno = 0; - *value = strtoll(pToken->z, endPtr, radix); + *value = strtoll(pToken->z, endPtr, 0); + if (**endPtr == 'e' || **endPtr == 'E' || **endPtr == '.') { + errno = 0; + double v = round(strtod(pToken->z, endPtr)); + if (v > INT64_MAX || v <= INT64_MIN) { + errno = ERANGE; + } else { + *value = v; + } + } // not a valid integer number, return error - if ((pToken->type == TK_STRING || pToken->type == TK_ID) && ((*endPtr - pToken->z) != pToken->n)) { + if (*endPtr - pToken->z != pToken->n) { return TK_ILLEGAL; } @@ -73,11 +75,11 @@ static int32_t tscToDouble(SSQLToken *pToken, double *value, char **endPtr) { *value = strtod(pToken->z, endPtr); // not a valid integer number, return error - if ((pToken->type == TK_STRING || pToken->type == TK_ID) && ((*endPtr - pToken->z) != pToken->n)) { + if ((*endPtr - pToken->z) != pToken->n) { return TK_ILLEGAL; - } else { - return pToken->type; } + + return pToken->type; } int tsParseTime(SSQLToken *pToken, int64_t *time, char **next, char *error, int16_t timePrec) { From 8f1cee4aada27f9c9e77b9779d7e8dba84865290 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 22 Jun 2020 02:55:48 +0000 Subject: [PATCH 18/27] [TD-715] incorrect handling of error code when creating vnode failed --- src/mnode/src/mnodeVgroup.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index 9553ade3aa..e6759e90db 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -678,8 +678,9 @@ static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) { SMnodeMsg *mnodeMsg = rpcMsg->handle; mnodeMsg->received++; if (rpcMsg->code == TSDB_CODE_SUCCESS) { - mnodeMsg->code = rpcMsg->code; mnodeMsg->successed++; + } else { + mnodeMsg->code = rpcMsg->code; } SVgObj *pVgroup = mnodeMsg->pVgroup; @@ -702,7 +703,7 @@ static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) { code = TSDB_CODE_MND_SDB_ERROR; } - dnodeSendRpcMnodeWriteRsp(mnodeMsg, code); + dnodeSendRpcMnodeWriteRsp(mnodeMsg, mnodeMsg->code); } } From e1740422338fc3166aa077b948d3b22dfe1d1e10 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 22 Jun 2020 02:57:01 +0000 Subject: [PATCH 19/27] fix balance coredump error --- src/tsdb/src/tsdbMain.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index ac334e8ed1..92a5895f21 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -213,10 +213,10 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_ SFileGroup *pFGroup = taosbsearch(&fid, pFileH->pFGroup, pFileH->nFGroups, sizeof(SFileGroup), keyFGroupCompFunc, TD_GE); if (pFGroup->fileId == fid) { - strcpy(fname, pFGroup->files[(*index) % 3].fname); + fname = strdup(pFGroup->files[(*index) % 3].fname); } else { if (pFGroup->fileId * 3 + 2 < eindex) { - strcpy(fname, pFGroup->files[0].fname); + fname = strdup(pFGroup->files[0].fname); *index = pFGroup->fileId * 3; } else { tfree(sdup); From 2f341dabf258b061dad8d17fd2dd2cd6cff1737f Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 22 Jun 2020 11:05:52 +0800 Subject: [PATCH 20/27] remove coverage test from CI. --- .travis.yml | 86 ----------------------------------------------------- 1 file changed, 86 deletions(-) diff --git a/.travis.yml b/.travis.yml index fcb2e8f22f..bb9ee5d430 100644 --- a/.travis.yml +++ b/.travis.yml @@ -135,92 +135,6 @@ matrix: # https://scan.coverity.com/faq#frequency branch_pattern: coverity_scan - - os: linux - dist: bionic - language: c - compiler: gcc - env: ENV_COVER=true - - git: - - depth: 1 - - addons: - apt: - packages: - - build-essential - - cmake - - net-tools - - python-pip - - python-setuptools - - python3-pip - - python3-setuptools - - lcov - - psmisc - - before_script: - - cd ${TRAVIS_BUILD_DIR} - - mkdir debug - - cd debug - - script: - - cmake -DCOVER=true .. > /dev/null - - make > /dev/null - - after_success: - - |- - case $TRAVIS_OS_NAME in - linux) - cd ${TRAVIS_BUILD_DIR}/debug - make install > /dev/null || travis_terminate $? - - pip install numpy - pip install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python2/ - pip3 install numpy - pip3 install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python3/ - - cd ${TRAVIS_BUILD_DIR}/tests - - ./test-all.sh smoke COVER - - TEST_RESULT=$? - - pkill taosd - sleep 1 - - cd ${TRAVIS_BUILD_DIR} - lcov -d . --capture --rc lcov_branch_coverage=1 -o coverage.info - lcov --remove coverage.info '*/tests/*' '*/test/*' '*/deps/*' '*/plugins/*' -o coverage.info - lcov -l --rc lcov_branch_coverage=1 coverage.info || travis_terminate $? - - gem install coveralls-lcov - - # Color setting - RED='\033[0;31m' - GREEN='\033[1;32m' - GREEN_DARK='\033[0;32m' - GREEN_UNDERLINE='\033[4;32m' - NC='\033[0m' - - coveralls-lcov coverage.info - if [ "$?" -eq "0" ]; then - echo -e "${GREEN} ## Uploaded to Coveralls.io! ## ${NC}" - else - echo -e "${RED} ## Coveralls.io not collect coverage report! ## ${NC} " - fi - - bash <(curl -s https://codecov.io/bash) -y .codecov.yml -f coverage.info - if [ "$?" -eq "0" ]; then - echo -e "${GREEN} ## Uploaded to Codecov! ## ${NC} " - else - echo -e "${RED} ## Codecov did not collect coverage report! ## ${NC} " - fi - - if [ "$TEST_RESULT" -ne "0" ]; then - travis_terminate $? - fi - ;; - esac - - os: linux dist: trusty language: c From 9f8d29ce11469b8f2263e584f00d03af09cf2b26 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 22 Jun 2020 11:30:34 +0800 Subject: [PATCH 21/27] Add coverage test badge to README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index c6d92d0bd2..e6da690dab 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ [![Build Status](https://travis-ci.org/taosdata/TDengine.svg?branch=master)](https://travis-ci.org/taosdata/TDengine) [![Build status](https://ci.appveyor.com/api/projects/status/kf3pwh2or5afsgl9/branch/master?svg=true)](https://ci.appveyor.com/project/sangshuduo/tdengine-2n8ge/branch/master) +[![Coverage Status](https://coveralls.io/repos/github/taosdata/TDengine/badge.svg?branch=develop)](https://coveralls.io/github/taosdata/TDengine?branch=develop) [![TDengine](TDenginelogo.png)](https://www.taosdata.com) From 270e2bb96f48503d89c4c99a28af955fee746aa0 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 22 Jun 2020 05:26:24 +0000 Subject: [PATCH 22/27] fix TSDB memory leak --- src/tsdb/src/tsdbMemTable.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 7a360c25f9..1108340cf0 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -587,6 +587,7 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe goto _err; } + tfree(dataDir); tsdbCloseHelperFile(pHelper, 0); pthread_rwlock_wrlock(&(pFileH->fhlock)); From 41c7f722a679b0e93143c42631a65395c2615de8 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 22 Jun 2020 13:36:24 +0800 Subject: [PATCH 23/27] remove unused os-specified functions. [TD-716] --- src/os/linux/src/linuxPlatform.c | 77 -------------------------------- 1 file changed, 77 deletions(-) diff --git a/src/os/linux/src/linuxPlatform.c b/src/os/linux/src/linuxPlatform.c index 782a508b17..a8ecf43c46 100644 --- a/src/os/linux/src/linuxPlatform.c +++ b/src/os/linux/src/linuxPlatform.c @@ -56,70 +56,8 @@ void taosMsleep(int mseconds) { bool taosCheckPthreadValid(pthread_t thread) { return thread != 0; } -void taosResetPthread(pthread_t *thread) { *thread = 0; } - int64_t taosGetPthreadId() { return (int64_t)pthread_self(); } -/* -* Function to get the private ip address of current machine. If get IP -* successfully, return 0, else, return -1. The return values is ip. -* -* Use: -* if (taosGetPrivateIp(ip) != 0) { -* perror("Fail to get private IP address\n"); -* exit(EXIT_FAILURE); -* } -*/ -int taosGetPrivateIp(char *const ip) { - bool hasLoCard = false; - - struct ifaddrs *ifaddr, *ifa; - int family, s; - char host[NI_MAXHOST]; - - if (getifaddrs(&ifaddr) == -1) { - return -1; - } - - /* Walk through linked list, maintaining head pointer so we can free list later */ - int flag = 0; - for (ifa = ifaddr; ifa != NULL; ifa = ifa->ifa_next) { - if (ifa->ifa_addr == NULL) continue; - - family = ifa->ifa_addr->sa_family; - if (strcmp("lo", ifa->ifa_name) == 0) { - hasLoCard = true; - continue; - } - - if (family == AF_INET) { - /* printf("%-8s", ifa->ifa_name); */ - s = getnameinfo(ifa->ifa_addr, (family == AF_INET) ? sizeof(struct sockaddr_in) : sizeof(struct sockaddr_in6), - host, NI_MAXHOST, NULL, 0, NI_NUMERICHOST); - if (s != 0) { - freeifaddrs(ifaddr); - return -1; - } - - strcpy(ip, host); - flag = 1; - break; - } - } - - freeifaddrs(ifaddr); - if (flag) { - return 0; - } else { - if (hasLoCard) { - uPrint("no net card was found, use lo:127.0.0.1 as default"); - strcpy(ip, "127.0.0.1"); - return 0; - } - return -1; - } -} - int taosSetNonblocking(int sock, int on) { int flags = 0; if ((flags = fcntl(sock, F_GETFL, 0)) < 0) { @@ -294,21 +232,6 @@ ssize_t twrite(int fd, void *buf, size_t n) { return n; } -bool taosSkipSocketCheck() { - struct utsname buf; - if (uname(&buf)) { - uPrint("can't fetch os info"); - return false; - } - - if (strstr(buf.release, "Microsoft") != 0) { - uPrint("using WSLv1"); - return true; - } - - return false; -} - void taosBlockSIGPIPE() { sigset_t signal_mask; sigemptyset(&signal_mask); From 806f8f5b8cd127fe60e4038a575a915a60d4cccc Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Mon, 22 Jun 2020 14:16:46 +0800 Subject: [PATCH 24/27] fix bug: return buffer in stack update boundary testcase. --- src/client/src/tscParseInsert.c | 8 ++++---- tests/pytest/table/boundary.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 4020d201ce..6e922d566b 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -989,13 +989,11 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { } int validateTableName(char *tblName, int len, SSQLToken* psTblToken) { - char buf[TSDB_TABLE_ID_LEN] = {0}; - tstrncpy(buf, tblName, sizeof(buf)); + tstrncpy(psTblToken->z, tblName, TSDB_TABLE_ID_LEN); psTblToken->n = len; psTblToken->type = TK_ID; - psTblToken->z = buf; - tSQLGetToken(buf, &psTblToken->type); + tSQLGetToken(psTblToken->z, &psTblToken->type); return tscValidateName(psTblToken); } @@ -1081,7 +1079,9 @@ int tsParseInsertSql(SSqlObj *pSql) { } pCmd->curSql = sToken.z; + char buf[TSDB_TABLE_ID_LEN]; SSQLToken sTblToken; + sTblToken.z = buf; // Check if the table name available or not if (validateTableName(sToken.z, sToken.n, &sTblToken) != TSDB_CODE_SUCCESS) { code = tscInvalidSQLErrMsg(pCmd->payload, "table name invalid", sToken.z); diff --git a/tests/pytest/table/boundary.py b/tests/pytest/table/boundary.py index 50586b72ff..a1bf096499 100644 --- a/tests/pytest/table/boundary.py +++ b/tests/pytest/table/boundary.py @@ -141,7 +141,7 @@ class TDTestCase: tdSql.prepare() # 8 bytes for timestamp - maxRowSize = 65535 - 8 + maxRowSize = self.getLimitFromSourceCode('TSDB_MAX_BYTES_PER_ROW') - 8 maxCols = self.getLimitFromSourceCode('TSDB_MAX_COLUMNS') - 1 # for binary cols, 2 bytes are used for length From 10d09614f30102a0e54299ea8168b675458f7c3f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 22 Jun 2020 06:47:13 +0000 Subject: [PATCH 25/27] [TD-437] [TD-718] invalid read while drop db --- src/mnode/src/mnodeAcct.c | 2 +- tests/script/general/stable/refcount.sim | 132 +++++++++++++++++++++++ tests/script/unique/cluster/balance2.sim | 2 +- 3 files changed, 134 insertions(+), 2 deletions(-) create mode 100644 tests/script/general/stable/refcount.sim diff --git a/src/mnode/src/mnodeAcct.c b/src/mnode/src/mnodeAcct.c index ad52a4c3ac..419372e06d 100644 --- a/src/mnode/src/mnodeAcct.c +++ b/src/mnode/src/mnodeAcct.c @@ -126,8 +126,8 @@ int32_t mnodeInitAccts() { } void mnodeCleanupAccts() { - sdbCloseTable(tsAcctSdb); acctCleanUp(); + sdbCloseTable(tsAcctSdb); } void *mnodeGetAcct(char *name) { diff --git a/tests/script/general/stable/refcount.sim b/tests/script/general/stable/refcount.sim new file mode 100644 index 0000000000..99d943b4d9 --- /dev/null +++ b/tests/script/general/stable/refcount.sim @@ -0,0 +1,132 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 +system sh/exec.sh -n dnode1 -s start + +sleep 3000 +sql connect + +print =============== step1 +sql create database d1; +sql use d1; +sql create table d1.t1 (ts timestamp, i int); +sql create table d1.t2 (ts timestamp, i int); +sql create table d1.t3 (ts timestamp, i int); +sql insert into d1.t1 values(now, 1); +sql insert into d1.t2 values(now, 1); +sql drop table d1.t1; +sql drop database d1; + +sql show databases; +if $rows != 0 then + return -1 +endi + +print =============== step2 +sql create database d2; +sql use d2; +sql create table d2.t1 (ts timestamp, i int); +sql create table d2.t2 (ts timestamp, i int); +sql create table d2.t3 (ts timestamp, i int); +sql insert into d2.t1 values(now, 1); +sql insert into d2.t2 values(now, 1); +sql drop table d2.t1; +sql drop table d2.t2; +sql drop table d2.t3; + +sql show d2.tables; +if $rows != 0 then + return -1 +endi + +sql show d2.vgroups; +if $rows != 0 then + return -1 +endi + +sql drop database d2; + +sql show databases; +if $rows != 0 then + return -1 +endi + +print =============== step3 +sql create database d3; +sql use d3; +sql create table d3.st (ts timestamp, i int) tags (j int); +sql create table d3.t1 using d3.st tags(1); +sql create table d3.t2 using d3.st tags(1); +sql create table d3.t3 using d3.st tags(1); +sql insert into d3.t1 values(now, 1); +sql drop table d3.t1; +sql drop table d3.t2; +sql drop table d3.t3; + +sql show d3.tables; +if $rows != 0 then + return -1 +endi + +sql show d3.vgroups; +if $rows != 0 then + return -1 +endi + +sql drop database d3; + +sql show databases; +if $rows != 0 then + return -1 +endi + +print =============== step4 +sql create database d4; +sql use d4; +sql create table d4.st (ts timestamp, i int) tags (j int); +sql create table d4.t1 using d4.st tags(1); +sql create table d4.t2 using d4.st tags(1); +sql create table d4.t3 using d4.st tags(1); +sql insert into d4.t1 values(now, 1); +sql drop table d4.t1; +sql drop table d4.st; + +sql show d4.tables; +if $rows != 0 then + return -1 +endi + +sql show d4.stables; +if $rows != 0 then + return -1 +endi + +sql drop database d4; + +sql show databases; +if $rows != 0 then + return -1 +endi + +print =============== step5 +sql create database d5; +sql use d5; +sql create table d5.st (ts timestamp, i int) tags (j int); +sql create table d5.t1 using d5.st tags(1); +sql create table d5.t2 using d5.st tags(1); +sql create table d5.t3 using d5.st tags(1); +sql insert into d5.t1 values(now, 1); +sql drop table d5.t1; + +sql drop database d5; + +sql show databases; +if $rows != 0 then + return -1 +endi + +print =============== step6 +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/cluster/balance2.sim b/tests/script/unique/cluster/balance2.sim index 0b3e1374ec..4c4339a78b 100644 --- a/tests/script/unique/cluster/balance2.sim +++ b/tests/script/unique/cluster/balance2.sim @@ -328,7 +328,7 @@ $x = 0 show6: $x = $x + 1 sleep 2000 - if $x == 30 then + if $x == 10 then return -1 endi sql show dnodes -x show6 From 5bc3dc1bbb7abb22b6d3ac38856750e396aeee5d Mon Sep 17 00:00:00 2001 From: Hui Li Date: Mon, 22 Jun 2020 15:52:29 +0800 Subject: [PATCH 26/27] [modify for covrity scan] --- src/kit/shell/src/shellEngine.c | 7 +++++-- src/kit/shell/src/shellImport.c | 5 ++++- src/kit/shell/src/shellLinux.c | 2 +- src/kit/taosdemo/taosdemo.c | 33 +++++++++++++++++++-------------- src/kit/taosdump/taosdump.c | 26 ++++++++++++++++---------- src/os/linux/src/linuxSysPara.c | 10 +++++----- src/util/inc/tutil.h | 2 ++ src/util/src/tnote.c | 2 ++ src/util/src/tutil.c | 19 +++++++++++++++++++ 9 files changed, 73 insertions(+), 33 deletions(-) diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c index e28c691a89..044e167af7 100644 --- a/src/kit/shell/src/shellEngine.c +++ b/src/kit/shell/src/shellEngine.c @@ -474,7 +474,7 @@ static int dumpResultToFile(const char* fname, TAOS_RES* tres) { } while( row != NULL); result = NULL; - taos_free_result(tres); + //taos_free_result(tres); fclose(fp); return numOfRows; @@ -803,6 +803,7 @@ void source_file(TAOS *con, char *fptr) { char *fname = full_path.we_wordv[0]; + /* if (access(fname, F_OK) != 0) { fprintf(stderr, "ERROR: file %s is not exist\n", fptr); @@ -810,6 +811,7 @@ void source_file(TAOS *con, char *fptr) { free(cmd); return; } + */ FILE *f = fopen(fname, "r"); if (f == NULL) { @@ -849,7 +851,7 @@ void source_file(TAOS *con, char *fptr) { void shellGetGrantInfo(void *con) { return; - +#if 0 char sql[] = "show grants"; TAOS_RES* tres = taos_query(con, sql); @@ -900,4 +902,5 @@ void shellGetGrantInfo(void *con) { } fprintf(stdout, "\n"); + #endif } diff --git a/src/kit/shell/src/shellImport.c b/src/kit/shell/src/shellImport.c index 347f99671d..ba123ac2d4 100644 --- a/src/kit/shell/src/shellImport.c +++ b/src/kit/shell/src/shellImport.c @@ -73,7 +73,7 @@ static void shellParseDirectory(const char *directoryName, const char *prefix, c } int fileNum = 0; - while (fscanf(fp, "%s", fileArray[fileNum++])) { + while (fscanf(fp, "%128s", fileArray[fileNum++])) { if (strcmp(fileArray[fileNum-1], shellTablesSQLFile) == 0) { fileNum--; } @@ -150,9 +150,11 @@ static void shellSourceFile(TAOS *con, char *fptr) { char *fname = full_path.we_wordv[0]; if (fname == NULL) { fprintf(stderr, "ERROR: invalid filename\n"); + free(cmd); return; } + /* if (access(fname, F_OK) != 0) { fprintf(stderr, "ERROR: file %s is not exist\n", fptr); @@ -168,6 +170,7 @@ static void shellSourceFile(TAOS *con, char *fptr) { free(cmd); return; } + */ FILE *f = fopen(fname, "r"); if (f == NULL) { diff --git a/src/kit/shell/src/shellLinux.c b/src/kit/shell/src/shellLinux.c index 829ceb9e5f..0325fa34f9 100644 --- a/src/kit/shell/src/shellLinux.c +++ b/src/kit/shell/src/shellLinux.c @@ -162,7 +162,7 @@ void shellReadCommand(TAOS *con, char *command) { // Read input. char c; while (1) { - c = getchar(); + c = (char)getchar(); // getchar() return an 'int' value if (c < 0) { // For UTF-8 int count = countPrefixOnes(c); diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 4a455271ff..81a98e0fcc 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -82,7 +82,7 @@ typedef struct DemoArguments { bool insert_only; char *output_file; int mode; - char *datatype[MAX_NUM_DATATYPE]; + char *datatype[MAX_NUM_DATATYPE+1]; int len_of_binary; int num_of_CPR; int num_of_threads; @@ -432,7 +432,7 @@ int main(int argc, char *argv[]) { tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); printf("###################################################################\n\n"); printf("Press enter key to continue"); - getchar(); + (void)getchar(); fprintf(fp, "###################################################################\n"); fprintf(fp, "# Server IP: %s:%hu\n", ip_addr == NULL ? "localhost" : ip_addr, port); @@ -845,10 +845,10 @@ void *syncWrite(void *sarg) { pstr += sprintf(pstr, "insert into %s.%s%d values", winfo->db_name, winfo->tb_prefix, tID); int k; for (k = 0; k < winfo->nrecords_per_request;) { - int rand_num = rand() % 100; + int rand_num = trand() % 100; int len = -1; if (winfo->data_of_order ==1 && rand_num < winfo->data_of_rate) { - long d = tmp_time - rand() % 1000000 + rand_num; + long d = tmp_time - trand() % 1000000 + rand_num; len = generateData(data, data_type, ncols_per_record, d, len_of_binary); } else { len = generateData(data, data_type, ncols_per_record, tmp_time += 1000, len_of_binary); @@ -940,10 +940,10 @@ void callBack(void *param, TAOS_RES *res, int code) { pstr += sprintf(pstr, "insert into %s values", tb_info->tb_name); for (int i = 0; i < tb_info->nrecords_per_request; i++) { - int rand_num = rand() % 100; + int rand_num = trand() % 100; if (tb_info->data_of_order ==1 && rand_num < tb_info->data_of_rate) { - long d = tmp_time - rand() % 1000000 + rand_num; + long d = tmp_time - trand() % 1000000 + rand_num; generateData(data, datatype, ncols_per_record, d, len_of_binary); } else { @@ -985,22 +985,27 @@ int32_t generateData(char *res, char **data_type, int num_of_cols, int64_t times } } + if (0 == c) { + perror("data type error!"); + exit(-1); + } + for (int i = 0; i < num_of_cols; i++) { if (strcasecmp(data_type[i % c], "tinyint") == 0) { - pstr += sprintf(pstr, ", %d", (int)(rand() % 128)); + pstr += sprintf(pstr, ", %d", (int)(trand() % 128)); } else if (strcasecmp(data_type[i % c], "smallint") == 0) { - pstr += sprintf(pstr, ", %d", (int)(rand() % 32767)); + pstr += sprintf(pstr, ", %d", (int)(trand() % 32767)); } else if (strcasecmp(data_type[i % c], "int") == 0) { - pstr += sprintf(pstr, ", %d", (int)(rand() % 10)); + pstr += sprintf(pstr, ", %d", (int)(trand() % 10)); } else if (strcasecmp(data_type[i % c], "bigint") == 0) { - pstr += sprintf(pstr, ", %" PRId64, rand() % 2147483648); + pstr += sprintf(pstr, ", %" PRId64, trand() % 2147483648); } else if (strcasecmp(data_type[i % c], "float") == 0) { - pstr += sprintf(pstr, ", %10.4f", (float)(rand() / 1000)); + pstr += sprintf(pstr, ", %10.4f", (float)(trand() / 1000)); } else if (strcasecmp(data_type[i % c], "double") == 0) { - double t = (double)(rand() / 1000000); + double t = (double)(trand() / 1000000); pstr += sprintf(pstr, ", %20.8f", t); } else if (strcasecmp(data_type[i % c], "bool") == 0) { - bool b = rand() & 1; + bool b = trand() & 1; pstr += sprintf(pstr, ", %s", b ? "true" : "false"); } else if (strcasecmp(data_type[i % c], "binary") == 0) { char s[len_of_binary]; @@ -1026,7 +1031,7 @@ void rand_string(char *str, int size) { --size; int n; for (n = 0; n < size; n++) { - int key = rand() % (int)(sizeof charset - 1); + int key = trand() % (int)(sizeof charset - 1); str[n] = charset[key]; } str[n] = 0; diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index ccef783709..46b14d6e56 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -229,7 +229,7 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { fprintf(stderr, "Invalid path %s\n", arg); return -1; } - strcpy(arguments->output, full_path.we_wordv[0]); + tstrncpy(arguments->output, full_path.we_wordv[0], TSDB_FILENAME_LEN); wordfree(&full_path); break; case 'i': @@ -411,7 +411,7 @@ int taosGetTableRecordInfo(char *table, STableRecordInfo *pTableRecordInfo) { if ((row = taos_fetch_row(result)) != NULL) { isSet = true; pTableRecordInfo->isMetric = true; - strcpy(pTableRecordInfo->tableRecord.metric, table); + tstrncpy(pTableRecordInfo->tableRecord.metric, table, TSDB_TABLE_NAME_LEN); } taos_free_result(result); @@ -642,9 +642,12 @@ int taosDumpDb(SDbInfo *dbInfo, SDumpArguments *arguments, FILE *fp) { taos_free_result(result); - lseek(fd, 0, SEEK_SET); + (void)lseek(fd, 0, SEEK_SET); - while (read(fd, &tableRecord, sizeof(STableRecord)) > 0) { + while (1) { + ssize_t ret = read(fd, &tableRecord, sizeof(STableRecord)); + if (ret <= 0) break; + tableRecord.name[sizeof(tableRecord.name) - 1] = 0; tableRecord.metric[sizeof(tableRecord.metric) - 1] = 0; taosDumpTable(tableRecord.name, tableRecord.metric, arguments, fp); @@ -807,7 +810,7 @@ int taosGetTableDes(char *table, STableDef *tableDes) { TAOS_FIELD *fields = taos_fetch_fields(result); - strcpy(tableDes->name, table); + tstrncpy(tableDes->name, table, TSDB_COL_NAME_LEN); while ((row = taos_fetch_row(result)) != NULL) { strncpy(tableDes->cols[count].field, (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX], @@ -903,16 +906,19 @@ int32_t taosDumpMetric(char *metric, SDumpArguments *arguments, FILE *fp) { taos_free_result(result); result = NULL; - lseek(fd, 0, SEEK_SET); + (void)lseek(fd, 0, SEEK_SET); - while (read(fd, &tableRecord, sizeof(STableRecord)) > 0) { + while (1) { + ssize_t ret = read(fd, &tableRecord, sizeof(STableRecord)); + if (ret <= 0) break; + tableRecord.name[sizeof(tableRecord.name) - 1] = 0; tableRecord.metric[sizeof(tableRecord.metric) - 1] = 0; taosDumpTable(tableRecord.name, tableRecord.metric, arguments, fp); } tclose(fd); - remove(".table.tmp"); + (void)remove(".table.tmp"); return 0; } @@ -1004,7 +1010,7 @@ int taosDumpTableData(FILE *fp, char *tbname, SDumpArguments *arguments) { break; } } - pstr += sprintf(pstr, ")"); + sprintf(pstr, ")"); count++; fprintf(fp, "%s", buffer); @@ -1327,7 +1333,7 @@ int convertNCharToReadable(char *str, int size, char *buf, int bufsize) { if ((int)wc < 256) { pbuf = stpcpy(pbuf, ascii_literal_list[(int)wc]); - } else { + } else if (byte_width > 0) { memcpy(pbuf, pstr, byte_width); pbuf += byte_width; } diff --git a/src/os/linux/src/linuxSysPara.c b/src/os/linux/src/linuxSysPara.c index 0e76ab0046..31ae33cdfc 100644 --- a/src/os/linux/src/linuxSysPara.c +++ b/src/os/linux/src/linuxSysPara.c @@ -162,7 +162,7 @@ static void taosGetSystemTimezone() { FILE *f = fopen("/etc/timezone", "r"); char buf[65] = {0}; if (f != NULL) { - fread(buf, 64, 1, f); + (void)fread(buf, 64, 1, f); fclose(f); } @@ -547,7 +547,7 @@ void taosSetCoreDump() { struct rlimit rlim; struct rlimit rlim_new; if (getrlimit(RLIMIT_CORE, &rlim) == 0) { - uPrint("the old unlimited para: rlim_cur=%d, rlim_max=%d", rlim.rlim_cur, rlim.rlim_max); + uPrint("the old unlimited para: rlim_cur=%d, rlim_max=%" PRIu64, rlim.rlim_cur, rlim.rlim_max); rlim_new.rlim_cur = RLIM_INFINITY; rlim_new.rlim_max = RLIM_INFINITY; if (setrlimit(RLIMIT_CORE, &rlim_new) != 0) { @@ -559,7 +559,7 @@ void taosSetCoreDump() { } if (getrlimit(RLIMIT_CORE, &rlim) == 0) { - uPrint("the new unlimited para: rlim_cur=%d, rlim_max=%d", rlim.rlim_cur, rlim.rlim_max); + uPrint("the new unlimited para: rlim_cur=%d, rlim_max=%" PRIu64, rlim.rlim_cur, rlim.rlim_max); } #ifndef _TD_ARM_ @@ -586,7 +586,7 @@ void taosSetCoreDump() { uPrint("_sysctl(kern_core_uses_pid) set fail: %s", strerror(errno)); } - uPrint("The old core_uses_pid[%d]: %d", old_len, old_usespid); + uPrint("The old core_uses_pid[%" PRIu64 "]: %d", old_len, old_usespid); old_usespid = 0; @@ -603,7 +603,7 @@ void taosSetCoreDump() { uPrint("_sysctl(kern_core_uses_pid) get fail: %s", strerror(errno)); } - uPrint("The new core_uses_pid[%d]: %d", old_len, old_usespid); + uPrint("The new core_uses_pid[%" PRIu64 "]: %d", old_len, old_usespid); #endif #if 0 diff --git a/src/util/inc/tutil.h b/src/util/inc/tutil.h index a314f0e31d..94084b5638 100644 --- a/src/util/inc/tutil.h +++ b/src/util/inc/tutil.h @@ -119,6 +119,8 @@ extern "C" { uint32_t taosRand(void); +uint32_t trand(void); + size_t twcslen(const wchar_t *wcs); int32_t strdequote(char *src); diff --git a/src/util/src/tnote.c b/src/util/src/tnote.c index a8d9e8d416..20c77d4fbe 100644 --- a/src/util/src/tnote.c +++ b/src/util/src/tnote.c @@ -128,10 +128,12 @@ int taosOpenNewNote(taosNoteInfo * pNote) bool taosCheckNoteIsOpen(char *noteName, taosNoteInfo * pNote) { + /* int exist = access(noteName, F_OK); if (exist != 0) { return false; } + */ int fd = open(noteName, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); if (fd < 0) { diff --git a/src/util/src/tutil.c b/src/util/src/tutil.c index aa5bfe322a..ccdf7111a7 100644 --- a/src/util/src/tutil.c +++ b/src/util/src/tutil.c @@ -55,6 +55,25 @@ uint32_t taosRand(void) */ return rand(); } + +uint32_t trand(void) +{ + int fd; + int seed; + + fd = open("/dev/urandom", 0); + if (fd < 0) { + seed = time(0); + } else { + int len = read(fd, &seed, sizeof(seed)); + if (len < 0) { + seed = time(0); + } + close(fd); + } + + return (uint32_t)seed; +} #endif size_t twcslen(const wchar_t *wcs) { From 3c1850ac6ae688973bedac9836cf2dbf7ca04d8e Mon Sep 17 00:00:00 2001 From: Hui Li Date: Mon, 22 Jun 2020 15:56:30 +0800 Subject: [PATCH 27/27] [modify for covrity scan] --- src/kit/taosdemo/taosdemo.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 81a98e0fcc..1dd31779fe 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -34,6 +34,7 @@ #include #include "taos.h" +#include "tutil.h" extern char configDir[];