From 3d8f36b5d2bed00edf5bb4fdf9e8e00ae8d39031 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 13 Aug 2020 01:35:38 +0800 Subject: [PATCH 01/51] add snap package support. --- snap/local/launcher.sh | 21 ++++++++++++ snap/snapcraft.yaml | 76 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+) create mode 100755 snap/local/launcher.sh create mode 100644 snap/snapcraft.yaml diff --git a/snap/local/launcher.sh b/snap/local/launcher.sh new file mode 100755 index 0000000000..5986114931 --- /dev/null +++ b/snap/local/launcher.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# Wrapper to check for custom config in $SNAP_USER_COMMON or $SNAP_COMMON and +# use it otherwise fall back to the included basic config which will at least +# allow mosquitto to run and do something. +# This script will also copy the full example config in to SNAP_USER_COMMON or +# SNAP_COMMON so that people can refer to it. +# +# The decision about whether to use SNAP_USER_COMMON or SNAP_COMMON is taken +# based on the user that runs the command. If the user is root, it is assumed +# that mosquitto is being run as a system daemon, and SNAP_COMMON will be used. +# If a non-root user runs the command, then SNAP_USER_COMMON will be used. + +case "$SNAP_USER_COMMON" in + */root/snap/tdengine/common*) COMMON=$SNAP_COMMON ;; + *) COMMON=$SNAP_USER_COMMON ;; +esac + +CONFIG_FILE="$SNAP/etc/taos" + +# Launch the snap +$SNAP/usr/bin/taosd -c $CONFIG_FILE $@ diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml new file mode 100644 index 0000000000..031704393f --- /dev/null +++ b/snap/snapcraft.yaml @@ -0,0 +1,76 @@ +name: tdengine +base: core18 # the base snap is the execution environment for this snap +version: '2.0.0.6' # just for humans, typically '1.2+git' or '1.3.2' +summary: Single-line elevator pitch for your amazing snap # 79 char long summary +description: | + TDengine is an open-source big data platform designed and optimized for Internet of Things (IoT), Connected Vehicles, and Industrial IoT. Besides the 10x faster time-series database, it provides caching, stream computing, message queuing and other functionalities to reduce the complexity and costs of development and operations. + +grade: stable +confinement: strict + +apps: + tdengine: + command: launcher.sh + daemon: simple + restart-condition: always + plugs: + - network + - network-bind + - systemfiles + + taos: + command: usr/bin/taos + plugs: + - network + - systemfiles + +plugs: + systemfiles: + interface: system-files + read: + - /etc/taos + - /var/lib/taos + write: + - /var/log/taos + - /var/lib/taos + +parts: + script: + plugin: dump + source: snap/local/ + prime: + - launcher.sh + + tdengine: + source: . + source-type: local + plugin: cmake + build-packages: + - gcc + - g++ + - make + - cmake + override-build: | + snapcraftctl build + if [ ! -d $SNAPCRAFT_STAGE/usr ]; then + mkdir $SNAPCRAFT_STAGE/usr + fi + if [ ! -d $SNAPCRAFT_STAGE/etc/taos ]; then + mkdir -p $SNAPCRAFT_STAGE/etc/taos + fi + cp $SNAPCRAFT_PART_BUILD/build/* -rf $SNAPCRAFT_STAGE/usr/ + cp $SNAPCRAFT_PART_SRC/packaging/cfg/taos.cfg -rf $SNAPCRAFT_STAGE/etc/taos/ + + prime: + - etc/taos/taos.cfg + - usr/bin/taosd + - usr/bin/taos + - usr/bin/taosdemo + - usr/lib/libtaos.so.2.0.0.6 + - usr/lib/libtaos.so.1 + - usr/lib/libtaos.so + + + #layout: + # usr/bin/taosd: + #bind-file: debug/build/bin/taosd From ecdf1ce5ec799cf16079703195286a3589149352 Mon Sep 17 00:00:00 2001 From: zyyang Date: Thu, 13 Aug 2020 11:12:50 +0800 Subject: [PATCH 02/51] fix TD-1115 in master branch --- .../jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java index 1cf024f30e..d72ba20783 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java @@ -158,9 +158,10 @@ public class TSDBStatement implements Statement { throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); } else if (resultSetPointer == TSDBConstants.JNI_NULL_POINTER) { // no result set is retrieved + this.connecter.freeResultSet(pSql); res = false; } - this.connecter.freeResultSet(pSql); + return res; } From fbce7be9f952a4fcea84dfa38e8265d684f0b16e Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 14 Aug 2020 10:01:36 +0800 Subject: [PATCH 03/51] add /tmp directory for accessing and change confinement to classic. --- snap/snapcraft.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 031704393f..dfa68a9347 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -6,7 +6,7 @@ description: | TDengine is an open-source big data platform designed and optimized for Internet of Things (IoT), Connected Vehicles, and Industrial IoT. Besides the 10x faster time-series database, it provides caching, stream computing, message queuing and other functionalities to reduce the complexity and costs of development and operations. grade: stable -confinement: strict +confinement: classic apps: tdengine: @@ -30,9 +30,11 @@ plugs: read: - /etc/taos - /var/lib/taos + - /tmp write: - /var/log/taos - /var/lib/taos + - /tmp parts: script: From c71ab49d6bc8a595fb7ef62efbadbfaa27bb5ec9 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 14 Aug 2020 06:00:25 +0000 Subject: [PATCH 04/51] TD-1143 --- src/plugins/monitor/src/monitorMain.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/plugins/monitor/src/monitorMain.c b/src/plugins/monitor/src/monitorMain.c index 55c242763b..d76bb4bd82 100644 --- a/src/plugins/monitor/src/monitorMain.c +++ b/src/plugins/monitor/src/monitorMain.c @@ -234,17 +234,22 @@ static void monitorInitDatabaseCb(void *param, TAOS_RES *result, int32_t code) { } void monitorStopSystem() { - monitorInfo("monitor module is stopped"); - monitorExecuteSQLFp = NULL; + if (tsMonitorConn.state == MONITOR_STATE_STOPPED) return; tsMonitorConn.state = MONITOR_STATE_STOPPED; + monitorExecuteSQLFp = NULL; + + monitorInfo("monitor module is stopped"); + if (tsMonitorConn.initTimer != NULL) { taosTmrStopA(&(tsMonitorConn.initTimer)); } if (tsMonitorConn.timer != NULL) { taosTmrStopA(&(tsMonitorConn.timer)); } - - taos_close(tsMonitorConn.conn); + if (tsMonitorConn.conn != NULL) { + taos_close(tsMonitorConn.conn); + tsMonitorConn.conn = NULL; + } } void monitorCleanUpSystem() { From faabd6e5cbdf02564c4c62ba81ff6e82c6a8e69d Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 14 Aug 2020 16:59:41 +0800 Subject: [PATCH 05/51] TD-1057 compile error in vs2019 --- src/client/src/TSDBJNIConnector.c | 2 +- src/client/src/tscSQLParser.c | 2 +- src/common/src/ttimezone.c | 2 +- src/kit/taosnetwork/client.c | 4 ++-- src/kit/taosnetwork/server.c | 6 +++--- src/os/inc/osSocket.h | 4 ++++ src/os/inc/osWindows.h | 22 +++++++++++++++++++--- src/os/src/detail/osSocket.c | 12 ++++++++++++ src/os/src/detail/osTime.c | 7 +++++++ src/os/src/windows/w64Socket.c | 22 +++++++++++++++++++++- src/plugins/http/src/httpServer.c | 6 +++--- src/rpc/src/rpcTcp.c | 4 ++-- src/util/src/tconfig.c | 2 +- 13 files changed, 77 insertions(+), 18 deletions(-) diff --git a/src/client/src/TSDBJNIConnector.c b/src/client/src/TSDBJNIConnector.c index 5036983424..34204f96bf 100644 --- a/src/client/src/TSDBJNIConnector.c +++ b/src/client/src/TSDBJNIConnector.c @@ -583,7 +583,7 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_consumeImp(JNIEn return 0l; } - return (long)res; + return (jlong)res; } JNIEXPORT void JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_unsubscribeImp(JNIEnv *env, jobject jobj, jlong sub, diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index f933366711..e5bb516ee6 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -2390,7 +2390,7 @@ bool validateIpAddress(const char* ip, size_t size) { strncpy(tmp, ip, size); - in_addr_t epAddr = inet_addr(tmp); + in_addr_t epAddr = taosInetAddr(tmp); return epAddr != INADDR_NONE; } diff --git a/src/common/src/ttimezone.c b/src/common/src/ttimezone.c index edb6aea7f9..62b1e0bb5c 100644 --- a/src/common/src/ttimezone.c +++ b/src/common/src/ttimezone.c @@ -50,7 +50,7 @@ void tsSetTimeZone() { #endif #endif - int32_t tz = (-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR; + int32_t tz = (int32_t)((-timezone * MILLISECOND_PER_SECOND) / MILLISECOND_PER_HOUR); tz += daylight; /* diff --git a/src/kit/taosnetwork/client.c b/src/kit/taosnetwork/client.c index 102f9b9d89..74c161bf67 100644 --- a/src/kit/taosnetwork/client.c +++ b/src/kit/taosnetwork/client.c @@ -91,7 +91,7 @@ int checkTcpPort(info_s *info) { serverAddr.sin_family = AF_INET; serverAddr.sin_port = htons(port); - serverAddr.sin_addr.s_addr = inet_addr(host); + serverAddr.sin_addr.s_addr = taosInetAddr(host); //printf("=================================\n"); if (connect(clientSocket, (struct sockaddr *)&serverAddr, sizeof(serverAddr)) < 0) { @@ -157,7 +157,7 @@ int checkUdpPort(info_s *info) { serverAddr.sin_family = AF_INET; serverAddr.sin_port = htons(port); - serverAddr.sin_addr.s_addr = inet_addr(host); + serverAddr.sin_addr.s_addr = taosInetAddr(host); memset(sendbuf, 0, BUFFER_SIZE); memset(recvbuf, 0, BUFFER_SIZE); diff --git a/src/kit/taosnetwork/server.c b/src/kit/taosnetwork/server.c index 1c3bc6fa09..a2bb2b2b65 100644 --- a/src/kit/taosnetwork/server.c +++ b/src/kit/taosnetwork/server.c @@ -128,7 +128,7 @@ static void *bindTcpPort(void *sarg) { if (errno == EINTR) { continue; } else { - printf("recv Client: %s pkg from TCP port: %d fail:%s.\n", inet_ntoa(clientAddr.sin_addr), port, strerror(errno)); + printf("recv Client: %s pkg from TCP port: %d fail:%s.\n", taosInetNtoa(clientAddr.sin_addr), port, strerror(errno)); close(serverSocket); return NULL; } @@ -139,7 +139,7 @@ static void *bindTcpPort(void *sarg) { } } - printf("recv Client: %s pkg from TCP port: %d, pkg len: %d\n", inet_ntoa(clientAddr.sin_addr), port, iDataNum); + printf("recv Client: %s pkg from TCP port: %d, pkg len: %d\n", taosInetNtoa(clientAddr.sin_addr), port, iDataNum); if (iDataNum > 0) { send(client, buffer, iDataNum, 0); break; @@ -188,7 +188,7 @@ static void *bindUdpPort(void *sarg) { continue; } if (iDataNum > 0) { - printf("recv Client: %s pkg from UDP port: %d, pkg len: %d\n", inet_ntoa(clientAddr.sin_addr), port, iDataNum); + printf("recv Client: %s pkg from UDP port: %d, pkg len: %d\n", taosInetNtoa(clientAddr.sin_addr), port, iDataNum); //printf("Read msg from udp:%s ... %s\n", buffer, buffer+iDataNum-16); sendto(serverSocket, buffer, iDataNum, 0, (struct sockaddr *)&clientAddr, (int)sin_size); diff --git a/src/os/inc/osSocket.h b/src/os/inc/osSocket.h index 58f95c3c2d..e5c3806d1b 100644 --- a/src/os/inc/osSocket.h +++ b/src/os/inc/osSocket.h @@ -65,6 +65,10 @@ void taosBlockSIGPIPE(); // TAOS_OS_FUNC_SOCKET_SETSOCKETOPT int taosSetSockOpt(SOCKET socketfd, int level, int optname, void *optval, int optlen); +// TAOS_OS_FUNC_SOCKET_INET +uint32_t taosInetAddr(char *ipAddr); +const char *taosInetNtoa(IN_ADDR ipInt); + #ifdef __cplusplus } #endif diff --git a/src/os/inc/osWindows.h b/src/os/inc/osWindows.h index 6665dcd920..994e3b991c 100644 --- a/src/os/inc/osWindows.h +++ b/src/os/inc/osWindows.h @@ -164,9 +164,25 @@ int gettimeofday(struct timeval *ptv, void *pTimeZone); #define MSG_NOSIGNAL 0 #define SO_NO_CHECK 0x1234 #define SOL_TCP 0x1234 -#define TCP_KEEPCNT 0x1234 -#define TCP_KEEPIDLE 0x1234 -#define TCP_KEEPINTVL 0x1234 + +#ifndef TCP_KEEPCNT + #define TCP_KEEPCNT 0x1234 +#endif + +#ifndef TCP_KEEPIDLE + #define TCP_KEEPIDLE 0x1234 +#endif + +#ifndef TCP_KEEPINTVL + #define TCP_KEEPINTVL 0x1234 +#endif + +#ifdef _MSC_VER +#if _MSC_VER >= 1900 + #define TAOS_OS_FUNC_SOCKET_INET +#endif +#endif + #define SHUT_RDWR SD_BOTH #define SHUT_RD SD_RECEIVE #define SHUT_WR SD_SEND diff --git a/src/os/src/detail/osSocket.c b/src/os/src/detail/osSocket.c index c8ad6a5acc..bcacd8575a 100644 --- a/src/os/src/detail/osSocket.c +++ b/src/os/src/detail/osSocket.c @@ -57,4 +57,16 @@ int taosSetSockOpt(SOCKET socketfd, int level, int optname, void *optval, int op return setsockopt(socketfd, level, optname, optval, (socklen_t)optlen); } +#endif + +#ifndef TAOS_OS_FUNC_SOCKET_INET + +uint32_t taosInetAddr(char *ipAddr) { + return inet_addr(ipAddr); +} + +const char *taosInetNtoa(IN_ADDR ipInt) { + return inet_ntoa(ipInt); +} + #endif \ No newline at end of file diff --git a/src/os/src/detail/osTime.c b/src/os/src/detail/osTime.c index ced1643d2b..57634e468a 100644 --- a/src/os/src/detail/osTime.c +++ b/src/os/src/detail/osTime.c @@ -61,8 +61,15 @@ int64_t user_mktime64(const unsigned int year0, const unsigned int mon0, res = res*24; res = ((res + hour) * 60 + min) * 60 + sec; +#ifdef _MSC_VER +#if _MSC_VER >= 1900 + int64_t timezone = _timezone; +#endif +#endif + return (res + timezone); } + // ==== mktime() kernel code =================// static int64_t m_deltaUtc = 0; void deltaToUtcInitOnce() { diff --git a/src/os/src/windows/w64Socket.c b/src/os/src/windows/w64Socket.c index dd8961da40..6514c2f851 100644 --- a/src/os/src/windows/w64Socket.c +++ b/src/os/src/windows/w64Socket.c @@ -62,4 +62,24 @@ int taosSetSockOpt(SOCKET socketfd, int level, int optname, void *optval, int op } return setsockopt(socketfd, level, optname, optval, optlen); -} \ No newline at end of file +} + +#ifdef TAOS_OS_FUNC_SOCKET_INET + +uint32_t taosInetAddr(char *ipAddr) { + uint32_t value; + int ret = inet_pton(AF_INET, ipAddr, &value); + if (ret <= 0) { + return INADDR_NONE; + } else { + return value; + } +} + +const char *taosInetNtoa(IN_ADDR ipInt) { + // not thread safe, only for debug usage while print log + static char tmpDstStr[16]; + return inet_ntop(AF_INET, &ipInt, tmpDstStr, INET6_ADDRSTRLEN); +} + +#endif \ No newline at end of file diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c index 8ee92be31c..21949e93af 100644 --- a/src/plugins/http/src/httpServer.c +++ b/src/plugins/http/src/httpServer.c @@ -302,7 +302,7 @@ static void *httpAcceptHttpConnection(void *arg) { #if 0 if (totalFds > tsHttpCacheSessions * 100) { httpError("fd:%d, ip:%s:%u, totalFds:%d larger than httpCacheSessions:%d*100, refuse connection", connFd, - inet_ntoa(clientAddr.sin_addr), htons(clientAddr.sin_port), totalFds, tsHttpCacheSessions); + taosInetNtoa(clientAddr.sin_addr), htons(clientAddr.sin_port), totalFds, tsHttpCacheSessions); taosCloseSocket(connFd); continue; } @@ -316,14 +316,14 @@ static void *httpAcceptHttpConnection(void *arg) { pContext = httpCreateContext(connFd); if (pContext == NULL) { - httpError("fd:%d, ip:%s:%u, no enough resource to allocate http context", connFd, inet_ntoa(clientAddr.sin_addr), + httpError("fd:%d, ip:%s:%u, no enough resource to allocate http context", connFd, taosInetNtoa(clientAddr.sin_addr), htons(clientAddr.sin_port)); taosCloseSocket(connFd); continue; } pContext->pThread = pThread; - sprintf(pContext->ipstr, "%s:%u", inet_ntoa(clientAddr.sin_addr), htons(clientAddr.sin_port)); + sprintf(pContext->ipstr, "%s:%u", taosInetAddr(clientAddr.sin_addr), htons(clientAddr.sin_port)); struct epoll_event event; event.events = EPOLLIN | EPOLLPRI | EPOLLWAKEUP | EPOLLERR | EPOLLHUP | EPOLLRDHUP; diff --git a/src/rpc/src/rpcTcp.c b/src/rpc/src/rpcTcp.c index 9da11831e5..3475e0f317 100644 --- a/src/rpc/src/rpcTcp.c +++ b/src/rpc/src/rpcTcp.c @@ -254,11 +254,11 @@ static void *taosAcceptTcpConnection(void *arg) { pFdObj->ip = caddr.sin_addr.s_addr; pFdObj->port = htons(caddr.sin_port); tDebug("%s new TCP connection from %s:%hu, fd:%d FD:%p numOfFds:%d", pServerObj->label, - inet_ntoa(caddr.sin_addr), pFdObj->port, connFd, pFdObj, pThreadObj->numOfFds); + taosInetNtoa(caddr.sin_addr), pFdObj->port, connFd, pFdObj, pThreadObj->numOfFds); } else { taosCloseSocket(connFd); tError("%s failed to malloc FdObj(%s) for connection from:%s:%hu", pServerObj->label, strerror(errno), - inet_ntoa(caddr.sin_addr), htons(caddr.sin_port)); + taosInetNtoa(caddr.sin_addr), htons(caddr.sin_port)); } // pick up next thread for next connection diff --git a/src/util/src/tconfig.c b/src/util/src/tconfig.c index ccf4ea7317..704af2017e 100644 --- a/src/util/src/tconfig.c +++ b/src/util/src/tconfig.c @@ -133,7 +133,7 @@ static void taosReadDirectoryConfig(SGlobalCfg *cfg, char *input_value) { } static void taosReadIpStrConfig(SGlobalCfg *cfg, char *input_value) { - uint32_t value = inet_addr(input_value); + uint32_t value = taosInetAddr(input_value); char * option = (char *)cfg->ptr; if (value == INADDR_NONE) { uError("config option:%s, input value:%s, is not a valid ip address, use default value:%s", From 8ac389c675802a81e6048d5e77aaa640ca3e3f74 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 14 Aug 2020 17:08:04 +0800 Subject: [PATCH 06/51] TD-1057 compile error after merge feature/query --- src/client/src/tscLocalMerge.c | 2 +- tests/tsim/src/simExe.c | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c index 186c2871a1..99c3bc4fb3 100644 --- a/src/client/src/tscLocalMerge.c +++ b/src/client/src/tscLocalMerge.c @@ -972,7 +972,7 @@ static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneO } if (pRes->numOfRows > 0) { - int32_t currentTotal = pRes->numOfRowsGroup + pRes->numOfRows; + int32_t currentTotal = (int32_t)(pRes->numOfRowsGroup + pRes->numOfRows); if (pQueryInfo->limit.limit >= 0 && currentTotal > pQueryInfo->limit.limit) { int32_t overflow = (int32_t)(currentTotal - pQueryInfo->limit.limit); diff --git a/tests/tsim/src/simExe.c b/tests/tsim/src/simExe.c index 50d1a9b5be..0529808b6b 100644 --- a/tests/tsim/src/simExe.c +++ b/tests/tsim/src/simExe.c @@ -748,11 +748,7 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { sprintf(value, "%d", *((int *)row[i])); break; case TSDB_DATA_TYPE_BIGINT: -#ifdef _TD_ARM_32_ - sprintf(value, "%lld", *((int64_t *)row[i])); -#else - sprintf(value, "%ld", *((int64_t *)row[i])); -#endif + sprintf(value, "%" PRId64, *((int64_t *)row[i])); break; case TSDB_DATA_TYPE_FLOAT:{ #ifdef _TD_ARM_32_ From 7f885cb3b3f137ef0b3a13eeb91f448793d59bdc Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 14 Aug 2020 09:39:56 +0000 Subject: [PATCH 07/51] TD-1057 --- src/kit/taosnetwork/client.c | 4 ++-- src/kit/taosnetwork/server.c | 6 +++--- src/os/inc/osSocket.h | 2 +- src/os/src/detail/osSocket.c | 2 +- src/os/src/windows/w64Socket.c | 2 +- src/plugins/http/src/httpServer.c | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/kit/taosnetwork/client.c b/src/kit/taosnetwork/client.c index 74c161bf67..102f9b9d89 100644 --- a/src/kit/taosnetwork/client.c +++ b/src/kit/taosnetwork/client.c @@ -91,7 +91,7 @@ int checkTcpPort(info_s *info) { serverAddr.sin_family = AF_INET; serverAddr.sin_port = htons(port); - serverAddr.sin_addr.s_addr = taosInetAddr(host); + serverAddr.sin_addr.s_addr = inet_addr(host); //printf("=================================\n"); if (connect(clientSocket, (struct sockaddr *)&serverAddr, sizeof(serverAddr)) < 0) { @@ -157,7 +157,7 @@ int checkUdpPort(info_s *info) { serverAddr.sin_family = AF_INET; serverAddr.sin_port = htons(port); - serverAddr.sin_addr.s_addr = taosInetAddr(host); + serverAddr.sin_addr.s_addr = inet_addr(host); memset(sendbuf, 0, BUFFER_SIZE); memset(recvbuf, 0, BUFFER_SIZE); diff --git a/src/kit/taosnetwork/server.c b/src/kit/taosnetwork/server.c index a2bb2b2b65..1c3bc6fa09 100644 --- a/src/kit/taosnetwork/server.c +++ b/src/kit/taosnetwork/server.c @@ -128,7 +128,7 @@ static void *bindTcpPort(void *sarg) { if (errno == EINTR) { continue; } else { - printf("recv Client: %s pkg from TCP port: %d fail:%s.\n", taosInetNtoa(clientAddr.sin_addr), port, strerror(errno)); + printf("recv Client: %s pkg from TCP port: %d fail:%s.\n", inet_ntoa(clientAddr.sin_addr), port, strerror(errno)); close(serverSocket); return NULL; } @@ -139,7 +139,7 @@ static void *bindTcpPort(void *sarg) { } } - printf("recv Client: %s pkg from TCP port: %d, pkg len: %d\n", taosInetNtoa(clientAddr.sin_addr), port, iDataNum); + printf("recv Client: %s pkg from TCP port: %d, pkg len: %d\n", inet_ntoa(clientAddr.sin_addr), port, iDataNum); if (iDataNum > 0) { send(client, buffer, iDataNum, 0); break; @@ -188,7 +188,7 @@ static void *bindUdpPort(void *sarg) { continue; } if (iDataNum > 0) { - printf("recv Client: %s pkg from UDP port: %d, pkg len: %d\n", taosInetNtoa(clientAddr.sin_addr), port, iDataNum); + printf("recv Client: %s pkg from UDP port: %d, pkg len: %d\n", inet_ntoa(clientAddr.sin_addr), port, iDataNum); //printf("Read msg from udp:%s ... %s\n", buffer, buffer+iDataNum-16); sendto(serverSocket, buffer, iDataNum, 0, (struct sockaddr *)&clientAddr, (int)sin_size); diff --git a/src/os/inc/osSocket.h b/src/os/inc/osSocket.h index e5c3806d1b..ecc69ec3d3 100644 --- a/src/os/inc/osSocket.h +++ b/src/os/inc/osSocket.h @@ -67,7 +67,7 @@ int taosSetSockOpt(SOCKET socketfd, int level, int optname, void *optval, int op // TAOS_OS_FUNC_SOCKET_INET uint32_t taosInetAddr(char *ipAddr); -const char *taosInetNtoa(IN_ADDR ipInt); +const char *taosInetNtoa(struct in_addr ipInt); #ifdef __cplusplus } diff --git a/src/os/src/detail/osSocket.c b/src/os/src/detail/osSocket.c index bcacd8575a..8a51c389e9 100644 --- a/src/os/src/detail/osSocket.c +++ b/src/os/src/detail/osSocket.c @@ -65,7 +65,7 @@ uint32_t taosInetAddr(char *ipAddr) { return inet_addr(ipAddr); } -const char *taosInetNtoa(IN_ADDR ipInt) { +const char *taosInetNtoa(struct in_addr ipInt) { return inet_ntoa(ipInt); } diff --git a/src/os/src/windows/w64Socket.c b/src/os/src/windows/w64Socket.c index 6514c2f851..8fd198ba80 100644 --- a/src/os/src/windows/w64Socket.c +++ b/src/os/src/windows/w64Socket.c @@ -76,7 +76,7 @@ uint32_t taosInetAddr(char *ipAddr) { } } -const char *taosInetNtoa(IN_ADDR ipInt) { +const char *taosInetNtoa(struct in_addr ipInt) { // not thread safe, only for debug usage while print log static char tmpDstStr[16]; return inet_ntop(AF_INET, &ipInt, tmpDstStr, INET6_ADDRSTRLEN); diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c index 21949e93af..f4aca91cba 100644 --- a/src/plugins/http/src/httpServer.c +++ b/src/plugins/http/src/httpServer.c @@ -323,7 +323,7 @@ static void *httpAcceptHttpConnection(void *arg) { } pContext->pThread = pThread; - sprintf(pContext->ipstr, "%s:%u", taosInetAddr(clientAddr.sin_addr), htons(clientAddr.sin_port)); + sprintf(pContext->ipstr, "%s:%u", taosInetNtoa(clientAddr.sin_addr), htons(clientAddr.sin_port)); struct epoll_event event; event.events = EPOLLIN | EPOLLPRI | EPOLLWAKEUP | EPOLLERR | EPOLLHUP | EPOLLRDHUP; From 75b91d88164bf7c678cfa3b0ceb0d414907e5434 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Fri, 14 Aug 2020 18:06:02 +0800 Subject: [PATCH 08/51] udpate taos sql md --- documentation20/webdocs/markdowndocs/TAOS SQL-ch.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md b/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md index 6c6a3afb60..293aac8d23 100644 --- a/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md +++ b/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md @@ -31,7 +31,7 @@ taos> DESCRIBE meters; - 时间格式为```YYYY-MM-DD HH:mm:ss.MS```, 默认时间分辨率为毫秒。比如：```2017-08-12 18:25:58.128``` - 内部函数now是服务器的当前时间 -- 插入记录时，如果时间戳为0，插入数据时使用服务器当前时间 +- 插入记录时，如果时间戳为now，插入数据时使用服务器当前时间 - Epoch Time: 时间戳也可以是一个长整数，表示从1970-01-01 08:00:00.000开始的毫秒数 - 时间可以加减，比如 now-2h，表明查询时刻向前推2个小时(最近2小时)。数字后面的时间单位：a(毫秒), s(秒), m(分), h(小时), d(天)，w(周), n(月), y(年)。比如select * from t1 where ts > now-2w and ts <= now-1w, 表示查询两周前整整一周的数据 - TDengine暂不支持时间窗口按照自然年和自然月切分。Where条件中的时间窗口单位的换算关系如下：interval(1y) 等效于 interval(365d), interval(1n) 等效于 interval(30d), interval(1w) 等效于 interval(7d) @@ -994,4 +994,4 @@ SELECT AVG(current),MAX(current),LEASTSQUARES(current, start_val, step_val), PER - 列名最大长度为65，最多允许1024列，最少需要2列，第一列必须是时间戳 - 标签最多允许128个，可以0个，标签总长度不超过16k个字符 - SQL语句最大长度65480个字符，但可通过系统配置参数maxSQLLength修改，最长可配置为8M -- 库的数目，超级表的数目、表的数目，系统不做限制，仅受系统资源限制 \ No newline at end of file +- 库的数目，超级表的数目、表的数目，系统不做限制，仅受系统资源限制 From c4f1db3ec7f8dd5f3059549908ea526a683336a0 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 14 Aug 2020 18:50:29 +0800 Subject: [PATCH 09/51] TD-1057 not show error if create file failed --- src/kit/shell/src/shellEngine.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c index abc4f7a02c..692b5e49a0 100644 --- a/src/kit/shell/src/shellEngine.c +++ b/src/kit/shell/src/shellEngine.c @@ -751,7 +751,9 @@ void read_history() { FILE *f = fopen(f_history, "r"); if (f == NULL) { +#ifndef WINDOWS fprintf(stderr, "Failed to open file %s\n", f_history); +#endif return; } @@ -776,7 +778,9 @@ void write_history() { FILE *f = fopen(f_history, "w"); if (f == NULL) { +#ifndef WINDOWS fprintf(stderr, "Failed to open file %s for write\n", f_history); +#endif return; } From aa33984c28770c477b655a5886888b04176da7a3 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 14 Aug 2020 18:51:06 +0800 Subject: [PATCH 10/51] add influxdbTestWriteLoop.sh --- .../perftest-scripts/influxdbTestWriteLoop.sh | 45 ++++++++++++++----- 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/tests/perftest-scripts/influxdbTestWriteLoop.sh b/tests/perftest-scripts/influxdbTestWriteLoop.sh index a9b10ac45d..2cbb456a0f 100755 --- a/tests/perftest-scripts/influxdbTestWriteLoop.sh +++ b/tests/perftest-scripts/influxdbTestWriteLoop.sh @@ -4,7 +4,7 @@ DATA_DIR=/mnt/root/testdata NUM_LOOP=1 NUM_OF_FILES=100 -rowsPerRequest=(1 100 500 1000 2000) +rowsPerRequest=(1 100 1000 10000 20000 50000 100000) function printTo { if $verbose ; then @@ -13,17 +13,21 @@ function printTo { } function runTest { - printf "R/R, " - for c in `seq 1 $clients`; do - if [ "$c" == "1" ]; then - printf "$c client, " - else - printf "$c clients, " - fi + declare -A avgRPR + + for r in ${!rowsPerRequest[@]}; do + for c in `seq 1 $clients`; do + avgRPR[$r, $c]=0 + done done - printf "\n" for r in ${rowsPerRequest[@]}; do + if [ "$r" == "1" ] || [ "$r" == "100" ] || [ "$r" == "1000" ]; then + NUM_OF_FILES=$clients + else + NUM_OF_FILES=100 + fi + printf "$r, " for c in `seq 1 $clients`; do totalRPR=0 @@ -39,13 +43,30 @@ function runTest { -numOfFiles $NUM_OF_FILES \ -writeClients $c \ -rowsPerRequest $r 2>&1 \ - | tee $OUTPUT_FILE + > $OUTPUT_FILE RPR=`cat $OUTPUT_FILE | grep speed | awk '{print $(NF-1)}'` totalRPR=`echo "scale=4; $totalRPR + $RPR" | bc` printTo "rows:$r, clients:$c, i:$i RPR:$RPR" done - avgRPR=`echo "scale=4; $totalRPR / $NUM_LOOP" | bc` - printf "$avgRPR, " + avgRPR[$r,$c]=`echo "scale=4; $totalRPR / $NUM_LOOP" | bc` + printTo "r:$r c:$c avgRPR:${avgRPR[$r, $c]}" + done + done + + printf "R/R, " + for c in `seq 1 $clients`; do + if [ "$c" == "1" ]; then + printf "$c client, " + else + printf "$c clients, " + fi + done + printf "\n" + + for r in ${!rowsPerRequest[@]}; do + printf "${rowsPerRequest[$r]}, " + for c in `seq 1 $clients`; do + printf "${avgRPR[$r,$c]}, " done printf "\n" done From b13a1dda0080f1ff43a573dc0960f0a08c786278 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 15 Aug 2020 04:25:34 +0000 Subject: [PATCH 11/51] TD-1147 --- .../pytest/tag_lite/too_many_tag_condtions.py | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 tests/pytest/tag_lite/too_many_tag_condtions.py diff --git a/tests/pytest/tag_lite/too_many_tag_condtions.py b/tests/pytest/tag_lite/too_many_tag_condtions.py new file mode 100644 index 0000000000..a40de405ff --- /dev/null +++ b/tests/pytest/tag_lite/too_many_tag_condtions.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + + tdLog.info('======================== dnode1 start') + tbPrefix = "ta_cr_tb" + mtPrefix = "ta_cr_mt" + tbNum = 2 + rowNum = 10 + totalNum = 200 + tagCondsLimit = 1024 + tdLog.info('=============== step1: create tbl and prepare data') + i = 0 + i = 2 + mt = "%s%d" % (mtPrefix, i) + tb = "%s%d" % (tbPrefix, i) + + sql ='create table %s (ts timestamp, tbcol int) TAGS(tgcol int)'% (mt) + tdLog.info(sql) + tdSql.execute(sql) + for i in range(0, tbNum): + tblName = "%s%d"%(tbPrefix, i) + sql = 'create table %s using %s TAGS(%d)'%(tblName, mt, i) + tdSql.execute(sql) + for j in range(0, rowNum): + sql = "insert into %s values(now, %d)"%(tblName, j) + tdSql.execute(sql) + + sqlPrefix = "select * from %s where "%(mt) + for i in range(2, 2048, 1): + conds = "tgcol=1 and "* (i - 1) + conds = "%stgcol=1"%(conds) + sql = "%s%s"%(sqlPrefix, conds) + if i >= tagCondsLimit: + tdSql.error(sql) + else: + tdSql.query(sql) + #tdSql.checkRows(1) + + for i in range(2, 2048, 1): + conds = "" + for j in range(0, i - 1): + conds = conds + "tgcol=%d or "%(j%tbNum) + conds += "tgcol=%d"%(i%tbNum) + sql = sqlPrefix + conds + if i >= tagCondsLimit: + tdSql.error(sql) + else: + tdSql.query(sql) + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From fae6f1f97583e8c174ab1368c6ec0dcf424f1a03 Mon Sep 17 00:00:00 2001 From: zyyang Date: Sat, 15 Aug 2020 12:51:34 +0800 Subject: [PATCH 12/51] update doc: add jdbc example for data subscribe --- .../markdowndocs/advanced features-ch.md | 136 ++++++++++++++++++ 1 file changed, 136 insertions(+) diff --git a/documentation20/webdocs/markdowndocs/advanced features-ch.md b/documentation20/webdocs/markdowndocs/advanced features-ch.md index 690f2a6268..6f8ceca071 100644 --- a/documentation20/webdocs/markdowndocs/advanced features-ch.md +++ b/documentation20/webdocs/markdowndocs/advanced features-ch.md @@ -295,6 +295,142 @@ $ taos 这时，因为电流超过了10A，您应该可以看到示例程序将它输出到了屏幕上。 您可以继续插入一些数据观察示例程序的输出。 +### jdbc使用数据订阅功能 + +（1）使用订阅功能前的数据准备 + +```shell +# 创建power库 +taos> create database power; +# 切换库 +taos> use power; +# 创建超级表 +taos> create table meters(ts timestamp, current float, voltage int, phase int) tags(location binary(64), groupI +d int); +# 创建表 +taos> create table d1001 using meters tags ("Beijing.Chaoyang",2); +taos> create table d1002 using meters tags ("Beijing.Haidian",2); +# 插入测试数据 +taos> insert into d1001 values("2020-08-15 12:00:00.000", 12, 220, 1),("2020-08-15 12:10:00.000", 12.3, 220, 2),("2020-08-15 12:20:00.000", 12.2, 220, 1); +taos> insert into d1002 values("2020-08-15 12:00:00.000", 9.9, 220, 1),("2020-08-15 12:10:00.000", 10.3, 220, 1),("2020-08-15 12:20:00.000", 11.2, 220, 1); +# 从超级表meters查询current大于10的数据 +taos> select * from meters where current > 10; + ts | current | voltage | phase| location | groupid | +=========================================================================================================== + 2020-08-15 12:10:00.000 | 10.30000 | 220 | 1 | Beijing.Haidian | 2 | + 2020-08-15 12:20:00.000 | 11.20000 | 220 | 1 | Beijing.Haidian | 2 | + 2020-08-15 12:00:00.000 | 12.00000 | 220 | 1 | Beijing.Chaoyang | 2 | + 2020-08-15 12:10:00.000 | 12.30000 | 220 | 2 | Beijing.Chaoyang | 2 | + 2020-08-15 12:20:00.000 | 12.20000 | 220 | 1 | Beijing.Chaoyang | 2 | +Query OK, 5 row(s) in set (0.004896s) +``` + +（2）使用jdbc提供的订阅功能 + +```java +public class SubscribeDemo { + private static final String topic = "topic_meter_current_bg_10"; + private static final String sql = "select * from meters where current > 10"; + + public static void main(String[] args) { + Connection connection = null; + Statement statement = null; + TSDBSubscribe subscribe = null; + long subscribeId = 0; + + try { + // 加载驱动 + Class.forName("com.taosdata.jdbc.TSDBDriver"); + // 获取Connectin + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + String jdbcUrl = "jdbc:TAOS://127.0.0.1:6030/power?user=root&password=taosdata"; + connection = DriverManager.getConnection(jdbcUrl, properties); + System.out.println("create the connection"); + // 创建Subscribe + subscribe = ((TSDBConnection) connection).createSubscribe(); + // subscribe订阅topic，topic为主题名称，sql为查询语句，restart代表是否每次订阅接受历史数据 + subscribeId = subscribe.subscribe(topic, sql, true); + System.out.println("create a subscribe topic: " + topic + "@[" + subscribeId + "]"); + int count = 0; + while (true) { + // 消费数据 + TSDBResultSet resultSet = subscribe.consume(subscribeId); + // 打印结果集 + if (resultSet != null) { + ResultSetMetaData metaData = resultSet.getMetaData(); + while (resultSet.next()) { + int columnCount = metaData.getColumnCount(); + for (int i = 1; i <= columnCount; i++) { + System.out.print(metaData.getColumnLabel(i) + " : " + resultSet.getString(i) + "\t"); + } + System.out.println("\n===================="); + count++; + } + } + if (count > 10) + break; + TimeUnit.SECONDS.sleep(1); + } + } catch (Exception e) { + e.printStackTrace(); + } finally { + try { + if (null != subscribe && subscribeId != 0) { + subscribe.unsubscribe(subscribeId, true); + System.out.println("unsubscribe the top@[" + subscribeId + "]"); + } + if (statement != null) { + statement.close(); + System.out.println("close the statement."); + } + if (connection != null) { + connection.close(); + System.out.println("close the connection."); + } + } catch (SQLException throwables) { + throwables.printStackTrace(); + } + } + } +} +``` + +（3）订阅功能演示 + +运行demo，首先，subscribe会将满足情况的历史数据消费 + +```shell +# java -jar subscribe.jar + +ts : 1597464000000 current : 12.0 voltage : 220 phase : 1 location : Beijing.Chaoyang groupid : 2 +==================== +ts : 1597464600000 current : 12.3 voltage : 220 phase : 2 location : Beijing.Chaoyang groupid : 2 +==================== +ts : 1597465200000 current : 12.2 voltage : 220 phase : 1 location : Beijing.Chaoyang groupid : 2 +==================== +ts : 1597464600000 current : 10.3 voltage : 220 phase : 1 location : Beijing.Haidian groupid : 2 +==================== +ts : 1597465200000 current : 11.2 voltage : 220 phase : 1 location : Beijing.Haidian groupid : 2 +==================== +``` + +接着，使用taos客户端向表中新增数据 + +```shell +# taos +taos> use power； +taos> insert into d1001 values("2020-08-15 12:40:00.000", 12.4, 220, 1); +``` + +查看数据消费情况 + +```shell +ts : 1597466400000 current : 12.4 voltage : 220 phase : 1 location : Beijing.Chaoyang groupid : 2 +==================== +``` + ## 缓存(Cache) From a6f1583f8132c933b731916d472867590a5aee0d Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 15 Aug 2020 13:33:58 +0800 Subject: [PATCH 13/51] move macro definition position for random test. --- src/tsdb/src/tsdbFile.c | 5 +++-- src/tsdb/src/tsdbRWHelper.c | 6 ++++-- src/util/src/tkvstore.c | 4 +++- src/wal/src/walMain.c | 4 +++- 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index 8bc4be6d67..d960dfb7ba 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -15,13 +15,14 @@ #define _DEFAULT_SOURCE #include +#define TAOS_RANDOM_FILE_FAIL_TEST + #include "os.h" #include "talgo.h" #include "tchecksum.h" #include "tsdbMain.h" #include "tutil.h" -#define TAOS_RANDOM_FILE_FAIL_TEST const char *tsdbFileSuffix[] = {".head", ".data", ".last", ".stat", ".h", ".d", ".l", ".s"}; @@ -529,4 +530,4 @@ static void tsdbInitFileGroup(SFileGroup *pFGroup, STsdbRepo *pRepo) { terrno = TSDB_CODE_TDB_FILE_CORRUPTED; } } -} \ No newline at end of file +} diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index ed851a08c2..12199f491f 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -14,13 +14,15 @@ */ #define _DEFAULT_SOURCE + +#define TAOS_RANDOM_FILE_FAIL_TEST + #include "os.h" #include "talgo.h" #include "tchecksum.h" #include "tcoding.h" #include "tscompression.h" #include "tsdbMain.h" -#define TAOS_RANDOM_FILE_FAIL_TEST #define TSDB_GET_COMPCOL_LEN(nCols) (sizeof(SCompData) + sizeof(SCompCol) * (nCols) + sizeof(TSCKSUM)) #define TSDB_KEY_COL_OFFSET 0 @@ -1651,4 +1653,4 @@ static int tsdbWriteBlockToProperFile(SRWHelper *pHelper, SDataCols *pDataCols, if (tsdbWriteBlockToFile(pHelper, pFile, pDataCols, pCompBlock, isLast, true) < 0) return -1; return 0; -} \ No newline at end of file +} diff --git a/src/util/src/tkvstore.c b/src/util/src/tkvstore.c index dd0600ec3c..9fab4a5936 100644 --- a/src/util/src/tkvstore.c +++ b/src/util/src/tkvstore.c @@ -14,6 +14,9 @@ */ #define _DEFAULT_SOURCE + +#define TAOS_RANDOM_FILE_FAIL_TEST + #include "os.h" #include "hash.h" #include "taoserror.h" @@ -21,7 +24,6 @@ #include "tcoding.h" #include "tkvstore.h" #include "tulog.h" -#define TAOS_RANDOM_FILE_FAIL_TEST #define TD_KVSTORE_HEADER_SIZE 512 #define TD_KVSTORE_MAJOR_VERSION 1 diff --git a/src/wal/src/walMain.c b/src/wal/src/walMain.c index 4ac8a096c6..bebad69f32 100644 --- a/src/wal/src/walMain.c +++ b/src/wal/src/walMain.c @@ -14,6 +14,9 @@ */ #define _DEFAULT_SOURCE + +#define TAOS_RANDOM_FILE_FAIL_TEST + #include "os.h" #include "tlog.h" #include "tchecksum.h" @@ -22,7 +25,6 @@ #include "taoserror.h" #include "twal.h" #include "tqueue.h" -#define TAOS_RANDOM_FILE_FAIL_TEST #define walPrefix "wal" From c5ae6d2aafee5b6090f863eb5d636dbbad597559 Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Sat, 15 Aug 2020 13:43:20 +0800 Subject: [PATCH 14/51] fix td-1150 --- src/client/src/tscStream.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index 60517a2f5c..9dd47888d2 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -136,7 +136,11 @@ static void tscProcessStreamTimer(void *handle, void *tmrId) { } pQueryInfo->window.ekey = etime; if (pQueryInfo->window.skey >= pQueryInfo->window.ekey) { - tscSetRetryTimer(pStream, pSql, pStream->slidingTime); + int64_t timer = pStream->slidingTime; + if (pStream->precision == TSDB_TIME_PRECISION_MICRO) { + timer /= 1000l; + } + tscSetRetryTimer(pStream, pSql, timer); return; } } From 6143e72b1c4d9c637f929e393d36acb814f784df Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 15 Aug 2020 14:20:20 +0800 Subject: [PATCH 15/51] TD-1047 iconv compile options --- cmake/define.inc | 1 + tests/script/jenkins/basic.txt | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cmake/define.inc b/cmake/define.inc index 2e3c639ecb..d4f3b3c8bf 100755 --- a/cmake/define.inc +++ b/cmake/define.inc @@ -50,6 +50,7 @@ IF (TD_ARM_64) ADD_DEFINITIONS(-D_M_X64) ADD_DEFINITIONS(-D_TD_ARM_64_) ADD_DEFINITIONS(-D_TD_ARM_) + ADD_DEFINITIONS(-DUSE_LIBICONV) SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") ENDIF () diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index e0f6b7b197..a48584b0ed 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -135,7 +135,6 @@ cd ../../../debug; make ./test.sh -f general/parser/limit2.sim ./test.sh -f general/parser/fill.sim ./test.sh -f general/parser/fill_stb.sim -#./test.sh -f general/parser/fill_us.sim ./test.sh -f general/parser/where.sim ./test.sh -f general/parser/slimit.sim ./test.sh -f general/parser/select_with_tags.sim @@ -143,7 +142,6 @@ cd ../../../debug; make ./test.sh -f general/parser/tags_dynamically_specifiy.sim ./test.sh -f general/parser/groupby.sim ./test.sh -f general/parser/set_tag_vals.sim -#./test.sh -f general/parser/sliding.sim ./test.sh -f general/parser/tags_filter.sim ./test.sh -f general/parser/slimit_alter_tags.sim ./test.sh -f general/parser/join.sim @@ -151,6 +149,8 @@ cd ../../../debug; make ./test.sh -f general/parser/binary_escapeCharacter.sim ./test.sh -f general/parser/bug.sim ./test.sh -f general/parser/repeatAlter.sim +./test.sh -f general/parser/union.sim +./test.sh -f general/parser/topbot.sim ./test.sh -f general/stable/disk.sim ./test.sh -f general/stable/dnode3.sim From 22cb8287b54ba0c7b69bcffc16fc9db0d22c805e Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 15 Aug 2020 14:29:35 +0800 Subject: [PATCH 16/51] TD-1047 minor changes for libiconv --- cmake/define.inc | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/cmake/define.inc b/cmake/define.inc index d4f3b3c8bf..c72995159f 100755 --- a/cmake/define.inc +++ b/cmake/define.inc @@ -33,11 +33,7 @@ IF (TD_LINUX_64) ADD_DEFINITIONS(-D_M_X64) ADD_DEFINITIONS(-D_TD_LINUX_64) SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g3 -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") - - FIND_PATH(ICONV_INCLUDE_EXIST iconv.h /usr/include/ /usr/local/include/) - IF (ICONV_INCLUDE_EXIST) - ADD_DEFINITIONS(-DUSE_LIBICONV) - ENDIF () + ADD_DEFINITIONS(-DUSE_LIBICONV) ENDIF () IF (TD_LINUX_32) @@ -134,6 +130,7 @@ ENDIF () IF (TD_WINDOWS_32) ADD_DEFINITIONS(-D_TD_WINDOWS_32) + ADD_DEFINITIONS(-DUSE_LIBICONV) ENDIF () INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) From acdcd47243de8d0e5ccde73fac3edb36f92e73db Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 15 Aug 2020 14:52:38 +0800 Subject: [PATCH 17/51] a crash generated by dn3_mn1_replica_change.sim --- src/tsdb/src/tsdbRead.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index c89ae0698a..ac3a6dac07 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -1795,19 +1795,22 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { if (pQueryHandle->checkFiles) { bool exists = true; + int32_t code = getDataBlocksInFiles(pQueryHandle, &exists); if (code != TSDB_CODE_SUCCESS) { - return code; + pQueryHandle->activeIndex = 0; + pQueryHandle->checkFiles = false; + + return false; } if (exists) { - elapsedTime = taosGetTimestampUs() - stime; - pQueryHandle->cost.checkForNextTime += elapsedTime; + pQueryHandle->cost.checkForNextTime += (taosGetTimestampUs() - stime); return exists; } pQueryHandle->activeIndex = 0; - pQueryHandle->checkFiles = false; + pQueryHandle->checkFiles = false; } // TODO: opt by consider the scan order From 40772c1e1f3ce691b247c2a70cfa92df206bffd2 Mon Sep 17 00:00:00 2001 From: zyyang Date: Sat, 15 Aug 2020 12:51:34 +0800 Subject: [PATCH 18/51] update doc: add jdbc example for data subscribe --- .../markdowndocs/advanced features-ch.md | 122 ++++++++++++++++++ 1 file changed, 122 insertions(+) diff --git a/documentation20/webdocs/markdowndocs/advanced features-ch.md b/documentation20/webdocs/markdowndocs/advanced features-ch.md index 690f2a6268..858f92b604 100644 --- a/documentation20/webdocs/markdowndocs/advanced features-ch.md +++ b/documentation20/webdocs/markdowndocs/advanced features-ch.md @@ -295,6 +295,128 @@ $ taos 这时，因为电流超过了10A，您应该可以看到示例程序将它输出到了屏幕上。 您可以继续插入一些数据观察示例程序的输出。 +### jdbc使用数据订阅功能 + +（1）使用订阅功能前的数据准备 + +```shell +# 创建power库 +taos> create database power; +# 切换库 +taos> use power; +# 创建超级表 +taos> create table meters(ts timestamp, current float, voltage int, phase int) tags(location binary(64), groupI +d int); +# 创建表 +taos> create table d1001 using meters tags ("Beijing.Chaoyang",2); +taos> create table d1002 using meters tags ("Beijing.Haidian",2); +# 插入测试数据 +taos> insert into d1001 values("2020-08-15 12:00:00.000", 12, 220, 1),("2020-08-15 12:10:00.000", 12.3, 220, 2),("2020-08-15 12:20:00.000", 12.2, 220, 1); +taos> insert into d1002 values("2020-08-15 12:00:00.000", 9.9, 220, 1),("2020-08-15 12:10:00.000", 10.3, 220, 1),("2020-08-15 12:20:00.000", 11.2, 220, 1); +# 从超级表meters查询current大于10的数据 +taos> select * from meters where current > 10; + ts | current | voltage | phase| location | groupid | +=========================================================================================================== + 2020-08-15 12:10:00.000 | 10.30000 | 220 | 1 | Beijing.Haidian | 2 | + 2020-08-15 12:20:00.000 | 11.20000 | 220 | 1 | Beijing.Haidian | 2 | + 2020-08-15 12:00:00.000 | 12.00000 | 220 | 1 | Beijing.Chaoyang | 2 | + 2020-08-15 12:10:00.000 | 12.30000 | 220 | 2 | Beijing.Chaoyang | 2 | + 2020-08-15 12:20:00.000 | 12.20000 | 220 | 1 | Beijing.Chaoyang | 2 | +Query OK, 5 row(s) in set (0.004896s) +``` + +（2）使用jdbc提供的订阅功能 + +```java +public class SubscribeDemo { + private static final String topic = "topic-meter-current-bg-10"; + private static final String sql = "select * from meters where current > 10"; + + public static void main(String[] args) { + Connection connection = null; + TSDBSubscribe subscribe = null; + + try { + // 加载驱动 + Class.forName("com.taosdata.jdbc.TSDBDriver"); + // 获取Connectin + Properties properties = new Properties(); + properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + String jdbcUrl = "jdbc:TAOS://127.0.0.1:6030/power?user=root&password=taosdata"; + connection = DriverManager.getConnection(jdbcUrl, properties); + // 创建Subscribe，topic为主题名称，sql为查询语句，restar为true代表每次订阅消费历史数据 + subscribe = ((TSDBConnection) connection).subscribe(topic, sql, true); + int count = 0; + while (true) { + // 消费数据 + TSDBResultSet resultSet = subscribe.consume(); + // 打印结果集 + if (resultSet != null) { + ResultSetMetaData metaData = resultSet.getMetaData(); + while (resultSet.next()) { + int columnCount = metaData.getColumnCount(); + for (int i = 1; i <= columnCount; i++) { + System.out.print(metaData.getColumnLabel(i) + " : " + resultSet.getString(i) + "\t"); + } + System.out.println("\n===================="); + count++; + } + } + if (count > 10) + break; + TimeUnit.SECONDS.sleep(1); + } + } catch (Exception e) { + e.printStackTrace(); + } finally { + try { + if (null != subscribe) + subscribe.close(true); + if (connection != null) + connection.close(); + } catch (SQLException throwables) { + throwables.printStackTrace(); + } + } + } +} +``` + +（3）订阅功能演示 + +运行demo，首先，subscribe会将满足情况的历史数据消费 + +```shell +# java -jar subscribe.jar + +ts : 1597464000000 current : 12.0 voltage : 220 phase : 1 location : Beijing.Chaoyang groupid : 2 +==================== +ts : 1597464600000 current : 12.3 voltage : 220 phase : 2 location : Beijing.Chaoyang groupid : 2 +==================== +ts : 1597465200000 current : 12.2 voltage : 220 phase : 1 location : Beijing.Chaoyang groupid : 2 +==================== +ts : 1597464600000 current : 10.3 voltage : 220 phase : 1 location : Beijing.Haidian groupid : 2 +==================== +ts : 1597465200000 current : 11.2 voltage : 220 phase : 1 location : Beijing.Haidian groupid : 2 +==================== +``` + +接着，使用taos客户端向表中新增数据 + +```shell +# taos +taos> use power； +taos> insert into d1001 values("2020-08-15 12:40:00.000", 12.4, 220, 1); +``` + +查看数据消费情况 + +```shell +ts : 1597466400000 current : 12.4 voltage : 220 phase : 1 location : Beijing.Chaoyang groupid : 2 +==================== +``` + ## 缓存(Cache) From cef379d992b0bf6e708342e87894f6f1288a7b77 Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Sat, 15 Aug 2020 16:24:20 +0800 Subject: [PATCH 19/51] update java document for subscription --- .../markdowndocs/advanced features-ch.md | 91 ++++++++----------- 1 file changed, 40 insertions(+), 51 deletions(-) diff --git a/documentation20/webdocs/markdowndocs/advanced features-ch.md b/documentation20/webdocs/markdowndocs/advanced features-ch.md index 858f92b604..b1d050c8cc 100644 --- a/documentation20/webdocs/markdowndocs/advanced features-ch.md +++ b/documentation20/webdocs/markdowndocs/advanced features-ch.md @@ -295,27 +295,30 @@ $ taos 这时，因为电流超过了10A，您应该可以看到示例程序将它输出到了屏幕上。 您可以继续插入一些数据观察示例程序的输出。 -### jdbc使用数据订阅功能 +### Java 使用数据订阅功能 -（1）使用订阅功能前的数据准备 +订阅功能也提供了 Java 开发接口，相关说明请见 [Java Connector](https://www.taosdata.com/cn/documentation20/connector/)。需要注意的是，目前 Java 接口没有提供异步订阅模式，但用户程序可以通过创建 `TimerTask` 等方式达到同样的效果。 -```shell -# 创建power库 +下面以一个示例程序介绍其具体使用方法。它所完成的功能与前面介绍的 C 语言示例基本相同，也是订阅数据库中所有电流超过 10A 的记录。 + +#### 准备数据 + +```sql +# 创建 power 库 taos> create database power; # 切换库 taos> use power; # 创建超级表 -taos> create table meters(ts timestamp, current float, voltage int, phase int) tags(location binary(64), groupI -d int); +taos> create table meters(ts timestamp, current float, voltage int, phase int) tags(location binary(64), groupId int); # 创建表 -taos> create table d1001 using meters tags ("Beijing.Chaoyang",2); -taos> create table d1002 using meters tags ("Beijing.Haidian",2); +taos> create table d1001 using meters tags ("Beijing.Chaoyang", 2); +taos> create table d1002 using meters tags ("Beijing.Haidian", 2); # 插入测试数据 taos> insert into d1001 values("2020-08-15 12:00:00.000", 12, 220, 1),("2020-08-15 12:10:00.000", 12.3, 220, 2),("2020-08-15 12:20:00.000", 12.2, 220, 1); taos> insert into d1002 values("2020-08-15 12:00:00.000", 9.9, 220, 1),("2020-08-15 12:10:00.000", 10.3, 220, 1),("2020-08-15 12:20:00.000", 11.2, 220, 1); -# 从超级表meters查询current大于10的数据 +# 从超级表 meters 查询电流大于 10A 的记录 taos> select * from meters where current > 10; - ts | current | voltage | phase| location | groupid | + ts | current | voltage | phase | location | groupid | =========================================================================================================== 2020-08-15 12:10:00.000 | 10.30000 | 220 | 1 | Beijing.Haidian | 2 | 2020-08-15 12:20:00.000 | 11.20000 | 220 | 1 | Beijing.Haidian | 2 | @@ -325,7 +328,7 @@ taos> select * from meters where current > 10; Query OK, 5 row(s) in set (0.004896s) ``` -（2）使用jdbc提供的订阅功能 +#### 示例程序 ```java public class SubscribeDemo { @@ -337,42 +340,36 @@ public class SubscribeDemo { TSDBSubscribe subscribe = null; try { - // 加载驱动 Class.forName("com.taosdata.jdbc.TSDBDriver"); - // 获取Connectin Properties properties = new Properties(); properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); String jdbcUrl = "jdbc:TAOS://127.0.0.1:6030/power?user=root&password=taosdata"; connection = DriverManager.getConnection(jdbcUrl, properties); - // 创建Subscribe，topic为主题名称，sql为查询语句，restar为true代表每次订阅消费历史数据 - subscribe = ((TSDBConnection) connection).subscribe(topic, sql, true); + subscribe = ((TSDBConnection) connection).subscribe(topic, sql, true); // 创建订阅 int count = 0; - while (true) { - // 消费数据 - TSDBResultSet resultSet = subscribe.consume(); - // 打印结果集 - if (resultSet != null) { - ResultSetMetaData metaData = resultSet.getMetaData(); - while (resultSet.next()) { - int columnCount = metaData.getColumnCount(); - for (int i = 1; i <= columnCount; i++) { - System.out.print(metaData.getColumnLabel(i) + " : " + resultSet.getString(i) + "\t"); - } - System.out.println("\n===================="); - count++; - } + while (count < 10) { + TimeUnit.SECONDS.sleep(1); // 等待1秒，避免频繁调用 consume，给服务端造成压力 + TSDBResultSet resultSet = subscribe.consume(); // 消费数据 + if (resultSet == null) { + continue; + } + ResultSetMetaData metaData = resultSet.getMetaData(); + while (resultSet.next()) { + int columnCount = metaData.getColumnCount(); + for (int i = 1; i <= columnCount; i++) { + System.out.print(metaData.getColumnLabel(i) + ": " + resultSet.getString(i) + "\t"); + } + System.out.println(); + count++; } - if (count > 10) - break; - TimeUnit.SECONDS.sleep(1); } } catch (Exception e) { e.printStackTrace(); } finally { try { if (null != subscribe) - subscribe.close(true); + subscribe.close(true); // 关闭订阅 if (connection != null) connection.close(); } catch (SQLException throwables) { @@ -383,38 +380,30 @@ public class SubscribeDemo { } ``` -（3）订阅功能演示 - -运行demo，首先，subscribe会将满足情况的历史数据消费 +运行示例程序，首先，它会消费符合查询条件的所有历史数据： ```shell # java -jar subscribe.jar -ts : 1597464000000 current : 12.0 voltage : 220 phase : 1 location : Beijing.Chaoyang groupid : 2 -==================== -ts : 1597464600000 current : 12.3 voltage : 220 phase : 2 location : Beijing.Chaoyang groupid : 2 -==================== -ts : 1597465200000 current : 12.2 voltage : 220 phase : 1 location : Beijing.Chaoyang groupid : 2 -==================== -ts : 1597464600000 current : 10.3 voltage : 220 phase : 1 location : Beijing.Haidian groupid : 2 -==================== -ts : 1597465200000 current : 11.2 voltage : 220 phase : 1 location : Beijing.Haidian groupid : 2 -==================== +ts: 1597464000000 current: 12.0 voltage: 220 phase: 1 location: Beijing.Chaoyang groupid : 2 +ts: 1597464600000 current: 12.3 voltage: 220 phase: 2 location: Beijing.Chaoyang groupid : 2 +ts: 1597465200000 current: 12.2 voltage: 220 phase: 1 location: Beijing.Chaoyang groupid : 2 +ts: 1597464600000 current: 10.3 voltage: 220 phase: 1 location: Beijing.Haidian groupid : 2 +ts: 1597465200000 current: 11.2 voltage: 220 phase: 1 location: Beijing.Haidian groupid : 2 ``` -接着，使用taos客户端向表中新增数据 +接着，使用 taos 客户端向表中新增一条数据： -```shell +```sql # taos taos> use power； taos> insert into d1001 values("2020-08-15 12:40:00.000", 12.4, 220, 1); ``` -查看数据消费情况 +因为这条数据的电流大于10A，示例程序会将其消费： ```shell -ts : 1597466400000 current : 12.4 voltage : 220 phase : 1 location : Beijing.Chaoyang groupid : 2 -==================== +ts: 1597466400000 current: 12.4 voltage: 220 phase: 1 location: Beijing.Chaoyang groupid: 2 ``` From 8a9ebd8a21eb214dba9a3d57212ec6715cdf1c95 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 13 Aug 2020 01:35:38 +0800 Subject: [PATCH 20/51] add snap package support. --- snap/local/launcher.sh | 21 ++++++++++++ snap/snapcraft.yaml | 76 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+) create mode 100755 snap/local/launcher.sh create mode 100644 snap/snapcraft.yaml diff --git a/snap/local/launcher.sh b/snap/local/launcher.sh new file mode 100755 index 0000000000..5986114931 --- /dev/null +++ b/snap/local/launcher.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# Wrapper to check for custom config in $SNAP_USER_COMMON or $SNAP_COMMON and +# use it otherwise fall back to the included basic config which will at least +# allow mosquitto to run and do something. +# This script will also copy the full example config in to SNAP_USER_COMMON or +# SNAP_COMMON so that people can refer to it. +# +# The decision about whether to use SNAP_USER_COMMON or SNAP_COMMON is taken +# based on the user that runs the command. If the user is root, it is assumed +# that mosquitto is being run as a system daemon, and SNAP_COMMON will be used. +# If a non-root user runs the command, then SNAP_USER_COMMON will be used. + +case "$SNAP_USER_COMMON" in + */root/snap/tdengine/common*) COMMON=$SNAP_COMMON ;; + *) COMMON=$SNAP_USER_COMMON ;; +esac + +CONFIG_FILE="$SNAP/etc/taos" + +# Launch the snap +$SNAP/usr/bin/taosd -c $CONFIG_FILE $@ diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml new file mode 100644 index 0000000000..031704393f --- /dev/null +++ b/snap/snapcraft.yaml @@ -0,0 +1,76 @@ +name: tdengine +base: core18 # the base snap is the execution environment for this snap +version: '2.0.0.6' # just for humans, typically '1.2+git' or '1.3.2' +summary: Single-line elevator pitch for your amazing snap # 79 char long summary +description: | + TDengine is an open-source big data platform designed and optimized for Internet of Things (IoT), Connected Vehicles, and Industrial IoT. Besides the 10x faster time-series database, it provides caching, stream computing, message queuing and other functionalities to reduce the complexity and costs of development and operations. + +grade: stable +confinement: strict + +apps: + tdengine: + command: launcher.sh + daemon: simple + restart-condition: always + plugs: + - network + - network-bind + - systemfiles + + taos: + command: usr/bin/taos + plugs: + - network + - systemfiles + +plugs: + systemfiles: + interface: system-files + read: + - /etc/taos + - /var/lib/taos + write: + - /var/log/taos + - /var/lib/taos + +parts: + script: + plugin: dump + source: snap/local/ + prime: + - launcher.sh + + tdengine: + source: . + source-type: local + plugin: cmake + build-packages: + - gcc + - g++ + - make + - cmake + override-build: | + snapcraftctl build + if [ ! -d $SNAPCRAFT_STAGE/usr ]; then + mkdir $SNAPCRAFT_STAGE/usr + fi + if [ ! -d $SNAPCRAFT_STAGE/etc/taos ]; then + mkdir -p $SNAPCRAFT_STAGE/etc/taos + fi + cp $SNAPCRAFT_PART_BUILD/build/* -rf $SNAPCRAFT_STAGE/usr/ + cp $SNAPCRAFT_PART_SRC/packaging/cfg/taos.cfg -rf $SNAPCRAFT_STAGE/etc/taos/ + + prime: + - etc/taos/taos.cfg + - usr/bin/taosd + - usr/bin/taos + - usr/bin/taosdemo + - usr/lib/libtaos.so.2.0.0.6 + - usr/lib/libtaos.so.1 + - usr/lib/libtaos.so + + + #layout: + # usr/bin/taosd: + #bind-file: debug/build/bin/taosd From f1bd15fa40527ada3ba8eb563fec63ef85608343 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 14 Aug 2020 10:01:36 +0800 Subject: [PATCH 21/51] add /tmp directory for accessing and change confinement to classic. --- snap/snapcraft.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 031704393f..dfa68a9347 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -6,7 +6,7 @@ description: | TDengine is an open-source big data platform designed and optimized for Internet of Things (IoT), Connected Vehicles, and Industrial IoT. Besides the 10x faster time-series database, it provides caching, stream computing, message queuing and other functionalities to reduce the complexity and costs of development and operations. grade: stable -confinement: strict +confinement: classic apps: tdengine: @@ -30,9 +30,11 @@ plugs: read: - /etc/taos - /var/lib/taos + - /tmp write: - /var/log/taos - /var/lib/taos + - /tmp parts: script: From e62bf8fc1ad15ee0bb88e05fb4e85beb01bc9684 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 15 Aug 2020 16:44:17 +0800 Subject: [PATCH 22/51] add taoswrapper and install hook. now it works. --- snap/hooks/install | 17 +++++++++++++++ snap/local/launcher.sh | 6 +++++- snap/local/taoswrapper.sh | 25 ++++++++++++++++++++++ snap/snapcraft.yaml | 45 +++++++++++++++++++++++++++++++++------ 4 files changed, 86 insertions(+), 7 deletions(-) create mode 100755 snap/hooks/install create mode 100755 snap/local/taoswrapper.sh diff --git a/snap/hooks/install b/snap/hooks/install new file mode 100755 index 0000000000..e58918d2c3 --- /dev/null +++ b/snap/hooks/install @@ -0,0 +1,17 @@ +#!/bin/sh + +if [ ! -d /var/lib/taos ]; then + mkdir -p /var/lib/taos +fi + +if [ ! -d /var/log/taos ]; then + mkdir -p -m777 /var/log/taos +fi + +if [ ! -d /etc/taos ]; then + mkdir -p /etc/taos +fi + +if [ ! -f /etc/taos/taos.cfg ]; then + cp $SNAP/etc/taos/taos.cfg /etc/taos/taos.cfg +fi diff --git a/snap/local/launcher.sh b/snap/local/launcher.sh index 5986114931..52b3e4ce5c 100755 --- a/snap/local/launcher.sh +++ b/snap/local/launcher.sh @@ -15,7 +15,11 @@ case "$SNAP_USER_COMMON" in *) COMMON=$SNAP_USER_COMMON ;; esac -CONFIG_FILE="$SNAP/etc/taos" +if [ -d /etc/taos ]; then + CONFIG_FILE="/etc/taos" +else + CONFIG_FILE="$SNAP/etc/taos" +fi # Launch the snap $SNAP/usr/bin/taosd -c $CONFIG_FILE $@ diff --git a/snap/local/taoswrapper.sh b/snap/local/taoswrapper.sh new file mode 100755 index 0000000000..c7e5c1b856 --- /dev/null +++ b/snap/local/taoswrapper.sh @@ -0,0 +1,25 @@ +#!/bin/sh +# Wrapper to check for custom config in $SNAP_USER_COMMON or $SNAP_COMMON and +# use it otherwise fall back to the included basic config which will at least +# allow mosquitto to run and do something. +# This script will also copy the full example config in to SNAP_USER_COMMON or +# SNAP_COMMON so that people can refer to it. +# +# The decision about whether to use SNAP_USER_COMMON or SNAP_COMMON is taken +# based on the user that runs the command. If the user is root, it is assumed +# that mosquitto is being run as a system daemon, and SNAP_COMMON will be used. +# If a non-root user runs the command, then SNAP_USER_COMMON will be used. + +case "$SNAP_USER_COMMON" in + */root/snap/tdengine/common*) COMMON=$SNAP_COMMON ;; + *) COMMON=$SNAP_USER_COMMON ;; +esac + +if [ -d /etc/taos ]; then + CONFIG_FILE="/etc/taos" +else + CONFIG_FILE="$SNAP/etc/taos" +fi + +# Launch the snap +$SNAP/usr/bin/taos -c $CONFIG_FILE $@ diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index dfa68a9347..b49298e828 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -1,7 +1,7 @@ name: tdengine base: core18 # the base snap is the execution environment for this snap version: '2.0.0.6' # just for humans, typically '1.2+git' or '1.3.2' -summary: Single-line elevator pitch for your amazing snap # 79 char long summary +summary: an open-source big data platform designed and optimized for IoT. description: | TDengine is an open-source big data platform designed and optimized for Internet of Things (IoT), Connected Vehicles, and Industrial IoT. Besides the 10x faster time-series database, it provides caching, stream computing, message queuing and other functionalities to reduce the complexity and costs of development and operations. @@ -16,19 +16,25 @@ apps: plugs: - network - network-bind + - system-observe - systemfiles taos: - command: usr/bin/taos + command: taoswrapper.sh plugs: - network - systemfiles + taosdemo: + command: usr/bin/taosdemo + plugs: + - network + plugs: systemfiles: interface: system-files read: - - /etc/taos + - /etc - /var/lib/taos - /tmp write: @@ -42,6 +48,7 @@ parts: source: snap/local/ prime: - launcher.sh + - taoswrapper.sh tdengine: source: . @@ -57,12 +64,21 @@ parts: if [ ! -d $SNAPCRAFT_STAGE/usr ]; then mkdir $SNAPCRAFT_STAGE/usr fi + if [ ! -d $SNAPCRAFT_STAGE/etc/taos ]; then mkdir -p $SNAPCRAFT_STAGE/etc/taos fi + cp $SNAPCRAFT_PART_BUILD/build/* -rf $SNAPCRAFT_STAGE/usr/ cp $SNAPCRAFT_PART_SRC/packaging/cfg/taos.cfg -rf $SNAPCRAFT_STAGE/etc/taos/ + if [ ! -d $SNAPCRAFT_STAGE/var/lib/taos ]; then + mkdir -p $SNAPCRAFT_STAGE/var/lib/taos + fi + if [ ! -d $SNAPCRAFT_STAGE/var/log/taos ]; then + mkdir -p $SNAPCRAFT_STAGE/var/log/taos + fi + prime: - etc/taos/taos.cfg - usr/bin/taosd @@ -72,7 +88,24 @@ parts: - usr/lib/libtaos.so.1 - usr/lib/libtaos.so + override-prime: | + snapcraftctl prime + if [ ! -d $SNAPCRAFT_STAGE/var/lib/taos ]; then + cp -rf $SNAPCRAFT_STAGE/var/lib/taos $SNAPCRAFT_PRIME + fi + if [ ! -d $SNAPCRAFT_STAGE/var/log/taos ]; then + cp -rf $SNAPCRAFT_STAGE/var/log/taos $SNAPCRAFT_PRIME + fi - #layout: - # usr/bin/taosd: - #bind-file: debug/build/bin/taosd +layout: + /var/lib/taos: + bind: $SNAP_DATA/var/lib/taos + /var/log/taos: + bind: $SNAP_DATA/var/log/taos + /etc/taos/taos.cfg: + bind-file: $SNAP_DATA/etc/taos/taos.cfg + + +hooks: + install: + plugs: [systemfiles] From af768fb57ee070445972e04180b3e5bf1ca3e93e Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 15 Aug 2020 16:53:16 +0800 Subject: [PATCH 23/51] add logo. --- snap/local/t-dengine.svg | 1 + snap/snapcraft.yaml | 1 + 2 files changed, 2 insertions(+) create mode 100644 snap/local/t-dengine.svg diff --git a/snap/local/t-dengine.svg b/snap/local/t-dengine.svg new file mode 100644 index 0000000000..e5f29a860b --- /dev/null +++ b/snap/local/t-dengine.svg @@ -0,0 +1 @@ +TDengine logo \ No newline at end of file diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index b49298e828..2c67e5d1a0 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -1,6 +1,7 @@ name: tdengine base: core18 # the base snap is the execution environment for this snap version: '2.0.0.6' # just for humans, typically '1.2+git' or '1.3.2' +icon: snap/local/t-dengine.svg summary: an open-source big data platform designed and optimized for IoT. description: | TDengine is an open-source big data platform designed and optimized for Internet of Things (IoT), Connected Vehicles, and Industrial IoT. Besides the 10x faster time-series database, it provides caching, stream computing, message queuing and other functionalities to reduce the complexity and costs of development and operations. From b0c9fc3201fefcaba6643f4ac2c77ffdc4e0155c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 16 Aug 2020 15:32:09 +0800 Subject: [PATCH 24/51] fix bug --- src/tsdb/src/tsdbMain.c | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index dd647ddd9b..733bca6af5 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -246,22 +246,18 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_ } } strcpy(name, fname + prefixLen); - } else { // get the named file at the specified index. If not there, return 0 + } else { // get the named file at the specified index. If not there, return 0 + fname = malloc(prefixLen + strlen(name) + 2); + sprintf(fname, "%s/%s", prefix, name); + if (access(fname, F_OK) != 0) return 0; if (*index == TSDB_META_FILE_INDEX) { // get meta file - fname = malloc(prefixLen + strlen(name) + 2); - sprintf(fname, "%s/%s", prefix, name); tsdbGetStoreInfo(fname, &magic, size); - taosFree(fname); - taosFree(sdup); - return magic; } else { - fname = malloc(prefixLen + strlen(name) + 2); - sprintf(fname, "%s/%s", prefix, name); tsdbGetFileInfoImpl(fname, &magic, size); - taosFree(fname); - taosFree(sdup); - return magic; } + taosFree(fname); + taosFree(sdup); + return magic; } if (stat(fname, &fState) < 0) { From df288e779e5e64b05d56878f4109a8bde03f1084 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 16 Aug 2020 15:35:51 +0800 Subject: [PATCH 25/51] fix sync memory leak --- src/tsdb/src/tsdbMain.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 733bca6af5..e2d7d03eda 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -249,7 +249,11 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_ } else { // get the named file at the specified index. If not there, return 0 fname = malloc(prefixLen + strlen(name) + 2); sprintf(fname, "%s/%s", prefix, name); - if (access(fname, F_OK) != 0) return 0; + if (access(fname, F_OK) != 0) { + taosFree(fname); + taosFree(sdup) + return 0; + } if (*index == TSDB_META_FILE_INDEX) { // get meta file tsdbGetStoreInfo(fname, &magic, size); } else { From 051d2c33c64b96b728ee60a01aa77ae1f2b6c8fe Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 16 Aug 2020 15:37:12 +0800 Subject: [PATCH 26/51] fix compile error --- src/tsdb/src/tsdbMain.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index e2d7d03eda..bc979cca84 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -251,7 +251,7 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_ sprintf(fname, "%s/%s", prefix, name); if (access(fname, F_OK) != 0) { taosFree(fname); - taosFree(sdup) + taosFree(sdup); return 0; } if (*index == TSDB_META_FILE_INDEX) { // get meta file From ff75c2cf118ac6824bdc503946c691ec7b072638 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 16 Aug 2020 15:39:31 +0000 Subject: [PATCH 27/51] TD-1124 --- packaging/cfg/taos.cfg | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/packaging/cfg/taos.cfg b/packaging/cfg/taos.cfg index 5d33da5acc..93a6936da3 100644 --- a/packaging/cfg/taos.cfg +++ b/packaging/cfg/taos.cfg @@ -14,12 +14,9 @@ # local fully qualified domain name (FQDN) # fqdn hostname -# first port number for the connection (10 continuous UDP/TCP port number are used) +# first port number for the connection (12 continuous UDP/TCP port number are used) # serverPort 6030 -# http service port, default tcp [6041] -# httpPort 6041 - # log file's directory # logDir /var/log/taos From 100e4d3f982fe09e2aca9730c986bf85b7f5c7fe Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sun, 16 Aug 2020 23:41:38 +0800 Subject: [PATCH 28/51] update influxdb test scripts --- tests/comparisonTest/influxdb/q1.txt | 1 - tests/comparisonTest/influxdb/q2.txt | 1 - tests/comparisonTest/influxdb/q3.txt | 1 - tests/comparisonTest/influxdb/q4.txt | 1 - tests/perftest-scripts/influxdbTestQ1Loop.sh | 97 ++++++ tests/perftest-scripts/influxdbTestQ2Loop.sh | 318 ++++++++++++++++++ tests/perftest-scripts/influxdbTestQ3Loop.sh | 94 ++++++ tests/perftest-scripts/influxdbTestQ4Loop.sh | 101 ++++++ .../perftest-scripts/influxdbTestWriteLoop.sh | 52 ++- 9 files changed, 647 insertions(+), 19 deletions(-) create mode 100755 tests/perftest-scripts/influxdbTestQ1Loop.sh create mode 100755 tests/perftest-scripts/influxdbTestQ2Loop.sh create mode 100755 tests/perftest-scripts/influxdbTestQ3Loop.sh create mode 100755 tests/perftest-scripts/influxdbTestQ4Loop.sh diff --git a/tests/comparisonTest/influxdb/q1.txt b/tests/comparisonTest/influxdb/q1.txt index 4bff11d200..05076529fe 100644 --- a/tests/comparisonTest/influxdb/q1.txt +++ b/tests/comparisonTest/influxdb/q1.txt @@ -8,4 +8,3 @@ select * from devices where devgroup='60'; select * from devices where devgroup='70'; select * from devices where devgroup='80'; select * from devices where devgroup='90'; - diff --git a/tests/comparisonTest/influxdb/q2.txt b/tests/comparisonTest/influxdb/q2.txt index c271837ede..10feba922c 100644 --- a/tests/comparisonTest/influxdb/q2.txt +++ b/tests/comparisonTest/influxdb/q2.txt @@ -58,4 +58,3 @@ select spread(temperature) from devices where devgroup=~/[1-7][0-9]/; select spread(temperature) from devices where devgroup=~/[1-8][0-9]/; select spread(temperature) from devices where devgroup=~/[1-9][0-9]/; select spread(temperature) from devices; - diff --git a/tests/comparisonTest/influxdb/q3.txt b/tests/comparisonTest/influxdb/q3.txt index fd32f3ea66..ecfe80d401 100644 --- a/tests/comparisonTest/influxdb/q3.txt +++ b/tests/comparisonTest/influxdb/q3.txt @@ -8,4 +8,3 @@ select count(temperature), sum(temperature), mean(temperature) from devices wher select count(temperature), sum(temperature), mean(temperature) from devices where devgroup=~/[1-8][0-9]/ group by devgroup; select count(temperature), sum(temperature), mean(temperature) from devices where devgroup=~/[1-9][0-9]/ group by devgroup; select count(temperature), sum(temperature), mean(temperature) from devices group by devgroup; - diff --git a/tests/comparisonTest/influxdb/q4.txt b/tests/comparisonTest/influxdb/q4.txt index 5b1524d607..d18869ebda 100644 --- a/tests/comparisonTest/influxdb/q4.txt +++ b/tests/comparisonTest/influxdb/q4.txt @@ -8,4 +8,3 @@ select count(temperature), sum(temperature), mean(temperature) from devices wher select count(temperature), sum(temperature), mean(temperature) from devices where devgroup=~/[1-8][0-9]/ group by time(1m); select count(temperature), sum(temperature), mean(temperature) from devices where devgroup=~/[1-9][0-9]/ group by time(1m); select count(temperature), sum(temperature), mean(temperature) from devices group by time(1m); - diff --git a/tests/perftest-scripts/influxdbTestQ1Loop.sh b/tests/perftest-scripts/influxdbTestQ1Loop.sh new file mode 100755 index 0000000000..4ba40f6887 --- /dev/null +++ b/tests/perftest-scripts/influxdbTestQ1Loop.sh @@ -0,0 +1,97 @@ +#!/bin/bash + +NUM_LOOP=5 + +function printTo { + if $verbose ; then + echo $1 + fi +} + +INFLUXDBTESTQ1OUT=influxdbTestQ1.out + +function runTest { + totalG0=0 + totalG10=0 + totalG20=0 + totalG30=0 + totalG40=0 + totalG50=0 + totalG60=0 + totalG70=0 + totalG80=0 + totalG90=0 + for i in `seq 1 $NUM_LOOP`; do + printTo "loop i:$i, $INFLUXDBTEST_DIR/infludbTest \ + -sql $INFLUXDBTEST_DIR/q1.txt" + $INFLUXDBTEST_DIR/influxdbTest \ + -sql $INFLUXDBTEST_DIR/q1.txt 2>&1 \ + | tee $INFLUXDBTESTQ1OUT + G0=`grep "devgroup='0'" $INFLUXDBTESTQ1OUT| awk '{print $5}'` + totalG0=`echo "scale=4; $totalG0 + $G0" | bc` + G10=`grep "devgroup='10'" $INFLUXDBTESTQ1OUT| awk '{print $5}'` + totalG10=`echo "scale=4; $totalG10 + $G10" | bc` + G20=`grep "devgroup='20'" $INFLUXDBTESTQ1OUT| awk '{print $5}'` + totalG20=`echo "scale=4; $totalG20 + $G20" | bc` + G30=`grep "devgroup='30'" $INFLUXDBTESTQ1OUT| awk '{print $5}'` + totalG30=`echo "scale=4; $totalG30 + $G30" | bc` + G40=`grep "devgroup='40'" $INFLUXDBTESTQ1OUT| awk '{print $5}'` + totalG40=`echo "scale=4; $totalG40 + $G40" | bc` + G50=`grep "devgroup='50'" $INFLUXDBTESTQ1OUT| awk '{print $5}'` + totalG50=`echo "scale=4; $totalG50 + $G50" | bc` + G60=`grep "devgroup='60'" $INFLUXDBTESTQ1OUT| awk '{print $5}'` + totalG60=`echo "scale=4; $totalG60 + $G60" | bc` + G70=`grep "devgroup='70'" $INFLUXDBTESTQ1OUT| awk '{print $5}'` + totalG70=`echo "scale=4; $totalG70 + $G70" | bc` + G80=`grep "devgroup='80'" $INFLUXDBTESTQ1OUT| awk '{print $5}'` + totalG80=`echo "scale=4; $totalG80 + $G80" | bc` + G90=`grep "devgroup='90'" $INFLUXDBTESTQ1OUT| awk '{print $5}'` + totalG90=`echo "scale=4; $totalG90 + $G90" | bc` + done + avgG0=`echo "scale=4; x = $totalG0 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG10=`echo "scale=4; x = $totalG10 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG20=`echo "scale=4; x = $totalG20 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG30=`echo "scale=4; x = $totalG30 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG40=`echo "scale=4; x = $totalG40 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG50=`echo "scale=4; x = $totalG50 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG60=`echo "scale=4; x = $totalG60 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG70=`echo "scale=4; x = $totalG70 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG80=`echo "scale=4; x = $totalG80 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG90=`echo "scale=4; x = $totalG90 / $NUM_LOOP; if(x<1) print 0; x" | bc` + echo "Latency, G-0, G-10, G-20, G-30, G-40, G-50, G-60, G-70, G-80, G-90" + echo "OpenINFLUXDB, $avgG0, $avgG10, $avgG20, $avgG30, $avgG40, $avgG50, $avgG60, $avgG70, $avgG80, $avgG90" +} + +################ Main ################ + +master=false +develop=true +verbose=false + +clients=1 + +while : ; do + case $1 in + -v) + verbose=true + shift ;; + + -c) + clients=$2 + shift 2;; + + -n) + NUM_LOOP=$2 + shift 2;; + + *) + break ;; + esac +done + +WORK_DIR=/mnt/root/TDengine +INFLUXDBTEST_DIR=$WORK_DIR/tests/comparisonTest/influxdb + +runTest + +printTo "Test done!" diff --git a/tests/perftest-scripts/influxdbTestQ2Loop.sh b/tests/perftest-scripts/influxdbTestQ2Loop.sh new file mode 100755 index 0000000000..394b47b125 --- /dev/null +++ b/tests/perftest-scripts/influxdbTestQ2Loop.sh @@ -0,0 +1,318 @@ +#!/bin/bash + +NUM_LOOP=5 + +function printTo { + if $verbose ; then + echo $1 + fi +} + +INFLUXDBTESTQ2OUT=influxdbTestQ2.out + +function runTest { + totalCount10=0 + totalCount20=0 + totalCount30=0 + totalCount40=0 + totalCount50=0 + totalCount60=0 + totalCount70=0 + totalCount80=0 + totalCount90=0 + totalCount100=0 + + totalMean10=0 + totalMean20=0 + totalMean30=0 + totalMean40=0 + totalMean50=0 + totalMean60=0 + totalMean70=0 + totalMean80=0 + totalMean90=0 + totalMean100=0 + + totalSum10=0 + totalSum20=0 + totalSum30=0 + totalSum40=0 + totalSum50=0 + totalSum60=0 + totalSum70=0 + totalSum80=0 + totalSum90=0 + totalSum100=0 + + totalMax10=0 + totalMax20=0 + totalMax30=0 + totalMax40=0 + totalMax50=0 + totalMax60=0 + totalMax70=0 + totalMax80=0 + totalMax90=0 + totalMax100=0 + + totalMin10=0 + totalMin20=0 + totalMin30=0 + totalMin40=0 + totalMin50=0 + totalMin60=0 + totalMin70=0 + totalMin80=0 + totalMin90=0 + totalMin100=0 + + totalSpread10=0 + totalSpread20=0 + totalSpread30=0 + totalSpread40=0 + totalSpread50=0 + totalSpread60=0 + totalSpread70=0 + totalSpread80=0 + totalSpread90=0 + totalSpread100=0 + + for i in `seq 1 $NUM_LOOP`; do + printTo "loop i:$i, $INFLUXDBTEST_DIR/influxdbTest \ + -sql $INFLUXDBTEST_DIR/q2.txt" + $INFLUXDBTEST_DIR/influxdbTest \ + -sql $INFLUXDBTEST_DIR/q2.txt 2>&1 \ + | tee $INFLUXDBTESTQ2OUT + + Count10=`cat $INFLUXDBTESTQ2OUT | grep count | grep "devgroup=~\/\[1-1\]" | awk '{print $5}'` + totalCount10=`echo "scale=4; $totalCount10 + $Count10" | bc` + Count20=`cat $INFLUXDBTESTQ2OUT | grep count | grep "devgroup=~\/\[1-2\]" | awk '{print $5}'` + totalCount20=`echo "scale=4; $totalCount20 + $Count20" | bc` + Count30=`cat $INFLUXDBTESTQ2OUT | grep count | grep "devgroup=~\/\[1-3\]" | awk '{print $5}'` + totalCount30=`echo "scale=4; $totalCount30 + $Count30" | bc` + Count40=`cat $INFLUXDBTESTQ2OUT | grep count | grep "devgroup=~\/\[1-4\]" | awk '{print $5}'` + totalCount40=`echo "scale=4; $totalCount40 + $Count40" | bc` + Count50=`cat $INFLUXDBTESTQ2OUT | grep count | grep "devgroup=~\/\[1-5\]" | awk '{print $5}'` + totalCount50=`echo "scale=4; $totalCount50 + $Count50" | bc` + Count60=`cat $INFLUXDBTESTQ2OUT | grep count | grep "devgroup=~\/\[1-6\]" | awk '{print $5}'` + totalCount60=`echo "scale=4; $totalCount60 + $Count60" | bc` + Count70=`cat $INFLUXDBTESTQ2OUT | grep count | grep "devgroup=~\/\[1-7\]" | awk '{print $5}'` + totalCount70=`echo "scale=4; $totalCount70 + $Count70" | bc` + Count80=`cat $INFLUXDBTESTQ2OUT | grep count | grep "devgroup=~\/\[1-8\]" | awk '{print $5}'` + totalCount80=`echo "scale=4; $totalCount80 + $Count80" | bc` + Count90=`cat $INFLUXDBTESTQ2OUT | grep count | grep "devgroup=~\/\[1-9\]" | awk '{print $5}'` + totalCount90=`echo "scale=4; $totalCount90 + $Count90" | bc` + Count100=`cat $INFLUXDBTESTQ2OUT | grep count | grep "devices;" | awk '{print $5}'` + totalCount100=`echo "scale=4; $totalCount100 + $Count100" | bc` + + Mean10=`cat $INFLUXDBTESTQ2OUT | grep mean | grep "devgroup=~\/\[1-1\]" | awk '{print $5}'` + totalMean10=`echo "scale=4; $totalMean10 + $Mean10" | bc` + Mean20=`cat $INFLUXDBTESTQ2OUT | grep mean | grep "devgroup=~\/\[1-2\]" | awk '{print $5}'` + totalMean20=`echo "scale=4; $totalMean20 + $Mean20" | bc` + Mean30=`cat $INFLUXDBTESTQ2OUT | grep mean | grep "devgroup=~\/\[1-3\]" | awk '{print $5}'` + totalMean30=`echo "scale=4; $totalMean30 + $Mean30" | bc` + Mean40=`cat $INFLUXDBTESTQ2OUT | grep mean | grep "devgroup=~\/\[1-4\]" | awk '{print $5}'` + totalMean40=`echo "scale=4; $totalMean40 + $Mean40" | bc` + Mean50=`cat $INFLUXDBTESTQ2OUT | grep mean | grep "devgroup=~\/\[1-5\]" | awk '{print $5}'` + totalMean50=`echo "scale=4; $totalMean50 + $Mean50" | bc` + Mean60=`cat $INFLUXDBTESTQ2OUT | grep mean | grep "devgroup=~\/\[1-6\]" | awk '{print $5}'` + totalMean60=`echo "scale=4; $totalMean60 + $Mean60" | bc` + Mean70=`cat $INFLUXDBTESTQ2OUT | grep mean | grep "devgroup=~\/\[1-7\]" | awk '{print $5}'` + totalMean70=`echo "scale=4; $totalMean70 + $Mean70" | bc` + Mean80=`cat $INFLUXDBTESTQ2OUT | grep mean | grep "devgroup=~\/\[1-8\]" | awk '{print $5}'` + totalMean80=`echo "scale=4; $totalMean80 + $Mean80" | bc` + Mean90=`cat $INFLUXDBTESTQ2OUT | grep mean | grep "devgroup=~\/\[1-9\]" | awk '{print $5}'` + totalMean90=`echo "scale=4; $totalMean90 + $Mean90" | bc` + Mean100=`cat $INFLUXDBTESTQ2OUT | grep mean | grep "devices;" | awk '{print $5}'` + totalMean100=`echo "scale=4; $totalMean100 + $Mean100" | bc` + + Sum10=`cat $INFLUXDBTESTQ2OUT | grep sum | grep "devgroup=~\/\[1-1\]" | awk '{print $5}'` + totalSum10=`echo "scale=4; $totalSum10 + $Sum10" | bc` + Sum20=`cat $INFLUXDBTESTQ2OUT | grep sum | grep "devgroup=~\/\[1-2\]" | awk '{print $5}'` + totalSum20=`echo "scale=4; $totalSum20 + $Sum20" | bc` + Sum30=`cat $INFLUXDBTESTQ2OUT | grep sum | grep "devgroup=~\/\[1-3\]" | awk '{print $5}'` + totalSum30=`echo "scale=4; $totalSum30 + $Sum30" | bc` + Sum40=`cat $INFLUXDBTESTQ2OUT | grep sum | grep "devgroup=~\/\[1-4\]" | awk '{print $5}'` + totalSum40=`echo "scale=4; $totalSum40 + $Sum40" | bc` + Sum50=`cat $INFLUXDBTESTQ2OUT | grep sum | grep "devgroup=~\/\[1-5\]" | awk '{print $5}'` + totalSum50=`echo "scale=4; $totalSum50 + $Sum50" | bc` + Sum60=`cat $INFLUXDBTESTQ2OUT | grep sum | grep "devgroup=~\/\[1-6\]" | awk '{print $5}'` + totalSum60=`echo "scale=4; $totalSum60 + $Sum60" | bc` + Sum70=`cat $INFLUXDBTESTQ2OUT | grep sum | grep "devgroup=~\/\[1-7\]" | awk '{print $5}'` + totalSum70=`echo "scale=4; $totalSum70 + $Sum70" | bc` + Sum80=`cat $INFLUXDBTESTQ2OUT | grep sum | grep "devgroup=~\/\[1-8\]" | awk '{print $5}'` + totalSum80=`echo "scale=4; $totalSum80 + $Sum80" | bc` + Sum90=`cat $INFLUXDBTESTQ2OUT | grep sum | grep "devgroup=~\/\[1-9\]" | awk '{print $5}'` + totalSum90=`echo "scale=4; $totalSum90 + $Sum90" | bc` + Sum100=`cat $INFLUXDBTESTQ2OUT | grep sum | grep "devices;" | awk '{print $5}'` + totalSum100=`echo "scale=4; $totalSum100 + $Sum100" | bc` + + Max10=`cat $INFLUXDBTESTQ2OUT | grep max | grep "devgroup=~\/\[1-1\]" | awk '{print $5}'` + totalMax10=`echo "scale=4; $totalMax10 + $Max10" | bc` + Max20=`cat $INFLUXDBTESTQ2OUT | grep max | grep "devgroup=~\/\[1-2\]" | awk '{print $5}'` + totalMax20=`echo "scale=4; $totalMax20 + $Max20" | bc` + Max30=`cat $INFLUXDBTESTQ2OUT | grep max | grep "devgroup=~\/\[1-3\]" | awk '{print $5}'` + totalMax30=`echo "scale=4; $totalMax30 + $Max30" | bc` + Max40=`cat $INFLUXDBTESTQ2OUT | grep max | grep "devgroup=~\/\[1-4\]" | awk '{print $5}'` + totalMax40=`echo "scale=4; $totalMax40 + $Max40" | bc` + Max50=`cat $INFLUXDBTESTQ2OUT | grep max | grep "devgroup=~\/\[1-5\]" | awk '{print $5}'` + totalMax50=`echo "scale=4; $totalMax50 + $Max50" | bc` + Max60=`cat $INFLUXDBTESTQ2OUT | grep max | grep "devgroup=~\/\[1-6\]" | awk '{print $5}'` + totalMax60=`echo "scale=4; $totalMax60 + $Max60" | bc` + Max70=`cat $INFLUXDBTESTQ2OUT | grep max | grep "devgroup=~\/\[1-7\]" | awk '{print $5}'` + totalMax70=`echo "scale=4; $totalMax70 + $Max70" | bc` + Max80=`cat $INFLUXDBTESTQ2OUT | grep max | grep "devgroup=~\/\[1-8\]" | awk '{print $5}'` + totalMax80=`echo "scale=4; $totalMax80 + $Max80" | bc` + Max90=`cat $INFLUXDBTESTQ2OUT | grep max | grep "devgroup=~\/\[1-9\]" | awk '{print $5}'` + totalMax90=`echo "scale=4; $totalMax90 + $Max90" | bc` + Max100=`cat $INFLUXDBTESTQ2OUT | grep max | grep "devices;" | awk '{print $5}'` + totalMax100=`echo "scale=4; $totalMax100 + $Max100" | bc` + + Min10=`cat $INFLUXDBTESTQ2OUT | grep min | grep "devgroup=~\/\[1-1\]" | awk '{print $5}'` + totalMin10=`echo "scale=4; $totalMin10 + $Min10" | bc` + Min20=`cat $INFLUXDBTESTQ2OUT | grep min | grep "devgroup=~\/\[1-2\]" | awk '{print $5}'` + totalMin20=`echo "scale=4; $totalMin20 + $Min20" | bc` + Min30=`cat $INFLUXDBTESTQ2OUT | grep min | grep "devgroup=~\/\[1-3\]" | awk '{print $5}'` + totalMin30=`echo "scale=4; $totalMin30 + $Min30" | bc` + Min40=`cat $INFLUXDBTESTQ2OUT | grep min | grep "devgroup=~\/\[1-4\]" | awk '{print $5}'` + totalMin40=`echo "scale=4; $totalMin40 + $Min40" | bc` + Min50=`cat $INFLUXDBTESTQ2OUT | grep min | grep "devgroup=~\/\[1-5\]" | awk '{print $5}'` + totalMin50=`echo "scale=4; $totalMin50 + $Min50" | bc` + Min60=`cat $INFLUXDBTESTQ2OUT | grep min | grep "devgroup=~\/\[1-6\]" | awk '{print $5}'` + totalMin60=`echo "scale=4; $totalMin60 + $Min60" | bc` + Min70=`cat $INFLUXDBTESTQ2OUT | grep min | grep "devgroup=~\/\[1-7\]" | awk '{print $5}'` + totalMin70=`echo "scale=4; $totalMin70 + $Min70" | bc` + Min80=`cat $INFLUXDBTESTQ2OUT | grep min | grep "devgroup=~\/\[1-8\]" | awk '{print $5}'` + totalMin80=`echo "scale=4; $totalMin80 + $Min80" | bc` + Min90=`cat $INFLUXDBTESTQ2OUT | grep min | grep "devgroup=~\/\[1-9\]" | awk '{print $5}'` + totalMin90=`echo "scale=4; $totalMin90 + $Min90" | bc` + Min100=`cat $INFLUXDBTESTQ2OUT | grep min | grep "devices;" | awk '{print $5}'` + totalMin100=`echo "scale=4; $totalMin100 + $Min100" | bc` + + Spread10=`cat $INFLUXDBTESTQ2OUT | grep spread | grep "devgroup=~\/\[1-1\]" | awk '{print $5}'` + totalSpread10=`echo "scale=4; $totalSpread10 + $Spread10" | bc` + Spread20=`cat $INFLUXDBTESTQ2OUT | grep spread | grep "devgroup=~\/\[1-2\]" | awk '{print $5}'` + totalSpread20=`echo "scale=4; $totalSpread20 + $Spread20" | bc` + Spread30=`cat $INFLUXDBTESTQ2OUT | grep spread | grep "devgroup=~\/\[1-3\]" | awk '{print $5}'` + totalSpread30=`echo "scale=4; $totalSpread30 + $Spread30" | bc` + Spread40=`cat $INFLUXDBTESTQ2OUT | grep spread | grep "devgroup=~\/\[1-4\]" | awk '{print $5}'` + totalSpread40=`echo "scale=4; $totalSpread40 + $Spread40" | bc` + Spread50=`cat $INFLUXDBTESTQ2OUT | grep spread | grep "devgroup=~\/\[1-5\]" | awk '{print $5}'` + totalSpread50=`echo "scale=4; $totalSpread50 + $Spread50" | bc` + Spread60=`cat $INFLUXDBTESTQ2OUT | grep spread | grep "devgroup=~\/\[1-6\]" | awk '{print $5}'` + totalSpread60=`echo "scale=4; $totalSpread60 + $Spread60" | bc` + Spread70=`cat $INFLUXDBTESTQ2OUT | grep spread | grep "devgroup=~\/\[1-7\]" | awk '{print $5}'` + totalSpread70=`echo "scale=4; $totalSpread70 + $Spread70" | bc` + Spread80=`cat $INFLUXDBTESTQ2OUT | grep spread | grep "devgroup=~\/\[1-8\]" | awk '{print $5}'` + totalSpread80=`echo "scale=4; $totalSpread80 + $Spread80" | bc` + Spread90=`cat $INFLUXDBTESTQ2OUT | grep spread | grep "devgroup=~\/\[1-9\]" | awk '{print $5}'` + totalSpread90=`echo "scale=4; $totalSpread90 + $Spread90" | bc` + Spread100=`cat $INFLUXDBTESTQ2OUT | grep spread | grep "devices;" | awk '{print $5}'` + totalSpread100=`echo "scale=4; $totalSpread100 + $Spread100" | bc` + + done + avgCount10=`echo "scale=4; x = $totalCount10 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgCount20=`echo "scale=4; x = $totalCount20 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgCount30=`echo "scale=4; x = $totalCount30 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgCount40=`echo "scale=4; x = $totalCount40 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgCount50=`echo "scale=4; x = $totalCount50 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgCount60=`echo "scale=4; x = $totalCount60 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgCount70=`echo "scale=4; x = $totalCount70 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgCount80=`echo "scale=4; x = $totalCount80 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgCount90=`echo "scale=4; x = $totalCount90 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgCount100=`echo "scale=4; x = $totalCount100 / $NUM_LOOP; if(x<1) print 0; x" | bc` + + avgMean10=`echo "scale=4; x = $totalMean10 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMean20=`echo "scale=4; x = $totalMean20 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMean30=`echo "scale=4; x = $totalMean30 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMean40=`echo "scale=4; x = $totalMean40 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMean50=`echo "scale=4; x = $totalMean50 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMean60=`echo "scale=4; x = $totalMean60 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMean70=`echo "scale=4; x = $totalMean70 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMean80=`echo "scale=4; x = $totalMean80 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMean90=`echo "scale=4; x = $totalMean90 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMean100=`echo "scale=4; x = $totalMean100 / $NUM_LOOP; if(x<1) print 0; x" | bc` + + avgSum10=`echo "scale=4; x = $totalSum10 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgSum20=`echo "scale=4; x = $totalSum20 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgSum30=`echo "scale=4; x = $totalSum30 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgSum40=`echo "scale=4; x = $totalSum40 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgSum50=`echo "scale=4; x = $totalSum50 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgSum60=`echo "scale=4; x = $totalSum60 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgSum70=`echo "scale=4; x = $totalSum70 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgSum80=`echo "scale=4; x = $totalSum80 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgSum90=`echo "scale=4; x = $totalSum90 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgSum100=`echo "scale=4; x = $totalSum100 / $NUM_LOOP; if(x<1) print 0; x" | bc` + + avgMax10=`echo "scale=4; x = $totalMax10 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMax20=`echo "scale=4; x = $totalMax20 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMax30=`echo "scale=4; x = $totalMax30 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMax40=`echo "scale=4; x = $totalMax40 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMax50=`echo "scale=4; x = $totalMax50 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMax60=`echo "scale=4; x = $totalMax60 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMax70=`echo "scale=4; x = $totalMax70 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMax80=`echo "scale=4; x = $totalMax80 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMax90=`echo "scale=4; x = $totalMax90 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMax100=`echo "scale=4; x = $totalMax100 / $NUM_LOOP; if(x<1) print 0; x" | bc` + + avgMin10=`echo "scale=4; x = $totalMin10 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMin20=`echo "scale=4; x = $totalMin20 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMin30=`echo "scale=4; x = $totalMin30 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMin40=`echo "scale=4; x = $totalMin40 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMin50=`echo "scale=4; x = $totalMin50 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMin60=`echo "scale=4; x = $totalMin60 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMin70=`echo "scale=4; x = $totalMin70 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMin80=`echo "scale=4; x = $totalMin80 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMin90=`echo "scale=4; x = $totalMin90 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMin100=`echo "scale=4; x = $totalMin100 / $NUM_LOOP; if(x<1) print 0; x" | bc` + + avgSpread10=`echo "scale=4; x = $totalSpread10 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgSpread20=`echo "scale=4; x = $totalSpread20 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgSpread30=`echo "scale=4; x = $totalSpread30 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgSpread40=`echo "scale=4; x = $totalSpread40 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgSpread50=`echo "scale=4; x = $totalSpread50 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgSpread60=`echo "scale=4; x = $totalSpread60 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgSpread70=`echo "scale=4; x = $totalSpread70 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgSpread80=`echo "scale=4; x = $totalSpread80 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgSpread90=`echo "scale=4; x = $totalSpread90 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgSpread100=`echo "scale=4; x = $totalSpread100 / $NUM_LOOP; if(x<1) print 0; x" | bc` + + echo "Latency, 10%, 20%, 30%, 40%, 50%, 60%, 70%, 80%, 90%, 100%" + echo "Count, $avgCount10, $avgCount20, $avgCount30, $avgCount40, $avgCount50, $avgCount60, $avgCount70, $avgCount80, $avgCount90, $avgCount100" + echo "Mean, $avgMean10, $avgMean20, $avgMean30, $avgMean40, $avgMean50, $avgMean60, $avgMean70, $avgMean80, $avgMean90, $avgMean100" + echo "Sum, $avgSum10, $avgSum20, $avgSum30, $avgSum40, $avgSum50, $avgSum60, $avgSum70, $avgSum80, $avgSum90, $avgSum100" + echo "Max, $avgMax10, $avgMax20, $avgMax30, $avgMax40, $avgMax50, $avgMax60, $avgMax70, $avgMax80, $avgMax90, $avgMax100" + echo "Min, $avgMin10, $avgMin20, $avgMin30, $avgMin40, $avgMin50, $avgMin60, $avgMin70, $avgMin80, $avgMin90, $avgMin100" + echo "Spread, $avgSpread10, $avgSpread20, $avgSpread30, $avgSpread40, $avgSpread50, $avgSpread60, $avgSpread70, $avgSpread80, $avgSpread90, $avgSpread100" +} + +################ Main ################ + +verbose=false + +for arg in "$@" +do + case $arg in + -v) + verbose=true + shift ;; + + -c) + clients=$2 + shift 2;; + + -n) + NUM_LOOP=$2 + shift 2;; + + *) + ;; + esac +done + +WORK_DIR=/mnt/root/TDengine +INFLUXDBTEST_DIR=$WORK_DIR/tests/comparisonTest/influxdb + +runTest + +printTo "Test done!" diff --git a/tests/perftest-scripts/influxdbTestQ3Loop.sh b/tests/perftest-scripts/influxdbTestQ3Loop.sh new file mode 100755 index 0000000000..e1cfb848fa --- /dev/null +++ b/tests/perftest-scripts/influxdbTestQ3Loop.sh @@ -0,0 +1,94 @@ +#!/bin/bash + +NUM_LOOP=5 + +function printTo { + if $verbose ; then + echo $1 + fi +} + +INFLUXDBTESTQ3OUT=opentsdbTestQ3.out + +function runTest { + totalG10=0 + totalG20=0 + totalG30=0 + totalG40=0 + totalG50=0 + totalG60=0 + totalG70=0 + totalG80=0 + totalG90=0 + totalG100=0 + for i in `seq 1 $NUM_LOOP`; do + printTo "loop i:$i, $INFLUXDBTEST_DIR/influxdbTest \ + -sql $INFLUXDBTEST_DIR/q3.txt" + $INFLUXDBTEST_DIR/influxdbTest \ + -sql $INFLUXDBTEST_DIR/q3.txt 2>&1 \ + | tee $INFLUXDBTESTQ3OUT + G10=`grep -w "devgroup=~\/\[1-1\]" $INFLUXDBTESTQ3OUT| awk '{print $5}'` + totalG10=`echo "scale=4; $totalG10 + $G10" | bc` + G20=`grep -w "devgroup=~\/\[1-2\]" $INFLUXDBTESTQ3OUT| awk '{print $5}'` + totalG20=`echo "scale=4; $totalG20 + $G20" | bc` + G30=`grep -w "devgroup=~\/\[1-3\]" $INFLUXDBTESTQ3OUT| awk '{print $5}'` + totalG30=`echo "scale=4; $totalG30 + $G30" | bc` + G40=`grep -w "devgroup=~\/\[1-4\]" $INFLUXDBTESTQ3OUT| awk '{print $5}'` + totalG40=`echo "scale=4; $totalG40 + $G40" | bc` + G50=`grep -w "devgroup=~\/\[1-5\]" $INFLUXDBTESTQ3OUT| awk '{print $5}'` + totalG50=`echo "scale=4; $totalG50 + $G50" | bc` + G60=`grep -w "devgroup=~\/\[1-6\]" $INFLUXDBTESTQ3OUT| awk '{print $5}'` + totalG60=`echo "scale=4; $totalG60 + $G60" | bc` + G70=`grep -w "devgroup=~\/\[1-7\]" $INFLUXDBTESTQ3OUT| awk '{print $5}'` + totalG70=`echo "scale=4; $totalG70 + $G70" | bc` + G80=`grep -w "devgroup=~\/\[1-8\]" $INFLUXDBTESTQ3OUT| awk '{print $5}'` + totalG80=`echo "scale=4; $totalG80 + $G80" | bc` + G90=`grep -w "devgroup=~\/\[1-9\]" $INFLUXDBTESTQ3OUT| awk '{print $5}'` + totalG90=`echo "scale=4; $totalG90 + $G90" | bc` + G100=`grep -w "devices group by devgroup;" $INFLUXDBTESTQ3OUT| awk '{print $5}'` + totalG100=`echo "scale=4; $totalG100 + $G100" | bc` + done + avgG10=`echo "scale=4; x = $totalG10 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG20=`echo "scale=4; x = $totalG20 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG30=`echo "scale=4; x = $totalG30 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG40=`echo "scale=4; x = $totalG40 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG50=`echo "scale=4; x = $totalG50 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG60=`echo "scale=4; x = $totalG60 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG70=`echo "scale=4; x = $totalG70 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG80=`echo "scale=4; x = $totalG80 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG90=`echo "scale=4; x = $totalG90 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG100=`echo "scale=4; x = $totalG100 / $NUM_LOOP; if(x<1) print 0; x" | bc` + echo "Latency, 10%, 20%, 30%, 40%, 50%, 60%, 70%, 80%, 90%, 100%" + echo "InfluxDB, $avgG10, $avgG20, $avgG30, $avgG40, $avgG50, $avgG60, $avgG70, $avgG80, $avgG90, $avgG100" +} + +################ Main ################ + +verbose=false + +for arg in "$@" +do + case $arg in + -v) + verbose=true + shift ;; + + -c) + clients=$2 + shift 2;; + + -n) + NUM_LOOP=$2 + shift 2;; + + *) + ;; + esac +done + +WORK_DIR=/mnt/root/TDengine +INFLUXDBTEST_DIR=$WORK_DIR/tests/comparisonTest/influxdb + +runTest + +printTo "Test done!" diff --git a/tests/perftest-scripts/influxdbTestQ4Loop.sh b/tests/perftest-scripts/influxdbTestQ4Loop.sh new file mode 100755 index 0000000000..56af3157e7 --- /dev/null +++ b/tests/perftest-scripts/influxdbTestQ4Loop.sh @@ -0,0 +1,101 @@ +#!/bin/bash -x + +DATA_DIR=/mnt/root/testdata +NUM_LOOP=5 + +function printTo { + if $verbose ; then + echo $1 + fi +} + +INFLUXDBTESTQ4OUT=influxdbTestQ4.out + +function runTest { + totalG10=0 + totalG20=0 + totalG30=0 + totalG40=0 + totalG50=0 + totalG60=0 + totalG70=0 + totalG80=0 + totalG90=0 + totalG100=0 + for i in `seq 1 $NUM_LOOP`; do + printTo "loop i:$i, $INFLUXDBTEST_DIR/influxdbTest \ + -sql $INFLUXDBTEST_DIR/q4.txt" + + $INFLUXDBTEST_DIR/influxdbTest \ + -sql $INFLUXDBTEST_DIR/q4.txt 2>&1 | + tee $INFLUXDBTESTQ4OUT + G10=`grep -w "devgroup=~\/\[1-1\]" $INFLUXDBTESTQ4OUT| awk '{print $5}'` + totalG10=`echo "scale=4; $totalG10 + $G10" | bc` + G20=`grep -w "devgroup=~\/\[1-2\]" $INFLUXDBTESTQ4OUT| awk '{print $5}'` + totalG20=`echo "scale=4; $totalG20 + $G20" | bc` + G30=`grep -w "devgroup=~\/\[1-3\]" $INFLUXDBTESTQ4OUT| awk '{print $5}'` + totalG30=`echo "scale=4; $totalG30 + $G30" | bc` + G40=`grep -w "devgroup=~\/\[1-4\]" $INFLUXDBTESTQ4OUT| awk '{print $5}'` + totalG40=`echo "scale=4; $totalG40 + $G40" | bc` + G50=`grep -w "devgroup=~\/\[1-5\]" $INFLUXDBTESTQ4OUT| awk '{print $5}'` + totalG50=`echo "scale=4; $totalG50 + $G50" | bc` + G60=`grep -w "devgroup=~\/\[1-6\]" $INFLUXDBTESTQ4OUT| awk '{print $5}'` + totalG60=`echo "scale=4; $totalG60 + $G60" | bc` + G70=`grep -w "devgroup=~\/\[1-7\]" $INFLUXDBTESTQ4OUT| awk '{print $5}'` + totalG70=`echo "scale=4; $totalG70 + $G70" | bc` + G80=`grep -w "devgroup=~\/\[1-8\]" $INFLUXDBTESTQ4OUT| awk '{print $5}'` + totalG80=`echo "scale=4; $totalG80 + $G80" | bc` + G90=`grep -w "devgroup=~\/\[1-9\]" $INFLUXDBTESTQ4OUT| awk '{print $5}'` + totalG90=`echo "scale=4; $totalG90 + $G90" | bc` + G100=`grep -w "devices group by time" $INFLUXDBTESTQ4OUT| awk '{print $5}'` + totalG100=`echo "scale=4; $totalG100 + $G100" | bc` + done + avgG10=`echo "scale=4; x = $totalG10 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG20=`echo "scale=4; x = $totalG20 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG30=`echo "scale=4; x = $totalG30 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG40=`echo "scale=4; x = $totalG40 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG50=`echo "scale=4; x = $totalG50 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG60=`echo "scale=4; x = $totalG60 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG70=`echo "scale=4; x = $totalG70 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG80=`echo "scale=4; x = $totalG80 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG90=`echo "scale=4; x = $totalG90 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG100=`echo "scale=4; x = $totalG100 / $NUM_LOOP; if(x<1) print 0; x" | bc` + echo "Latency, 10%, 20%, 30%, 40%, 50%, 60%, 70%, 80%, 90%, 100%" + echo "OpenINFLUXDB, $avgG10, $avgG20, $avgG30, $avgG40, $avgG50, $avgG60, $avgG70, $avgG80, $avgG90, $avgG100" +} + +################ Main ################ + +verbose=false +regeneratedata=false + +for arg in "$@" +do + case $arg in + -v) + verbose=true + shift ;; + + -c) + clients=$2 + shift 2;; + + -r) + regeneratedata=true + ;; + + -n) + NUM_LOOP=$2 + shift 2;; + + *) + ;; + esac +done + +WORK_DIR=/mnt/root/TDengine +INFLUXDBTEST_DIR=$WORK_DIR/tests/comparisonTest/influxdb + +runTest + +printTo "Test done!" diff --git a/tests/perftest-scripts/influxdbTestWriteLoop.sh b/tests/perftest-scripts/influxdbTestWriteLoop.sh index a9b10ac45d..b2c35791b4 100755 --- a/tests/perftest-scripts/influxdbTestWriteLoop.sh +++ b/tests/perftest-scripts/influxdbTestWriteLoop.sh @@ -4,7 +4,7 @@ DATA_DIR=/mnt/root/testdata NUM_LOOP=1 NUM_OF_FILES=100 -rowsPerRequest=(1 100 500 1000 2000) +rowsPerRequest=(1 100 1000 10000 20000 50000 100000) function printTo { if $verbose ; then @@ -13,21 +13,26 @@ function printTo { } function runTest { - printf "R/R, " - for c in `seq 1 $clients`; do - if [ "$c" == "1" ]; then - printf "$c client, " - else - printf "$c clients, " - fi - done - printf "\n" + declare -A avgRPR + + for r in ${!rowsPerRequest[@]}; do + for c in `seq 1 $clients`; do + avgRPR[$r, $c]=0 + done + done + + + for r in ${!rowsPerRequest[@]}; do + if [ "$r" == "1" ] || [ "$r" == "100" ] || [ "$r" == "1000" ]; then + NUM_OF_FILES=$clients + else + NUM_OF_FILES=100 + fi - for r in ${rowsPerRequest[@]}; do printf "$r, " for c in `seq 1 $clients`; do totalRPR=0 - OUTPUT_FILE=influxdbTestWrite-RPR$r-clients$c.out + OUTPUT_FILE=influxdbTestWrite-RPR${rowsPerRequest[$r]}-clients$c.out for i in `seq 1 $NUM_LOOP`; do printTo "loop i:$i, $INF_TEST_DIR/influxdbTest \ -dataDir $DATA_DIR \ @@ -39,13 +44,30 @@ function runTest { -numOfFiles $NUM_OF_FILES \ -writeClients $c \ -rowsPerRequest $r 2>&1 \ - | tee $OUTPUT_FILE + | tee $OUTPUT_FILE RPR=`cat $OUTPUT_FILE | grep speed | awk '{print $(NF-1)}'` totalRPR=`echo "scale=4; $totalRPR + $RPR" | bc` printTo "rows:$r, clients:$c, i:$i RPR:$RPR" done - avgRPR=`echo "scale=4; $totalRPR / $NUM_LOOP" | bc` - printf "$avgRPR, " + avgRPR[$r,$c]=`echo "scale=4; $totalRPR / $NUM_LOOP" | bc` + printTo "r:$r c:$c avgRPR:${avgRPR[$r, $c]}" + done + done + + printf "R/R, " + for c in `seq 1 $clients`; do + if [ "$c" == "1" ]; then + printf "$c client, " + else + printf "$c clients, " + fi + done + printf "\n" + + for r in ${!rowsPerRequest[@]}; do + printf "${rowsPerRequest[$r]}, " + for c in `seq 1 $clients`; do + printf "${avgRPR[$r,$c]}, " done printf "\n" done From 3c046d8904ccf1acd7ea50c707c48e2ebf2e5609 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 17 Aug 2020 00:01:12 +0800 Subject: [PATCH 29/51] fix typo. --- tests/perftest-scripts/influxdbTestQ1Loop.sh | 2 +- tests/perftest-scripts/influxdbTestQ4Loop.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/perftest-scripts/influxdbTestQ1Loop.sh b/tests/perftest-scripts/influxdbTestQ1Loop.sh index 4ba40f6887..86735899c8 100755 --- a/tests/perftest-scripts/influxdbTestQ1Loop.sh +++ b/tests/perftest-scripts/influxdbTestQ1Loop.sh @@ -59,7 +59,7 @@ function runTest { avgG80=`echo "scale=4; x = $totalG80 / $NUM_LOOP; if(x<1) print 0; x" | bc` avgG90=`echo "scale=4; x = $totalG90 / $NUM_LOOP; if(x<1) print 0; x" | bc` echo "Latency, G-0, G-10, G-20, G-30, G-40, G-50, G-60, G-70, G-80, G-90" - echo "OpenINFLUXDB, $avgG0, $avgG10, $avgG20, $avgG30, $avgG40, $avgG50, $avgG60, $avgG70, $avgG80, $avgG90" + echo "InfluxDB, $avgG0, $avgG10, $avgG20, $avgG30, $avgG40, $avgG50, $avgG60, $avgG70, $avgG80, $avgG90" } ################ Main ################ diff --git a/tests/perftest-scripts/influxdbTestQ4Loop.sh b/tests/perftest-scripts/influxdbTestQ4Loop.sh index 56af3157e7..eb7cfc5556 100755 --- a/tests/perftest-scripts/influxdbTestQ4Loop.sh +++ b/tests/perftest-scripts/influxdbTestQ4Loop.sh @@ -61,7 +61,7 @@ function runTest { avgG90=`echo "scale=4; x = $totalG90 / $NUM_LOOP; if(x<1) print 0; x" | bc` avgG100=`echo "scale=4; x = $totalG100 / $NUM_LOOP; if(x<1) print 0; x" | bc` echo "Latency, 10%, 20%, 30%, 40%, 50%, 60%, 70%, 80%, 90%, 100%" - echo "OpenINFLUXDB, $avgG10, $avgG20, $avgG30, $avgG40, $avgG50, $avgG60, $avgG70, $avgG80, $avgG90, $avgG100" + echo "InfluxDB, $avgG10, $avgG20, $avgG30, $avgG40, $avgG50, $avgG60, $avgG70, $avgG80, $avgG90, $avgG100" } ################ Main ################ From 5994bed21b94a876b3cbfb15334f2e3a1e83a52f Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Mon, 17 Aug 2020 11:18:48 +0800 Subject: [PATCH 30/51] fix td-851 --- src/client/src/tscParseInsert.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index e97d6c32e0..cb49bd80b7 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -904,6 +904,11 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { sToken = tStrGetToken(sql, &index, true, numOfIgnoreToken, &ignoreTokenTypes); sql += index; + if (TK_ILLEGAL == sToken.type) { + tdDestroyKVRowBuilder(&kvRowBuilder); + return TSDB_CODE_TSC_INVALID_SQL; + } + if (sToken.n == 0 || sToken.type == TK_RP) { break; } From 5fa71e9848b5ced44a8f2e13be3d4cc452e447df Mon Sep 17 00:00:00 2001 From: Hui Li Date: Mon, 17 Aug 2020 11:21:28 +0800 Subject: [PATCH 31/51] [update version] --- src/util/src/version.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/util/src/version.c b/src/util/src/version.c index d541b1afd3..90c3b5452d 100644 --- a/src/util/src/version.c +++ b/src/util/src/version.c @@ -1,7 +1,7 @@ -char version[12] = "2.0.0.6"; +char version[12] = "2.0.1.0"; char compatible_version[12] = "2.0.0.0"; -char gitinfo[48] = "e9a20fafbe9e3b0b12cbdf55604163b4b9a41b41"; -char gitinfoOfInternal[48] = "dd679db0b9edeedad68574c1e031544711a9831f"; -char buildinfo[64] = "Built by at 2020-08-12 07:59"; +char gitinfo[48] = "7ac6c2b8de3cd66e180132fc1cf77715237308a1"; +char gitinfoOfInternal[48] = "e1e64838ece2b6dbe964ec3a39953455f354d930"; +char buildinfo[64] = "Built by root at 2020-08-17 11:13"; -void libtaos_2_0_0_6_Linux_x64() {}; +void libtaos_2_0_1_0_Linux_x64() {}; From b9672029cf68aa25a20545e4a1869d394216f2a6 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 17 Aug 2020 03:45:33 +0000 Subject: [PATCH 32/51] TD-1155 #2977 query result from rest interface is different from taos console --- src/plugins/http/src/httpSql.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/plugins/http/src/httpSql.c b/src/plugins/http/src/httpSql.c index e0e0d1aa8b..041fbdb92a 100644 --- a/src/plugins/http/src/httpSql.c +++ b/src/plugins/http/src/httpSql.c @@ -210,10 +210,14 @@ void httpProcessSingleSqlRetrieveCallBack(void *param, TAOS_RES *result, int num } } +#if 0 // todo refactor if (tscResultsetFetchCompleted(result)) { + httpDebug("context:%p, fd:%d, ip:%s, user:%s, resultset fetch completed", pContext, pContext->fd, pContext->ipstr, + pContext->user); isContinue = false; } +#endif if (isContinue) { // retrieve next batch of rows From ac537df0d0195b662f0d661d3f1a35f8cd16a33d Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 17 Aug 2020 14:28:46 +0800 Subject: [PATCH 33/51] fix part of coverity scan problem --- src/tsdb/src/tsdbFile.c | 4 +++- src/tsdb/src/tsdbMemTable.c | 4 ++-- src/util/src/tkvstore.c | 2 ++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index d960dfb7ba..154d728024 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -132,7 +132,7 @@ int tsdbOpenFileH(STsdbRepo *pRepo) { char *fname = malloc(strlen(tDataDir) + strlen(dp->d_name) + 2); if (fname == NULL) goto _err; sprintf(fname, "%s/%s", tDataDir, dp->d_name); - remove(fname); + (void)remove(fname); free(fname); } else if (code == REG_NOMATCH) { tsdbError("vgId:%d invalid file %s exists, ignore it", REPO_ID(pRepo), dp->d_name); @@ -447,6 +447,8 @@ void tsdbGetFileInfoImpl(char *fname, uint32_t *magic, int32_t *size) { *magic = info.magic; *size = (int32_t)offset; + return; + _err: if (fd >= 0) close(fd); *magic = TSDB_FILE_INIT_MAGIC; diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 2df8ff26bd..f6a7f1b35c 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -683,11 +683,11 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe pthread_rwlock_wrlock(&(pFileH->fhlock)); - rename(helperNewHeadF(pHelper)->fname, helperHeadF(pHelper)->fname); + (void)rename(helperNewHeadF(pHelper)->fname, helperHeadF(pHelper)->fname); pGroup->files[TSDB_FILE_TYPE_HEAD].info = helperNewHeadF(pHelper)->info; if (newLast) { - rename(helperNewLastF(pHelper)->fname, helperLastF(pHelper)->fname); + (void)rename(helperNewLastF(pHelper)->fname, helperLastF(pHelper)->fname); pGroup->files[TSDB_FILE_TYPE_LAST].info = helperNewLastF(pHelper)->info; } else { pGroup->files[TSDB_FILE_TYPE_LAST].info = helperLastF(pHelper)->info; diff --git a/src/util/src/tkvstore.c b/src/util/src/tkvstore.c index 9fab4a5936..6f7b2ffb88 100644 --- a/src/util/src/tkvstore.c +++ b/src/util/src/tkvstore.c @@ -351,6 +351,8 @@ void tsdbGetStoreInfo(char *fname, uint32_t *magic, int32_t *size) { *magic = info.magic; *size = (int32_t)offset; + return; + _err: if (fd >= 0) close(fd); *magic = TD_KVSTORE_INIT_MAGIC; From e80c5f3282bf399e61625d99a6e9597a1dba94df Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 17 Aug 2020 14:44:50 +0800 Subject: [PATCH 34/51] fix part of coverity scan problem --- src/tsdb/src/tsdbFile.c | 4 +++- src/tsdb/src/tsdbMemTable.c | 4 ++-- src/util/src/tkvstore.c | 2 ++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index d960dfb7ba..154d728024 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -132,7 +132,7 @@ int tsdbOpenFileH(STsdbRepo *pRepo) { char *fname = malloc(strlen(tDataDir) + strlen(dp->d_name) + 2); if (fname == NULL) goto _err; sprintf(fname, "%s/%s", tDataDir, dp->d_name); - remove(fname); + (void)remove(fname); free(fname); } else if (code == REG_NOMATCH) { tsdbError("vgId:%d invalid file %s exists, ignore it", REPO_ID(pRepo), dp->d_name); @@ -447,6 +447,8 @@ void tsdbGetFileInfoImpl(char *fname, uint32_t *magic, int32_t *size) { *magic = info.magic; *size = (int32_t)offset; + return; + _err: if (fd >= 0) close(fd); *magic = TSDB_FILE_INIT_MAGIC; diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 2df8ff26bd..f6a7f1b35c 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -683,11 +683,11 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe pthread_rwlock_wrlock(&(pFileH->fhlock)); - rename(helperNewHeadF(pHelper)->fname, helperHeadF(pHelper)->fname); + (void)rename(helperNewHeadF(pHelper)->fname, helperHeadF(pHelper)->fname); pGroup->files[TSDB_FILE_TYPE_HEAD].info = helperNewHeadF(pHelper)->info; if (newLast) { - rename(helperNewLastF(pHelper)->fname, helperLastF(pHelper)->fname); + (void)rename(helperNewLastF(pHelper)->fname, helperLastF(pHelper)->fname); pGroup->files[TSDB_FILE_TYPE_LAST].info = helperNewLastF(pHelper)->info; } else { pGroup->files[TSDB_FILE_TYPE_LAST].info = helperLastF(pHelper)->info; diff --git a/src/util/src/tkvstore.c b/src/util/src/tkvstore.c index 9fab4a5936..6f7b2ffb88 100644 --- a/src/util/src/tkvstore.c +++ b/src/util/src/tkvstore.c @@ -351,6 +351,8 @@ void tsdbGetStoreInfo(char *fname, uint32_t *magic, int32_t *size) { *magic = info.magic; *size = (int32_t)offset; + return; + _err: if (fd >= 0) close(fd); *magic = TD_KVSTORE_INIT_MAGIC; From 920cfc5981d839fa1df6877a5adf50b873b37c39 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 17 Aug 2020 08:41:55 +0000 Subject: [PATCH 35/51] TD-1154 #1112 How to modify root's password when I have forgot it? --- src/client/src/taos.def | 1 + src/client/src/tscSql.c | 91 ++++++++++++++++++++++---------- src/client/src/tscSystem.c | 8 ++- src/common/inc/tglobal.h | 1 + src/common/src/tglobal.c | 1 + src/dnode/src/dnodeSystem.c | 2 + src/kit/shell/inc/shell.h | 1 + src/kit/shell/src/shellDarwin.c | 6 +-- src/kit/shell/src/shellEngine.c | 9 +++- src/kit/shell/src/shellLinux.c | 10 ++-- src/kit/shell/src/shellWindows.c | 15 ++++-- src/kit/taosdemo/taosdemo.c | 8 +-- src/mnode/src/mnodeUser.c | 32 +++++++++++ 13 files changed, 137 insertions(+), 48 deletions(-) diff --git a/src/client/src/taos.def b/src/client/src/taos.def index df84ac657e..49d7290ce7 100644 --- a/src/client/src/taos.def +++ b/src/client/src/taos.def @@ -3,6 +3,7 @@ taos_init taos_cleanup taos_options taos_connect +taos_connect_auth taos_close taos_stmt_init taos_stmt_prepare diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 29c8aa0a56..7046bc7725 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -16,6 +16,7 @@ #include "hash.h" #include "os.h" #include "qAst.h" +#include "tkey.h" #include "tcache.h" #include "tnote.h" #include "trpc.h" @@ -47,18 +48,37 @@ static bool validPassword(const char* passwd) { return validImpl(passwd, TSDB_PASSWORD_LEN - 1); } -SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pass, const char *db, uint16_t port, - void (*fp)(void *, TAOS_RES *, int), void *param, void **taos) { +SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pass, const char *auth, const char *db, + uint16_t port, void (*fp)(void *, TAOS_RES *, int), void *param, void **taos) { taos_init(); - + if (!validUserName(user)) { terrno = TSDB_CODE_TSC_INVALID_USER_LENGTH; return NULL; } - if (!validPassword(pass)) { - terrno = TSDB_CODE_TSC_INVALID_PASS_LENGTH; - return NULL; + char secretEncrypt[32] = {0}; + int secretEncryptLen = 0; + if (auth == NULL) { + if (!validPassword(pass)) { + terrno = TSDB_CODE_TSC_INVALID_PASS_LENGTH; + return NULL; + } + taosEncryptPass((uint8_t *)pass, strlen(pass), secretEncrypt); + } else { + int outlen = 0; + int len = strlen(auth); + char *base64 = (char *)base64_decode(auth, len, &outlen); + if (base64 == NULL || outlen == 0) { + tscError("invalid auth info:%s", auth); + free(base64); + terrno = TSDB_CODE_TSC_INVALID_PASS_LENGTH; + return NULL; + } else { + memcpy(secretEncrypt, base64, outlen); + free(base64); + } + secretEncryptLen = outlen; } if (ip) { @@ -67,7 +87,7 @@ SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con } void *pDnodeConn = NULL; - if (tscInitRpc(user, pass, &pDnodeConn) != 0) { + if (tscInitRpc(user, secretEncrypt, &pDnodeConn) != 0) { terrno = TSDB_CODE_RPC_NETWORK_UNAVAIL; return NULL; } @@ -82,7 +102,8 @@ SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con pObj->signature = pObj; tstrncpy(pObj->user, user, sizeof(pObj->user)); - taosEncryptPass((uint8_t *)pass, strlen(pass), pObj->pass); + secretEncryptLen = MIN(secretEncryptLen, sizeof(pObj->pass)); + memcpy(pObj->pass, secretEncrypt, secretEncryptLen); if (db) { int32_t len = (int32_t)strlen(db); @@ -144,20 +165,17 @@ static void syncConnCallback(void *param, TAOS_RES *tres, int code) { tsem_post(&pSql->rspSem); } -TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port) { - tscDebug("try to create a connection to %s:%u, user:%s db:%s", ip, port, user, db); - if (user == NULL) user = TSDB_DEFAULT_USER; - if (pass == NULL) pass = TSDB_DEFAULT_PASS; - - STscObj* pObj = NULL; - SSqlObj *pSql = taosConnectImpl(ip, user, pass, db, port, syncConnCallback, NULL, (void**) &pObj); +TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, const char *auth, const char *db, + uint16_t port) { + STscObj *pObj = NULL; + SSqlObj *pSql = taosConnectImpl(ip, user, pass, auth, db, port, syncConnCallback, NULL, (void **)&pObj); if (pSql != NULL) { pSql->fp = syncConnCallback; pSql->param = pSql; - + tscProcessSql(pSql); tsem_wait(&pSql->rspSem); - + if (pSql->res.code != TSDB_CODE_SUCCESS) { terrno = pSql->res.code; taos_free_result(pSql); @@ -182,23 +200,38 @@ TAOS *taos_connect(const char *ip, const char *user, const char *pass, const cha return NULL; } -TAOS *taos_connect_c(const char *ip, uint8_t ipLen, const char *user, uint8_t userLen, - const char *pass, uint8_t passLen, const char *db, uint8_t dbLen, uint16_t port) { - char ipBuf[TSDB_EP_LEN] = {0}; - char userBuf[TSDB_USER_LEN] = {0}; - char passBuf[TSDB_PASSWORD_LEN] = {0}; - char dbBuf[TSDB_DB_NAME_LEN] = {0}; - strncpy(ipBuf, ip, MIN(TSDB_EP_LEN - 1, ipLen)); - strncpy(userBuf, user, MIN(TSDB_USER_LEN - 1, userLen)); - strncpy(passBuf, pass, MIN(TSDB_PASSWORD_LEN - 1,passLen)); - strncpy(dbBuf, db, MIN(TSDB_DB_NAME_LEN - 1, dbLen)); - return taos_connect(ipBuf, userBuf, passBuf, dbBuf, port); +TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port) { + tscDebug("try to create a connection to %s:%u, user:%s db:%s", ip, port, user, db); + if (user == NULL) user = TSDB_DEFAULT_USER; + if (pass == NULL) pass = TSDB_DEFAULT_PASS; + + return taos_connect_internal(ip, user, pass, NULL, db, port); } +TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port) { + tscDebug("try to create a connection to %s:%u by auth, user:%s db:%s", ip, port, user, db); + if (user == NULL) user = TSDB_DEFAULT_USER; + if (auth == NULL) return NULL; + + return taos_connect_internal(ip, user, NULL, auth, db, port); +} + +TAOS *taos_connect_c(const char *ip, uint8_t ipLen, const char *user, uint8_t userLen, const char *pass, + uint8_t passLen, const char *db, uint8_t dbLen, uint16_t port) { + char ipBuf[TSDB_EP_LEN] = {0}; + char userBuf[TSDB_USER_LEN] = {0}; + char passBuf[TSDB_PASSWORD_LEN] = {0}; + char dbBuf[TSDB_DB_NAME_LEN] = {0}; + strncpy(ipBuf, ip, MIN(TSDB_EP_LEN - 1, ipLen)); + strncpy(userBuf, user, MIN(TSDB_USER_LEN - 1, userLen)); + strncpy(passBuf, pass, MIN(TSDB_PASSWORD_LEN - 1, passLen)); + strncpy(dbBuf, db, MIN(TSDB_DB_NAME_LEN - 1, dbLen)); + return taos_connect(ipBuf, userBuf, passBuf, dbBuf, port); +} TAOS *taos_connect_a(char *ip, char *user, char *pass, char *db, uint16_t port, void (*fp)(void *, TAOS_RES *, int), void *param, void **taos) { - SSqlObj* pSql = taosConnectImpl(ip, user, pass, db, port, fp, param, taos); + SSqlObj* pSql = taosConnectImpl(ip, user, pass, NULL, db, port, fp, param, taos); if (pSql == NULL) { return NULL; } diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index 211e673754..72f23881d2 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -47,10 +47,8 @@ void tscCheckDiskUsage(void *UNUSED_PARAM(para), void* UNUSED_PARAM(param)) { taosTmrReset(tscCheckDiskUsage, 1000, NULL, tscTmr, &tscCheckDiskUsageTmr); } -int32_t tscInitRpc(const char *user, const char *secret, void** pDnodeConn) { +int32_t tscInitRpc(const char *user, const char *secretEncrypt, void **pDnodeConn) { SRpcInit rpcInit; - char secretEncrypt[32] = {0}; - taosEncryptPass((uint8_t *)secret, strlen(secret), secretEncrypt); if (*pDnodeConn == NULL) { memset(&rpcInit, 0, sizeof(rpcInit)); @@ -60,11 +58,11 @@ int32_t tscInitRpc(const char *user, const char *secret, void** pDnodeConn) { rpcInit.cfp = tscProcessMsgFromServer; rpcInit.sessions = tsMaxConnections; rpcInit.connType = TAOS_CONN_CLIENT; - rpcInit.user = (char*)user; + rpcInit.user = (char *)user; rpcInit.idleTime = 2000; rpcInit.ckey = "key"; rpcInit.spi = 1; - rpcInit.secret = secretEncrypt; + rpcInit.secret = (char *)secretEncrypt; *pDnodeConn = rpcOpen(&rpcInit); if (*pDnodeConn == NULL) { diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index a04b5f32b9..4ffc631566 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -113,6 +113,7 @@ extern char tsInternalPass[]; extern int32_t tsMonitorInterval; // internal +extern int32_t tsPrintAuth; extern int32_t tscEmbedded; extern char configDir[]; extern char tsVnodeDir[]; diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 391e5f5935..e9d7a71477 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -146,6 +146,7 @@ char tsInternalPass[] = "secretkey"; int32_t tsMonitorInterval = 30; // seconds // internal +int32_t tsPrintAuth = 0; int32_t tscEmbedded = 0; char configDir[TSDB_FILENAME_LEN] = {0}; char tsVnodeDir[TSDB_FILENAME_LEN] = {0}; diff --git a/src/dnode/src/dnodeSystem.c b/src/dnode/src/dnodeSystem.c index 543e1c9639..6f32bc0f7a 100644 --- a/src/dnode/src/dnodeSystem.c +++ b/src/dnode/src/dnodeSystem.c @@ -52,6 +52,8 @@ int32_t main(int32_t argc, char *argv[]) { } else if (strcmp(argv[i], "-k") == 0) { grantParseParameter(); exit(EXIT_SUCCESS); + } else if (strcmp(argv[i], "-A") == 0) { + tsPrintAuth = 1; } #ifdef TAOS_MEM_CHECK else if (strcmp(argv[i], "--alloc-random-fail") == 0) { diff --git a/src/kit/shell/inc/shell.h b/src/kit/shell/inc/shell.h index 54ac45583c..d47f87965b 100644 --- a/src/kit/shell/inc/shell.h +++ b/src/kit/shell/inc/shell.h @@ -39,6 +39,7 @@ typedef struct SShellArguments { char* host; char* password; char* user; + char* auth; char* database; char* timezone; bool is_raw_time; diff --git a/src/kit/shell/src/shellDarwin.c b/src/kit/shell/src/shellDarwin.c index ce41827462..ffe537dd91 100644 --- a/src/kit/shell/src/shellDarwin.c +++ b/src/kit/shell/src/shellDarwin.c @@ -32,16 +32,16 @@ void insertChar(Command *cmd, char *c, int size); void printHelp() { char indent[10] = " "; - printf("taos shell is used to test the TDEngine database\n"); + printf("taos shell is used to test the TDengine database\n"); printf("%s%s\n", indent, "-h"); - printf("%s%s%s\n", indent, indent, "TDEngine server IP address to connect. The default host is localhost."); + printf("%s%s%s\n", indent, indent, "TDengine server IP address to connect. The default host is localhost."); printf("%s%s\n", indent, "-p"); printf("%s%s%s\n", indent, indent, "The password to use when connecting to the server."); printf("%s%s\n", indent, "-P"); printf("%s%s%s\n", indent, indent, "The TCP/IP port number to use for the connection"); printf("%s%s\n", indent, "-u"); - printf("%s%s%s\n", indent, indent, "The TDEngine user name to use when connecting to the server."); + printf("%s%s%s\n", indent, indent, "The user name to use when connecting to the server."); printf("%s%s\n", indent, "-c"); printf("%s%s%s\n", indent, indent, "Configuration directory."); printf("%s%s\n", indent, "-s"); diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c index 692b5e49a0..d82ddf8871 100644 --- a/src/kit/shell/src/shellEngine.c +++ b/src/kit/shell/src/shellEngine.c @@ -38,6 +38,7 @@ SShellHistory history; #define DEFAULT_MAX_BINARY_DISPLAY_WIDTH 30 extern int32_t tsMaxBinaryDisplayWidth; +extern TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port); /* * FUNCTION: Initialize the shell. @@ -70,7 +71,13 @@ TAOS *shellInit(SShellArguments *args) { tsTableMetaKeepTimer = 3000; // Connect to the database. - TAOS *con = taos_connect(args->host, args->user, args->password, args->database, args->port); + TAOS *con = NULL; + if (args->auth == NULL) { + con = taos_connect(args->host, args->user, args->password, args->database, args->port); + } else { + con = taos_connect_auth(args->host, args->user, args->auth, args->database, args->port); + } + if (con == NULL) { printf("taos connect failed, reason: %s.\n\n", tstrerror(terrno)); fflush(stdout); diff --git a/src/kit/shell/src/shellLinux.c b/src/kit/shell/src/shellLinux.c index 963afe346d..c74eeb7f59 100644 --- a/src/kit/shell/src/shellLinux.c +++ b/src/kit/shell/src/shellLinux.c @@ -33,10 +33,11 @@ const char *argp_program_bug_address = ""; static char doc[] = ""; static char args_doc[] = ""; static struct argp_option options[] = { - {"host", 'h', "HOST", 0, "TDEngine server IP address to connect. The default host is localhost."}, + {"host", 'h', "HOST", 0, "TDengine server IP address to connect. The default host is localhost."}, {"password", 'p', "PASSWORD", OPTION_ARG_OPTIONAL, "The password to use when connecting to the server."}, {"port", 'P', "PORT", 0, "The TCP/IP port number to use for the connection."}, - {"user", 'u', "USER", 0, "The TDEngine user name to use when connecting to the server."}, + {"user", 'u', "USER", 0, "The user name to use when connecting to the server."}, + {"user", 'A', "Auth", 0, "The user auth to use when connecting to the server."}, {"config-dir", 'c', "CONFIG_DIR", 0, "Configuration directory."}, {"commands", 's', "COMMANDS", 0, "Commands to run without enter the shell."}, {"raw-time", 'r', 0, 0, "Output time as uint64_t."}, @@ -76,11 +77,14 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { case 'u': arguments->user = arg; break; + case 'A': + arguments->auth = arg; + break; case 'c': if (wordexp(arg, &full_path, 0) != 0) { fprintf(stderr, "Invalid path %s\n", arg); return -1; - } + } if (strlen(full_path.we_wordv[0]) >= TSDB_FILENAME_LEN) { fprintf(stderr, "config file path: %s overflow max len %d\n", full_path.we_wordv[0], TSDB_FILENAME_LEN - 1); wordfree(&full_path); diff --git a/src/kit/shell/src/shellWindows.c b/src/kit/shell/src/shellWindows.c index 279d3b9cdd..67174827d4 100644 --- a/src/kit/shell/src/shellWindows.c +++ b/src/kit/shell/src/shellWindows.c @@ -21,16 +21,18 @@ extern char configDir[]; void printHelp() { char indent[10] = " "; - printf("taos shell is used to test the TDEngine database\n"); + printf("taos shell is used to test the TDengine database\n"); printf("%s%s\n", indent, "-h"); - printf("%s%s%s\n", indent, indent, "TDEngine server IP address to connect. The default host is localhost."); + printf("%s%s%s\n", indent, indent, "TDengine server IP address to connect. The default host is localhost."); printf("%s%s\n", indent, "-p"); printf("%s%s%s\n", indent, indent, "The password to use when connecting to the server."); printf("%s%s\n", indent, "-P"); printf("%s%s%s\n", indent, indent, "The TCP/IP port number to use for the connection"); printf("%s%s\n", indent, "-u"); - printf("%s%s%s\n", indent, indent, "The TDEngine user name to use when connecting to the server."); + printf("%s%s%s\n", indent, indent, "The user name to use when connecting to the server."); + printf("%s%s\n", indent, "-A"); + printf("%s%s%s\n", indent, indent, "The user auth to use when connecting to the server."); printf("%s%s\n", indent, "-c"); printf("%s%s%s\n", indent, indent, "Configuration directory."); printf("%s%s\n", indent, "-s"); @@ -79,6 +81,13 @@ void shellParseArgument(int argc, char *argv[], SShellArguments *arguments) { fprintf(stderr, "option -u requires an argument\n"); exit(EXIT_FAILURE); } + } else if (strcmp(argv[i], "-A") == 0) { + if (i < argc - 1) { + arguments->auth = argv[++i]; + } else { + fprintf(stderr, "option -A requires an argument\n"); + exit(EXIT_FAILURE); + } } else if (strcmp(argv[i], "-c") == 0) { if (i < argc - 1) { if (strlen(argv[++i]) >= TSDB_FILENAME_LEN) { diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 192cb3145c..74bdf6f074 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -85,9 +85,9 @@ typedef struct DemoArguments { #ifdef LINUX /* The options we understand. */ static struct argp_option options[] = { - {0, 'h', "host", 0, "The host to connect to TDEngine. Default is localhost.", 0}, + {0, 'h', "host", 0, "The host to connect to TDengine. Default is localhost.", 0}, {0, 'p', "port", 0, "The TCP/IP port number to use for the connection. Default is 0.", 1}, - {0, 'u', "user", 0, "The TDEngine user name to use when connecting to the server. Default is 'root'.", 2}, + {0, 'u', "user", 0, "The TDengine user name to use when connecting to the server. Default is 'root'.", 2}, {0, 'P', "password", 0, "The password to use when connecting to the server. Default is 'taosdata'.", 3}, {0, 'd', "database", 0, "Destination database. Default is 'test'.", 3}, {0, 'm', "table_prefix", 0, "Table prefix name. Default is 't'.", 3}, @@ -264,11 +264,11 @@ typedef struct DemoArguments { void printHelp() { char indent[10] = " "; printf("%s%s\n", indent, "-h"); - printf("%s%s%s\n", indent, indent, "host, The host to connect to TDEngine. Default is localhost."); + printf("%s%s%s\n", indent, indent, "host, The host to connect to TDengine. Default is localhost."); printf("%s%s\n", indent, "-p"); printf("%s%s%s\n", indent, indent, "port, The TCP/IP port number to use for the connection. Default is 0."); printf("%s%s\n", indent, "-u"); - printf("%s%s%s\n", indent, indent, "user, The TDEngine user name to use when connecting to the server. Default is 'root'."); + printf("%s%s%s\n", indent, indent, "user, The user name to use when connecting to the server. Default is 'root'."); printf("%s%s\n", indent, "-p"); printf("%s%s%s\n", indent, indent, "password, The password to use when connecting to the server. Default is 'taosdata'."); printf("%s%s\n", indent, "-d"); diff --git a/src/mnode/src/mnodeUser.c b/src/mnode/src/mnodeUser.c index 765661ac71..6e63a8f2de 100644 --- a/src/mnode/src/mnodeUser.c +++ b/src/mnode/src/mnodeUser.c @@ -20,6 +20,7 @@ #include "tglobal.h" #include "tgrant.h" #include "tdataformat.h" +#include "tkey.h" #include "mnode.h" #include "dnode.h" #include "mnodeDef.h" @@ -100,6 +101,32 @@ static int32_t mnodeUserActionDecode(SSdbOper *pOper) { return TSDB_CODE_SUCCESS; } +static void mnodePrintUserAuth() { + FILE *fp = fopen("auth.txt", "w"); + if (!fp) { + mDebug("failed to auth.txt for write"); + return; + } + + void * pIter = NULL; + SUserObj *pUser = NULL; + + while (1) { + pIter = mnodeGetNextUser(pIter, &pUser); + if (pUser == NULL) break; + + char *base64 = base64_encode((const unsigned char *)pUser->pass, TSDB_KEY_LEN * 2); + fprintf(fp, "user:%24s auth:%s\n", pUser->user, base64); + free(base64); + + mnodeDecUserRef(pUser); + } + + fflush(fp); + sdbFreeIter(pIter); + fclose(fp); +} + static int32_t mnodeUserActionRestored() { int32_t numOfRows = sdbGetNumOfRows(tsUserSdb); if (numOfRows <= 0 && dnodeIsFirstDeploy()) { @@ -111,6 +138,11 @@ static int32_t mnodeUserActionRestored() { mnodeDecAcctRef(pAcct); } + if (tsPrintAuth != 0) { + mInfo("print user auth, for -A parameter is set"); + mnodePrintUserAuth(); + } + return TSDB_CODE_SUCCESS; } From 3595d06a6437a062023e10f867a362f51d076a78 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 17 Aug 2020 09:47:00 +0000 Subject: [PATCH 36/51] TD-1165 fix defects found by coverity scan --- src/balance/src/balance.c | 3 ++- src/common/src/ttimezone.c | 4 +++- src/dnode/src/dnodeMgmt.c | 8 ++++---- src/mnode/src/mnodeDb.c | 4 ++-- src/os/src/detail/osSysinfo.c | 1 + src/plugins/mqtt/src/mqttSystem.c | 29 +++++++++++++++++++++-------- src/util/src/tsocket.c | 4 ++-- 7 files changed, 35 insertions(+), 18 deletions(-) diff --git a/src/balance/src/balance.c b/src/balance/src/balance.c index 53638f1025..0d2d9fc778 100644 --- a/src/balance/src/balance.c +++ b/src/balance/src/balance.c @@ -118,6 +118,7 @@ static void balanceSwapVnodeGid(SVnodeGid *pVnodeGid1, SVnodeGid *pVnodeGid2) { } int32_t balanceAllocVnodes(SVgObj *pVgroup) { + static int32_t randIndex = 0; int32_t dnode = 0; int32_t vnodes = 0; @@ -160,7 +161,7 @@ int32_t balanceAllocVnodes(SVgObj *pVgroup) { */ if (pVgroup->numOfVnodes == 1) { } else if (pVgroup->numOfVnodes == 2) { - if (rand() % 2 == 0) { + if (randIndex++ % 2 == 0) { balanceSwapVnodeGid(pVgroup->vnodeGid, pVgroup->vnodeGid + 1); } } else { diff --git a/src/common/src/ttimezone.c b/src/common/src/ttimezone.c index 62b1e0bb5c..62d4768db8 100644 --- a/src/common/src/ttimezone.c +++ b/src/common/src/ttimezone.c @@ -23,7 +23,9 @@ // TODO refactor to set the tz value through parameter void tsSetTimeZone() { SGlobalCfg *cfg_timezone = taosGetConfigOption("timezone"); - uInfo("timezone is set to %s by %s", tsTimezone, tsCfgStatusStr[cfg_timezone->cfgStatus]); + if (cfg_timezone != NULL) { + uInfo("timezone is set to %s by %s", tsTimezone, tsCfgStatusStr[cfg_timezone->cfgStatus]); + } #ifdef WINDOWS char winStr[TSDB_LOCALE_LEN * 2]; diff --git a/src/dnode/src/dnodeMgmt.c b/src/dnode/src/dnodeMgmt.c index 532f03bcb5..e2b32145a3 100644 --- a/src/dnode/src/dnodeMgmt.c +++ b/src/dnode/src/dnodeMgmt.c @@ -710,10 +710,10 @@ static void dnodeSendStatusMsg(void *handle, void *tmrId) { pStatus->clusterCfg.statusInterval = htonl(tsStatusInterval); pStatus->clusterCfg.maxtablesPerVnode = htonl(tsMaxTablePerVnode); pStatus->clusterCfg.maxVgroupsPerDb = htonl(tsMaxVgroupsPerDb); - strcpy(pStatus->clusterCfg.arbitrator, tsArbitrator); - strcpy(pStatus->clusterCfg.timezone, tsTimezone); - strcpy(pStatus->clusterCfg.locale, tsLocale); - strcpy(pStatus->clusterCfg.charset, tsCharset); + tstrncpy(pStatus->clusterCfg.arbitrator, tsArbitrator, TSDB_EP_LEN); + tstrncpy(pStatus->clusterCfg.timezone, tsTimezone, 64); + tstrncpy(pStatus->clusterCfg.locale, tsLocale, TSDB_LOCALE_LEN); + tstrncpy(pStatus->clusterCfg.charset, tsCharset, TSDB_LOCALE_LEN); vnodeBuildStatusMsg(pStatus); contLen = sizeof(SDMStatusMsg) + pStatus->openVnodes * sizeof(SVnodeLoad); diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index f6249986fd..f828bc5a3d 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -73,9 +73,9 @@ static int32_t mnodeDbActionInsert(SSdbOper *pOper) { pthread_mutex_lock(&pDb->mutex); pDb->vgListSize = VG_LIST_SIZE; pDb->vgList = calloc(pDb->vgListSize, sizeof(SVgObj *)); + pDb->numOfVgroups = 0; pthread_mutex_unlock(&pDb->mutex); - pDb->numOfVgroups = 0; pDb->numOfTables = 0; pDb->numOfSuperTables = 0; @@ -927,7 +927,7 @@ static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SCMAlterDbMsg *pAlter) { if (quorum >= 0 && quorum != pDb->cfg.quorum) { mDebug("db:%s, quorum:%d change to %d", pDb->name, pDb->cfg.quorum, quorum); - newCfg.compression = quorum; + newCfg.quorum = quorum; } return newCfg; diff --git a/src/os/src/detail/osSysinfo.c b/src/os/src/detail/osSysinfo.c index 7c01239490..edf8fe6945 100644 --- a/src/os/src/detail/osSysinfo.c +++ b/src/os/src/detail/osSysinfo.c @@ -170,6 +170,7 @@ static void taosGetSystemTimezone() { fclose(f); + buf[sizeof(buf) - 1] = 0; char *lineEnd = strstr(buf, "\n"); if (lineEnd != NULL) { *lineEnd = 0; diff --git a/src/plugins/mqtt/src/mqttSystem.c b/src/plugins/mqtt/src/mqttSystem.c index 0259ea23eb..ab3f3f5d84 100644 --- a/src/plugins/mqtt/src/mqttSystem.c +++ b/src/plugins/mqtt/src/mqttSystem.c @@ -70,14 +70,27 @@ int32_t mqttInitSystem() { recntStatus.port = strbetween("'1883'", "'", "'"); } - topicPath = strbetween(strstr(url, strstr(_begin_hostname, ":") != NULL ? recntStatus.port : recntStatus.hostname), - "/", "/"); - char* _topic = "+/+/+/"; - int _tpsize = strlen(topicPath) + strlen(_topic) + 1; - recntStatus.topic = calloc(1, _tpsize); - sprintf(recntStatus.topic, "/%s/%s", topicPath, _topic); - recntStatus.client_id = strlen(tsMqttBrokerClientId) < 3 ? tsMqttBrokerClientId : "taos_mqtt"; - mqttConnect = NULL; + char* portStr = recntStatus.hostname; + if (_begin_hostname != NULL) { + char* colonStr = strstr(_begin_hostname, ":"); + if (colonStr != NULL) { + portStr = recntStatus.port; + } + } + + char* topicStr = strstr(url, portStr); + if (topicStr != NULL) { + topicPath = strbetween(topicStr, "/", "/"); + char* _topic = "+/+/+/"; + int _tpsize = strlen(topicPath) + strlen(_topic) + 1; + recntStatus.topic = calloc(1, _tpsize); + sprintf(recntStatus.topic, "/%s/%s", topicPath, _topic); + recntStatus.client_id = strlen(tsMqttBrokerClientId) < 3 ? tsMqttBrokerClientId : "taos_mqtt"; + mqttConnect = NULL; + } else { + topicPath = NULL; + } + return rc; } diff --git a/src/util/src/tsocket.c b/src/util/src/tsocket.c index 9abda0e196..23f2e5f00a 100644 --- a/src/util/src/tsocket.c +++ b/src/util/src/tsocket.c @@ -371,7 +371,7 @@ SOCKET taosOpenTcpServerSocket(uint32_t ip, uint16_t port) { serverAdd.sin_addr.s_addr = ip; serverAdd.sin_port = (uint16_t)htons(port); - if ((sockFd = (int)socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) < 0) { + if ((sockFd = (int)socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) < 2) { uError("failed to open TCP socket: %d (%s)", errno, strerror(errno)); return -1; } @@ -382,7 +382,7 @@ SOCKET taosOpenTcpServerSocket(uint32_t ip, uint16_t port) { uError("setsockopt SO_REUSEADDR failed: %d (%s)", errno, strerror(errno)); taosCloseSocket(sockFd); return -1; - }; + } /* bind socket to server address */ if (bind(sockFd, (struct sockaddr *)&serverAdd, sizeof(serverAdd)) < 0) { From a2fbca353af50b0608c63e71d16bccff32c837ab Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 17 Aug 2020 17:56:43 +0800 Subject: [PATCH 37/51] TD-1154 minor changes for windows compile --- src/client/src/tscSql.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 7046bc7725..1bd885466c 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -67,7 +67,7 @@ SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con taosEncryptPass((uint8_t *)pass, strlen(pass), secretEncrypt); } else { int outlen = 0; - int len = strlen(auth); + int len = (int)strlen(auth); char *base64 = (char *)base64_decode(auth, len, &outlen); if (base64 == NULL || outlen == 0) { tscError("invalid auth info:%s", auth); From 8d8e339dd6c4b9e01ffe3d8403b0022b84f5895b Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 17 Aug 2020 18:05:21 +0800 Subject: [PATCH 38/51] update tdengine write test script. --- .../perftest-scripts/tdengineTestWriteLoop.sh | 79 +++++++++++-------- 1 file changed, 48 insertions(+), 31 deletions(-) diff --git a/tests/perftest-scripts/tdengineTestWriteLoop.sh b/tests/perftest-scripts/tdengineTestWriteLoop.sh index 5cb2a7199a..79ae8e52e3 100755 --- a/tests/perftest-scripts/tdengineTestWriteLoop.sh +++ b/tests/perftest-scripts/tdengineTestWriteLoop.sh @@ -1,7 +1,7 @@ #!/bin/bash DATA_DIR=/mnt/root/testdata -NUM_LOOP=1 +NUM_LOOP=5 NUM_OF_FILES=100 rowsPerRequest=(1 100 500 1000 2000) @@ -13,6 +13,42 @@ function printTo { } function runTest { + declare -A avgRPR + + for r in ${!rowsPerRequest[@]}; do + for c in `seq 1 $clients`; do + avgRPR[$r, $c]=0 + done + done + + for r in ${!rowsPerRequest[@]}; do + for c in `seq 1 $clients`; do + totalRPR=0 + OUTPUT_FILE=tdengineTestWrite-RPR${rowsPerRequest[$r]}-clients$c.out + + for i in `seq 1 $NUM_LOOP`; do + restartTaosd + $TAOSD_DIR/taos -s "drop database db" > /dev/null 2>&1 + printTo "loop i:$i, $TDTEST_DIR/tdengineTest \ + -dataDir $DATA_DIR \ + -numOfFiles $NUM_OF_FILES \ + -w -clients $c \ + -rowsPerRequest $r" + $TDTEST_DIR/tdengineTest \ + -dataDir $DATA_DIR \ + -numOfFiles 1 \ + -w -clients $c \ + -rowsPerRequest $r \ + | tee $OUTPUT_FILE + RPR=`cat $OUTPUT_FILE | grep speed | awk '{print $(NF-1)}'` + totalRPR=`echo "scale=4; $totalRPR + $RPR" | bc` + printTo "rows:$r, clients:$c, i:$i RPR:$RPR" + done + avgRPR[$r,$c]=`echo "scale=4; $totalRPR / $NUM_LOOP" | bc` + printTo "r:$r c:$c avgRPR:${avgRPR[$r, $c]}" + done + done + printf "R/R, " for c in `seq 1 $clients`; do if [ "$c" == "1" ]; then @@ -23,29 +59,10 @@ function runTest { done printf "\n" - for r in ${rowsPerRequest[@]}; do - printf "$r, " + for r in ${!rowsPerRequest[@]}; do + printf "${rowsPerRequest[$r]}, " for c in `seq 1 $clients`; do - totalRPR=0 - for i in `seq 1 $NUM_LOOP`; do - restartTaosd - $TAOSD_DIR/taos -s "drop database db" > /dev/null 2>&1 - printTo "loop i:$i, $TDTEST_DIR/tdengineTest \ - -dataDir $DATA_DIR \ - -numOfFiles $NUM_OF_FILES \ - -w -clients $c \ - -rowsPerRequest $r" - RPR=`$TDTEST_DIR/tdengineTest \ - -dataDir $DATA_DIR \ - -numOfFiles $NUM_OF_FILES \ - -w -clients $c \ - -rowsPerRequest $r \ - | grep speed | awk '{print $(NF-1)}'` - totalRPR=`echo "scale=4; $totalRPR + $RPR" | bc` - printTo "rows:$r, clients:$c, i:$i RPR:$RPR" - done - avgRPR=`echo "scale=4; $totalRPR / $NUM_LOOP" | bc` - printf "$avgRPR, " + printf "${avgRPR[$r,$c]}, " done printf "\n" done @@ -80,10 +97,6 @@ while : ; do verbose=true shift ;; - -n) - NUM_LOOP=$2 - shift 2;; - master) master=true develop=false @@ -98,18 +111,22 @@ while : ; do clients=$2 shift 2;; + -n) + NUM_LOOP=$2 + shift 2;; + *) break ;; esac done if $master ; then - printTo "Test master branch.." + echo "Test master branch.." cp /mnt/root/cfg/master/taos.cfg /etc/taos/taos.cfg WORK_DIR=/mnt/root/TDengine.master else - printTo "Test develop branch.." - cp /mnt/root/cfg/perftest/taos.cfg /etc/taos/taos.cfg + echo "Test develop branch.." + cp /mnt/root/cfg/10billion/taos.cfg /etc/taos/taos.cfg WORK_DIR=/mnt/root/TDengine fi @@ -118,4 +135,4 @@ TDTEST_DIR=$WORK_DIR/tests/comparisonTest/tdengine runTest -printTo "Test done!" +echo "Test done!" From e14f821b00b16cb61b83c8e030ede3310187ee05 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Mon, 17 Aug 2020 18:37:20 +0800 Subject: [PATCH 39/51] TD-1169: fix failed python cases --- tests/pytest/functions/function_operations.py | 6 +----- tests/pytest/query/queryNormal.py | 3 ++- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/tests/pytest/functions/function_operations.py b/tests/pytest/functions/function_operations.py index 36810621cb..930c23d8a6 100644 --- a/tests/pytest/functions/function_operations.py +++ b/tests/pytest/functions/function_operations.py @@ -49,11 +49,7 @@ class TDTestCase: tdSql.query("select col1 + col2 from test1") tdSql.checkRows(10) - tdSql.checkData(0, 0, 2.0) - - tdSql.query("select col1 + col2 * col3 from test1") - tdSql.checkRows(10) - tdSql.checkData(1, 0, 6.0) + tdSql.checkData(0, 0, 2.0) tdSql.query("select col1 + col2 * col3 + col3 / col4 + col5 + col6 from test1") tdSql.checkRows(10) diff --git a/tests/pytest/query/queryNormal.py b/tests/pytest/query/queryNormal.py index 712a56d2d7..1ab285bbad 100644 --- a/tests/pytest/query/queryNormal.py +++ b/tests/pytest/query/queryNormal.py @@ -36,7 +36,8 @@ class TDTestCase: "insert into tb2 using stb1 tags(2,'tb2', '表2') values ('2020-04-18 15:00:02.000', 3, 2.1), ('2020-04-18 15:00:03.000', 4, 2.2)") # inner join --- bug - tdSql.error("select * from tb1 a, tb2 b where a.ts = b.ts") + tdSql.query("select * from tb1 a, tb2 b where a.ts = b.ts") + tdSql.checkRows(0) # join 3 tables -- bug exists tdSql.error("select stb_t.ts, stb_t.dscrption, stb_t.temperature, stb_p.id, stb_p.dscrption, stb_p.pressure,stb_v.velocity from stb_p, stb_t, stb_v where stb_p.ts=stb_t.ts and stb_p.ts=stb_v.ts and stb_p.id = stb_t.id") From 1f77b851f38ebb7201c9c017d01d6e515dd94374 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 17 Aug 2020 18:41:20 +0800 Subject: [PATCH 40/51] fix array varible bug. --- tests/perftest-scripts/tdengineTestWriteLoop.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/perftest-scripts/tdengineTestWriteLoop.sh b/tests/perftest-scripts/tdengineTestWriteLoop.sh index 79ae8e52e3..12602f7708 100755 --- a/tests/perftest-scripts/tdengineTestWriteLoop.sh +++ b/tests/perftest-scripts/tdengineTestWriteLoop.sh @@ -17,7 +17,7 @@ function runTest { for r in ${!rowsPerRequest[@]}; do for c in `seq 1 $clients`; do - avgRPR[$r, $c]=0 + avgRPR[$r,$c]=0 done done @@ -33,19 +33,19 @@ function runTest { -dataDir $DATA_DIR \ -numOfFiles $NUM_OF_FILES \ -w -clients $c \ - -rowsPerRequest $r" + -rowsPerRequest ${rowsPerRequest[$r]}" $TDTEST_DIR/tdengineTest \ -dataDir $DATA_DIR \ -numOfFiles 1 \ -w -clients $c \ - -rowsPerRequest $r \ + -rowsPerRequest ${rowsPerRequest[$r]} \ | tee $OUTPUT_FILE RPR=`cat $OUTPUT_FILE | grep speed | awk '{print $(NF-1)}'` totalRPR=`echo "scale=4; $totalRPR + $RPR" | bc` - printTo "rows:$r, clients:$c, i:$i RPR:$RPR" + printTo "rows:${rowsPerRequest[$r]}, clients:$c, i:$i RPR:$RPR" done avgRPR[$r,$c]=`echo "scale=4; $totalRPR / $NUM_LOOP" | bc` - printTo "r:$r c:$c avgRPR:${avgRPR[$r, $c]}" + printTo "r:${rowsPerRequest[$r]} c:$c avgRPR:${avgRPR[$r,$c]}" done done From d74e39a9787b08172e147d81af111cdf2d898bc0 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 17 Aug 2020 19:14:56 +0800 Subject: [PATCH 41/51] fix minor mistake. --- tests/perftest-scripts/tdengineTestWriteLoop.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/perftest-scripts/tdengineTestWriteLoop.sh b/tests/perftest-scripts/tdengineTestWriteLoop.sh index 12602f7708..e1ee4418dc 100755 --- a/tests/perftest-scripts/tdengineTestWriteLoop.sh +++ b/tests/perftest-scripts/tdengineTestWriteLoop.sh @@ -36,7 +36,7 @@ function runTest { -rowsPerRequest ${rowsPerRequest[$r]}" $TDTEST_DIR/tdengineTest \ -dataDir $DATA_DIR \ - -numOfFiles 1 \ + -numOfFiles $NUM_OF_FILES \ -w -clients $c \ -rowsPerRequest ${rowsPerRequest[$r]} \ | tee $OUTPUT_FILE @@ -126,7 +126,7 @@ if $master ; then WORK_DIR=/mnt/root/TDengine.master else echo "Test develop branch.." - cp /mnt/root/cfg/10billion/taos.cfg /etc/taos/taos.cfg + cp /mnt/root/cfg/develop/taos.cfg /etc/taos/taos.cfg WORK_DIR=/mnt/root/TDengine fi From 3d47697e031cb3402380fc2e856ca80a7190b3d1 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 17 Aug 2020 13:00:38 +0000 Subject: [PATCH 42/51] TD-1167, fix coverity problem about client --- src/client/src/tscSQLParser.c | 5 ++++- src/client/src/tscServer.c | 1 + src/client/src/tscSubquery.c | 28 +++++++++++++++++----------- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index e5bb516ee6..c74a403a04 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -1158,8 +1158,9 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t int32_t ret = exprTreeFromSqlExpr(pCmd, &pNode, pItem->pNode, pQueryInfo->exprList, pQueryInfo, colList); if (ret != TSDB_CODE_SUCCESS) { - tExprTreeDestroy(&pNode, NULL); taosTFree(arithmeticExprStr); + taosArrayDestroy(colList); + tExprTreeDestroy(&pNode, NULL); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } @@ -1168,6 +1169,8 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t SColIndex* pIndex = taosArrayGet(colList, k); if (pIndex->flag == 1) { taosTFree(arithmeticExprStr); + taosArrayDestroy(colList); + tExprTreeDestroy(&pNode, NULL); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } } diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 6b75b680b1..4c73c6d968 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -158,6 +158,7 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { if (pRsp->killConnection) { tscKillConnection(pObj); + return; } else { if (pRsp->queryId) tscKillQuery(pObj, htonl(pRsp->queryId)); if (pRsp->streamId) tscKillStream(pObj, htonl(pRsp->streamId)); diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 75644c355c..d2b5439f8d 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -684,6 +684,8 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow freeJoinSubqueryObj(pParentSql); pParentSql->res.code = code; tscQueueAsyncRes(pParentSql); + taosArrayDestroy(s1); + taosArrayDestroy(s2); return; } @@ -1296,7 +1298,9 @@ int32_t tscHandleMasterJoinQuery(SSqlObj* pSql) { tscError("%p tableIndex:%d, failed to allocate join support object, abort further query", pSql, i); pState->numOfRemain = i; pSql->res.code = TSDB_CODE_TSC_OUT_OF_MEMORY; - + if (0 == i) { + taosTFree(pState); + } return pSql->res.code; } @@ -1304,7 +1308,9 @@ int32_t tscHandleMasterJoinQuery(SSqlObj* pSql) { if (code != TSDB_CODE_SUCCESS) { // failed to create subquery object, quit query tscDestroyJoinSupporter(pSupporter); pSql->res.code = TSDB_CODE_TSC_OUT_OF_MEMORY; - + if (0 == i) { + taosTFree(pState); + } break; } } @@ -2091,17 +2097,17 @@ void tscBuildResFromSubqueries(SSqlObj *pSql) { // return; // } - tscFetchDatablockFromSubquery(pSql); - if (pRes->code != TSDB_CODE_SUCCESS) { - return; - } +// tscFetchDatablockFromSubquery(pSql); +// if (pRes->code != TSDB_CODE_SUCCESS) { +// return; +// } } - if (pSql->res.code == TSDB_CODE_SUCCESS) { - (*pSql->fp)(pSql->param, pSql, pRes->numOfRows); - } else { - tscQueueAsyncRes(pSql); - } +// if (pSql->res.code == TSDB_CODE_SUCCESS) { +// (*pSql->fp)(pSql->param, pSql, pRes->numOfRows); +// } else { +// tscQueueAsyncRes(pSql); +// } } static void transferNcharData(SSqlObj *pSql, int32_t columnIndex, TAOS_FIELD *pField) { From 255c0c3ccc8177b80844e0cd43e613bd1dd4f34d Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Mon, 17 Aug 2020 23:16:35 +0800 Subject: [PATCH 43/51] change icon path. --- snap/{local => gui}/t-dengine.svg | 0 snap/snapcraft.yaml | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename snap/{local => gui}/t-dengine.svg (100%) diff --git a/snap/local/t-dengine.svg b/snap/gui/t-dengine.svg similarity index 100% rename from snap/local/t-dengine.svg rename to snap/gui/t-dengine.svg diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 8be1511683..cf4f993394 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -1,7 +1,7 @@ name: tdengine base: core18 # the base snap is the execution environment for this snap version: '2.0.0.6' # just for humans, typically '1.2+git' or '1.3.2' -icon: snap/local/t-dengine.svg +icon: snap/gui/t-dengine.svg summary: an open-source big data platform designed and optimized for IoT. description: | TDengine is an open-source big data platform designed and optimized for Internet of Things (IoT), Connected Vehicles, and Industrial IoT. Besides the 10x faster time-series database, it provides caching, stream computing, message queuing and other functionalities to reduce the complexity and costs of development and operations. From 25fe2350d05b678ee19a7abecc494aa2e7b40d0e Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 17 Aug 2020 15:35:36 +0000 Subject: [PATCH 44/51] TD-1144 fix bug while redirect status msg --- src/dnode/src/dnodeMgmt.c | 7 ++++++- src/mnode/src/mnodePeer.c | 3 ++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/dnode/src/dnodeMgmt.c b/src/dnode/src/dnodeMgmt.c index e2b32145a3..4b06626753 100644 --- a/src/dnode/src/dnodeMgmt.c +++ b/src/dnode/src/dnodeMgmt.c @@ -449,7 +449,12 @@ static int32_t dnodeProcessConfigDnodeMsg(SRpcMsg *pMsg) { } void dnodeUpdateMnodeEpSetForPeer(SRpcEpSet *pEpSet) { - dInfo("mnode EP list for is changed, numOfEps:%d inUse:%d", pEpSet->numOfEps, pEpSet->inUse); + if (pEpSet->numOfEps <= 0) { + dError("mnode EP list for peer is changed, but content is invalid, discard it"); + return; + } + + dInfo("mnode EP list for peer is changed, numOfEps:%d inUse:%d", pEpSet->numOfEps, pEpSet->inUse); for (int i = 0; i < pEpSet->numOfEps; ++i) { pEpSet->port[i] -= TSDB_PORT_DNODEDNODE; dInfo("mnode index:%d %s:%u", i, pEpSet->fqdn[i], pEpSet->port[i]) diff --git a/src/mnode/src/mnodePeer.c b/src/mnode/src/mnodePeer.c index 7b4b4e4343..8b368a33f4 100644 --- a/src/mnode/src/mnodePeer.c +++ b/src/mnode/src/mnodePeer.c @@ -58,7 +58,8 @@ int32_t mnodeProcessPeerReq(SMnodeMsg *pMsg) { rpcRsp->rsp = epSet; rpcRsp->len = sizeof(SRpcEpSet); - mDebug("%p, msg:%s in mpeer queue, will be redireced inUse:%d", pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType], epSet->inUse); + mDebug("%p, msg:%s in mpeer queue, will be redireced, numOfEps:%d inUse:%d", pMsg->rpcMsg.ahandle, + taosMsg[pMsg->rpcMsg.msgType], epSet->numOfEps, epSet->inUse); for (int32_t i = 0; i < epSet->numOfEps; ++i) { mDebug("mnode index:%d ep:%s:%d", i, epSet->fqdn[i], htons(epSet->port[i])); } From ab5693295e2424d53b736248317823864dc94ce2 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 18 Aug 2020 02:11:33 +0000 Subject: [PATCH 45/51] TD-1165 --- src/balance/src/balance.c | 2 +- src/plugins/mqtt/src/mqttSystem.c | 6 +++++- src/util/src/tsocket.c | 8 ++++---- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/balance/src/balance.c b/src/balance/src/balance.c index 0d2d9fc778..2b1888042c 100644 --- a/src/balance/src/balance.c +++ b/src/balance/src/balance.c @@ -165,7 +165,7 @@ int32_t balanceAllocVnodes(SVgObj *pVgroup) { balanceSwapVnodeGid(pVgroup->vnodeGid, pVgroup->vnodeGid + 1); } } else { - int32_t randVal = rand() % 6; + int32_t randVal = randIndex++ % 6; if (randVal == 1) { // 1, 0, 2 balanceSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 1); } else if (randVal == 2) { // 1, 2, 0 diff --git a/src/plugins/mqtt/src/mqttSystem.c b/src/plugins/mqtt/src/mqttSystem.c index ab3f3f5d84..b64c4715e8 100644 --- a/src/plugins/mqtt/src/mqttSystem.c +++ b/src/plugins/mqtt/src/mqttSystem.c @@ -47,7 +47,11 @@ int32_t mqttInitSystem() { recntStatus.recvbufsz = sizeof(recvbuf); char* url = tsMqttBrokerAddress; recntStatus.user_name = strstr(url, "@") != NULL ? strbetween(url, "//", ":") : NULL; - recntStatus.password = strstr(url, "@") != NULL ? strbetween(strstr(url, recntStatus.user_name), ":", "@") : NULL; + + char * passStr = strstr(url, recntStatus.user_name); + if (passStr != NULL) { + recntStatus.password = strstr(url, "@") != NULL ? strbetween(passStr, ":", "@") : NULL; + } if (strlen(url) == 0) { mqttDebug("mqtt module not init, url is null"); diff --git a/src/util/src/tsocket.c b/src/util/src/tsocket.c index 23f2e5f00a..94e6d0e4e1 100644 --- a/src/util/src/tsocket.c +++ b/src/util/src/tsocket.c @@ -231,7 +231,7 @@ SOCKET taosOpenUdpSocket(uint32_t ip, uint16_t port) { localAddr.sin_addr.s_addr = ip; localAddr.sin_port = (uint16_t)htons(port); - if ((sockFd = (int)socket(AF_INET, SOCK_DGRAM, 0)) < 0) { + if ((sockFd = (int)socket(AF_INET, SOCK_DGRAM, 0)) <= 2) { uError("failed to open udp socket: %d (%s)", errno, strerror(errno)); return -1; } @@ -265,7 +265,7 @@ SOCKET taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clie sockFd = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP); - if (sockFd < 0) { + if (sockFd <= 2) { uError("failed to open the socket: %d (%s)", errno, strerror(errno)); return -1; } @@ -276,7 +276,7 @@ SOCKET taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clie uError("setsockopt SO_REUSEADDR failed: %d (%s)", errno, strerror(errno)); taosCloseSocket(sockFd); return -1; - }; + } if (clientIp != 0) { memset((char *)&clientAddr, 0, sizeof(clientAddr)); @@ -371,7 +371,7 @@ SOCKET taosOpenTcpServerSocket(uint32_t ip, uint16_t port) { serverAdd.sin_addr.s_addr = ip; serverAdd.sin_port = (uint16_t)htons(port); - if ((sockFd = (int)socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) < 2) { + if ((sockFd = (int)socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) <= 2) { uError("failed to open TCP socket: %d (%s)", errno, strerror(errno)); return -1; } From 4de2164b5f60d77edd22fb4c7b67b07e417189f3 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 18 Aug 2020 06:10:21 +0000 Subject: [PATCH 46/51] TD-1167 fix coverity problem --- src/client/src/tscServer.c | 4 ++-- src/query/src/qExecutor.c | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 4c73c6d968..0785dd8b53 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -100,7 +100,7 @@ static void tscDumpEpSetFromVgroupInfo(SCMCorVgroupInfo *pVgroupInfo, SRpcEpSet pEpSet->inUse = (inUse >= 0 && inUse < TSDB_MAX_REPLICA) ? inUse: 0; pEpSet->numOfEps = pVgroupInfo->numOfEps; for (int32_t i = 0; i < pVgroupInfo->numOfEps; ++i) { - strncpy(pEpSet->fqdn[i], pVgroupInfo->epAddr[i].fqdn, TSDB_FQDN_LEN); + tstrncpy(pEpSet->fqdn[i], pVgroupInfo->epAddr[i].fqdn, sizeof(pEpSet->fqdn[i])); pEpSet->port[i] = pVgroupInfo->epAddr[i].port; } taosCorEndRead(&pVgroupInfo->version); @@ -117,7 +117,7 @@ static void tscUpdateVgroupInfo(SSqlObj *pObj, SRpcEpSet *pEpSet) { pVgroupInfo->inUse = pEpSet->inUse; pVgroupInfo->numOfEps = pEpSet->numOfEps; for (int32_t i = 0; i < pVgroupInfo->numOfEps; i++) { - strncpy(pVgroupInfo->epAddr[i].fqdn, pEpSet->fqdn[i], TSDB_FQDN_LEN); + tstrncpy(pVgroupInfo->epAddr[i].fqdn, pEpSet->fqdn[i], sizeof(pEpSet->fqdn[i])); pVgroupInfo->epAddr[i].port = pEpSet->port[i]; } tscDebug("after: EndPoint in use: %d", pVgroupInfo->inUse); diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index c8efee03cd..e0eaf5eedf 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -5242,7 +5242,6 @@ static int32_t getColumnIndexInSource(SQueryTableMsg *pQueryMsg, SSqlFuncMsg *pE j += 1; } } - assert(0); } @@ -5933,6 +5932,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList, if (p1 == NULL) { goto _cleanup; } + taosArrayPush(pQInfo->tableqinfoGroupInfo.pGroupList, &p1); for(int32_t j = 0; j < s; ++j) { STableKeyInfo* info = taosArrayGet(pa, j); @@ -5956,8 +5956,6 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList, taosHashPut(pQInfo->tableqinfoGroupInfo.map, &id->tid, sizeof(id->tid), &item, POINTER_BYTES); index += 1; } - - taosArrayPush(pQInfo->tableqinfoGroupInfo.pGroupList, &p1); } pQInfo->arrTableIdInfo = taosArrayInit(tableIndex, sizeof(STableIdInfo)); From 7969af493a896e4bf3998beda867d0d5bf992458 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 18 Aug 2020 06:31:52 +0000 Subject: [PATCH 47/51] TD-1170 forbbiden func on tbname --- src/client/src/tscSQLParser.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index c74a403a04..d108349085 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -1652,10 +1652,12 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col } SColumnIndex index = COLUMN_INDEX_INITIALIZER; - if ((getColumnIndexByName(pCmd, &pParamElem->pNode->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS) || - index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { + if ((getColumnIndexByName(pCmd, &pParamElem->pNode->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } + if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6); + } // 2. check if sql function can be applied on this column data type pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); @@ -1864,7 +1866,10 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col if (getColumnIndexByName(pCmd, &pParamElem->pNode->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } - + if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6); + } + pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); SSchema* pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); From 0698465236fb1f68e34f1e4033af939868fe2568 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 18 Aug 2020 06:40:33 +0000 Subject: [PATCH 48/51] The mqtt test has not passed, there are memory leaks, waiting for reconstruction --- src/plugins/mqtt/src/mqttSystem.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/plugins/mqtt/src/mqttSystem.c b/src/plugins/mqtt/src/mqttSystem.c index b64c4715e8..69810e2785 100644 --- a/src/plugins/mqtt/src/mqttSystem.c +++ b/src/plugins/mqtt/src/mqttSystem.c @@ -39,6 +39,7 @@ int mttIsRuning = 1; int32_t mqttInitSystem() { int rc = 0; +#if 0 uint8_t sendbuf[2048]; uint8_t recvbuf[1024]; recntStatus.sendbuf = sendbuf; @@ -95,11 +96,13 @@ int32_t mqttInitSystem() { topicPath = NULL; } +#endif return rc; } int32_t mqttStartSystem() { int rc = 0; +#if 0 if (recntStatus.user_name != NULL && recntStatus.password != NULL) { mqttInfo("connecting to mqtt://%s:%s@%s:%s/%s/", recntStatus.user_name, recntStatus.password, recntStatus.hostname, recntStatus.port, topicPath); @@ -116,18 +119,22 @@ int32_t mqttStartSystem() { } else { mqttInfo("listening for '%s' messages.", recntStatus.topic); } +#endif return rc; } void mqttStopSystem() { +#if 0 mqttClient.error = MQTT_ERROR_SOCKET_ERROR; mttIsRuning = 0; usleep(300000U); mqttCleanup(EXIT_SUCCESS, mqttClient.socketfd, &clientDaemonThread); mqttInfo("mqtt is stoped"); +#endif } void mqttCleanUpSystem() { +#if 0 mqttInfo("starting to cleanup mqtt"); free(recntStatus.user_name); free(recntStatus.password); @@ -136,6 +143,7 @@ void mqttCleanUpSystem() { free(recntStatus.topic); free(topicPath); mqttInfo("mqtt is cleaned up"); +#endif } void mqtt_PublishCallback(void** unused, struct mqtt_response_publish* published) { @@ -187,9 +195,11 @@ void* mqttClientRefresher(void* client) { } void mqttCleanup(int status, int sockfd, pthread_t* client_daemon) { +#if 0 mqttInfo("clean up mqtt module"); if (sockfd != -1) close(sockfd); if (client_daemon != NULL) pthread_cancel(*client_daemon); +#endif } void mqttInitConnCb(void* param, TAOS_RES* result, int32_t code) { From 152867cb2ddf11444233f56846e8476fa027d36a Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 18 Aug 2020 06:49:24 +0000 Subject: [PATCH 49/51] minor changes --- src/util/src/tsocket.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/util/src/tsocket.c b/src/util/src/tsocket.c index 94e6d0e4e1..2de3490459 100644 --- a/src/util/src/tsocket.c +++ b/src/util/src/tsocket.c @@ -233,6 +233,7 @@ SOCKET taosOpenUdpSocket(uint32_t ip, uint16_t port) { if ((sockFd = (int)socket(AF_INET, SOCK_DGRAM, 0)) <= 2) { uError("failed to open udp socket: %d (%s)", errno, strerror(errno)); + close(sockFd); return -1; } @@ -267,6 +268,7 @@ SOCKET taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clie if (sockFd <= 2) { uError("failed to open the socket: %d (%s)", errno, strerror(errno)); + close(sockFd); return -1; } @@ -373,6 +375,7 @@ SOCKET taosOpenTcpServerSocket(uint32_t ip, uint16_t port) { if ((sockFd = (int)socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) <= 2) { uError("failed to open TCP socket: %d (%s)", errno, strerror(errno)); + close(sockFd); return -1; } From 4954164e32134d88faff32a44a647bd278cc375a Mon Sep 17 00:00:00 2001 From: Hui Li Date: Tue, 18 Aug 2020 16:12:53 +0800 Subject: [PATCH 50/51] [TD-1171] --- src/dnode/src/dnodeMain.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dnode/src/dnodeMain.c b/src/dnode/src/dnodeMain.c index dded10bd1c..d199236a74 100644 --- a/src/dnode/src/dnodeMain.c +++ b/src/dnode/src/dnodeMain.c @@ -49,8 +49,8 @@ typedef struct { } SDnodeComponent; static const SDnodeComponent tsDnodeComponents[] = { - {"check", dnodeInitCheck, dnodeCleanupCheck}, // NOTES: dnodeInitCheck must be first component !!! {"storage", dnodeInitStorage, dnodeCleanupStorage}, + {"check", dnodeInitCheck, dnodeCleanupCheck}, // NOTES: dnodeInitCheck must be behind the dnodeInitStorage component !!! {"vread", dnodeInitVnodeRead, dnodeCleanupVnodeRead}, {"vwrite", dnodeInitVnodeWrite, dnodeCleanupVnodeWrite}, {"mread", dnodeInitMnodeRead, dnodeCleanupMnodeRead}, From 1a9d37b5976c68324615e3125206e3ddbc432e4b Mon Sep 17 00:00:00 2001 From: Hui Li Date: Tue, 18 Aug 2020 16:26:11 +0800 Subject: [PATCH 51/51] [TD-1171] --- src/dnode/src/dnodeCheck.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/dnode/src/dnodeCheck.c b/src/dnode/src/dnodeCheck.c index dfdc3fa53f..9b68fc1f6c 100644 --- a/src/dnode/src/dnodeCheck.c +++ b/src/dnode/src/dnodeCheck.c @@ -131,6 +131,8 @@ static int dnodeCheckCpu() { } static int dnodeCheckDisk() { + taosGetDisk(); + if (tsAvailDataDirGB < tsMinimalDataDirGB) { dError("free disk size: %f GB, too little, quit", tsAvailDataDirGB); return -1;