diff --git a/.travis.yml b/.travis.yml index 6a4acce451..4afdb6154e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -52,7 +52,8 @@ matrix: cd ${TRAVIS_BUILD_DIR}/debug make install > /dev/null || travis_terminate $? - pip3 install numpy + py3ver=`python3 --version|awk '{print $2}'|cut -d "." -f 1,2` && apt install python$py3ver-dev + pip3 install psutil pip3 install --user ${TRAVIS_BUILD_DIR}/src/connector/python/linux/python3/ cd ${TRAVIS_BUILD_DIR}/tests diff --git a/CMakeLists.txt b/CMakeLists.txt index 500b2ff1eb..8dd99f0b58 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,8 +16,8 @@ MESSAGE(STATUS "Community directory: " ${TD_COMMUNITY_DIR}) INCLUDE(cmake/input.inc) INCLUDE(cmake/platform.inc) -INCLUDE(cmake/env.inc) INCLUDE(cmake/define.inc) +INCLUDE(cmake/env.inc) INCLUDE(cmake/install.inc) ADD_SUBDIRECTORY(deps) diff --git a/cmake/define.inc b/cmake/define.inc index d68f2230f1..1bb1692a2b 100755 --- a/cmake/define.inc +++ b/cmake/define.inc @@ -28,3 +28,107 @@ ENDIF () IF (TD_RANDOM_NETWORK_FAIL) ADD_DEFINITIONS(-DTAOS_RANDOM_NETWORK_FAIL) ENDIF () + +IF (TD_ARM) + ADD_DEFINITIONS(-D_TD_ARM_) + IF (TD_ARM_64) + ADD_DEFINITIONS(-D_TD_ARM_64_) + ENDIF () + IF (TD_ARM_32) + ADD_DEFINITIONS(-D_TD_ARM_32_) + ENDIF () +ENDIF () + +IF (TD_MIPS) + ADD_DEFINITIONS(-D_TD_MIPS_) + IF (TD_MIPS_64) + ADD_DEFINITIONS(-D_TD_MIPS_64_) + ENDIF () + IF (TD_MIPS_32) + ADD_DEFINITIONS(-D_TD_MIPS_32_) + ENDIF () +ENDIF () + +IF (TD_LINUX_64) + ADD_DEFINITIONS(-D_M_X64) + ADD_DEFINITIONS(-D_TD_LINUX_64) + IF (NOT TD_ARM) + IF (${CMAKE_CXX_COMPILER_ID} MATCHES "Clang") + SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -Wno-missing-braces -fPIC -g3 -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + ELSE () + SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -malign-double -g3 -gdwarf-2 -malign-stringops -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + ENDIF () + ELSE () + SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + ENDIF () + FIND_PATH(ICONV_INCLUDE_EXIST iconv.h /usr/include/ /usr/local/include/) + IF (ICONV_INCLUDE_EXIST) + ADD_DEFINITIONS(-DUSE_LIBICONV) + ENDIF () +ENDIF () + +IF (TD_LINUX_32) + ADD_DEFINITIONS(-D_TD_LINUX_32) + ADD_DEFINITIONS(-DUSE_LIBICONV) + SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g -fsigned-char -munaligned-access -fpack-struct=8 -latomic -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + IF (NOT TD_ARM) + EXIT () + ENDIF () +ENDIF () + +IF (TD_LINUX) + ADD_DEFINITIONS(-DLINUX) + ADD_DEFINITIONS(-D_TD_LINUX) + ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT) + + SET(DEBUG_FLAGS "-O0 -DDEBUG") + SET(RELEASE_FLAGS "-O0") + + IF (${COVER} MATCHES "true") + MESSAGE(STATUS "Test coverage mode, add extra flags") + SET(GCC_COVERAGE_COMPILE_FLAGS "-fprofile-arcs -ftest-coverage") + SET(GCC_COVERAGE_LINK_FLAGS "-lgcov --coverage") + SET(COMMON_FLAGS "${COMMON_FLAGS} ${GCC_COVERAGE_COMPILE_FLAGS} ${GCC_COVERAGE_LINK_FLAGS}") + ENDIF () +ENDIF () + +IF (TD_APLHINE) + SET(COMMON_FLAGS "${COMMON_FLAGS} -largp") + link_libraries(/usr/lib/libargp.a) + ADD_DEFINITIONS(-D_ALPINE) +ENDIF () + +IF (TD_DARWIN_64) + ADD_DEFINITIONS(-D_TD_DARWIN_64) + ADD_DEFINITIONS(-DDARWIN) + ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT) + ADD_DEFINITIONS(-DUSE_LIBICONV) + SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -Wno-missing-braces -fPIC -g -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + SET(DEBUG_FLAGS "-O0 -DDEBUG") + SET(RELEASE_FLAGS "-O0") +ENDIF () + +IF (TD_WINDOWS) + ADD_DEFINITIONS(-DWINDOWS) + ADD_DEFINITIONS(-D__CLEANUP_C) + ADD_DEFINITIONS(-DPTW32_STATIC_LIB) + ADD_DEFINITIONS(-DPTW32_BUILD) + ADD_DEFINITIONS(-D_MBCS -D_CRT_SECURE_NO_DEPRECATE -D_CRT_NONSTDC_NO_DEPRECATE) + SET(CMAKE_GENERATOR "NMake Makefiles" CACHE INTERNAL "" FORCE) + IF (NOT TD_GODLL) + SET(COMMON_FLAGS "/nologo /WX /Oi /Oy- /Gm- /EHsc /MT /GS /Gy /fp:precise /Zc:wchar_t /Zc:forScope /Gd /errorReport:prompt /analyze-") + SET(DEBUG_FLAGS "/Zi /W3 /GL") + SET(RELEASE_FLAGS "/W0 /GL") + ENDIF () +ENDIF () + +IF (TD_WINDOWS_64) + ADD_DEFINITIONS(-D_M_X64) + ADD_DEFINITIONS(-D_TD_WINDOWS_64) + ADD_DEFINITIONS(-DUSE_LIBICONV) +ENDIF () + +IF (TD_WINDOWS_32) + ADD_DEFINITIONS(-D_TD_WINDOWS_32) +ENDIF () + diff --git a/cmake/platform.inc b/cmake/platform.inc index 7a371df70b..a0668de7b5 100755 --- a/cmake/platform.inc +++ b/cmake/platform.inc @@ -14,6 +14,7 @@ PROJECT(TDengine) # # Set macro definitions according to os platform +SET(TD_LINUX FALSE) SET(TD_LINUX_64 FALSE) SET(TD_LINUX_32 FALSE) SET(TD_ARM FALSE) @@ -22,8 +23,11 @@ SET(TD_ARM_32 FALSE) SET(TD_MIPS FALSE) SET(TD_MIPS_64 FALSE) SET(TD_MIPS_32 FALSE) -SET(TD_DARWIN_64 FALSE) +SET(TD_WINDOWS FALSE) SET(TD_WINDOWS_64 FALSE) +SET(TD_WINDOWS_32 FALSE) +SET(TD_DARWIN_64 FALSE) +SET(TD_APLHINE FALSE) # if generate ARM version: # cmake -DCPUTYPE=aarch32 .. or cmake -DCPUTYPE=aarch64 @@ -31,27 +35,21 @@ IF (${CPUTYPE} MATCHES "aarch32") SET(TD_ARM TRUE) SET(TD_ARM_32 TRUE) SET(TD_PAGMODE_LITE TRUE) - ADD_DEFINITIONS(-D_TD_ARM_) - ADD_DEFINITIONS(-D_TD_ARM_32_) MESSAGE(STATUS "input cpuType: aarch32") ELSEIF (${CPUTYPE} MATCHES "aarch64") SET(TD_ARM TRUE) SET(TD_ARM_64 TRUE) - ADD_DEFINITIONS(-D_TD_ARM_) - ADD_DEFINITIONS(-D_TD_ARM_64_) - MESSAGE(STATUS "input cpuType: aarch64") + MESSAGE(STATUS "input cpuType: aarch64") ELSEIF (${CPUTYPE} MATCHES "mips64") SET(TD_MIPS TRUE) SET(TD_MIPS_64 TRUE) - ADD_DEFINITIONS(-D_TD_MIPS_) - ADD_DEFINITIONS(-D_TD_MIPS_64_) MESSAGE(STATUS "input cpuType: mips64") ELSEIF (${CPUTYPE} MATCHES "x64") MESSAGE(STATUS "input cpuType: x64") ELSEIF (${CPUTYPE} MATCHES "x86") MESSAGE(STATUS "input cpuType: x86") ELSE () - MESSAGE(STATUS "input cpuType: " ${CPUTYPE}) + MESSAGE(STATUS "input cpuType unknown " ${CPUTYPE}) ENDIF () # @@ -62,16 +60,13 @@ execute_process(COMMAND ${TD_COMMUNITY_DIR}/packaging/tools/get_os.sh "" OUTPUT_ MESSAGE(STATUS "The current os is " ${TD_OS_INFO}) IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux") + SET(TD_LINUX TRUE) IF (${CMAKE_SIZEOF_VOID_P} MATCHES 8) SET(TD_LINUX_64 TRUE) - ADD_DEFINITIONS(-D_M_X64) - ADD_DEFINITIONS(-D_TD_LINUX_64) MESSAGE(STATUS "The current platform is Linux 64-bit") ELSEIF (${CMAKE_SIZEOF_VOID_P} MATCHES 4) SET(TD_LINUX_32 TRUE) - ADD_DEFINITIONS(-D_TD_LINUX_32) IF (TD_ARM) - #ADD_DEFINITIONS(-D_M_IX86) MESSAGE(STATUS "The current platform is Linux 32-bit") ELSE () MESSAGE(FATAL_ERROR "The current platform is Linux 32-bit, but no ARM not supported yet") @@ -81,24 +76,26 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux") MESSAGE(FATAL_ERROR "The current platform is Linux neither 32-bit nor 64-bit, not supported yet") EXIT () ENDIF () + + IF (${TD_OS_INFO} MATCHES "Alpine") + SET(TD_APLHINE TRUE) + MESSAGE(STATUS "The current OS is Alpine, append extra flags") + ENDIF() ELSEIF (${CMAKE_SYSTEM_NAME} MATCHES "Darwin") IF (${CMAKE_SIZEOF_VOID_P} MATCHES 8) SET(TD_DARWIN_64 TRUE) - ADD_DEFINITIONS(-D_TD_DARWIN_64) MESSAGE(STATUS "The current platform is Darwin 64-bit") ELSE () MESSAGE(FATAL_ERROR "The current platform is Darwin 32-bit, not supported yet") EXIT () ENDIF () ELSEIF (${CMAKE_SYSTEM_NAME} MATCHES "Windows") + SET(TD_WINDOWS TRUE) IF (${CMAKE_SIZEOF_VOID_P} MATCHES 8) SET(TD_WINDOWS_64 TRUE) - ADD_DEFINITIONS(-D_M_X64) - ADD_DEFINITIONS(-D_TD_WINDOWS_64) MESSAGE(STATUS "The current platform is Windows 64-bit") ELSE () SET(TD_WINDOWS_32 TRUE) - ADD_DEFINITIONS(-D_TD_WINDOWS_32) MESSAGE(FATAL_ERROR "The current platform is Windows 32-bit, not supported yet") EXIT () ENDIF () @@ -106,74 +103,3 @@ ELSE() MESSAGE(FATAL_ERROR "The current platform is not Linux/Darwin/Windows, stop compile") EXIT () ENDIF () - -IF (TD_LINUX_64) - SET(DEBUG_FLAGS "-O0 -DDEBUG") - SET(RELEASE_FLAGS "-O0") - IF (NOT TD_ARM) - IF (${CMAKE_CXX_COMPILER_ID} MATCHES "Clang") - SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -Wno-missing-braces -fPIC -g3 -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") - ELSE () - SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -malign-double -g3 -gdwarf-2 -malign-stringops -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") - ENDIF () - ELSE () - SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") - ENDIF () - ADD_DEFINITIONS(-DLINUX) - ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT) - IF (${TD_OS_INFO} MATCHES "Alpine") - MESSAGE(STATUS "The current OS is Alpine, append extra flags") - SET(COMMON_FLAGS "${COMMON_FLAGS} -largp") - link_libraries(/usr/lib/libargp.a) - ADD_DEFINITIONS(-D_ALPINE) - ENDIF () - IF (${COVER} MATCHES "true") - MESSAGE(STATUS "Test coverage mode, add extra flags") - SET(GCC_COVERAGE_COMPILE_FLAGS "-fprofile-arcs -ftest-coverage") - SET(GCC_COVERAGE_LINK_FLAGS "-lgcov --coverage") - SET(COMMON_FLAGS "${COMMON_FLAGS} ${GCC_COVERAGE_COMPILE_FLAGS} ${GCC_COVERAGE_LINK_FLAGS}") - ENDIF () - ELSEIF (TD_LINUX_32) - IF (NOT TD_ARM) - EXIT () - ENDIF () - SET(DEBUG_FLAGS "-O0 -DDEBUG") - SET(RELEASE_FLAGS "-O0") - SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g -fsigned-char -munaligned-access -fpack-struct=8 -latomic -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") - ADD_DEFINITIONS(-DLINUX) - ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT) - ADD_DEFINITIONS(-DUSE_LIBICONV) - IF (${TD_OS_INFO} MATCHES "Alpine") - MESSAGE(STATUS "The current OS is Alpine, add extra flags") - SET(COMMON_FLAGS "${COMMON_FLAGS} -largp") - link_library(/usr/lib/libargp.a) - ADD_DEFINITIONS(-D_ALPINE) - ENDIF () - IF (${COVER} MATCHES "true") - MESSAGE(STATUS "Test coverage mode, add extra flags") - SET(GCC_COVERAGE_COMPILE_FLAGS "-fprofile-arcs -ftest-coverage") - SET(GCC_COVERAGE_LINK_FLAGS "-lgcov --coverage") - SET(COMMON_FLAGS "${COMMON_FLAGS} ${GCC_COVERAGE_COMPILE_FLAGS} ${GCC_COVERAGE_LINK_FLAGS}") - ENDIF () - ELSEIF (TD_WINDOWS_64) - SET(CMAKE_GENERATOR "NMake Makefiles" CACHE INTERNAL "" FORCE) - IF (NOT TD_GODLL) - SET(COMMON_FLAGS "/nologo /WX /Oi /Oy- /Gm- /EHsc /MT /GS /Gy /fp:precise /Zc:wchar_t /Zc:forScope /Gd /errorReport:prompt /analyze-") - SET(DEBUG_FLAGS "/Zi /W3 /GL") - SET(RELEASE_FLAGS "/W0 /GL") - ENDIF () - ADD_DEFINITIONS(-DWINDOWS) - ADD_DEFINITIONS(-D__CLEANUP_C) - ADD_DEFINITIONS(-DPTW32_STATIC_LIB) - ADD_DEFINITIONS(-DPTW32_BUILD) - ADD_DEFINITIONS(-D_MBCS -D_CRT_SECURE_NO_DEPRECATE -D_CRT_NONSTDC_NO_DEPRECATE) - ELSEIF (TD_DARWIN_64) - SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -Wno-missing-braces -fPIC -g -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") - SET(DEBUG_FLAGS "-O0 -DDEBUG") - SET(RELEASE_FLAGS "-O0") - ADD_DEFINITIONS(-DDARWIN) - ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT) - ELSE () - MESSAGE(FATAL_ERROR "The current platform is not support yet, stop compile") - EXIT () - ENDIF () diff --git a/documentation/webdocs/markdowndocs/Super Table-ch.md b/documentation/webdocs/markdowndocs/Super Table-ch.md index 5f592ec339..f45f2f91b1 100644 --- a/documentation/webdocs/markdowndocs/Super Table-ch.md +++ b/documentation/webdocs/markdowndocs/Super Table-ch.md @@ -4,7 +4,7 @@ TDengine要求每个数据采集点单独建表。独立建表的模式能够避 ## 什么是超级表 -STable是同一类型数据采集点的抽象，是同类型采集实例的集合，包含多张数据结构一样的子表。每个STable为其子表定义了表结构和一组标签：表结构即表中记录的数据列及其数据类型；标签名和数据类型由STable定义，标签值记录着每个子表的静态信息，用以对子表进行分组过滤。子表本质上就是普通的表，由一个时间戳主键和若干个数据列组成，每行记录着具体的数据，数据查询操作与普通表完全相同；但子表与普通表的区别在于每个子表从属于一张超级表，并带有一组由STable定义的标签值。每种类型的采集设备可以定义一个STable。数据模型定义表的每列数据的类型，如温度、压力、电压、电流、GPS实时位置等，而标签信息属于Meta Data，如采集设备的序列号、型号、位置等，是静态的，是表的元数据。用户在创建表（数据采集点）时指定STable(采集类型)外，还可以指定标签的值，也可事后增加或修改。 +超级表是同一类型数据采集点的抽象，是同类型采集实例的集合，包含多张数据结构一样的子表。每个STable为其子表定义了表结构和一组标签：表结构即表中记录的数据列及其数据类型；标签名和数据类型由STable定义，标签值记录着每个子表的静态信息，用以对子表进行分组过滤。子表本质上就是普通的表，由一个时间戳主键和若干个数据列组成，每行记录着具体的数据，数据查询操作与普通表完全相同；但子表与普通表的区别在于每个子表从属于一张超级表，并带有一组由STable定义的标签值。每种类型的采集设备可以定义一个STable。数据模型定义表的每列数据的类型，如温度、压力、电压、电流、GPS实时位置等，而标签信息属于Meta Data，如采集设备的序列号、型号、位置等，是静态的，是表的元数据。用户在创建表（数据采集点）时指定STable(采集类型)外，还可以指定标签的值，也可事后增加或修改。 TDengine扩展标准SQL语法用于定义STable，使用关键词tags指定标签信息。语法如下： diff --git a/documentation/webdocs/markdowndocs/administrator-ch.md b/documentation/webdocs/markdowndocs/administrator-ch.md index 289f2dbf91..b48cf23995 100644 --- a/documentation/webdocs/markdowndocs/administrator-ch.md +++ b/documentation/webdocs/markdowndocs/administrator-ch.md @@ -16,9 +16,9 @@ TDengine的所有可执行文件默认存放在 _/usr/local/taos/bin_ 目录下。其中包括： -- _taosd_：TDengine服务端可执行文件 -- _taos_： TDengine Shell可执行文件 -- _taosdump_：数据导出工具 +- _taosd_：TDengine服务端可执行文件。 +- _taos_： TDengine Shell可执行文件。 +- _taosdump_：数据导出工具。 - *rmtaos*： 卸载TDengine的脚本, 该脚本会删除全部的程序和数据文件。请务必谨慎执行，如非必须不建议使用。 您可以通过修改系统配置文件taos.cfg来配置不同的数据目录和日志目录 diff --git a/packaging/cfg/taos.cfg b/packaging/cfg/taos.cfg index 2d4ebb4095..f2dbc04353 100644 --- a/packaging/cfg/taos.cfg +++ b/packaging/cfg/taos.cfg @@ -11,9 +11,6 @@ # second full-qualified domain name (FQDN) for TDengine system, for cluster edition only # second cluster_hostname2:6030 -# the arbitrator's full-qualified domain name (FQDN) for TDengine system, for cluster edition only -# arbitrator arbitrator_hostname:6030 - # the full-qualified domain name (FQDN) of dnode # fqdn hostname @@ -23,45 +20,57 @@ # http service port, default tcp [6020] # httpPort 6020 -# data file's directory -# dataDir /var/lib/taos - # log file's directory # logDir /var/log/taos -# number of management nodes in the system -# numOfMnodes 3 +# scriptDir file's directory +# scriptDir /var/log/taos -# optional roles for dnode. 0 - any, 1 - mnode, 2 - dnode -# role 0 +# data file's directory +# dataDir /var/lib/taos + +# the arbitrator's full-qualified domain name (FQDN) for TDengine system, for cluster edition only +# arbitrator arbitrator_hostname:6030 # number of threads per CPU core # numOfThreadsPerCore 1.0 -# max number of vgroups per db -# maxVgroupsPerDb 0 - -# max number of tables per vnode -# maxTablesPerVnode 1000000 - # the ratio of threads responsible for querying in the total thread # ratioOfQueryThreads 0.5 -# interval of DNode report status to MNode, unit is Second, for cluster version only -# statusInterval 1 +# number of management nodes in the system +# numOfMnodes 3 -# interval of Shell send HB to MNode, unit is Second -# shellActivityTimer 3 +# if backup vnode directory when remove dnode +# vnodeBak 1 + +# Whether to start load balancing +# balance 1 + +# optional roles for dnode. 0 - any, 1 - mnode, 2 - dnode +# role 0 + +# max timer control block +# maxTmrCtrl 512 # interval of system monitor # monitorInterval 30 +# number of seconds allowed for a dnode to be offline, for cluster version only +# offlineThreshold 8640000 + # RPC re-try timer, millisecond # rpcTimer 300 # RPC maximum time for ack, seconds # rpcMaxTime 600 +# interval of DNode report status to MNode, unit is Second, for cluster version only +# statusInterval 1 + +# interval of Shell send HB to MNode, unit is Second +# shellActivityTimer 3 + # duration of to keep tableMeta kept in Cache, seconds # tableMetaKeepTimer 7200 @@ -71,38 +80,38 @@ # Time window minimum # minIntervalTime 10 -# max length of an SQL -# maxSQLLength 65480 +# the max allowed delayed time for launching continuous query. 20ms by default +# maxStreamCompDelay 20000 -# Support the maximum number of records allowed for super table time sorting -# maxNumOfOrderedRes 100000 +# The minimum time to wait before the first stream execution +# maxFirstStreamCompDelay 10000 -# system locale -# locale en_US.UTF-8 +# Retry wait time benchmark +# retryStreamCompDelay 10 -# default system charset -# charset UTF-8 +# the delayed time for launching each continuous query. 10% of the whole computing time window by default. +# streamCompDelayRatio 0.1 -# system time zone -# timezone Asia/Shanghai (CST, +0800) +# max number of vgroups per db +# maxVgroupsPerDb 0 -# set write ahead log (WAL) level -# walLevel 1 +# max number of tables per vnode +# maxTablesPerVnode 1000000 -# enable/disable async log -# asyncLog 1 - -# the compressed rpc message, option: -# -1 (no compression) -# 0 (all message compressed), -# > 0 (rpc message body which larger than this value will be compressed) -# compressMsgSize -1 +# Step size of increasing table number in vnode +# tableIncStepPerVnode 1000 # cache block size (Mbyte) -# cache 16 +# cache 16 # number of cache blocks per vnode -# blocks 4 +# blocks 6 + +# number of days per DB file +# days 10 + +# number of days to keep DB file +# keep 3650 # min row of records in file block # minRows 100 @@ -111,17 +120,44 @@ # maxRows 4096 # enable/disable compression -# comp 1 +# comp 2 -# number of days per DB file -# days 10 +# set write ahead log (WAL) level +# walLevel 1 -# number of days to keep DB file -# keep 3650 +# When walLevel is set to 2, the cycle of fsync is executed +# fsync 3000 # number of replications, for cluster version only # replica 1 +# mqtt uri +# mqttBrokerAddress mqtt://username:password@hostname:1883/taos/ + +# mqtt client name +# mqttBrokerClientId taos_mqtt + +# the compressed rpc message, option: +# -1 (no compression) +# 0 (all message compressed), +# > 0 (rpc message body which larger than this value will be compressed) +# compressMsgSize -1 + +# max length of an SQL +# maxSQLLength 65480 + +# Support the maximum number of records allowed for super table time sorting +# maxNumOfOrderedRes 100000 + +# system time zone +# timezone Asia/Shanghai (CST, +0800) + +# system locale +# locale en_US.UTF-8 + +# default system charset +# charset UTF-8 + # max number of connections from client for dnode # maxShellConns 5000 @@ -137,50 +173,36 @@ # Stop writing data when the disk size of the log folder is less than this value # minimalDataDirGB 0.1 -# number of seconds allowed for a dnode to be offline, for cluster version only -# offlineThreshold 8640000 - # start http service # http 1 -# start system monitor module -# monitor 1 - # start muqq service # mqtt 0 -# mqtt uri -# mqttBrokerAddress mqtt://username:password@hostname:1883/taos/ +# start system monitor module +# monitor 1 -# mqtt client name -# mqttBrokerClientId taos_mqtt - -# maximum number of rows returned by the restful interface -# restfulRowLimit 10240 +# Record the SQL through restful interface +# httpEnableRecordSql 0 # number of threads used to process http requests # httpMaxThreads 2 -# The minimum time to wait before the first stream execution -# maxFirstStreamCompDelay 10000 - -# Retry wait time benchmark -# retryStreamCompDelay 10 - -# the delayed time for launching each continuous query. 10% of the whole computing time window by default. -# streamCompDelayRatio 0.1 - -# the max allowed delayed time for launching continuous query. 20ms by default -# maxStreamCompDelay 20000 +# maximum number of rows returned by the restful interface +# restfulRowLimit 10240 # The following parameter is used to limit the maximum number of lines in log files. # max number of rows per log filters # numOfLogLines 10000000 +# enable/disable async log +# asyncLog 1 + # The following parameters are used for debug purpose only. # debugFlag 8 bits mask: FILE-SCREEN-UNUSED-HeartBeat-DUMP-TRACE_WARN-ERROR -# 131: output warning and error，135: output info, warning and error to log. -# 199: output debug, info, warning and error to both screen and file +# 131: output warning and error, 135: output debug, warning and error, 143 : output trace, debug, warning and error to log. +# 199: output debug, warning and error to both screen and file +# 207: output trace, debug, warning and error to both screen and file # debug flag for all log type, take effect when non-zero value # debugFlag 0 @@ -236,9 +258,6 @@ # debug flag for http server # tsdbDebugFlag 131 -# Record the SQL through restful interface -# httpEnableRecordSql 0 - # Record the SQL in taos client # tscEnableRecordSql 0 diff --git a/packaging/deb/tarbitratord b/packaging/deb/tarbitratord new file mode 100644 index 0000000000..bf4dba7258 --- /dev/null +++ b/packaging/deb/tarbitratord @@ -0,0 +1,88 @@ +#!/bin/bash +# +# Modified from original source: Elastic Search +# https://github.com/elasticsearch/elasticsearch +# Thank you to the Elastic Search authors +# +# chkconfig: 2345 99 01 +# +### BEGIN INIT INFO +# Provides: TDEngine +# Required-Start: $local_fs $network $syslog +# Required-Stop: $local_fs $network $syslog +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Starts TDEngine tarbitrator +# Description: Starts TDEngine tarbitrator, a arbitrator +### END INIT INFO + +set -e + +PATH="/bin:/usr/bin:/sbin:/usr/sbin" +NAME="tarbitrator" +USER="root" +GROUP="root" +DAEMON="/usr/local/taos/bin/tarbitrator" +DAEMON_OPTS="" +PID_FILE="/var/run/$NAME.pid" +APPARGS="" + +# Maximum number of open files +MAX_OPEN_FILES=65535 + +. /lib/lsb/init-functions + +case "$1" in + start) + + log_action_begin_msg "Starting tarbitrator..." + if start-stop-daemon --test --start --chuid "$USER:$GROUP" --background --make-pidfile --pidfile "$PID_FILE" --exec "$DAEMON" -- $APPARGS &> /dev/null; then + + touch "$PID_FILE" && chown "$USER":"$GROUP" "$PID_FILE" + + if [ -n "$MAX_OPEN_FILES" ]; then + ulimit -n $MAX_OPEN_FILES + fi + + start-stop-daemon --start --chuid "$USER:$GROUP" --background --make-pidfile --pidfile "$PID_FILE" --exec "$DAEMON" -- $APPARGS + + log_end_msg $? + fi + ;; + + stop) + log_action_begin_msg "Stopping tarbitrator..." + set +e + if [ -f "$PID_FILE" ]; then + start-stop-daemon --stop --pidfile "$PID_FILE" --user "$USER" --retry=TERM/120/KILL/5 > /dev/null + if [ $? -eq 1 ]; then + log_action_cont_msg "TSD is not running but pid file exists, cleaning up" + elif [ $? -eq 3 ]; then + PID="`cat $PID_FILE`" + log_failure_msg "Failed to stop tarbitrator (pid $PID)" + exit 1 + fi + rm -f "$PID_FILE" + else + log_action_cont_msg "tarbitrator was not running" + fi + log_action_end_msg 0 + set -e + ;; + + restart|force-reload) + if [ -f "$PID_FILE" ]; then + $0 stop + sleep 1 + fi + $0 start + ;; + status) + status_of_proc -p "$PID_FILE" "$DAEMON" "$NAME" + ;; + *) + exit 1 + ;; +esac + +exit 0 diff --git a/packaging/rpm/tarbitratord b/packaging/rpm/tarbitratord new file mode 100644 index 0000000000..5d3f2b9c79 --- /dev/null +++ b/packaging/rpm/tarbitratord @@ -0,0 +1,141 @@ +#!/bin/bash +# +# tarbitratord This shell script takes care of starting and stopping tarbitrator. +# +# chkconfig: 2345 99 01 +# description: tarbitrator is a arbitrator used in TDengine cluster. +# +# +### BEGIN INIT INFO +# Provides: TDEngine +# Required-Start: $network $local_fs $remote_fs +# Required-Stop: $network $local_fs $remote_fs +# Short-Description: start and stop taosd +# Description: tarbitrator is a arbitrator used in TDengine cluster. +### END INIT INFO + +# Source init functions +. /etc/init.d/functions + +# Maximum number of open files +MAX_OPEN_FILES=65535 + +# Default program options +NAME=tarbitrator +PROG=/usr/local/taos/bin/tarbitrator +USER=root +GROUP=root + +# Default directories +LOCK_DIR=/var/lock/subsys +PID_DIR=/var/run/$NAME + +# Set file names +LOCK_FILE=$LOCK_DIR/$NAME +PID_FILE=$PID_DIR/$NAME.pid + +[ -e $PID_DIR ] || mkdir -p $PID_DIR + +PROG_OPTS="" + +start() { + echo -n "Starting ${NAME}: " + # check identity + curid="`id -u -n`" + if [ "$curid" != root ] && [ "$curid" != "$USER" ] ; then + echo "Must be run as root or $USER, but was run as $curid" + return 1 + fi + # Sets the maximum number of open file descriptors allowed. + ulimit -n $MAX_OPEN_FILES + curulimit="`ulimit -n`" + if [ "$curulimit" -lt $MAX_OPEN_FILES ] ; then + echo "'ulimit -n' must be greater than or equal to $MAX_OPEN_FILES, is $curulimit" + return 1 + fi + + if [ "`id -u -n`" == root ] ; then + # Changes the owner of the lock, and the pid files to allow + # non-root OpenTSDB daemons to run /usr/share/opentsdb/bin/opentsdb_restart.py. + touch $LOCK_FILE && chown $USER:$GROUP $LOCK_FILE + touch $PID_FILE && chown $USER:$GROUP $PID_FILE + daemon --user $USER --pidfile $PID_FILE "$PROG $PROG_OPTS &> /dev/null &" + else + # Don't have to change user. + daemon --pidfile $PID_FILE "$PROG $PROG_OPTS &> /dev/null &" + fi + retval=$? + sleep 2 + echo + [ $retval -eq 0 ] && (findproc > $PID_FILE && touch $LOCK_FILE) + return $retval +} + +stop() { + echo -n "Stopping ${NAME}: " + killproc -p $PID_FILE $NAME + retval=$? + echo + # Non-root users don't have enough permission to remove pid and lock files. + # So, the opentsdb_restart.py cannot get rid of the files, and the command + # "service opentsdb status" will complain about the existing pid file. + # Makes the pid file empty. + echo > $PID_FILE + [ $retval -eq 0 ] && (rm -f $PID_FILE && rm -f $LOCK_FILE) + return $retval +} + +restart() { + stop + start +} + +reload() { + restart +} + +force_reload() { + restart +} + +rh_status() { + # run checks to determine if the service is running or use generic status + status -p $PID_FILE -l $LOCK_FILE $NAME +} + +rh_status_q() { + rh_status >/dev/null 2>&1 +} + +case "$1" in + start) + rh_status_q && exit 0 + $1 + ;; + stop) + rh_status_q || exit 0 + $1 + ;; + restart) + $1 + ;; + reload) + rh_status_q || exit 7 + $1 + ;; + force-reload) + force_reload + ;; + status) + rh_status + ;; + condrestart|try-restart) + rh_status_q || exit 0 + restart + ;; + *) + echo "Usage: $0 {start|stop|status|restart|condrestart|try-restart|reload|force-reload}" + exit 2 +esac + +exit $? diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index c573a2086f..b2613a7cb6 100644 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -99,6 +99,44 @@ else os_type=1 fi + +# ============================= get input parameters ================================================= + +# install.sh -v [server | client] -e [yes | no] -i [systemd | service | ...] + +# set parameters by default value +interactiveFqdn=yes # [yes | no] +verType=server # [server | client] +initType=systemd # [systemd | service | ...] + +while getopts "hv:e:i:" arg +do + case $arg in + e) + #echo "interactiveFqdn=$OPTARG" + interactiveFqdn=$( echo $OPTARG ) + ;; + v) + #echo "verType=$OPTARG" + verType=$(echo $OPTARG) + ;; + i) + #echo "initType=$OPTARG" + initType=$(echo $OPTARG) + ;; + h) + echo "Usage: `basename $0` -v [server | client] -e [yes | no]" + exit 0 + ;; + ?) #unknow option + echo "unkonw argument" + exit 1 + ;; + esac +done + +echo "verType=${verType} interactiveFqdn=${interactiveFqdn}" + function kill_taosd() { pid=$(ps -ef | grep "taosd" | grep -v "grep" | awk '{print $2}') if [ -n "$pid" ]; then @@ -129,15 +167,17 @@ function install_bin() { ${csudo} rm -f ${bin_link_dir}/taosdemo || : ${csudo} rm -f ${bin_link_dir}/taosdump || : ${csudo} rm -f ${bin_link_dir}/rmtaos || : + ${csudo} rm -f ${bin_link_dir}/tarbitrator || : ${csudo} cp -r ${script_dir}/bin/* ${install_main_dir}/bin && ${csudo} chmod 0555 ${install_main_dir}/bin/* #Make link - [ -x ${install_main_dir}/bin/taos ] && ${csudo} ln -s ${install_main_dir}/bin/taos ${bin_link_dir}/taos || : - [ -x ${install_main_dir}/bin/taosd ] && ${csudo} ln -s ${install_main_dir}/bin/taosd ${bin_link_dir}/taosd || : - [ -x ${install_main_dir}/bin/taosdump ] && ${csudo} ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump || : - [ -x ${install_main_dir}/bin/taosdemo ] && ${csudo} ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || : - [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/rmtaos || : + [ -x ${install_main_dir}/bin/taos ] && ${csudo} ln -s ${install_main_dir}/bin/taos ${bin_link_dir}/taos || : + [ -x ${install_main_dir}/bin/taosd ] && ${csudo} ln -s ${install_main_dir}/bin/taosd ${bin_link_dir}/taosd || : + [ -x ${install_main_dir}/bin/taosdump ] && ${csudo} ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump || : + [ -x ${install_main_dir}/bin/taosdemo ] && ${csudo} ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || : + [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/rmtaos || : + [ -x ${install_main_dir}/bin/tarbitrator ] && ${csudo} ln -s ${install_main_dir}/bin/tarbitrator ${bin_link_dir}/tarbitrator || : if [ "$verMode" == "cluster" ]; then ${csudo} cp -r ${script_dir}/nginxd/* ${nginx_dir} && ${csudo} chmod 0555 ${nginx_dir}/* @@ -189,50 +229,55 @@ function install_config() { if ((${update_flag}==1)); then return 0 fi + + if [ "$interactiveFqdn" == "no" ]; then + return 0 + fi - IP_FORMAT="(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)" - IP_PATTERN="\b$IP_FORMAT\.$IP_FORMAT\.$IP_FORMAT\.$IP_FORMAT\b" + #FQDN_FORMAT="(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)" + #FQDN_FORMAT="(:[1-6][0-9][0-9][0-9][0-9]$)" + #PORT_FORMAT="(/[1-6][0-9][0-9][0-9][0-9]?/)" + #FQDN_PATTERN=":[0-9]{1,5}$" + # first full-qualified domain name (FQDN) for TDengine cluster system echo - echo -e -n "${GREEN}Enter the IP address of an existing TDengine cluster node to join${NC} OR ${GREEN}leave it blank to build one${NC} :" - read masterIp + echo -e -n "${GREEN}Enter the FQDN of an existing TDengine cluster node to join${NC} OR ${GREEN}leave it blank to build one${NC} :" + read firstFqdn while true; do - if [ ! -z "$masterIp" ]; then - # check the format of the masterIp - if [[ $masterIp =~ $IP_PATTERN ]]; then - # Write the first IP to configuration file - sudo sed -i -r "s/#*\s*(masterIp\s*).*/\1$masterIp/" ${cfg_dir}/taos.cfg - - # Get the second IP address + if [ ! -z "$firstFqdn" ]; then + # check the format of the firstFqdn + #if [[ $firstFqdn == $FQDN_PATTERN ]]; then + # Write the first FQDN to configuration file + ${csudo} sed -i -r "s/#*\s*(first\s*).*/\1$firstFqdn/" ${cfg_install_dir}/taos.cfg + # Get the second FQDN echo - echo -e -n "${GREEN}Enter the IP address of another node in cluster${NC} OR ${GREEN}leave it blank to skip${NC}: " - read secondIp + echo -e -n "${GREEN}Enter the FQDN of another node in cluster${NC} OR ${GREEN}leave it blank to skip${NC}: " + read secondFqdn while true; do - - if [ ! -z "$secondIp" ]; then - if [[ $secondIp =~ $IP_PATTERN ]]; then - # Write the second IP to configuration file - sudo sed -i -r "s/#*\s*(secondIp\s*).*/\1$secondIp/" ${cfg_dir}/taos.cfg + if [ ! -z "$secondFqdn" ]; then + #if [[ $secondFqdn == $FQDN_PATTERN ]]; then + # Write the second FQDN to configuration file + ${csudo} sed -i -r "s/#*\s*(second\s*).*/\1$secondFqdn/" ${cfg_install_dir}/taos.cfg break - else - read -p "Please enter the correct IP address: " secondIp - fi + #else + # read -p "Please enter the correct FQDN: " secondFqdn + #fi else break fi done break - else - read -p "Please enter the correct IP address: " masterIp - fi + #else + # read -p "Please enter the correct FQDN: " firstFqdn + #fi else break fi done - fi + fi } @@ -266,16 +311,24 @@ function clean_service_on_sysvinit() { if pidof taosd &> /dev/null; then ${csudo} service taosd stop || : fi + + if pidof tarbitrator &> /dev/null; then + ${csudo} service tarbitratord stop || : + fi if ((${initd_mod}==1)); then ${csudo} chkconfig --del taosd || : + ${csudo} chkconfig --del tarbitratord || : elif ((${initd_mod}==2)); then ${csudo} insserv -r taosd || : + ${csudo} insserv -r tarbitratord || : elif ((${initd_mod}==3)); then ${csudo} update-rc.d -f taosd remove || : + ${csudo} update-rc.d -f tarbitratord remove || : fi ${csudo} rm -f ${service_config_dir}/taosd || : + ${csudo} rm -f ${service_config_dir}/tarbitratord || : if $(which init &> /dev/null); then ${csudo} init q || : @@ -292,9 +345,13 @@ function install_service_on_sysvinit() { if ((${os_type}==1)); then ${csudo} cp -f ${script_dir}/init.d/taosd.deb ${install_main_dir}/init.d/taosd ${csudo} cp ${script_dir}/init.d/taosd.deb ${service_config_dir}/taosd && ${csudo} chmod a+x ${service_config_dir}/taosd + ${csudo} cp -f ${script_dir}/init.d/tarbitratord.deb ${install_main_dir}/init.d/tarbitratord + ${csudo} cp ${script_dir}/init.d/tarbitratord.deb ${service_config_dir}/tarbitratord && ${csudo} chmod a+x ${service_config_dir}/tarbitratord elif ((${os_type}==2)); then ${csudo} cp -f ${script_dir}/init.d/taosd.rpm ${install_main_dir}/init.d/taosd ${csudo} cp ${script_dir}/init.d/taosd.rpm ${service_config_dir}/taosd && ${csudo} chmod a+x ${service_config_dir}/taosd + ${csudo} cp -f ${script_dir}/init.d/tarbitratord.rpm ${install_main_dir}/init.d/tarbitratord + ${csudo} cp ${script_dir}/init.d/tarbitratord.rpm ${service_config_dir}/tarbitratord && ${csudo} chmod a+x ${service_config_dir}/tarbitratord fi #restart_config_str="taos:2345:respawn:${service_config_dir}/taosd start" @@ -303,11 +360,16 @@ function install_service_on_sysvinit() { if ((${initd_mod}==1)); then ${csudo} chkconfig --add taosd || : ${csudo} chkconfig --level 2345 taosd on || : + ${csudo} chkconfig --add tarbitratord || : + ${csudo} chkconfig --level 2345 tarbitratord on || : elif ((${initd_mod}==2)); then ${csudo} insserv taosd || : ${csudo} insserv -d taosd || : + ${csudo} insserv tarbitratord || : + ${csudo} insserv -d tarbitratord || : elif ((${initd_mod}==3)); then ${csudo} update-rc.d taosd defaults || : + ${csudo} update-rc.d tarbitratord defaults || : fi } @@ -331,8 +393,17 @@ function clean_service_on_systemd() { fi ${csudo} systemctl disable nginxd &> /dev/null || echo &> /dev/null - ${csudo} rm -f ${nginx_service_config} - fi + ${csudo} rm -f ${nginx_service_config} + + tarbitratord_service_config="${service_config_dir}/tarbitratord.service" + if systemctl is-active --quiet tarbitratord; then + echo "tarbitrator is running, stopping it..." + ${csudo} systemctl stop tarbitratord &> /dev/null || echo &> /dev/null + fi + ${csudo} systemctl disable tarbitratord &> /dev/null || echo &> /dev/null + + ${csudo} rm -f ${tarbitratord_service_config} + fi } # taos:2345:respawn:/etc/init.d/taosd start @@ -364,28 +435,52 @@ function install_service_on_systemd() { ${csudo} systemctl enable taosd if [ "$verMode" == "cluster" ]; then + + tarbitratord_service_config="${service_config_dir}/tarbitratord.service" + + ${csudo} bash -c "echo '[Unit]' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Description=TDengine arbitrator service' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'After=network-online.target' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Wants=network-online.target' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo '[Service]' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Type=simple' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'ExecStart=/usr/bin/tarbitrator' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'LimitNOFILE=infinity' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'LimitNPROC=infinity' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'LimitCORE=infinity' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'TimeoutStartSec=0' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'StandardOutput=null' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Restart=always' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'StartLimitBurst=3' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'StartLimitInterval=60s' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo '[Install]' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'WantedBy=multi-user.target' >> ${tarbitratord_service_config}" + ${csudo} systemctl enable tarbitratord + nginx_service_config="${service_config_dir}/nginxd.service" - ${csudo} bash -c "echo '[Unit]' >> ${nginx_service_config}" - ${csudo} bash -c "echo 'Description=Nginx For TDengine Service' >> ${nginx_service_config}" - ${csudo} bash -c "echo 'After=network-online.target' >> ${nginx_service_config}" - ${csudo} bash -c "echo 'Wants=network-online.target' >> ${nginx_service_config}" - ${csudo} bash -c "echo >> ${nginx_service_config}" - ${csudo} bash -c "echo '[Service]' >> ${nginx_service_config}" - ${csudo} bash -c "echo 'Type=forking' >> ${nginx_service_config}" - ${csudo} bash -c "echo 'PIDFile=/usr/local/nginxd/logs/nginx.pid' >> ${nginx_service_config}" - ${csudo} bash -c "echo 'ExecStart=/usr/local/nginxd/sbin/nginx' >> ${nginx_service_config}" - ${csudo} bash -c "echo 'ExecStop=/usr/local/nginxd/sbin/nginx -s stop' >> ${nginx_service_config}" - ${csudo} bash -c "echo 'LimitNOFILE=infinity' >> ${nginx_service_config}" - ${csudo} bash -c "echo 'LimitNPROC=infinity' >> ${nginx_service_config}" - ${csudo} bash -c "echo 'LimitCORE=infinity' >> ${nginx_service_config}" - ${csudo} bash -c "echo 'TimeoutStartSec=0' >> ${nginx_service_config}" - ${csudo} bash -c "echo 'StandardOutput=null' >> ${nginx_service_config}" - ${csudo} bash -c "echo 'Restart=always' >> ${nginx_service_config}" - ${csudo} bash -c "echo 'StartLimitBurst=3' >> ${nginx_service_config}" - ${csudo} bash -c "echo 'StartLimitInterval=60s' >> ${nginx_service_config}" - ${csudo} bash -c "echo >> ${nginx_service_config}" - ${csudo} bash -c "echo '[Install]' >> ${nginx_service_config}" - ${csudo} bash -c "echo 'WantedBy=multi-user.target' >> ${nginx_service_config}" + ${csudo} bash -c "echo '[Unit]' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'Description=Nginx For TDengine Service' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'After=network-online.target' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'Wants=network-online.target' >> ${nginx_service_config}" + ${csudo} bash -c "echo >> ${nginx_service_config}" + ${csudo} bash -c "echo '[Service]' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'Type=forking' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'PIDFile=/usr/local/nginxd/logs/nginx.pid' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'ExecStart=/usr/local/nginxd/sbin/nginx' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'ExecStop=/usr/local/nginxd/sbin/nginx -s stop' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'LimitNOFILE=infinity' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'LimitNPROC=infinity' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'LimitCORE=infinity' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'TimeoutStartSec=0' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'StandardOutput=null' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'Restart=always' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'StartLimitBurst=3' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'StartLimitInterval=60s' >> ${nginx_service_config}" + ${csudo} bash -c "echo >> ${nginx_service_config}" + ${csudo} bash -c "echo '[Install]' >> ${nginx_service_config}" + ${csudo} bash -c "echo 'WantedBy=multi-user.target' >> ${nginx_service_config}" if ! ${csudo} systemctl enable nginxd &> /dev/null; then ${csudo} systemctl daemon-reexec ${csudo} systemctl enable nginxd @@ -600,8 +695,8 @@ function install_TDengine() { echo -e "${GREEN_DARK}To access TDengine ${NC}: use ${GREEN_UNDERLINE}taos${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${nginx_port}${NC}" else echo -e "${GREEN_DARK}To access TDengine ${NC}: use ${GREEN_UNDERLINE}taos${NC} in shell${NC}" - fi - else + fi + else echo -e "${GREEN_DARK}To access TDengine ${NC}: use ${GREEN_UNDERLINE}taos${NC} in shell${NC}" fi @@ -620,7 +715,7 @@ function install_TDengine() { ## ==============================Main program starts from here============================ -if [ -z $1 ]; then +if [ "$verType" == "server" ]; then # Install server and client if [ -x ${bin_dir}/taosd ]; then update_flag=1 @@ -628,7 +723,8 @@ if [ -z $1 ]; then else install_TDengine fi -else +elif [ "$verType" == "client" ]; then + interactiveFqdn=no # Only install client if [ -x ${bin_dir}/taos ]; then update_flag=1 @@ -636,4 +732,6 @@ else else install_TDengine client fi +else + echo "please input correct verType" fi diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index c214626efd..1cb412d9dd 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -36,7 +36,7 @@ if [ "$pagMode" == "lite" ]; then strip ${build_dir}/bin/taos bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${script_dir}/remove.sh" else - bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${build_dir}/bin/taosdemo ${build_dir}/bin/taosdump ${script_dir}/remove.sh" + bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${build_dir}/bin/taosdemo ${build_dir}/bin/taosdump ${build_dir}/bin/tarbitrator ${script_dir}/remove.sh" fi lib_files="${build_dir}/lib/libtaos.so.${version}" @@ -54,14 +54,18 @@ nginx_dir="${code_dir}/../../enterprise/src/plugins/web" # temp use rpm's taosd. TODO: later modify according to os type init_file_deb=${script_dir}/../deb/taosd init_file_rpm=${script_dir}/../rpm/taosd +init_file_tarbitrator_deb=${script_dir}/../deb/tarbitratord +init_file_tarbitrator_rpm=${script_dir}/../rpm/tarbitratord # make directories. mkdir -p ${install_dir} mkdir -p ${install_dir}/inc && cp ${header_files} ${install_dir}/inc mkdir -p ${install_dir}/cfg && cp ${cfg_dir}/taos.cfg ${install_dir}/cfg/taos.cfg -mkdir -p ${install_dir}/bin && cp ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* +mkdir -p ${install_dir}/bin && cp ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* || : mkdir -p ${install_dir}/init.d && cp ${init_file_deb} ${install_dir}/init.d/taosd.deb mkdir -p ${install_dir}/init.d && cp ${init_file_rpm} ${install_dir}/init.d/taosd.rpm +mkdir -p ${install_dir}/init.d && cp ${init_file_tarbitrator_deb} ${install_dir}/init.d/tarbitratord.deb || : +mkdir -p ${install_dir}/init.d && cp ${init_file_tarbitrator_rpm} ${install_dir}/init.d/tarbitratord.rpm || : if [ "$verMode" == "cluster" ]; then sed 's/verMode=edge/verMode=cluster/g' ${install_dir}/bin/remove.sh >> remove_temp.sh diff --git a/packaging/tools/remove.sh b/packaging/tools/remove.sh index 28cc835f30..9fb8731449 100755 --- a/packaging/tools/remove.sh +++ b/packaging/tools/remove.sh @@ -26,6 +26,7 @@ v15_java_app_dir="/usr/local/lib/taos" service_config_dir="/etc/systemd/system" taos_service_name="taosd" +tarbitrator_service_name="tarbitratord" nginx_service_name="nginxd" csudo="" if command -v sudo > /dev/null; then @@ -59,6 +60,12 @@ function kill_taosd() { fi } +function kill_tarbitrator() { + pid=$(ps -ef | grep "tarbitrator" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo} kill -9 $pid || : + fi +} function clean_bin() { # Remove link ${csudo} rm -f ${bin_link_dir}/taos || : @@ -102,9 +109,18 @@ function clean_service_on_systemd() { ${csudo} rm -f ${taosd_service_config} if [ "$verMode" == "cluster" ]; then - nginx_service_config="${service_config_dir}/${nginx_service_name}.service" - - if [ -d ${bin_dir}/web ]; then + tarbitratord_service_config="${service_config_dir}/${tarbitrator_service_name}.service" + + if systemctl is-active --quiet ${tarbitrator_service_name}; then + echo "TDengine tarbitrator is running, stopping it..." + ${csudo} systemctl stop ${tarbitrator_service_name} &> /dev/null || echo &> /dev/null + fi + ${csudo} systemctl disable ${tarbitrator_service_name} &> /dev/null || echo &> /dev/null + + ${csudo} rm -f ${tarbitratord_service_config} + + nginx_service_config="${service_config_dir}/${nginx_service_name}.service" + if [ -d ${bin_dir}/web ]; then if systemctl is-active --quiet ${nginx_service_name}; then echo "Nginx for TDengine is running, stopping it..." ${csudo} systemctl stop ${nginx_service_name} &> /dev/null || echo &> /dev/null @@ -124,16 +140,25 @@ function clean_service_on_sysvinit() { echo "TDengine taosd is running, stopping it..." ${csudo} service taosd stop || : fi - + + if pidof tarbitrator &> /dev/null; then + echo "TDengine tarbitrator is running, stopping it..." + ${csudo} service tarbitratord stop || : + fi + if ((${initd_mod}==1)); then ${csudo} chkconfig --del taosd || : + ${csudo} chkconfig --del tarbitratord || : elif ((${initd_mod}==2)); then ${csudo} insserv -r taosd || : + ${csudo} insserv -r tarbitratord || : elif ((${initd_mod}==3)); then ${csudo} update-rc.d -f taosd remove || : + ${csudo} update-rc.d -f tarbitratord remove || : fi ${csudo} rm -f ${service_config_dir}/taosd || : + ${csudo} rm -f ${service_config_dir}/tarbitratord || : if $(which init &> /dev/null); then ${csudo} init q || : @@ -148,6 +173,7 @@ function clean_service() { else # must manual stop taosd kill_taosd + kill_tarbitrator fi } diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 7efcd54cfd..7a44870938 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -60,15 +60,14 @@ typedef struct SCMCorVgroupInfo { } SCMCorVgroupInfo; typedef struct STableMeta { - STableComInfo tableInfo; - uint8_t tableType; - int16_t sversion; - int16_t tversion; + STableComInfo tableInfo; + uint8_t tableType; + int16_t sversion; + int16_t tversion; SCMVgroupInfo vgroupInfo; SCMCorVgroupInfo corVgroupInfo; - int32_t sid; // the index of one table in a virtual node - uint64_t uid; // unique id of a table - SSchema schema[]; // if the table is TSDB_CHILD_TABLE, schema is acquired by super table meta info + STableId id; + SSchema schema[]; // if the table is TSDB_CHILD_TABLE, schema is acquired by super table meta info } STableMeta; typedef struct STableMetaInfo { diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 232ea52ded..ae2370cd56 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -629,8 +629,8 @@ int32_t tscAllocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int3 } static void tsSetBlockInfo(SSubmitBlk *pBlocks, const STableMeta *pTableMeta, int32_t numOfRows) { - pBlocks->tid = pTableMeta->sid; - pBlocks->uid = pTableMeta->uid; + pBlocks->tid = pTableMeta->id.tid; + pBlocks->uid = pTableMeta->id.uid; pBlocks->sversion = pTableMeta->sversion; pBlocks->numOfRows += numOfRows; } @@ -686,7 +686,7 @@ static int32_t doParseInsertStatement(SSqlObj *pSql, void *pTableList, char **st STableComInfo tinfo = tscGetTableInfo(pTableMeta); STableDataBlocks *dataBuf = NULL; - int32_t ret = tscGetDataBlockFromList(pTableList, pCmd->pDataBlocks, pTableMeta->uid, TSDB_DEFAULT_PAYLOAD_SIZE, + int32_t ret = tscGetDataBlockFromList(pTableList, pCmd->pDataBlocks, pTableMeta->id.uid, TSDB_DEFAULT_PAYLOAD_SIZE, sizeof(SSubmitBlk), tinfo.rowSize, pTableMetaInfo->name, pTableMeta, &dataBuf); if (ret != TSDB_CODE_SUCCESS) { diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index f58431849c..9488b37fe3 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -635,7 +635,7 @@ int32_t parseIntervalClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQ int32_t tableIndex = COLUMN_INDEX_INITIAL_VAL; for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); - if (pTableMetaInfo->pTableMeta->uid == uid) { + if (pTableMetaInfo->pTableMeta->id.uid == uid) { tableIndex = i; break; } @@ -3053,7 +3053,7 @@ static int32_t getJoinCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr* STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); SSchema* pTagSchema1 = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, index.columnIndex); - pLeft->uid = pTableMetaInfo->pTableMeta->uid; + pLeft->uid = pTableMetaInfo->pTableMeta->id.uid; pLeft->tagColId = pTagSchema1->colId; strcpy(pLeft->tableId, pTableMetaInfo->name); @@ -3065,7 +3065,7 @@ static int32_t getJoinCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); SSchema* pTagSchema2 = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, index.columnIndex); - pRight->uid = pTableMetaInfo->pTableMeta->uid; + pRight->uid = pTableMetaInfo->pTableMeta->id.uid; pRight->tagColId = pTagSchema2->colId; strcpy(pRight->tableId, pTableMetaInfo->name); @@ -3603,7 +3603,7 @@ static int32_t setTableCondForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableCondIndex); STagCond* pTagCond = &pQueryInfo->tagCond; - pTagCond->tbnameCond.uid = pTableMetaInfo->pTableMeta->uid; + pTagCond->tbnameCond.uid = pTableMetaInfo->pTableMeta->id.uid; assert(pExpr->nSQLOptr == TK_LIKE || pExpr->nSQLOptr == TK_IN); @@ -3840,7 +3840,7 @@ static int32_t getTagQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondE // add to source column list STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, i); - int64_t uid = pTableMetaInfo->pTableMeta->uid; + int64_t uid = pTableMetaInfo->pTableMeta->id.uid; int32_t numOfCols = tscGetNumOfColumns(pTableMetaInfo->pTableMeta); size_t num = taosArrayGetSize(colList); @@ -4506,8 +4506,8 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { SUpdateTableTagValMsg* pUpdateMsg = (SUpdateTableTagValMsg*) pCmd->payload; pUpdateMsg->head.vgId = htonl(pTableMeta->vgroupInfo.vgId); - pUpdateMsg->tid = htonl(pTableMeta->sid); - pUpdateMsg->uid = htobe64(pTableMeta->uid); + pUpdateMsg->tid = htonl(pTableMeta->id.tid); + pUpdateMsg->uid = htobe64(pTableMeta->id.uid); pUpdateMsg->colId = htons(pTagsSchema->colId); pUpdateMsg->type = pTagsSchema->type; pUpdateMsg->bytes = htons(pTagsSchema->bytes); @@ -5045,7 +5045,7 @@ void addGroupInfoForSubquery(SSqlObj* pParentObj, SSqlObj* pSql, int32_t subClau if (pExpr->functionId != TSDB_FUNC_TAG) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex); - int16_t columnInfo = tscGetJoinTagColIdByUid(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->uid); + int16_t columnInfo = tscGetJoinTagColIdByUid(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->id.uid); SColumnIndex index = {.tableIndex = 0, .columnIndex = columnInfo}; SSchema* pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); diff --git a/src/client/src/tscSchemaUtil.c b/src/client/src/tscSchemaUtil.c index 9b8f48b109..63607b7a86 100644 --- a/src/client/src/tscSchemaUtil.c +++ b/src/client/src/tscSchemaUtil.c @@ -162,8 +162,8 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size .numOfColumns = pTableMetaMsg->numOfColumns, }; - pTableMeta->sid = pTableMetaMsg->sid; - pTableMeta->uid = pTableMetaMsg->uid; + pTableMeta->id.tid = pTableMetaMsg->sid; + pTableMeta->id.uid = pTableMetaMsg->uid; pTableMeta->vgroupInfo = pTableMetaMsg->vgroup; tscInitCorVgroupInfo(&pTableMeta->corVgroupInfo, &pTableMeta->vgroupInfo); diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 6c1faf6f9d..521280af87 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -605,9 +605,9 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char } STableIdInfo *pTableIdInfo = (STableIdInfo *)pMsg; - pTableIdInfo->tid = htonl(pTableMeta->sid); - pTableIdInfo->uid = htobe64(pTableMeta->uid); - pTableIdInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pTableMeta->uid, dfltKey)); + pTableIdInfo->tid = htonl(pTableMeta->id.tid); + pTableIdInfo->uid = htobe64(pTableMeta->id.uid); + pTableIdInfo->key = htobe64(tscGetSubscriptionProgress(pSql->pSubscription, pTableMeta->id.uid, dfltKey)); pQueryMsg->numOfTables = htonl(1); // set the number of tables pMsg += sizeof(STableIdInfo); @@ -640,7 +640,7 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char } tscDebug("%p vgId:%d, query on table:%s, tid:%d, uid:%" PRIu64, pSql, htonl(pQueryMsg->head.vgId), pTableMetaInfo->name, - pTableMeta->sid, pTableMeta->uid); + pTableMeta->id.tid, pTableMeta->id.uid); return pMsg; } @@ -714,8 +714,8 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { if (pCol->colIndex.columnIndex >= tscGetNumOfColumns(pTableMeta) || pColSchema->type < TSDB_DATA_TYPE_BOOL || pColSchema->type > TSDB_DATA_TYPE_NCHAR) { - tscError("%p sid:%d uid:%" PRIu64" id:%s, column index out of range, numOfColumns:%d, index:%d, column name:%s", - pSql, pTableMeta->sid, pTableMeta->uid, pTableMetaInfo->name, tscGetNumOfColumns(pTableMeta), pCol->colIndex.columnIndex, + tscError("%p tid:%d uid:%" PRIu64" id:%s, column index out of range, numOfColumns:%d, index:%d, column name:%s", + pSql, pTableMeta->id.tid, pTableMeta->id.uid, pTableMetaInfo->name, tscGetNumOfColumns(pTableMeta), pCol->colIndex.columnIndex, pColSchema->name); return TSDB_CODE_TSC_INVALID_SQL; @@ -833,8 +833,8 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { if ((pCol->colIndex.columnIndex >= numOfTagColumns || pCol->colIndex.columnIndex < -1) || (pColSchema->type < TSDB_DATA_TYPE_BOOL || pColSchema->type > TSDB_DATA_TYPE_NCHAR)) { - tscError("%p sid:%d uid:%" PRIu64 " id:%s, tag index out of range, totalCols:%d, numOfTags:%d, index:%d, column name:%s", - pSql, pTableMeta->sid, pTableMeta->uid, pTableMetaInfo->name, total, numOfTagColumns, + tscError("%p tid:%d uid:%" PRIu64 " id:%s, tag index out of range, totalCols:%d, numOfTags:%d, index:%d, column name:%s", + pSql, pTableMeta->id.tid, pTableMeta->id.uid, pTableMetaInfo->name, total, numOfTagColumns, pCol->colIndex.columnIndex, pColSchema->name); return TSDB_CODE_TSC_INVALID_SQL; @@ -855,7 +855,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { if (pQueryInfo->tagCond.pCond != NULL && taosArrayGetSize(pQueryInfo->tagCond.pCond) > 0) { STagCond* pTagCond = &pQueryInfo->tagCond; - SCond *pCond = tsGetSTableQueryCond(pTagCond, pTableMeta->uid); + SCond *pCond = tsGetSTableQueryCond(pTagCond, pTableMeta->id.uid); if (pCond != NULL && pCond->cond != NULL) { pQueryMsg->tagCondLen = htons(pCond->len); memcpy(pMsg, pCond->cond, pCond->len); @@ -1739,7 +1739,7 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } - tscDebug("%p recv table meta, uid:%"PRId64 ", tid:%d, name:%s", pSql, pTableMeta->uid, pTableMeta->sid, pTableMetaInfo->name); + tscDebug("%p recv table meta, uid:%"PRId64 ", tid:%d, name:%s", pSql, pTableMeta->id.uid, pTableMeta->id.tid, pTableMetaInfo->name); free(pTableMeta); return TSDB_CODE_SUCCESS; @@ -2215,7 +2215,7 @@ int tscRenewTableMeta(SSqlObj *pSql, char *tableId) { STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; if (pTableMetaInfo->pTableMeta) { tscDebug("%p update table meta, old meta numOfTags:%d, numOfCols:%d, uid:%" PRId64 ", addr:%p", pSql, - tscGetNumOfTags(pTableMeta), tscGetNumOfColumns(pTableMeta), pTableMeta->uid, pTableMeta); + tscGetNumOfTags(pTableMeta), tscGetNumOfColumns(pTableMeta), pTableMeta->id.uid, pTableMeta); } taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pTableMeta), true); diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c index 8e554d62ff..6e572d94d1 100644 --- a/src/client/src/tscSub.c +++ b/src/client/src/tscSub.c @@ -241,7 +241,7 @@ static int tscUpdateSubscription(STscObj* pObj, SSub* pSub) { STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) { STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; - SSubscriptionProgress target = {.uid = pTableMeta->uid, .key = 0}; + SSubscriptionProgress target = {.uid = pTableMeta->id.uid, .key = 0}; SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress); if (p == NULL) { taosArrayClear(pSub->progress); @@ -334,7 +334,7 @@ void tscSaveSubscriptionProgress(void* sub) { char path[256]; sprintf(path, "%s/subscribe", tsDataDir); - if (tmkdir(path, 0777) != 0) { + if (taosMkDir(path, 0777) != 0) { tscError("failed to create subscribe dir: %s", path); } diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 95d65f4aff..15e02799aa 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -180,7 +180,7 @@ SJoinSupporter* tscCreateJoinSupporter(SSqlObj* pSql, SSubqueryState* pState, in pSupporter->limit = pQueryInfo->limit; STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, pSql->cmd.clauseIndex, index); - pSupporter->uid = pTableMetaInfo->pTableMeta->uid; + pSupporter->uid = pTableMetaInfo->pTableMeta->id.uid; assert (pSupporter->uid != 0); getTmpfilePath("join-", pSupporter->path); @@ -355,7 +355,7 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) { // set the join condition tag column info, to do extract method if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { assert(pQueryInfo->tagCond.joinInfo.hasJoin); - int16_t colId = tscGetJoinTagColIdByUid(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->uid); + int16_t colId = tscGetJoinTagColIdByUid(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->id.uid); pExpr->param[0].i64Key = colId; pExpr->numOfParams = 1; @@ -499,7 +499,7 @@ static void issueTSCompQuery(SSqlObj* pSql, SJoinSupporter* pSupporter, SSqlObj* // set the tags value for ts_comp function if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, 0); - int16_t tagColId = tscGetJoinTagColIdByUid(&pSupporter->tagCond, pTableMetaInfo->pTableMeta->uid); + int16_t tagColId = tscGetJoinTagColIdByUid(&pSupporter->tagCond, pTableMetaInfo->pTableMeta->id.uid); pExpr->param->i64Key = tagColId; pExpr->numOfParams = 1; } @@ -560,7 +560,7 @@ static int32_t getIntersectionOfTableTuple(SQueryInfo* pQueryInfo, SSqlObj* pPar qsort(p2->pIdTagList, p2->num, p2->tagSize, tscCompareTidTags); STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - int16_t tagColId = tscGetJoinTagColIdByUid(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->uid); + int16_t tagColId = tscGetJoinTagColIdByUid(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->id.uid); SSchema* pColSchema = tscGetTableColumnSchemaById(pTableMetaInfo->pTableMeta, tagColId); @@ -1034,7 +1034,7 @@ void tscSetupOutputColumnIndex(SSqlObj* pSql) { int32_t tableIndexOfSub = -1; for (int32_t j = 0; j < pQueryInfo->numOfTables; ++j) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, j); - if (pTableMetaInfo->pTableMeta->uid == pExpr->uid) { + if (pTableMetaInfo->pTableMeta->id.uid == pExpr->uid) { tableIndexOfSub = j; break; } @@ -1205,7 +1205,7 @@ int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter STagCond* pTagCond = &pSupporter->tagCond; assert(pTagCond->joinInfo.hasJoin); - int32_t tagColId = tscGetJoinTagColIdByUid(pTagCond, pTableMetaInfo->pTableMeta->uid); + int32_t tagColId = tscGetJoinTagColIdByUid(pTagCond, pTableMetaInfo->pTableMeta->id.uid); SSchema* s = tscGetTableColumnSchemaById(pTableMetaInfo->pTableMeta, tagColId); int16_t bytes = 0; @@ -1237,7 +1237,7 @@ int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter SSqlExpr *pExpr = tscSqlExprGet(pNewQueryInfo, 0); if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { - int16_t tagColId = tscGetJoinTagColIdByUid(&pSupporter->tagCond, pTableMetaInfo->pTableMeta->uid); + int16_t tagColId = tscGetJoinTagColIdByUid(&pSupporter->tagCond, pTableMetaInfo->pTableMeta->id.uid); pExpr->param->i64Key = tagColId; pExpr->numOfParams = 1; } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index e7fa2a84a9..73d6f0e592 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -955,7 +955,7 @@ static SSqlExpr* doBuildSqlExpr(SQueryInfo* pQueryInfo, int16_t functionId, SCol pExpr->interBytes = interSize; if (pTableMetaInfo->pTableMeta) { - pExpr->uid = pTableMetaInfo->pTableMeta->uid; + pExpr->uid = pTableMetaInfo->pTableMeta->id.uid; } return pExpr; @@ -1482,7 +1482,7 @@ STableMetaInfo* tscGetTableMetaInfoByUid(SQueryInfo* pQueryInfo, uint64_t uid, i int32_t k = -1; for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { - if (pQueryInfo->pTableMetaInfo[i]->pTableMeta->uid == uid) { + if (pQueryInfo->pTableMetaInfo[i]->pTableMeta->id.uid == uid) { k = i; break; } @@ -1760,7 +1760,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void TSDB_QUERY_SET_TYPE(pNewQueryInfo->type, TSDB_QUERY_TYPE_SUBQUERY);// it must be the subquery } - uint64_t uid = pTableMetaInfo->pTableMeta->uid; + uint64_t uid = pTableMetaInfo->pTableMeta->id.uid; tscSqlExprCopy(pNewQueryInfo->exprList, pQueryInfo->exprList, uid, true); int32_t numOfOutput = tscSqlExprNumOfExprs(pNewQueryInfo); diff --git a/src/dnode/src/dnodeVWrite.c b/src/dnode/src/dnodeVWrite.c index dc09a03e14..51bc8890fc 100644 --- a/src/dnode/src/dnodeVWrite.c +++ b/src/dnode/src/dnodeVWrite.c @@ -38,6 +38,8 @@ typedef struct { typedef struct { SRspRet rspRet; + int32_t processedCount; + int32_t code; void *pCont; int32_t contLen; SRpcMsg rpcMsg; @@ -187,13 +189,16 @@ void dnodeFreeVnodeWqueue(void *wqueue) { void dnodeSendRpcVnodeWriteRsp(void *pVnode, void *param, int32_t code) { SWriteMsg *pWrite = (SWriteMsg *)param; - if (code > 0) return; + if (code < 0) pWrite->code = code; + int32_t count = atomic_add_fetch_32(&pWrite->processedCount, 1); + + if (count <= 1) return; SRpcMsg rpcRsp = { .handle = pWrite->rpcMsg.handle, .pCont = pWrite->rspRet.rsp, .contLen = pWrite->rspRet.len, - .code = code, + .code = pWrite->code, }; rpcSendResponse(&rpcRsp); @@ -239,7 +244,10 @@ static void *dnodeProcessWriteQueue(void *param) { } int32_t code = vnodeProcessWrite(pVnode, type, pHead, pRspRet); - if (pWrite) pWrite->rpcMsg.code = code; + if (pWrite) { + pWrite->rpcMsg.code = code; + if (code <= 0) pWrite->processedCount = 1; + } } walFsync(vnodeGetWal(pVnode)); diff --git a/src/inc/query.h b/src/inc/query.h index ccff05bc1b..ec1e458b62 100644 --- a/src/inc/query.h +++ b/src/inc/query.h @@ -40,9 +40,6 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryTableMs */ bool qTableQuery(qinfo_t qinfo); -void* pGetRspMsg(qinfo_t qinfo); - - /** * Retrieve the produced results information, if current query is not paused or completed, * this function will be blocked to wait for the query execution completed or paused, @@ -65,6 +62,11 @@ int32_t qRetrieveQueryResultInfo(qinfo_t qinfo, bool* buildRes, void* pRspContex */ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp** pRsp, int32_t* contLen, bool* continueExec); +/** + * + * @param qinfo + * @return + */ void* qGetResultRetrieveMsg(qinfo_t qinfo); /** @@ -85,7 +87,7 @@ void qQueryMgmtNotifyClosed(void* pExecutor); void qCleanupQueryMgmt(void* pExecutor); void** qRegisterQInfo(void* pMgmt, uint64_t qInfo); void** qAcquireQInfo(void* pMgmt, uint64_t key); -void** qReleaseQInfo(void* pMgmt, void* pQInfo, bool needFree); +void** qReleaseQInfo(void* pMgmt, void* pQInfo, bool freeHandle); #ifdef __cplusplus } diff --git a/src/inc/tsync.h b/src/inc/tsync.h index 65b91d87e4..2dfac7ec32 100644 --- a/src/inc/tsync.h +++ b/src/inc/tsync.h @@ -26,6 +26,7 @@ extern "C" { typedef enum _TAOS_SYNC_ROLE { TAOS_SYNC_ROLE_OFFLINE, TAOS_SYNC_ROLE_UNSYNCED, + TAOS_SYNC_ROLE_SYNCING, TAOS_SYNC_ROLE_SLAVE, TAOS_SYNC_ROLE_MASTER, } ESyncRole; @@ -78,6 +79,9 @@ typedef void (*FConfirmForward)(void *ahandle, void *mhandle, int32_t code); // when role is changed, call this to notify app typedef void (*FNotifyRole)(void *ahandle, int8_t role); +// if a number of retrieving data failed, call this to start flow control +typedef void (*FNotifyFlowCtrl)(void *ahandle, int32_t mseconds); + // when data file is synced successfully, notity app typedef int (*FNotifyFileSynced)(void *ahandle, uint64_t fversion); @@ -93,6 +97,7 @@ typedef struct { FWriteToCache writeToCache; FConfirmForward confirmForward; FNotifyRole notifyRole; + FNotifyFlowCtrl notifyFlowCtrl; FNotifyFileSynced notifyFileSynced; } SSyncInfo; diff --git a/src/mnode/src/mnodeMnode.c b/src/mnode/src/mnodeMnode.c index d2f389ca0b..2fff885fca 100644 --- a/src/mnode/src/mnodeMnode.c +++ b/src/mnode/src/mnodeMnode.c @@ -190,18 +190,7 @@ void *mnodeGetNextMnode(void *pIter, SMnodeObj **pMnode) { } char *mnodeGetMnodeRoleStr(int32_t role) { - switch (role) { - case TAOS_SYNC_ROLE_OFFLINE: - return "offline"; - case TAOS_SYNC_ROLE_UNSYNCED: - return "unsynced"; - case TAOS_SYNC_ROLE_SLAVE: - return "slave"; - case TAOS_SYNC_ROLE_MASTER: - return "master"; - default: - return "undefined"; - } + return syncRole[role]; } void mnodeUpdateMnodeEpSet() { diff --git a/src/os/inc/osDarwin64.h b/src/os/inc/osDarwin64.h index 2bb0777ab8..3ceb7ea8f5 100644 --- a/src/os/inc/osDarwin64.h +++ b/src/os/inc/osDarwin64.h @@ -205,7 +205,6 @@ ssize_t tread(int fd, void *buf, size_t count); ssize_t twrite(int fd, void *buf, size_t n); -char *taosCharsetReplace(char *charsetstr); bool taosCheckPthreadValid(pthread_t thread); @@ -219,8 +218,6 @@ int taosSetSockOpt(int socketfd, int level, int optname, void *optval, int optle void taosPrintOsInfo(); -char *taosCharsetReplace(char *charsetstr); - void taosPrintOsInfo(); void taosGetSystemInfo(); diff --git a/src/os/inc/osSpec.h b/src/os/inc/osSpec.h index b7b7bd9ff4..48ec6ae04f 100644 --- a/src/os/inc/osSpec.h +++ b/src/os/inc/osSpec.h @@ -20,6 +20,89 @@ extern "C" { #endif +#define tclose(x) taosCloseSocket(x) +#define tfree(x) \ + do { \ + if (x) { \ + free((void *)(x)); \ + x = 0; \ + } \ + } while (0); + +#define tstrncpy(dst, src, size) \ + do { \ + strncpy((dst), (src), (size)); \ + (dst)[(size)-1] = 0; \ + } while (0); + +#ifndef STDERR_FILENO +#define STDERR_FILENO (2) +#endif + +#define FD_VALID(x) ((x) > STDERR_FILENO) +#define FD_INITIALIZER ((int32_t)-1) + +#define WCHAR wchar_t + +#define POINTER_SHIFT(p, b) ((void *)((char *)(p) + (b))) +#define POINTER_DISTANCE(p1, p2) ((char *)(p1) - (char *)(p2)) + +#ifndef NDEBUG +#define ASSERT(x) assert(x) +#else +#define ASSERT(x) +#endif + +#ifdef UNUSED +#undefine UNUSED +#endif +#define UNUSED(x) ((void)(x)) + +#ifdef UNUSED_FUNC +#undefine UNUSED_FUNC +#endif + +#ifdef UNUSED_PARAM +#undef UNUSED_PARAM +#endif + +#if defined(__GNUC__) +#define UNUSED_PARAM(x) _UNUSED##x __attribute__((unused)) +#define UNUSED_FUNC __attribute__((unused)) +#else +#define UNUSED_PARAM(x) x +#define UNUSED_FUNC +#endif + +#ifdef tListLen +#undefine tListLen +#endif +#define tListLen(x) (sizeof(x) / sizeof((x)[0])) + +#if defined(__GNUC__) +#define FORCE_INLINE inline __attribute__((always_inline)) +#else +#define FORCE_INLINE +#endif + +#define DEFAULT_UNICODE_ENCODEC "UCS-4LE" + +#define DEFAULT_COMP(x, y) \ + do { \ + if ((x) == (y)) { \ + return 0; \ + } else { \ + return (x) < (y) ? -1 : 1; \ + } \ + } while (0) + +#define ALIGN_NUM(n, align) (((n) + ((align)-1)) & (~((align)-1))) + +// align to 8bytes +#define ALIGN8(n) ALIGN_NUM(n, 8) + +#define POW2(x) ((x) * (x)) + #ifndef TAOS_OS_FUNC_MATH #define SWAP(a, b, c) \ do { \ @@ -46,6 +129,12 @@ extern "C" { #ifndef TAOS_OS_DEF_TIME #define MILLISECOND_PER_SECOND ((int64_t)1000L) #endif +#define MILLISECOND_PER_MINUTE (MILLISECOND_PER_SECOND * 60) +#define MILLISECOND_PER_HOUR (MILLISECOND_PER_MINUTE * 60) +#define MILLISECOND_PER_DAY (MILLISECOND_PER_HOUR * 24) +#define MILLISECOND_PER_WEEK (MILLISECOND_PER_DAY * 7) +#define MILLISECOND_PER_MONTH (MILLISECOND_PER_DAY * 30) +#define MILLISECOND_PER_YEAR (MILLISECOND_PER_DAY * 365) #ifndef TAOS_OS_FUNC_SEMPHONE #define tsem_t sem_t @@ -144,24 +233,13 @@ extern "C" { ssize_t taosTReadImp(int fd, void *buf, size_t count); ssize_t taosTWriteImp(int fd, void *buf, size_t count); ssize_t taosTSendFileImp(int dfd, int sfd, off_t *offset, size_t size); -#ifndef TAOS_OS_FUNC_FILE +#ifndef TAOS_OS_FUNC_FILE_OP #define taosTRead(fd, buf, count) taosTReadImp(fd, buf, count) #define taosTWrite(fd, buf, count) taosTWriteImp(fd, buf, count) #define taosLSeek(fd, offset, whence) lseek(fd, offset, whence) #define taosTSendFile(dfd, sfd, offset, size) taosTSendFileImp(dfd, sfd, offset, size) #endif -#ifdef TAOS_RANDOM_FILE_FAIL - void taosSetRandomFileFailFactor(int factor); - void taosSetRandomFileFailOutput(const char *path); - ssize_t taosReadFileRandomFail(int fd, void *buf, size_t count, const char *file, uint32_t line); - ssize_t taosWriteFileRandomFail(int fd, void *buf, size_t count, const char *file, uint32_t line); - off_t taosLSeekRandomFail(int fd, off_t offset, int whence, const char *file, uint32_t line); - #define taosTRead(fd, buf, count) taosReadFileRandomFail(fd, buf, count, __FILE__, __LINE__) - #define taosTWrite(fd, buf, count) taosWriteFileRandomFail(fd, buf, count, __FILE__, __LINE__) - #define taosLSeek(fd, offset, whence) taosLSeekRandomFail(fd, offset, whence, __FILE__, __LINE__) -#endif - #ifndef TAOS_OS_FUNC_NETWORK #define taosSend(sockfd, buf, len, flags) send(sockfd, buf, len, flags) #define taosSendto(sockfd, buf, len, flags, dest_addr, addrlen) sendto(sockfd, buf, len, flags, dest_addr, addrlen) @@ -176,17 +254,6 @@ ssize_t taosTSendFileImp(int dfd, int sfd, off_t *offset, size_t size); } #endif -#ifdef TAOS_RANDOM_NETWORK_FAIL - ssize_t taosSendRandomFail(int sockfd, const void *buf, size_t len, int flags); - ssize_t taosSendToRandomFail(int sockfd, const void *buf, size_t len, int flags, const struct sockaddr *dest_addr, socklen_t addrlen); - ssize_t taosReadSocketRandomFail(int fd, void *buf, size_t count); - ssize_t taosWriteSocketRandomFail(int fd, const void *buf, size_t count); - #define taosSend(sockfd, buf, len, flags) taosSendRandomFail(sockfd, buf, len, flags) - #define taosSendto(sockfd, buf, len, flags, dest_addr, addrlen) taosSendToRandomFail(sockfd, buf, len, flags, dest_addr, addrlen) - #define taosReadSocket(fd, buf, len) taosReadSocketRandomFail(fd, buf, len) - #define taosWriteSocket(fd, buf, len) taosWriteSocketRandomFail(fd, buf, len) -#endif - #ifndef TAOS_OS_FUNC_LZ4 #define BUILDIN_CLZL(val) __builtin_clzl(val) #define BUILDIN_CTZL(val) __builtin_ctzl(val) @@ -194,6 +261,10 @@ ssize_t taosTSendFileImp(int dfd, int sfd, off_t *offset, size_t size); #define BUILDIN_CTZ(val) __builtin_ctz(val) #endif +#ifndef TAOS_OS_FUNC_WCHAR + #define twcslen wcslen +#endif + #undef threadlocal #ifdef _ISOC11_SOURCE #define threadlocal _Thread_local @@ -233,6 +304,81 @@ void taosMsleep(int mseconds); int taosInitTimer(void (*callback)(int), int ms); void taosUninitTimer(); +// TAOS_OS_FUNC_RAND +uint32_t taosRand(void); +void taosRandStr(char* str, int32_t size); +uint32_t trand(void); + +// TAOS_OS_FUNC_FILE +void getTmpfilePath(const char *fileNamePrefix, char *dstPath); +int32_t taosFileRename(char *fullPath, char *suffix, char delimiter, char **dstPath); + +// USE_LIBICONV +int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs); +bool taosMbsToUcs4(char *mbs, size_t mbs_len, char *ucs4, int32_t ucs4_max_len, size_t *len); +int tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int bytes); +bool taosValidateEncodec(const char *encodec); +char * taosCharsetReplace(char *charsetstr); + +// TAOS_OS_FUNC_MALLOC +#define TAOS_ALLOC_MODE_DEFAULT 0 +#define TAOS_ALLOC_MODE_RANDOM_FAIL 1 +#define TAOS_ALLOC_MODE_DETECT_LEAK 2 +void taosSetAllocMode(int mode, const char *path, bool autoDump); +void taosDumpMemoryLeak(); +void * tmalloc(size_t size); +void * tcalloc(size_t nmemb, size_t size); +size_t tsizeof(void *ptr); +void tmemset(void *ptr, int c); +void * trealloc(void *ptr, size_t size); +void tzfree(void *ptr); + +// TAOS_OS_FUNC_DIR +void taosRemoveDir(char *rootDir); +int taosMkDir(const char *pathname, mode_t mode); +void taosMvDir(char* destDir, char *srcDir); + + + +#ifdef TAOS_RANDOM_FILE_FAIL + void taosSetRandomFileFailFactor(int factor); + void taosSetRandomFileFailOutput(const char *path); + ssize_t taosReadFileRandomFail(int fd, void *buf, size_t count, const char *file, uint32_t line); + ssize_t taosWriteFileRandomFail(int fd, void *buf, size_t count, const char *file, uint32_t line); + off_t taosLSeekRandomFail(int fd, off_t offset, int whence, const char *file, uint32_t line); + #define taosTRead(fd, buf, count) taosReadFileRandomFail(fd, buf, count, __FILE__, __LINE__) + #define taosTWrite(fd, buf, count) taosWriteFileRandomFail(fd, buf, count, __FILE__, __LINE__) + #define taosLSeek(fd, offset, whence) taosLSeekRandomFail(fd, offset, whence, __FILE__, __LINE__) +#endif + +#ifdef TAOS_RANDOM_NETWORK_FAIL + ssize_t taosSendRandomFail(int sockfd, const void *buf, size_t len, int flags); + ssize_t taosSendToRandomFail(int sockfd, const void *buf, size_t len, int flags, const struct sockaddr *dest_addr, socklen_t addrlen); + ssize_t taosReadSocketRandomFail(int fd, void *buf, size_t count); + ssize_t taosWriteSocketRandomFail(int fd, const void *buf, size_t count); + #define taosSend(sockfd, buf, len, flags) taosSendRandomFail(sockfd, buf, len, flags) + #define taosSendto(sockfd, buf, len, flags, dest_addr, addrlen) taosSendToRandomFail(sockfd, buf, len, flags, dest_addr, addrlen) + #define taosReadSocket(fd, buf, len) taosReadSocketRandomFail(fd, buf, len) + #define taosWriteSocket(fd, buf, len) taosWriteSocketRandomFail(fd, buf, len) +#endif + +#ifdef TAOS_MEM_CHECK + void * taos_malloc(size_t size, const char *file, uint32_t line); + void * taos_calloc(size_t num, size_t size, const char *file, uint32_t line); + void * taos_realloc(void *ptr, size_t size, const char *file, uint32_t line); + void taos_free(void *ptr, const char *file, uint32_t line); + char * taos_strdup(const char *str, const char *file, uint32_t line); + char * taos_strndup(const char *str, size_t size, const char *file, uint32_t line); + ssize_t taos_getline(char **lineptr, size_t *n, FILE *stream, const char *file, uint32_t line); + #define malloc(size) taos_malloc(size, __FILE__, __LINE__) + #define calloc(num, size) taos_calloc(num, size, __FILE__, __LINE__) + #define realloc(ptr, size) taos_realloc(ptr, size, __FILE__, __LINE__) + #define free(ptr) taos_free(ptr, __FILE__, __LINE__) + #define strdup(str) taos_strdup(str, __FILE__, __LINE__) + #define strndup(str, size) taos_strndup(str, size, __FILE__, __LINE__) + #define getline(lineptr, n, stream) taos_getline(lineptr, n, stream, __FILE__, __LINE__) +#endif // TAOS_MEM_CHECK + #ifdef __cplusplus } #endif diff --git a/src/os/inc/osWindows64.h b/src/os/inc/osWindows64.h index 8fd41fb2fd..b4687afb82 100644 --- a/src/os/inc/osWindows64.h +++ b/src/os/inc/osWindows64.h @@ -48,6 +48,8 @@ extern "C" { #endif +#define TAOS_OS_FUNC_WCHAR + // for function open in stat.h #define S_IRWXU _S_IREAD #define S_IRWXG _S_IWRITE @@ -359,7 +361,6 @@ int taosSetNonblocking(int sock, int on); int taosSetSockOpt(int socketfd, int level, int optname, void *optval, int optlen); -char *taosCharsetReplace(char *charsetstr); void taosPrintOsInfo(); diff --git a/src/os/src/detail/osCoredump.c b/src/os/src/detail/osCoredump.c index fe97000c7d..1ead88ab9e 100644 --- a/src/os/src/detail/osCoredump.c +++ b/src/os/src/detail/osCoredump.c @@ -18,7 +18,6 @@ #include "tconfig.h" #include "tglobal.h" #include "tulog.h" -#include "tutil.h" #include "tsystem.h" #ifndef TAOS_OS_FUNC_CORE diff --git a/src/os/src/detail/osDir.c b/src/os/src/detail/osDir.c new file mode 100644 index 0000000000..27555938c2 --- /dev/null +++ b/src/os/src/detail/osDir.c @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "tglobal.h" +#include "tulog.h" + +#ifndef TAOS_OS_FUNC_DIR + +void taosRemoveDir(char *rootDir) { + DIR *dir = opendir(rootDir); + if (dir == NULL) return; + + struct dirent *de = NULL; + while ((de = readdir(dir)) != NULL) { + if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue; + + char filename[1024]; + snprintf(filename, 1023, "%s/%s", rootDir, de->d_name); + if (de->d_type & DT_DIR) { + taosRemoveDir(filename); + } else { + (void)remove(filename); + uInfo("file:%s is removed", filename); + } + } + + closedir(dir); + rmdir(rootDir); + + uInfo("dir:%s is removed", rootDir); +} + +int taosMkDir(const char *path, mode_t mode) { + int code = mkdir(path, 0755); + if (code < 0 && errno == EEXIST) code = 0; + return code; +} + +void taosMvDir(char* destDir, char *srcDir) { + if (0 == tsEnableVnodeBak) { + uInfo("vnode backup not enabled"); + return; + } + + char shellCmd[1024+1] = {0}; + + //(void)snprintf(shellCmd, 1024, "cp -rf %s %s", srcDir, destDir); + (void)snprintf(shellCmd, 1024, "mv %s %s", srcDir, destDir); + tSystem(shellCmd); + uInfo("shell cmd:%s is executed", shellCmd); +} + + +#endif \ No newline at end of file diff --git a/src/os/src/detail/osFile.c b/src/os/src/detail/osFile.c index 897b6c3f03..5502dfb4a6 100644 --- a/src/os/src/detail/osFile.c +++ b/src/os/src/detail/osFile.c @@ -15,74 +15,54 @@ #define _DEFAULT_SOURCE #include "os.h" +#include "ttime.h" -ssize_t taosTReadImp(int fd, void *buf, size_t count) { - size_t leftbytes = count; - ssize_t readbytes; - char * tbuf = (char *)buf; +#ifndef TAOS_OS_FUNC_FILE - while (leftbytes > 0) { - readbytes = read(fd, (void *)tbuf, leftbytes); - if (readbytes < 0) { - if (errno == EINTR) { - continue; - } else { - return -1; - } - } else if (readbytes == 0) { - return (ssize_t)(count - leftbytes); - } - - leftbytes -= readbytes; - tbuf += readbytes; +void getTmpfilePath(const char *fileNamePrefix, char *dstPath) { + const char* tdengineTmpFileNamePrefix = "tdengine-"; + + char tmpPath[PATH_MAX]; + char *tmpDir = "/tmp/"; + + strcpy(tmpPath, tmpDir); + strcat(tmpPath, tdengineTmpFileNamePrefix); + if (strlen(tmpPath) + strlen(fileNamePrefix) + strlen("-%d-%s") < PATH_MAX) { + strcat(tmpPath, fileNamePrefix); + strcat(tmpPath, "-%d-%s"); } - - return (ssize_t)count; + + char rand[8] = {0}; + taosRandStr(rand, tListLen(rand) - 1); + snprintf(dstPath, PATH_MAX, tmpPath, getpid(), rand); } -ssize_t taosTSendFileImp(int dfd, int sfd, off_t *offset, size_t size) { - size_t leftbytes = size; - ssize_t sentbytes; +// rename file name +int32_t taosFileRename(char *fullPath, char *suffix, char delimiter, char **dstPath) { + int32_t ts = taosGetTimestampSec(); - while (leftbytes > 0) { - /* - * TODO : Think to check if file is larger than 1GB - */ - //if (leftbytes > 1000000000) leftbytes = 1000000000; - sentbytes = sendfile(dfd, sfd, offset, leftbytes); - if (sentbytes == -1) { - if (errno == EINTR) { - continue; - } - else { - return -1; - } - } else if (sentbytes == 0) { - return (ssize_t)(size - leftbytes); - } + char fname[PATH_MAX] = {0}; // max file name length must be less than 255 - leftbytes -= sentbytes; + char *delimiterPos = strrchr(fullPath, delimiter); + if (delimiterPos == NULL) return -1; + + int32_t fileNameLen = 0; + if (suffix) + fileNameLen = snprintf(fname, PATH_MAX, "%s.%d.%s", delimiterPos + 1, ts, suffix); + else + fileNameLen = snprintf(fname, PATH_MAX, "%s.%d", delimiterPos + 1, ts); + + size_t len = (size_t)((delimiterPos - fullPath) + fileNameLen + 1); + if (*dstPath == NULL) { + *dstPath = calloc(1, len + 1); + if (*dstPath == NULL) return -1; } - return size; + strncpy(*dstPath, fullPath, (size_t)(delimiterPos - fullPath + 1)); + strncat(*dstPath, fname, (size_t)fileNameLen); + (*dstPath)[len] = 0; + + return rename(fullPath, *dstPath); } -ssize_t taosTWriteImp(int fd, void *buf, size_t n) { - size_t nleft = n; - ssize_t nwritten = 0; - char *tbuf = (char *)buf; - - while (nleft > 0) { - nwritten = write(fd, (void *)tbuf, nleft); - if (nwritten < 0) { - if (errno == EINTR) { - continue; - } - return -1; - } - nleft -= nwritten; - tbuf += nwritten; - } - - return n; -} +#endif \ No newline at end of file diff --git a/src/os/src/detail/osFileOp.c b/src/os/src/detail/osFileOp.c new file mode 100644 index 0000000000..897b6c3f03 --- /dev/null +++ b/src/os/src/detail/osFileOp.c @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" + +ssize_t taosTReadImp(int fd, void *buf, size_t count) { + size_t leftbytes = count; + ssize_t readbytes; + char * tbuf = (char *)buf; + + while (leftbytes > 0) { + readbytes = read(fd, (void *)tbuf, leftbytes); + if (readbytes < 0) { + if (errno == EINTR) { + continue; + } else { + return -1; + } + } else if (readbytes == 0) { + return (ssize_t)(count - leftbytes); + } + + leftbytes -= readbytes; + tbuf += readbytes; + } + + return (ssize_t)count; +} + +ssize_t taosTSendFileImp(int dfd, int sfd, off_t *offset, size_t size) { + size_t leftbytes = size; + ssize_t sentbytes; + + while (leftbytes > 0) { + /* + * TODO : Think to check if file is larger than 1GB + */ + //if (leftbytes > 1000000000) leftbytes = 1000000000; + sentbytes = sendfile(dfd, sfd, offset, leftbytes); + if (sentbytes == -1) { + if (errno == EINTR) { + continue; + } + else { + return -1; + } + } else if (sentbytes == 0) { + return (ssize_t)(size - leftbytes); + } + + leftbytes -= sentbytes; + } + + return size; +} + +ssize_t taosTWriteImp(int fd, void *buf, size_t n) { + size_t nleft = n; + ssize_t nwritten = 0; + char *tbuf = (char *)buf; + + while (nleft > 0) { + nwritten = write(fd, (void *)tbuf, nleft); + if (nwritten < 0) { + if (errno == EINTR) { + continue; + } + return -1; + } + nleft -= nwritten; + tbuf += nwritten; + } + + return n; +} diff --git a/src/os/src/detail/osMalloc.c b/src/os/src/detail/osMalloc.c new file mode 100644 index 0000000000..66377827da --- /dev/null +++ b/src/os/src/detail/osMalloc.c @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" + +#ifndef TAOS_OS_FUNC_MALLOC + +void *tmalloc(size_t size) { + if (size <= 0) return NULL; + + void *ret = malloc(size + sizeof(size_t)); + if (ret == NULL) return NULL; + + *(size_t *)ret = size; + + return (void *)((char *)ret + sizeof(size_t)); +} + +void *tcalloc(size_t nmemb, size_t size) { + size_t tsize = nmemb * size; + void * ret = tmalloc(tsize); + if (ret == NULL) return NULL; + + tmemset(ret, 0); + return ret; +} + +size_t tsizeof(void *ptr) { return (ptr) ? (*(size_t *)((char *)ptr - sizeof(size_t))) : 0; } + +void tmemset(void *ptr, int c) { memset(ptr, c, tsizeof(ptr)); } + +void * trealloc(void *ptr, size_t size) { + if (ptr == NULL) return tmalloc(size); + + if (size <= tsizeof(ptr)) return ptr; + + void * tptr = (void *)((char *)ptr - sizeof(size_t)); + size_t tsize = size + sizeof(size_t); + tptr = realloc(tptr, tsize); + if (tptr == NULL) return NULL; + + *(size_t *)tptr = size; + + return (void *)((char *)tptr + sizeof(size_t)); +} + +void tzfree(void *ptr) { + if (ptr) { + free((void *)((char *)ptr - sizeof(size_t))); + } +} + +#endif \ No newline at end of file diff --git a/src/util/src/tmem.c b/src/os/src/detail/osMem.c similarity index 99% rename from src/util/src/tmem.c rename to src/os/src/detail/osMem.c index 9c512ad8dc..93943c6543 100644 --- a/src/util/src/tmem.c +++ b/src/os/src/detail/osMem.c @@ -13,13 +13,10 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "os.h" #include "tulog.h" -#define TAOS_MEM_CHECK_IMPL -#include "tutil.h" - - #ifdef TAOS_MEM_CHECK static int allocMode = TAOS_ALLOC_MODE_DEFAULT; diff --git a/src/os/src/detail/osRand.c b/src/os/src/detail/osRand.c new file mode 100644 index 0000000000..7e5f585634 --- /dev/null +++ b/src/os/src/detail/osRand.c @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" + +#ifndef TAOS_OS_FUNC_RAND + +uint32_t taosRand(void) { return rand(); } + +uint32_t trand(void) { + int fd; + int seed; + + fd = open("/dev/urandom", 0); + if (fd < 0) { + seed = time(0); + } else { + int len = read(fd, &seed, sizeof(seed)); + if (len < 0) { + seed = time(0); + } + close(fd); + } + + return (uint32_t)seed; +} + +#endif + +void taosRandStr(char* str, int32_t size) { + const char* set = "abcdefghijklmnopqrstuvwxyz0123456789-_."; + int32_t len = 39; + + for (int32_t i = 0; i < size; ++i) { + str[i] = set[taosRand() % len]; + } +} \ No newline at end of file diff --git a/src/os/src/detail/osSysinfo.c b/src/os/src/detail/osSysinfo.c index ac9b441555..7a395285af 100644 --- a/src/os/src/detail/osSysinfo.c +++ b/src/os/src/detail/osSysinfo.c @@ -18,8 +18,8 @@ #include "tconfig.h" #include "tglobal.h" #include "tulog.h" -#include "tutil.h" #include "tsystem.h" +#include "taosdef.h" #ifndef TAOS_OS_FUNC_SYSINFO diff --git a/src/os/src/detail/osTimer.c b/src/os/src/detail/osTimer.c index 843c29fcb0..e0a2e90314 100644 --- a/src/os/src/detail/osTimer.c +++ b/src/os/src/detail/osTimer.c @@ -13,13 +13,12 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "os.h" #include "taosdef.h" #include "tglobal.h" #include "ttimer.h" #include "tulog.h" -#include "tutil.h" -#include #ifndef TAOS_OS_FUNC_TIMER diff --git a/src/os/src/detail/osWchar.c b/src/os/src/detail/osWchar.c new file mode 100644 index 0000000000..4e02973e50 --- /dev/null +++ b/src/os/src/detail/osWchar.c @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "tglobal.h" +#include "taosdef.h" + +#ifndef TAOS_OS_FUNC_WCHAR + +int tasoUcs4Compare(void* f1_ucs4, void *f2_ucs4, int bytes) { + return wcsncmp((wchar_t *)f1_ucs4, (wchar_t *)f2_ucs4, bytes / TSDB_NCHAR_SIZE); +} + +#endif + +#ifdef USE_LIBICONV +#include "iconv.h" + +int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs) { + iconv_t cd = iconv_open(tsCharset, DEFAULT_UNICODE_ENCODEC); + size_t ucs4_input_len = ucs4_max_len; + size_t outLen = ucs4_max_len; + if (iconv(cd, (char **)&ucs4, &ucs4_input_len, &mbs, &outLen) == -1) { + iconv_close(cd); + return -1; + } + + iconv_close(cd); + return (int32_t)(ucs4_max_len - outLen); +} + +bool taosMbsToUcs4(char *mbs, size_t mbsLength, char *ucs4, int32_t ucs4_max_len, size_t *len) { + memset(ucs4, 0, ucs4_max_len); + iconv_t cd = iconv_open(DEFAULT_UNICODE_ENCODEC, tsCharset); + size_t ucs4_input_len = mbsLength; + size_t outLeft = ucs4_max_len; + if (iconv(cd, &mbs, &ucs4_input_len, &ucs4, &outLeft) == -1) { + iconv_close(cd); + return false; + } + + iconv_close(cd); + if (len != NULL) { + *len = ucs4_max_len - outLeft; + } + + return true; +} + +bool taosValidateEncodec(const char *encodec) { + iconv_t cd = iconv_open(encodec, DEFAULT_UNICODE_ENCODEC); + if (cd == (iconv_t)(-1)) { + return false; + } + + iconv_close(cd); + return true; +} + +#else + +int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs) { + mbstate_t state = {0}; + int32_t len = (int32_t)wcsnrtombs(NULL, (const wchar_t **)&ucs4, ucs4_max_len / 4, 0, &state); + if (len < 0) { + return -1; + } + + memset(&state, 0, sizeof(state)); + len = wcsnrtombs(mbs, (const wchar_t **)&ucs4, ucs4_max_len / 4, (size_t)len, &state); + if (len < 0) { + return -1; + } + + return len; +} + +bool taosMbsToUcs4(char *mbs, size_t mbsLength, char *ucs4, int32_t ucs4_max_len, size_t *len) { + memset(ucs4, 0, ucs4_max_len); + mbstate_t state = {0}; + int32_t retlen = mbsnrtowcs((wchar_t *)ucs4, (const char **)&mbs, mbsLength, ucs4_max_len / 4, &state); + *len = retlen; + + return retlen >= 0; +} + +bool taosValidateEncodec(const char *encodec) { + return true; +} + +#endif + +typedef struct CharsetPair { + char *oldCharset; + char *newCharset; +} CharsetPair; + +char *taosCharsetReplace(char *charsetstr) { + CharsetPair charsetRep[] = { + { "utf8", "UTF-8" }, { "936", "CP936" }, + }; + + for (int32_t i = 0; i < tListLen(charsetRep); ++i) { + if (strcasecmp(charsetRep[i].oldCharset, charsetstr) == 0) { + return strdup(charsetRep[i].newCharset); + } + } + + return strdup(charsetstr); +} \ No newline at end of file diff --git a/src/os/src/windows64/w64File.c b/src/os/src/windows64/w64File.c new file mode 100644 index 0000000000..140277f990 --- /dev/null +++ b/src/os/src/windows64/w64File.c @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" + +void getTmpfilePath(const char *fileNamePrefix, char *dstPath) { + const char* tdengineTmpFileNamePrefix = "tdengine-"; + char tmpPath[PATH_MAX]; + + char *tmpDir = getenv("tmp"); + if (tmpDir == NULL) { + tmpDir = ""; + } + + strcpy(tmpPath, tmpDir); + strcat(tmpPath, tdengineTmpFileNamePrefix); + if (strlen(tmpPath) + strlen(fileNamePrefix) + strlen("-%d-%s") < PATH_MAX) { + strcat(tmpPath, fileNamePrefix); + strcat(tmpPath, "-%d-%s"); + } + + char rand[8] = {0}; + taosRandStr(rand, tListLen(rand) - 1); + snprintf(dstPath, PATH_MAX, tmpPath, getpid(), rand); +} diff --git a/src/os/src/windows64/w64Wchar.c b/src/os/src/windows64/w64Wchar.c new file mode 100644 index 0000000000..d5930d1de9 --- /dev/null +++ b/src/os/src/windows64/w64Wchar.c @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "tglobal.h" +#include "tulog.h" + +size_t twcslen(const wchar_t *wcs) { + int *wstr = (int *)wcs; + if (NULL == wstr) { + return 0; + } + + size_t n = 0; + while (1) { + if (0 == *wstr++) { + break; + } + n++; + } + + return n; +} + +int tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int bytes) { + for (int i = 0; i < bytes; ++i) { + int32_t f1 = *(int32_t *)((char *)f1_ucs4 + i * 4); + int32_t f2 = *(int32_t *)((char *)f2_ucs4 + i * 4); + + if ((f1 == 0 && f2 != 0) || (f1 != 0 && f2 == 0)) { + return f1 - f2; + } else if (f1 == 0 && f2 == 0) { + return 0; + } + + if (f1 != f2) { + return f1 - f2; + } + } + + return 0; + +#if 0 + int32_t ucs4_max_len = bytes + 4; + char *f1_mbs = calloc(bytes, 1); + char *f2_mbs = calloc(bytes, 1); + if (taosUcs4ToMbs(f1_ucs4, ucs4_max_len, f1_mbs) < 0) { + return -1; + } + if (taosUcs4ToMbs(f2_ucs4, ucs4_max_len, f2_mbs) < 0) { + return -1; + } + int32_t ret = strcmp(f1_mbs, f2_mbs); + free(f1_mbs); + free(f2_mbs); + return ret; +#endif +} diff --git a/src/query/inc/qAst.h b/src/query/inc/qAst.h index 00049b486d..01b4c16ac1 100644 --- a/src/query/inc/qAst.h +++ b/src/query/inc/qAst.h @@ -42,7 +42,6 @@ typedef void (*__do_filter_suppl_fn_t)(void *, void *); /** * this structure is used to filter data in tags, so the offset of filtered tag column in tagdata string is required - * */ typedef struct tQueryInfo { uint8_t optr; // expression operator diff --git a/src/query/inc/tsqlfunction.h b/src/query/inc/tsqlfunction.h index 5ce9121cf1..58d2ff260e 100644 --- a/src/query/inc/tsqlfunction.h +++ b/src/query/inc/tsqlfunction.h @@ -22,7 +22,7 @@ extern "C" { #include "os.h" -#include "../../common/inc/tname.h" +#include "tname.h" #include "taosdef.h" #include "trpc.h" #include "tvariant.h" diff --git a/src/query/src/qAst.c b/src/query/src/qAst.c index e3c0c1dbb0..44051c7e3b 100644 --- a/src/query/src/qAst.c +++ b/src/query/src/qAst.c @@ -878,28 +878,6 @@ void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, free(pRightOutput); } -//void tSQLBinaryExprTrv(tExprNode *pExprs, SArray* res) { -// if (pExprs == NULL) { -// return; -// } -// -// tExprNode *pLeft = pExprs->_node.pLeft; -// tExprNode *pRight = pExprs->_node.pRight; -// -// // recursive traverse left child branch -// if (pLeft->nodeType == TSQL_NODE_EXPR) { -// tSQLBinaryExprTrv(pLeft, res); -// } else if (pLeft->nodeType == TSQL_NODE_COL) { -// taosArrayPush(res, &pLeft->pSchema->colId); -// } -// -// if (pRight->nodeType == TSQL_NODE_EXPR) { -// tSQLBinaryExprTrv(pRight, res); -// } else if (pRight->nodeType == TSQL_NODE_COL) { -// taosArrayPush(res, &pRight->pSchema->colId); -// } -//} - static void exprTreeToBinaryImpl(SBufferWriter* bw, tExprNode* expr) { tbufWriteUint8(bw, expr->nodeType); diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index ea0312b875..6efc8a827e 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -6287,10 +6287,17 @@ void qDestroyQueryInfo(qinfo_t qHandle) { freeQInfo(pQInfo); } -static void setQueryResultReady(SQInfo* pQInfo) { +static bool doBuildResCheck(SQInfo* pQInfo) { + bool buildRes = false; + pthread_mutex_lock(&pQInfo->lock); + pQInfo->dataReady = QUERY_RESULT_READY; + buildRes = (pQInfo->rspContext != NULL); + pthread_mutex_unlock(&pQInfo->lock); + + return buildRes; } bool qTableQuery(qinfo_t qinfo) { @@ -6303,16 +6310,13 @@ bool qTableQuery(qinfo_t qinfo) { if (IS_QUERY_KILLED(pQInfo)) { qDebug("QInfo:%p it is already killed, abort", pQInfo); - setQueryResultReady(pQInfo); - return false; + return doBuildResCheck(pQInfo); } if (pQInfo->tableqinfoGroupInfo.numOfTables == 0) { - setQueryStatus(pQInfo->runtimeEnv.pQuery, QUERY_COMPLETED); - setQueryResultReady(pQInfo); - qDebug("QInfo:%p no table exists for query, abort", pQInfo); - return false; + setQueryStatus(pQInfo->runtimeEnv.pQuery, QUERY_COMPLETED); + return doBuildResCheck(pQInfo); } // error occurs, record the error code and return to client @@ -6320,9 +6324,7 @@ bool qTableQuery(qinfo_t qinfo) { if (ret != TSDB_CODE_SUCCESS) { pQInfo->code = ret; qDebug("QInfo:%p query abort due to error/cancel occurs, code:%s", pQInfo, tstrerror(pQInfo->code)); - - setQueryResultReady(pQInfo); - return false; + return doBuildResCheck(pQInfo); } qDebug("QInfo:%p query task is launched", pQInfo); @@ -6347,17 +6349,7 @@ bool qTableQuery(qinfo_t qinfo) { pQInfo, pQuery->rec.rows, pQuery->rec.total + pQuery->rec.rows); } - bool buildRes = false; - pthread_mutex_lock(&pQInfo->lock); - pQInfo->dataReady = QUERY_RESULT_READY; - - if (pQInfo->rspContext != NULL) { - buildRes = true; - } - - - pthread_mutex_unlock(&pQInfo->lock); - return buildRes; + return doBuildResCheck(pQInfo); } int32_t qRetrieveQueryResultInfo(qinfo_t qinfo, bool* buildRes, void* pRspContext) { @@ -6484,7 +6476,6 @@ int32_t qKillQuery(qinfo_t qinfo) { return TSDB_CODE_QRY_INVALID_QHANDLE; } -// sem_post(&pQInfo->dataReady); setQueryKilled(pQInfo); return TSDB_CODE_SUCCESS; } @@ -6749,14 +6740,14 @@ void** qAcquireQInfo(void* pMgmt, uint64_t key) { } } -void** qReleaseQInfo(void* pMgmt, void* pQInfo, bool needFree) { +void** qReleaseQInfo(void* pMgmt, void* pQInfo, bool freeHandle) { SQueryMgmt *pQueryMgmt = pMgmt; if (pQueryMgmt->qinfoPool == NULL) { return NULL; } - taosCacheRelease(pQueryMgmt->qinfoPool, pQInfo, needFree); + taosCacheRelease(pQueryMgmt->qinfoPool, pQInfo, freeHandle); return 0; } diff --git a/src/query/tests/resultBufferTest.cpp b/src/query/tests/resultBufferTest.cpp index 3b74bf1b64..e9611a3232 100644 --- a/src/query/tests/resultBufferTest.cpp +++ b/src/query/tests/resultBufferTest.cpp @@ -10,7 +10,7 @@ namespace { // simple test void simpleTest() { SDiskbasedResultBuf* pResultBuf = NULL; - int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 1000, 64, 1024, 4, NULL); + int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 64, 1024, 4096, NULL); int32_t pageId = 0; int32_t groupId = 0; @@ -52,7 +52,7 @@ void simpleTest() { void writeDownTest() { SDiskbasedResultBuf* pResultBuf = NULL; - int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 1000, 64, 1024, 4, NULL); + int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 64, 1024, 4*1024, NULL); int32_t pageId = 0; int32_t writePageId = 0; @@ -99,7 +99,7 @@ void writeDownTest() { void recyclePageTest() { SDiskbasedResultBuf* pResultBuf = NULL; - int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 1000, 64, 1024, 4, NULL); + int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 64, 1024, 4*1024, NULL); int32_t pageId = 0; int32_t writePageId = 0; diff --git a/src/sync/inc/syncInt.h b/src/sync/inc/syncInt.h index d6d86064d6..be1e01cb23 100644 --- a/src/sync/inc/syncInt.h +++ b/src/sync/inc/syncInt.h @@ -125,6 +125,8 @@ typedef struct SsyncPeer { uint64_t sversion; // track the peer version in retrieve process int syncFd; int peerFd; // forward FD + int numOfRetrieves; // number of retrieves tried + int fileChanged; // a flag to indicate file is changed during retrieving process void *timer; void *pConn; int notifyFd; @@ -152,6 +154,7 @@ typedef struct SSyncNode { FWriteToCache writeToCache; FConfirmForward confirmForward; FNotifyRole notifyRole; + FNotifyFlowCtrl notifyFlowCtrl; FNotifyFileSynced notifyFileSynced; pthread_mutex_t mutex; } SSyncNode; diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index 93c4a9402f..cdbf5ab47d 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -71,6 +71,7 @@ static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo); char* syncRole[] = { "offline", "unsynced", + "syncing", "slave", "master" }; @@ -137,6 +138,7 @@ void *syncStart(const SSyncInfo *pInfo) pNode->writeToCache = pInfo->writeToCache; pNode->notifyRole = pInfo->notifyRole; pNode->confirmForward = pInfo->confirmForward; + pNode->notifyFlowCtrl = pInfo->notifyFlowCtrl; pNode->notifyFileSynced = pInfo->notifyFileSynced; pNode->selfIndex = -1; @@ -530,6 +532,16 @@ void syncBroadcastStatus(SSyncNode *pNode) } } +static void syncResetFlowCtrl(SSyncNode *pNode) { + + for (int i = 0; i < pNode->replica; ++i) { + pNode->peerInfo[i]->numOfRetrieves = 0; + } + + if (pNode->notifyFlowCtrl) + (*pNode->notifyFlowCtrl)(pNode->ahandle, 0); +} + static void syncChooseMaster(SSyncNode *pNode) { SSyncPeer *pPeer; int onlineNum = 0; @@ -575,6 +587,7 @@ static void syncChooseMaster(SSyncNode *pNode) { if (index == pNode->selfIndex) { sInfo("vgId:%d, start to work as master", pNode->vgId); nodeRole = TAOS_SYNC_ROLE_MASTER; + syncResetFlowCtrl(pNode); (*pNode->notifyRole)(pNode->ahandle, nodeRole); } else { pPeer = pNode->peerInfo[index]; @@ -706,6 +719,9 @@ static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus peersStatus[], int8_t ne if (peerOldRole != newRole || nodeRole != selfOldRole) syncBroadcastStatus(pNode); + + if (nodeRole != TAOS_SYNC_ROLE_MASTER) + syncResetFlowCtrl(pNode); } static void syncRestartPeer(SSyncPeer *pPeer) { diff --git a/src/sync/src/syncRestore.c b/src/sync/src/syncRestore.c index 34afbc4db6..07d99b916e 100644 --- a/src/sync/src/syncRestore.c +++ b/src/sync/src/syncRestore.c @@ -300,6 +300,8 @@ void *syncRestoreData(void *param) taosBlockSIGPIPE(); __sync_fetch_and_add(&tsSyncNum, 1); + (*pNode->notifyRole)(pNode->ahandle, TAOS_SYNC_ROLE_SYNCING); + if (syncOpenRecvBuffer(pNode) < 0) { sError("%s, failed to allocate recv buffer", pPeer->id); } else { @@ -307,7 +309,6 @@ void *syncRestoreData(void *param) sInfo("%s, it is synced successfully", pPeer->id); nodeRole = TAOS_SYNC_ROLE_SLAVE; syncBroadcastStatus(pNode); - (*pNode->notifyRole)(pNode->ahandle, nodeRole); } else { sError("%s, failed to restore data, restart connection", pPeer->id); nodeRole = TAOS_SYNC_ROLE_UNSYNCED; @@ -315,6 +316,8 @@ void *syncRestoreData(void *param) } } + (*pNode->notifyRole)(pNode->ahandle, nodeRole); + nodeSStatus = TAOS_SYNC_STATUS_INIT; tclose(pPeer->syncFd) syncCloseRecvBuffer(pNode); diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c index 74970017fa..8aa7d101e7 100644 --- a/src/sync/src/syncRetrieve.c +++ b/src/sync/src/syncRetrieve.c @@ -83,6 +83,7 @@ static int syncAreFilesModified(SSyncPeer *pPeer) int code = 0; if (len >0) { sDebug("%s, processed file is changed", pPeer->id); + pPeer->fileChanged = 1; code = 1; } @@ -454,9 +455,11 @@ static int syncRetrieveDataStepByStep(SSyncPeer *pPeer) void *syncRetrieveData(void *param) { - SSyncPeer *pPeer = (SSyncPeer *)param; + SSyncPeer *pPeer = (SSyncPeer *)param; + SSyncNode *pNode = pPeer->pSyncNode; taosBlockSIGPIPE(); + pPeer->fileChanged = 0; pPeer->syncFd = taosOpenTcpClientSocket(pPeer->ip, pPeer->port, 0); if (pPeer->syncFd < 0) { sError("%s, failed to open socket to sync", pPeer->id); @@ -471,6 +474,18 @@ void *syncRetrieveData(void *param) } } + if (pPeer->fileChanged) { + // if file is changed 3 times continuously, start flow control + pPeer->numOfRetrieves++; + if (pPeer->numOfRetrieves >= 2 && pNode->notifyFlowCtrl) + (*pNode->notifyFlowCtrl)(pNode->ahandle, 4 << (pPeer->numOfRetrieves - 2)); + } else { + pPeer->numOfRetrieves = 0; + if (pNode->notifyFlowCtrl) + (*pNode->notifyFlowCtrl)(pNode->ahandle, 0); + } + + pPeer->fileChanged = 0; tclose(pPeer->notifyFd); tclose(pPeer->syncFd); syncDecPeerRef(pPeer); diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 09bbbd8f4d..ddb935b7e5 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -69,8 +69,8 @@ int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg) { if (tid < pMeta->maxTables && pMeta->tables[tid] != NULL) { if (TABLE_UID(pMeta->tables[tid]) == pCfg->tableId.uid) { - tsdbError("vgId:%d table %s already exists, tid %d uid %" PRId64, REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), - TABLE_TID(pTable), TABLE_UID(pTable)); + tsdbError("vgId:%d table %s already exists, tid %d uid %" PRId64, REPO_ID(pRepo), TABLE_CHAR_NAME(pMeta->tables[tid]), + TABLE_TID(pMeta->tables[tid]), TABLE_UID(pMeta->tables[tid])); return TSDB_CODE_TDB_TABLE_ALREADY_EXIST; } else { tsdbError("vgId:%d table %s at tid %d uid %" PRIu64 @@ -1295,4 +1295,4 @@ static int tsdbAdjustMetaTables(STsdbRepo *pRepo, int tid) { tsdbDebug("vgId:%d tsdb meta maxTables is adjusted as %d", REPO_ID(pRepo), maxTables); return 0; -} \ No newline at end of file +} diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index a3907e4719..0e3a657fde 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -30,11 +30,6 @@ #define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) #define QH_GET_NUM_OF_COLS(handle) ((size_t)(taosArrayGetSize((handle)->pColumns))) -enum { - QUERY_RANGE_LESS_EQUAL = 0, - QUERY_RANGE_GREATER_EQUAL = 1, -}; - enum { TSDB_QUERY_TYPE_ALL = 1, TSDB_QUERY_TYPE_LAST = 2, diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index 1969b40443..2521c582d1 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -12,9 +12,10 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) AUX_SOURCE_DIRECTORY(src SRC) ADD_LIBRARY(tutil ${SRC}) TARGET_LINK_LIBRARIES(tutil pthread osdetail m rt lz4) + ADD_SUBDIRECTORY(tests) + FIND_PATH(ICONV_INCLUDE_EXIST iconv.h /usr/include/ /usr/local/include/) IF (ICONV_INCLUDE_EXIST) - ADD_DEFINITIONS(-DUSE_LIBICONV) FIND_PATH(ICONV_LIBRARY_A_EXIST libiconv.a /usr/lib/ /usr/local/lib/ /lib64) FIND_PATH(ICONV_LIBRARY_SO_EXIST libiconv.so /usr/lib/ /usr/local/lib/ /lib64) IF (ICONV_LIBRARY_A_EXIST OR ICONV_LIBRARY_SO_EXIST) @@ -28,9 +29,7 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) MESSAGE(STATUS "Failed to find iconv, use default encoding method") ENDIF () - ADD_SUBDIRECTORY(tests) ELSEIF (TD_WINDOWS_64) - ADD_DEFINITIONS(-DUSE_LIBICONV) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/pthread) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/iconv) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/regex) @@ -70,7 +69,6 @@ ELSEIF (TD_WINDOWS_64) ADD_LIBRARY(tutil ${SRC}) TARGET_LINK_LIBRARIES(tutil iconv regex pthread osdetail winmm IPHLPAPI ws2_32 lz4) ELSEIF(TD_DARWIN_64) - ADD_DEFINITIONS(-DUSE_LIBICONV) AUX_SOURCE_DIRECTORY(src SRC) ADD_LIBRARY(tutil ${SRC}) TARGET_LINK_LIBRARIES(tutil iconv pthread osdetail lz4) diff --git a/src/util/inc/talgo.h b/src/util/inc/talgo.h index 5de8a302b1..e71e340a21 100644 --- a/src/util/inc/talgo.h +++ b/src/util/inc/talgo.h @@ -20,8 +20,6 @@ extern "C" { #endif -#include "os.h" - #define TD_EQ 0x1 #define TD_GT 0x2 #define TD_LT 0x4 diff --git a/src/util/inc/ttime.h b/src/util/inc/ttime.h index 576c9a51f6..25d1bdb23e 100644 --- a/src/util/inc/ttime.h +++ b/src/util/inc/ttime.h @@ -20,9 +20,8 @@ extern "C" { #endif -#include -#include -#include "tutil.h" +#include "os.h" +#include "taosdef.h" //@return timestamp in second int32_t taosGetTimestampSec(); diff --git a/src/util/inc/tutil.h b/src/util/inc/tutil.h index 4c003eeb2c..e312513ff3 100644 --- a/src/util/inc/tutil.h +++ b/src/util/inc/tutil.h @@ -25,148 +25,24 @@ extern "C" { #include "tcrc32c.h" #include "taosdef.h" -#ifndef STDERR_FILENO -#define STDERR_FILENO (2) -#endif - -#define FD_VALID(x) ((x) > STDERR_FILENO) -#define FD_INITIALIZER ((int32_t)-1) - -#define WCHAR wchar_t - -#define tfree(x) \ - do { \ - if (x) { \ - free((void *)(x)); \ - x = 0; \ - } \ - } while(0); - -#define tstrncpy(dst, src, size) \ - do { \ - strncpy((dst), (src), (size)); \ - (dst)[(size)-1] = 0; \ - } while (0); - -#define tclose(x) taosCloseSocket(x) - -// Pointer p drift right by b bytes -#define POINTER_SHIFT(p, b) ((void *)((char *)(p) + (b))) -#define POINTER_DISTANCE(p1, p2) ((char *)(p1) - (char *)(p2)) - -#ifndef NDEBUG -#define ASSERT(x) assert(x) -#else -#define ASSERT(x) -#endif - -#ifdef UNUSED -#undefine UNUSED -#endif -#define UNUSED(x) ((void)(x)) - -#ifdef UNUSED_FUNC -#undefine UNUSED_FUNC -#endif - -#ifdef UNUSED_PARAM -#undef UNUSED_PARAM -#endif - -#if defined(__GNUC__) -#define UNUSED_PARAM(x) _UNUSED##x __attribute__((unused)) -#define UNUSED_FUNC __attribute__((unused)) -#else -#define UNUSED_PARAM(x) x -#define UNUSED_FUNC -#endif - -#ifdef tListLen -#undefine tListLen -#endif -#define tListLen(x) (sizeof(x) / sizeof((x)[0])) - -#if defined(__GNUC__) -#define FORCE_INLINE inline __attribute__((always_inline)) -#else -#define FORCE_INLINE -#endif - -#define DEFAULT_UNICODE_ENCODEC "UCS-4LE" - -#define DEFAULT_COMP(x, y) \ - do { \ - if ((x) == (y)) { \ - return 0; \ - } else { \ - return (x) < (y) ? -1 : 1; \ - } \ - } while (0) - -#define ALIGN_NUM(n, align) (((n) + ((align)-1)) & (~((align)-1))) - -// align to 8bytes -#define ALIGN8(n) ALIGN_NUM(n, 8) - -#define MILLISECOND_PER_MINUTE (MILLISECOND_PER_SECOND * 60) -#define MILLISECOND_PER_HOUR (MILLISECOND_PER_MINUTE * 60) -#define MILLISECOND_PER_DAY (MILLISECOND_PER_HOUR * 24) -#define MILLISECOND_PER_WEEK (MILLISECOND_PER_DAY * 7) -#define MILLISECOND_PER_MONTH (MILLISECOND_PER_DAY * 30) -#define MILLISECOND_PER_YEAR (MILLISECOND_PER_DAY * 365) - -#define POW2(x) ((x) * (x)) - -uint32_t taosRand(void); - -uint32_t trand(void); - -size_t twcslen(const wchar_t *wcs); - int32_t strdequote(char *src); - -size_t strtrim(char *src); - -char *strnchr(char *haystack, char needle, int32_t len, bool skipquote); - -char **strsplit(char *src, const char *delim, int32_t *num); - -char* strtolower(char *dst, const char *src); - -char* strntolower(char *dst, const char *src, int32_t n); - +size_t strtrim(char *src); +char * strnchr(char *haystack, char needle, int32_t len, bool skipquote); +char ** strsplit(char *src, const char *delim, int32_t *num); +char * strtolower(char *dst, const char *src); +char * strntolower(char *dst, const char *src, int32_t n); int64_t strnatoi(char *num, int32_t len); - -//char* strreplace(const char* str, const char* pattern, const char* rep); - -char *strbetween(char *string, char *begin, char *end); - -char *paGetToken(char *src, char **token, int32_t *tokenLen); +char * strbetween(char *string, char *begin, char *end); +char * paGetToken(char *src, char **token, int32_t *tokenLen); int32_t taosByteArrayToHexStr(char bytes[], int32_t len, char hexstr[]); - int32_t taosHexStrToByteArray(char hexstr[], char bytes[]); -int32_t taosFileRename(char *fullPath, char *suffix, char delimiter, char **dstPath); - -/** - * - * @param fileNamePattern - * @param dstPath - */ -void getTmpfilePath(const char *fileNamePattern, char *dstPath); - -bool taosMbsToUcs4(char *mbs, size_t mbs_len, char *ucs4, int32_t ucs4_max_len, size_t* len); - -int tasoUcs4Compare(void* f1_ucs4, void *f2_ucs4, int bytes); - -void taosRandStr(char* str, int32_t size); - -int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs); - -bool taosValidateEncodec(const char *encodec); - bool taosGetVersionNumber(char *versionStr, int *versionNubmer); +int taosCheckVersion(char *input_client_version, char *input_server_version, int compared_segments); + +char * taosIpStr(uint32_t ipInt); +uint32_t ip2uint(const char *const ip_addr); static FORCE_INLINE void taosEncryptPass(uint8_t *inBuf, unsigned int inLen, char *target) { MD5_CTX context; @@ -176,56 +52,6 @@ static FORCE_INLINE void taosEncryptPass(uint8_t *inBuf, unsigned int inLen, cha memcpy(target, context.digest, TSDB_KEY_LEN); } -int taosCheckVersion(char *input_client_version, char *input_server_version, int compared_segments); - -char *taosIpStr(uint32_t ipInt); - -uint32_t ip2uint(const char *const ip_addr); - -void taosRemoveDir(char *rootDir); -int tmkdir(const char *pathname, mode_t mode); -void taosMvDir(char* destDir, char *srcDir); - -#define TAOS_ALLOC_MODE_DEFAULT 0 -#define TAOS_ALLOC_MODE_RANDOM_FAIL 1 -#define TAOS_ALLOC_MODE_DETECT_LEAK 2 -void taosSetAllocMode(int mode, const char* path, bool autoDump); -void taosDumpMemoryLeak(); - -void * tmalloc(size_t size); -void * tcalloc(size_t nmemb, size_t size); -size_t tsizeof(void *ptr); -void tmemset(void *ptr, int c); -void * trealloc(void *ptr, size_t size); -void tzfree(void *ptr); - -#ifdef TAOS_MEM_CHECK - -void * taos_malloc(size_t size, const char *file, uint32_t line); -void * taos_calloc(size_t num, size_t size, const char *file, uint32_t line); -void * taos_realloc(void *ptr, size_t size, const char *file, uint32_t line); -void taos_free(void *ptr, const char *file, uint32_t line); -char * taos_strdup(const char *str, const char *file, uint32_t line); -char * taos_strndup(const char *str, size_t size, const char *file, uint32_t line); -ssize_t taos_getline(char **lineptr, size_t *n, FILE *stream, const char *file, uint32_t line); - -#ifndef TAOS_MEM_CHECK_IMPL - -#define malloc(size) taos_malloc(size, __FILE__, __LINE__) -#define calloc(num, size) taos_calloc(num, size, __FILE__, __LINE__) -#define realloc(ptr, size) taos_realloc(ptr, size, __FILE__, __LINE__) -#define free(ptr) taos_free(ptr, __FILE__, __LINE__) -#define strdup(str) taos_strdup(str, __FILE__, __LINE__) -#define strndup(str, size) taos_strndup(str, size, __FILE__, __LINE__) -#define getline(lineptr, n, stream) taos_getline(lineptr, n, stream, __FILE__, __LINE__) - -#endif // TAOS_MEM_CHECK_IMPL - -#endif // TAOS_MEM_CHECK - - -char *taosCharsetReplace(char *charsetstr); - #ifdef __cplusplus } #endif diff --git a/src/util/src/hash.c b/src/util/src/hash.c index 5dff6286f0..695529d02f 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -14,10 +14,9 @@ */ #include "os.h" - #include "hash.h" #include "tulog.h" -#include "tutil.h" +#include "taosdef.h" static FORCE_INLINE void __wr_lock(void *lock) { if (lock == NULL) { diff --git a/src/util/src/tconfig.c b/src/util/src/tconfig.c index d561e8ba5f..59bc09e877 100644 --- a/src/util/src/tconfig.c +++ b/src/util/src/tconfig.c @@ -118,7 +118,7 @@ static void taosReadDirectoryConfig(SGlobalCfg *cfg, char *input_value) { wordfree(&full_path); - int code = tmkdir(option, 0755); + int code = taosMkDir(option, 0755); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(errno); uError("config option:%s, input value:%s, directory not exist, create fail:%s", diff --git a/src/util/src/tsocket.c b/src/util/src/tsocket.c index 6c536a4303..6f4d2b8afd 100644 --- a/src/util/src/tsocket.c +++ b/src/util/src/tsocket.c @@ -304,10 +304,10 @@ int taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clientI //uError("failed to connect socket, ip:0x%x, port:%hu(%s)", destIp, destPort, strerror(errno)); close(sockFd); sockFd = -1; + } else { + taosKeepTcpAlive(sockFd); } - // taosKeepTcpAlive(sockFd); - return sockFd; } @@ -349,7 +349,7 @@ int taosKeepTcpAlive(int sockFd) { struct linger linger = {0}; linger.l_onoff = 1; - //linger.l_linger = 0; + linger.l_linger = 3; if (taosSetSockOpt(sockFd, SOL_SOCKET, SO_LINGER, (void *)&linger, sizeof(linger)) < 0) { uError("setsockopt SO_LINGER failed: %d (%s)", errno, strerror(errno)); close(sockFd); diff --git a/src/util/src/tutil.c b/src/util/src/tutil.c index 7b73b1e17c..eaa60d739b 100644 --- a/src/util/src/tutil.c +++ b/src/util/src/tutil.c @@ -14,11 +14,6 @@ */ #include "os.h" - -#ifdef USE_LIBICONV -#include "iconv.h" -#endif - #include "tcrc32c.h" #include "tglobal.h" #include "ttime.h" @@ -27,76 +22,6 @@ #include "tulog.h" #include "taoserror.h" - -#ifdef WINDOWS -uint32_t taosRand(void) -{ - return rand(); -} -#else -uint32_t taosRand(void) -{ -/* - int fd; - int seed; - - fd = open("/dev/urandom", 0); - if (fd < 0) { - seed = time(0); - } else { - int len = read(fd, &seed, sizeof(seed)); - if (len < 0) { - seed = time(0); - } - close(fd); - } - - return (uint32_t)seed; -*/ - return rand(); -} - -uint32_t trand(void) -{ - int fd; - int seed; - - fd = open("/dev/urandom", 0); - if (fd < 0) { - seed = time(0); - } else { - int len = read(fd, &seed, sizeof(seed)); - if (len < 0) { - seed = time(0); - } - close(fd); - } - - return (uint32_t)seed; -} -#endif - -size_t twcslen(const wchar_t *wcs) { -#ifdef WINDOWS - int *wstr = (int *)wcs; - if (NULL == wstr) { - return 0; - } - - size_t n = 0; - while (1) { - if (0 == *wstr++) { - break; - } - n++; - } - - return n; -#else - return wcslen(wcs); -#endif -} - int32_t strdequote(char *z) { if (z == NULL) { return 0; @@ -234,6 +159,7 @@ char* strtolower(char *dst, const char *src) { *p = 0; return dst; } + char* strntolower(char *dst, const char *src, int32_t n) { int esc = 0; char quote = 0, *p = dst, c; @@ -343,96 +269,6 @@ int64_t strnatoi(char *num, int32_t len) { return ret; } -#if 0 -FORCE_INLINE size_t getLen(size_t old, size_t size) { - if (old == 1) { - old = 2; - } - - while (old < size) { - old = (old * 1.5); - } - - return old; -} - -static char *ensureSpace(char *dest, size_t *curSize, size_t size) { - if (*curSize < size) { - *curSize = getLen(*curSize, size); - - char *tmp = realloc(dest, *curSize); - if (tmp == NULL) { - free(dest); - return NULL; - } - - return tmp; - } - - return dest; -} - -char *strreplace(const char *str, const char *pattern, const char *rep) { - if (str == NULL || pattern == NULL || rep == NULL) { - return NULL; - } - - const char *s = str; - - size_t oldLen = strlen(str); - size_t newLen = oldLen; - - size_t repLen = strlen(rep); - size_t patternLen = strlen(pattern); - - char *dest = calloc(1, oldLen + 1); - if (dest == NULL) { - return NULL; - } - - if (patternLen == 0) { - return strcpy(dest, str); - } - - int32_t start = 0; - - while (1) { - char *p = strstr(str, pattern); - if (p == NULL) { // remain does not contain pattern - size_t remain = (oldLen - (str - s)); - size_t size = remain + start + 1; - - dest = ensureSpace(dest, &newLen, size); - if (dest == NULL) { - return NULL; - } - - strcpy(dest + start, str); - dest[start + remain] = 0; - break; - } - - size_t len = p - str; - size_t size = start + len + repLen + 1; - - dest = ensureSpace(dest, &newLen, size); - if (dest == NULL) { - return NULL; - } - - memcpy(dest + start, str, len); - - str += (len + patternLen); - start += len; - - memcpy(dest + start, rep, repLen); - start += repLen; - } - - return dest; -} -#endif - char *strbetween(char *string, char *begin, char *end) { char *result = NULL; char *_begin = strstr(string, begin); @@ -491,170 +327,6 @@ int32_t taosHexStrToByteArray(char hexstr[], char bytes[]) { return 0; } -// rename file name -int32_t taosFileRename(char *fullPath, char *suffix, char delimiter, char **dstPath) { - int32_t ts = taosGetTimestampSec(); - - char fname[PATH_MAX] = {0}; // max file name length must be less than 255 - - char *delimiterPos = strrchr(fullPath, delimiter); - if (delimiterPos == NULL) return -1; - - int32_t fileNameLen = 0; - if (suffix) - fileNameLen = snprintf(fname, PATH_MAX, "%s.%d.%s", delimiterPos + 1, ts, suffix); - else - fileNameLen = snprintf(fname, PATH_MAX, "%s.%d", delimiterPos + 1, ts); - - size_t len = (size_t)((delimiterPos - fullPath) + fileNameLen + 1); - if (*dstPath == NULL) { - *dstPath = calloc(1, len + 1); - if (*dstPath == NULL) return -1; - } - - strncpy(*dstPath, fullPath, (size_t)(delimiterPos - fullPath + 1)); - strncat(*dstPath, fname, (size_t)fileNameLen); - (*dstPath)[len] = 0; - - return rename(fullPath, *dstPath); -} - -void getTmpfilePath(const char *fileNamePrefix, char *dstPath) { - const char* tdengineTmpFileNamePrefix = "tdengine-"; - - char tmpPath[PATH_MAX]; - -#ifdef WINDOWS - char *tmpDir = getenv("tmp"); - if (tmpDir == NULL) { - tmpDir = ""; - } -#else - char *tmpDir = "/tmp/"; -#endif - - strcpy(tmpPath, tmpDir); - strcat(tmpPath, tdengineTmpFileNamePrefix); - if (strlen(tmpPath) + strlen(fileNamePrefix) + strlen("-%d-%s") < PATH_MAX) { - strcat(tmpPath, fileNamePrefix); - strcat(tmpPath, "-%d-%s"); - } - - char rand[8] = {0}; - taosRandStr(rand, tListLen(rand) - 1); - snprintf(dstPath, PATH_MAX, tmpPath, getpid(), rand); -} - -void taosRandStr(char* str, int32_t size) { - const char* set = "abcdefghijklmnopqrstuvwxyz0123456789-_."; - int32_t len = 39; - - for (int32_t i = 0; i < size; ++i) { - str[i] = set[taosRand() % len]; - } -} - -int tasoUcs4Compare(void* f1_ucs4, void *f2_ucs4, int bytes) { -#if defined WINDOWS - for (int i = 0; i < bytes; ++i) { - int32_t f1 = *(int32_t*)((char*)f1_ucs4 + i * 4); - int32_t f2 = *(int32_t*)((char*)f2_ucs4 + i * 4); - - if ((f1 == 0 && f2 != 0) || (f1 != 0 && f2 == 0)) { - return f1 - f2; - } - else if (f1 == 0 && f2 == 0) { - return 0; - } - - if (f1 != f2) { - return f1 - f2; - } - } - return 0; - -#if 0 - int32_t ucs4_max_len = bytes + 4; - char *f1_mbs = calloc(bytes, 1); - char *f2_mbs = calloc(bytes, 1); - if (taosUcs4ToMbs(f1_ucs4, ucs4_max_len, f1_mbs) < 0) { - return -1; - } - if (taosUcs4ToMbs(f2_ucs4, ucs4_max_len, f2_mbs) < 0) { - return -1; - } - int32_t ret = strcmp(f1_mbs, f2_mbs); - free(f1_mbs); - free(f2_mbs); - return ret; -#endif - -#else - return wcsncmp((wchar_t *)f1_ucs4, (wchar_t *)f2_ucs4, bytes / TSDB_NCHAR_SIZE); -#endif -} - -int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs) { -#ifdef USE_LIBICONV - iconv_t cd = iconv_open(tsCharset, DEFAULT_UNICODE_ENCODEC); - size_t ucs4_input_len = ucs4_max_len; - size_t outLen = ucs4_max_len; - if (iconv(cd, (char **)&ucs4, &ucs4_input_len, &mbs, &outLen) == -1) { - iconv_close(cd); - return -1; - } - iconv_close(cd); - return (int32_t)(ucs4_max_len - outLen); -#else - mbstate_t state = {0}; - int32_t len = (int32_t) wcsnrtombs(NULL, (const wchar_t **) &ucs4, ucs4_max_len / 4, 0, &state); - if (len < 0) { - return -1; - } - memset(&state, 0, sizeof(state)); - len = wcsnrtombs(mbs, (const wchar_t **) &ucs4, ucs4_max_len / 4, (size_t) len, &state); - if (len < 0) { - return -1; - } - return len; -#endif -} - -bool taosMbsToUcs4(char *mbs, size_t mbsLength, char *ucs4, int32_t ucs4_max_len, size_t* len) { - memset(ucs4, 0, ucs4_max_len); -#ifdef USE_LIBICONV - iconv_t cd = iconv_open(DEFAULT_UNICODE_ENCODEC, tsCharset); - size_t ucs4_input_len = mbsLength; - size_t outLeft = ucs4_max_len; - if (iconv(cd, &mbs, &ucs4_input_len, &ucs4, &outLeft) == -1) { - iconv_close(cd); - return false; - } - - iconv_close(cd); - if (len != NULL) { - *len = ucs4_max_len - outLeft; - } - - return true; -#else - mbstate_t state = {0}; - int32_t len = mbsnrtowcs((wchar_t *) ucs4, (const char **) &mbs, mbsLength, ucs4_max_len / 4, &state); - return len >= 0; -#endif -} - -bool taosValidateEncodec(const char *encodec) { -#ifdef USE_LIBICONV - iconv_t cd = iconv_open(encodec, DEFAULT_UNICODE_ENCODEC); - if (cd == (iconv_t)(-1)) { - return false; - } - iconv_close(cd); -#endif - return true; -} - bool taosGetVersionNumber(char *versionStr, int *versionNubmer) { if (versionStr == NULL || versionNubmer == NULL) { return false; @@ -734,112 +406,3 @@ FORCE_INLINE double taos_align_get_double(const char* pBuf) { *(int64_t*)(&dv) = *(int64_t*)pBuf; return dv; } - -typedef struct CharsetPair { - char *oldCharset; - char *newCharset; -} CharsetPair; - -char *taosCharsetReplace(char *charsetstr) { - CharsetPair charsetRep[] = { - { "utf8", "UTF-8" }, { "936", "CP936" }, - }; - - for (int32_t i = 0; i < tListLen(charsetRep); ++i) { - if (strcasecmp(charsetRep[i].oldCharset, charsetstr) == 0) { - return strdup(charsetRep[i].newCharset); - } - } - - return strdup(charsetstr); -} - -void *tmalloc(size_t size) { - if (size <= 0) return NULL; - - void *ret = malloc(size + sizeof(size_t)); - if (ret == NULL) return NULL; - - *(size_t *)ret = size; - - return (void *)((char *)ret + sizeof(size_t)); -} - -void *tcalloc(size_t nmemb, size_t size) { - size_t tsize = nmemb * size; - void * ret = tmalloc(tsize); - if (ret == NULL) return NULL; - - tmemset(ret, 0); - return ret; -} - -size_t tsizeof(void *ptr) { return (ptr) ? (*(size_t *)((char *)ptr - sizeof(size_t))) : 0; } - -void tmemset(void *ptr, int c) { memset(ptr, c, tsizeof(ptr)); } - -void * trealloc(void *ptr, size_t size) { - if (ptr == NULL) return tmalloc(size); - - if (size <= tsizeof(ptr)) return ptr; - - void * tptr = (void *)((char *)ptr - sizeof(size_t)); - size_t tsize = size + sizeof(size_t); - tptr = realloc(tptr, tsize); - if (tptr == NULL) return NULL; - - *(size_t *)tptr = size; - - return (void *)((char *)tptr + sizeof(size_t)); -} - -void tzfree(void *ptr) { - if (ptr) { - free((void *)((char *)ptr - sizeof(size_t))); - } -} - -void taosRemoveDir(char *rootDir) { - DIR *dir = opendir(rootDir); - if (dir == NULL) return; - - struct dirent *de = NULL; - while ((de = readdir(dir)) != NULL) { - if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue; - - char filename[1024]; - snprintf(filename, 1023, "%s/%s", rootDir, de->d_name); - if (de->d_type & DT_DIR) { - taosRemoveDir(filename); - } else { - (void)remove(filename); - uInfo("file:%s is removed", filename); - } - } - - closedir(dir); - rmdir(rootDir); - - uInfo("dir:%s is removed", rootDir); -} - -int tmkdir(const char *path, mode_t mode) { - int code = mkdir(path, 0755); - if (code < 0 && errno == EEXIST) code = 0; - return code; -} - -void taosMvDir(char* destDir, char *srcDir) { - if (0 == tsEnableVnodeBak) { - uInfo("vnode backup not enabled"); - return; - } - - char shellCmd[1024+1] = {0}; - - //(void)snprintf(shellCmd, 1024, "cp -rf %s %s", srcDir, destDir); - (void)snprintf(shellCmd, 1024, "mv %s %s", srcDir, destDir); - tSystem(shellCmd); - uInfo("shell cmd:%s is executed", shellCmd); -} - diff --git a/src/vnode/inc/vnodeInt.h b/src/vnode/inc/vnodeInt.h index 74cfbf1e73..38f7c8e605 100644 --- a/src/vnode/inc/vnodeInt.h +++ b/src/vnode/inc/vnodeInt.h @@ -37,6 +37,7 @@ extern int32_t vDebugFlag; typedef struct { int32_t vgId; // global vnode group ID int32_t refCount; // reference count + int32_t delay; int8_t status; int8_t role; int8_t accessState; diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 77d4503d9d..8a56a2c500 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -44,6 +44,7 @@ static int vnodeProcessTsdbStatus(void *arg, int status); static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int32_t *size, uint64_t *fversion); static int vnodeGetWalInfo(void *ahandle, char *name, uint32_t *index); static void vnodeNotifyRole(void *ahandle, int8_t role); +static void vnodeCtrlFlow(void *handle, int32_t mseconds); static int vnodeNotifyFileSynced(void *ahandle, uint64_t fversion); int32_t vnodeInitResources() { @@ -277,6 +278,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { syncInfo.writeToCache = vnodeWriteToQueue; syncInfo.confirmForward = dnodeSendRpcVnodeWriteRsp; syncInfo.notifyRole = vnodeNotifyRole; + syncInfo.notifyFlowCtrl = vnodeCtrlFlow; syncInfo.notifyFileSynced = vnodeNotifyFileSynced; pVnode->sync = syncStart(&syncInfo); @@ -417,10 +419,7 @@ static void vnodeBuildVloadMsg(SVnodeObj *pVnode, SDMStatusMsg *pStatus) { if (pVnode->status != TAOS_VN_STATUS_READY) return; if (pStatus->openVnodes >= TSDB_MAX_VNODES) return; - // still need report status when unsynced - if (pVnode->syncCfg.replica > 1 && pVnode->role == TAOS_SYNC_ROLE_UNSYNCED) { - } else if (pVnode->tsdb == NULL) { - } else { + if (pVnode->tsdb) { tsdbReportStat(pVnode->tsdb, &pointsWritten, &totalStorage, &compStorage); } @@ -549,6 +548,13 @@ static void vnodeNotifyRole(void *ahandle, int8_t role) { cqStop(pVnode->cq); } +static void vnodeCtrlFlow(void *ahandle, int32_t mseconds) { + SVnodeObj *pVnode = ahandle; + if (pVnode->delay != mseconds) + vInfo("vgId:%d, sync flow control, mseconds:%d", pVnode->vgId, mseconds); + pVnode->delay = mseconds; +} + static int vnodeResetTsdb(SVnodeObj *pVnode) { char rootDir[128] = "\0"; diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index 6b9b8ca4fd..900ff1fbba 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -78,6 +78,8 @@ int32_t vnodeProcessWrite(void *param1, int qtype, void *param2, void *item) { // assign version pVnode->version++; pHead->version = pVnode->version; + if (pVnode->delay) usleep(pVnode->delay*1000); + } else { // from wal or forward // for data from WAL or forward, version may be smaller if (pHead->version <= pVnode->version) return 0; diff --git a/src/wal/src/walMain.c b/src/wal/src/walMain.c index 43869da477..bf3b0f63ea 100644 --- a/src/wal/src/walMain.c +++ b/src/wal/src/walMain.c @@ -101,7 +101,7 @@ void *walOpen(const char *path, const SWalCfg *pCfg) { } } - if (tmkdir(path, 0755) != 0) { + if (taosMkDir(path, 0755) != 0) { terrno = TAOS_SYSTEM_ERROR(errno); wError("wal:%s, failed to create directory(%s)", path, strerror(errno)); walRelease(pWal); @@ -420,7 +420,7 @@ int walHandleExistingFiles(const char *path) { if ( strncmp(ent->d_name, walPrefix, plen) == 0) { snprintf(oname, sizeof(oname), "%s/%s", path, ent->d_name); snprintf(nname, sizeof(nname), "%s/old/%s", path, ent->d_name); - if (tmkdir(opath, 0755) != 0) { + if (taosMkDir(opath, 0755) != 0) { wError("wal:%s, failed to create directory:%s(%s)", oname, opath, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); break; diff --git a/tests/comparisonTest/opentsdb/opentsdbtest/pom.xml b/tests/comparisonTest/opentsdb/opentsdbtest/pom.xml index 4e307db079..52067343ef 100644 --- a/tests/comparisonTest/opentsdb/opentsdbtest/pom.xml +++ b/tests/comparisonTest/opentsdb/opentsdbtest/pom.xml @@ -94,7 +94,7 @@ com.google.guava guava - 24.1.1 + 29.0-jre diff --git a/tests/perftest-scripts/cassandraTestQ1Loop.sh b/tests/perftest-scripts/cassandraTestQ1Loop.sh new file mode 100755 index 0000000000..e65ec2575c --- /dev/null +++ b/tests/perftest-scripts/cassandraTestQ1Loop.sh @@ -0,0 +1,101 @@ +#!/bin/bash + +NUM_LOOP=5 + +function printTo { + if $verbose ; then + echo $1 + fi +} + +CASTESTQ1OUT=cassandraTestQ1.out + +function runTest { + totalG0=0 + totalG10=0 + totalG20=0 + totalG30=0 + totalG40=0 + totalG50=0 + totalG60=0 + totalG70=0 + totalG80=0 + totalG90=0 + for i in `seq 1 $NUM_LOOP`; do + printTo "loop i:$i, java -jar \ + $CASTEST_DIR/cassandratest/target/cassandratest-1.0-SNAPSHOT-jar-with-dependencies.jar \ + -conf $CASTEST_DIR/application.conf \ + -sql $CASTEST_DIR/q1.txt" + java -jar \ + $CASTEST_DIR/cassandratest/target/cassandratest-1.0-SNAPSHOT-jar-with-dependencies.jar \ + -conf $CASTEST_DIR/application.conf \ + -sql $CASTEST_DIR/q1.txt \ + > $CASTESTQ1OUT + G0=`grep "devgroup=0" $CASTESTQ1OUT| awk '{print $2}'` + totalG0=`echo "scale=4; $totalG0 + $G0" | bc` + G10=`grep "devgroup=10" $CASTESTQ1OUT| awk '{print $2}'` + totalG10=`echo "scale=4; $totalG10 + $G10" | bc` + G20=`grep "devgroup=20" $CASTESTQ1OUT| awk '{print $2}'` + totalG20=`echo "scale=4; $totalG20 + $G20" | bc` + G30=`grep "devgroup=30" $CASTESTQ1OUT| awk '{print $2}'` + totalG30=`echo "scale=4; $totalG30 + $G30" | bc` + G40=`grep "devgroup=40" $CASTESTQ1OUT| awk '{print $2}'` + totalG40=`echo "scale=4; $totalG40 + $G40" | bc` + G50=`grep "devgroup=50" $CASTESTQ1OUT| awk '{print $2}'` + totalG50=`echo "scale=4; $totalG50 + $G50" | bc` + G60=`grep "devgroup=60" $CASTESTQ1OUT| awk '{print $2}'` + totalG60=`echo "scale=4; $totalG60 + $G60" | bc` + G70=`grep "devgroup=70" $CASTESTQ1OUT| awk '{print $2}'` + totalG70=`echo "scale=4; $totalG70 + $G70" | bc` + G80=`grep "devgroup=80" $CASTESTQ1OUT| awk '{print $2}'` + totalG80=`echo "scale=4; $totalG80 + $G80" | bc` + G90=`grep "devgroup=90" $CASTESTQ1OUT| awk '{print $2}'` + totalG90=`echo "scale=4; $totalG90 + $G90" | bc` + done + avgG0=`echo "scale=4; x = $totalG0 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG10=`echo "scale=4; x = $totalG10 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG20=`echo "scale=4; x = $totalG20 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG30=`echo "scale=4; x = $totalG30 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG40=`echo "scale=4; x = $totalG40 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG50=`echo "scale=4; x = $totalG50 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG60=`echo "scale=4; x = $totalG60 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG70=`echo "scale=4; x = $totalG70 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG80=`echo "scale=4; x = $totalG80 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG90=`echo "scale=4; x = $totalG90 / $NUM_LOOP; if(x<1) print 0; x" | bc` + echo "Latency, G-0, G-10, G-20, G-30, G-40, G-50, G-60, G-70, G-80, G-90" + echo "Cassandra, $avgG0, $avgG10, $avgG20, $avgG30, $avgG40, $avgG50, $avgG60, $avgG70, $avgG80, $avgG90" +} + +################ Main ################ + +master=false +develop=true +verbose=false + +clients=1 + +while : ; do + case $1 in + -v) + verbose=true + shift ;; + + -c) + clients=$2 + shift 2;; + + -n) + NUM_LOOP=$2 + shift 2;; + + *) + break ;; + esac +done + +WORK_DIR=/mnt/root/TDengine +CASTEST_DIR=$WORK_DIR/tests/comparisonTest/cassandra + +runTest + +printTo "Test done!" diff --git a/tests/perftest-scripts/cassandraTestQ2Loop.sh b/tests/perftest-scripts/cassandraTestQ2Loop.sh new file mode 100755 index 0000000000..76e42cf89d --- /dev/null +++ b/tests/perftest-scripts/cassandraTestQ2Loop.sh @@ -0,0 +1,278 @@ +#!/bin/bash + +NUM_LOOP=5 + +function printTo { + if $verbose ; then + echo $1 + fi +} + +CASTESTQ2OUT=cassandraTestQ2.out + +function runTest { + totalCount10=0 + totalCount20=0 + totalCount30=0 + totalCount40=0 + totalCount50=0 + totalCount60=0 + totalCount70=0 + totalCount80=0 + totalCount90=0 + totalCount100=0 + + totalAvg10=0 + totalAvg20=0 + totalAvg30=0 + totalAvg40=0 + totalAvg50=0 + totalAvg60=0 + totalAvg70=0 + totalAvg80=0 + totalAvg90=0 + totalAvg100=0 + + totalSum10=0 + totalSum20=0 + totalSum30=0 + totalSum40=0 + totalSum50=0 + totalSum60=0 + totalSum70=0 + totalSum80=0 + totalSum90=0 + totalSum100=0 + + totalMax10=0 + totalMax20=0 + totalMax30=0 + totalMax40=0 + totalMax50=0 + totalMax60=0 + totalMax70=0 + totalMax80=0 + totalMax90=0 + totalMax100=0 + + totalMin10=0 + totalMin20=0 + totalMin30=0 + totalMin40=0 + totalMin50=0 + totalMin60=0 + totalMin70=0 + totalMin80=0 + totalMin90=0 + totalMin100=0 + + for i in `seq 1 $NUM_LOOP`; do + printTo "loop i:$i, java -jar \ + $CASTEST_DIR/cassandratest/target/cassandratest-1.0-SNAPSHOT-jar-with-dependencies.jar \ + -conf $CASTEST_DIR/application.conf \ + -sql $CASTEST_DIR/q2.txt" + java -jar \ + $CASTEST_DIR/cassandratest/target/cassandratest-1.0-SNAPSHOT-jar-with-dependencies.jar \ + -conf $CASTEST_DIR/application.conf \ + -sql $CASTEST_DIR/q2.txt \ + > $CASTESTQ2OUT + + Count10=`cat $CASTESTQ2OUT | grep count | grep "devgroup<10" | awk '{print $2}'` + totalCount10=`echo "scale=4; $totalCount10 + $Count10" | bc` + Count20=`cat $CASTESTQ2OUT | grep count | grep "devgroup<20" | awk '{print $2}'` + totalCount20=`echo "scale=4; $totalCount20 + $Count20" | bc` + Count30=`cat $CASTESTQ2OUT | grep count | grep "devgroup<30" | awk '{print $2}'` + totalCount30=`echo "scale=4; $totalCount30 + $Count30" | bc` + Count40=`cat $CASTESTQ2OUT | grep count | grep "devgroup<40" | awk '{print $2}'` + totalCount40=`echo "scale=4; $totalCount40 + $Count40" | bc` + Count50=`cat $CASTESTQ2OUT | grep count | grep "devgroup<50" | awk '{print $2}'` + totalCount50=`echo "scale=4; $totalCount50 + $Count50" | bc` + Count60=`cat $CASTESTQ2OUT | grep count | grep "devgroup<60" | awk '{print $2}'` + totalCount60=`echo "scale=4; $totalCount60 + $Count60" | bc` + Count70=`cat $CASTESTQ2OUT | grep count | grep "devgroup<70" | awk '{print $2}'` + totalCount70=`echo "scale=4; $totalCount70 + $Count70" | bc` + Count80=`cat $CASTESTQ2OUT | grep count | grep "devgroup<80" | awk '{print $2}'` + totalCount80=`echo "scale=4; $totalCount80 + $Count80" | bc` + Count90=`cat $CASTESTQ2OUT | grep count | grep "devgroup<90" | awk '{print $2}'` + totalCount90=`echo "scale=4; $totalCount90 + $Count90" | bc` + Count100=`cat $CASTESTQ2OUT | grep count | grep "test allow filtering;" | awk '{print $2}'` + totalCount100=`echo "scale=4; $totalCount100 + $Count100" | bc` + + Avg10=`cat $CASTESTQ2OUT | grep avg | grep "devgroup<10" | awk '{print $2}'` + totalAvg10=`echo "scale=4; $totalAvg10 + $Avg10" | bc` + Avg20=`cat $CASTESTQ2OUT | grep avg | grep "devgroup<20" | awk '{print $2}'` + totalAvg20=`echo "scale=4; $totalAvg20 + $Avg20" | bc` + Avg30=`cat $CASTESTQ2OUT | grep avg | grep "devgroup<30" | awk '{print $2}'` + totalAvg30=`echo "scale=4; $totalAvg30 + $Avg30" | bc` + Avg40=`cat $CASTESTQ2OUT | grep avg | grep "devgroup<40" | awk '{print $2}'` + totalAvg40=`echo "scale=4; $totalAvg40 + $Avg40" | bc` + Avg50=`cat $CASTESTQ2OUT | grep avg | grep "devgroup<50" | awk '{print $2}'` + totalAvg50=`echo "scale=4; $totalAvg50 + $Avg50" | bc` + Avg60=`cat $CASTESTQ2OUT | grep avg | grep "devgroup<60" | awk '{print $2}'` + totalAvg60=`echo "scale=4; $totalAvg60 + $Avg60" | bc` + Avg70=`cat $CASTESTQ2OUT | grep avg | grep "devgroup<70" | awk '{print $2}'` + totalAvg70=`echo "scale=4; $totalAvg70 + $Avg70" | bc` + Avg80=`cat $CASTESTQ2OUT | grep avg | grep "devgroup<80" | awk '{print $2}'` + totalAvg80=`echo "scale=4; $totalAvg80 + $Avg80" | bc` + Avg90=`cat $CASTESTQ2OUT | grep avg | grep "devgroup<90" | awk '{print $2}'` + totalAvg90=`echo "scale=4; $totalAvg90 + $Avg90" | bc` + Avg100=`cat $CASTESTQ2OUT | grep avg | grep "test allow filtering;" | awk '{print $2}'` + totalAvg100=`echo "scale=4; $totalAvg100 + $Avg100" | bc` + + Sum10=`cat $CASTESTQ2OUT | grep sum | grep "devgroup<10" | awk '{print $2}'` + totalSum10=`echo "scale=4; $totalSum10 + $Sum10" | bc` + Sum20=`cat $CASTESTQ2OUT | grep sum | grep "devgroup<20" | awk '{print $2}'` + totalSum20=`echo "scale=4; $totalSum20 + $Sum20" | bc` + Sum30=`cat $CASTESTQ2OUT | grep sum | grep "devgroup<30" | awk '{print $2}'` + totalSum30=`echo "scale=4; $totalSum30 + $Sum30" | bc` + Sum40=`cat $CASTESTQ2OUT | grep sum | grep "devgroup<40" | awk '{print $2}'` + totalSum40=`echo "scale=4; $totalSum40 + $Sum40" | bc` + Sum50=`cat $CASTESTQ2OUT | grep sum | grep "devgroup<50" | awk '{print $2}'` + totalSum50=`echo "scale=4; $totalSum50 + $Sum50" | bc` + Sum60=`cat $CASTESTQ2OUT | grep sum | grep "devgroup<60" | awk '{print $2}'` + totalSum60=`echo "scale=4; $totalSum60 + $Sum60" | bc` + Sum70=`cat $CASTESTQ2OUT | grep sum | grep "devgroup<70" | awk '{print $2}'` + totalSum70=`echo "scale=4; $totalSum70 + $Sum70" | bc` + Sum80=`cat $CASTESTQ2OUT | grep sum | grep "devgroup<80" | awk '{print $2}'` + totalSum80=`echo "scale=4; $totalSum80 + $Sum80" | bc` + Sum90=`cat $CASTESTQ2OUT | grep sum | grep "devgroup<90" | awk '{print $2}'` + totalSum90=`echo "scale=4; $totalSum90 + $Sum90" | bc` + Sum100=`cat $CASTESTQ2OUT | grep sum | grep "test allow filtering;" | awk '{print $2}'` + totalSum100=`echo "scale=4; $totalSum100 + $Sum100" | bc` + + Max10=`cat $CASTESTQ2OUT | grep max | grep "devgroup<10" | awk '{print $2}'` + totalMax10=`echo "scale=4; $totalMax10 + $Max10" | bc` + Max20=`cat $CASTESTQ2OUT | grep max | grep "devgroup<20" | awk '{print $2}'` + totalMax20=`echo "scale=4; $totalMax20 + $Max20" | bc` + Max30=`cat $CASTESTQ2OUT | grep max | grep "devgroup<30" | awk '{print $2}'` + totalMax30=`echo "scale=4; $totalMax30 + $Max30" | bc` + Max40=`cat $CASTESTQ2OUT | grep max | grep "devgroup<40" | awk '{print $2}'` + totalMax40=`echo "scale=4; $totalMax40 + $Max40" | bc` + Max50=`cat $CASTESTQ2OUT | grep max | grep "devgroup<50" | awk '{print $2}'` + totalMax50=`echo "scale=4; $totalMax50 + $Max50" | bc` + Max60=`cat $CASTESTQ2OUT | grep max | grep "devgroup<60" | awk '{print $2}'` + totalMax60=`echo "scale=4; $totalMax60 + $Max60" | bc` + Max70=`cat $CASTESTQ2OUT | grep max | grep "devgroup<70" | awk '{print $2}'` + totalMax70=`echo "scale=4; $totalMax70 + $Max70" | bc` + Max80=`cat $CASTESTQ2OUT | grep max | grep "devgroup<80" | awk '{print $2}'` + totalMax80=`echo "scale=4; $totalMax80 + $Max80" | bc` + Max90=`cat $CASTESTQ2OUT | grep max | grep "devgroup<90" | awk '{print $2}'` + totalMax90=`echo "scale=4; $totalMax90 + $Max90" | bc` + Max100=`cat $CASTESTQ2OUT | grep max | grep "test allow filtering;" | awk '{print $2}'` + totalMax100=`echo "scale=4; $totalMax100 + $Max100" | bc` + + Min10=`cat $CASTESTQ2OUT | grep min | grep "devgroup<10" | awk '{print $2}'` + totalMin10=`echo "scale=4; $totalMin10 + $Min10" | bc` + Min20=`cat $CASTESTQ2OUT | grep min | grep "devgroup<20" | awk '{print $2}'` + totalMin20=`echo "scale=4; $totalMin20 + $Min20" | bc` + Min30=`cat $CASTESTQ2OUT | grep min | grep "devgroup<30" | awk '{print $2}'` + totalMin30=`echo "scale=4; $totalMin30 + $Min30" | bc` + Min40=`cat $CASTESTQ2OUT | grep min | grep "devgroup<40" | awk '{print $2}'` + totalMin40=`echo "scale=4; $totalMin40 + $Min40" | bc` + Min50=`cat $CASTESTQ2OUT | grep min | grep "devgroup<50" | awk '{print $2}'` + totalMin50=`echo "scale=4; $totalMin50 + $Min50" | bc` + Min60=`cat $CASTESTQ2OUT | grep min | grep "devgroup<60" | awk '{print $2}'` + totalMin60=`echo "scale=4; $totalMin60 + $Min60" | bc` + Min70=`cat $CASTESTQ2OUT | grep min | grep "devgroup<70" | awk '{print $2}'` + totalMin70=`echo "scale=4; $totalMin70 + $Min70" | bc` + Min80=`cat $CASTESTQ2OUT | grep min | grep "devgroup<80" | awk '{print $2}'` + totalMin80=`echo "scale=4; $totalMin80 + $Min80" | bc` + Min90=`cat $CASTESTQ2OUT | grep min | grep "devgroup<90" | awk '{print $2}'` + totalMin90=`echo "scale=4; $totalMin90 + $Min90" | bc` + Min100=`cat $CASTESTQ2OUT | grep min | grep "test allow filtering;" | awk '{print $2}'` + totalMin100=`echo "scale=4; $totalMin100 + $Min100" | bc` + + done + avgCount10=`echo "scale=4; x = $totalCount10 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgCount20=`echo "scale=4; x = $totalCount20 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgCount30=`echo "scale=4; x = $totalCount30 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgCount40=`echo "scale=4; x = $totalCount40 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgCount50=`echo "scale=4; x = $totalCount50 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgCount60=`echo "scale=4; x = $totalCount60 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgCount70=`echo "scale=4; x = $totalCount70 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgCount80=`echo "scale=4; x = $totalCount80 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgCount90=`echo "scale=4; x = $totalCount90 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgCount100=`echo "scale=4; x = $totalCount100 / $NUM_LOOP; if(x<1) print 0; x" | bc` + + avgAvg10=`echo "scale=4; x = $totalAvg10 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgAvg20=`echo "scale=4; x = $totalAvg20 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgAvg30=`echo "scale=4; x = $totalAvg30 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgAvg40=`echo "scale=4; x = $totalAvg40 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgAvg50=`echo "scale=4; x = $totalAvg50 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgAvg60=`echo "scale=4; x = $totalAvg60 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgAvg70=`echo "scale=4; x = $totalAvg70 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgAvg80=`echo "scale=4; x = $totalAvg80 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgAvg90=`echo "scale=4; x = $totalAvg90 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgAvg100=`echo "scale=4; x = $totalAvg100 / $NUM_LOOP; if(x<1) print 0; x" | bc` + + avgSum10=`echo "scale=4; x = $totalSum10 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgSum20=`echo "scale=4; x = $totalSum20 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgSum30=`echo "scale=4; x = $totalSum30 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgSum40=`echo "scale=4; x = $totalSum40 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgSum50=`echo "scale=4; x = $totalSum50 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgSum60=`echo "scale=4; x = $totalSum60 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgSum70=`echo "scale=4; x = $totalSum70 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgSum80=`echo "scale=4; x = $totalSum80 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgSum90=`echo "scale=4; x = $totalSum90 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgSum100=`echo "scale=4; x = $totalSum100 / $NUM_LOOP; if(x<1) print 0; x" | bc` + + avgMax10=`echo "scale=4; x = $totalMax10 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMax20=`echo "scale=4; x = $totalMax20 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMax30=`echo "scale=4; x = $totalMax30 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMax40=`echo "scale=4; x = $totalMax40 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMax50=`echo "scale=4; x = $totalMax50 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMax60=`echo "scale=4; x = $totalMax60 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMax70=`echo "scale=4; x = $totalMax70 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMax80=`echo "scale=4; x = $totalMax80 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMax90=`echo "scale=4; x = $totalMax90 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMax100=`echo "scale=4; x = $totalMax100 / $NUM_LOOP; if(x<1) print 0; x" | bc` + + avgMin10=`echo "scale=4; x = $totalMin10 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMin20=`echo "scale=4; x = $totalMin20 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMin30=`echo "scale=4; x = $totalMin30 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMin40=`echo "scale=4; x = $totalMin40 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMin50=`echo "scale=4; x = $totalMin50 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMin60=`echo "scale=4; x = $totalMin60 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMin70=`echo "scale=4; x = $totalMin70 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMin80=`echo "scale=4; x = $totalMin80 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMin90=`echo "scale=4; x = $totalMin90 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgMin100=`echo "scale=4; x = $totalMin100 / $NUM_LOOP; if(x<1) print 0; x" | bc` + + echo "Latency, 10%, 20%, 30%, 40%, 50%, 60%, 70%, 80%, 90%, 100%" + echo "Count, $avgCount10, $avgCount20, $avgCount30, $avgCount40, $avgCount50, $avgCount60, $avgCount70, $avgCount80, $avgCount90, $avgCount100" + echo "Avg, $avgAvg10, $avgAvg20, $avgAvg30, $avgAvg40, $avgAvg50, $avgAvg60, $avgAvg70, $avgAvg80, $avgAvg90, $avgAvg100" + echo "Sum, $avgSum10, $avgSum20, $avgSum30, $avgSum40, $avgSum50, $avgSum60, $avgSum70, $avgSum80, $avgSum90, $avgSum100" + echo "Max, $avgMax10, $avgMax20, $avgMax30, $avgMax40, $avgMax50, $avgMax60, $avgMax70, $avgMax80, $avgMax90, $avgMax100" + echo "Min, $avgMin10, $avgMin20, $avgMin30, $avgMin40, $avgMin50, $avgMin60, $avgMin70, $avgMin80, $avgMin90, $avgMin100" +} + +################ Main ################ + +verbose=false + +for arg in "$@" +do + case $arg in + -v) + verbose=true + shift ;; + + -c) + clients=$2 + shift 2;; + + -n) + NUM_LOOP=$2 + shift 2;; + + *) + ;; + esac +done + +WORK_DIR=/mnt/root/TDengine +CASTEST_DIR=$WORK_DIR/tests/comparisonTest/cassandra + +runTest + +printTo "Test done!" diff --git a/tests/perftest-scripts/cassandraTestQ3Loop.sh b/tests/perftest-scripts/cassandraTestQ3Loop.sh new file mode 100755 index 0000000000..ce81fcb818 --- /dev/null +++ b/tests/perftest-scripts/cassandraTestQ3Loop.sh @@ -0,0 +1,98 @@ +#!/bin/bash + +NUM_LOOP=5 + +function printTo { + if $verbose ; then + echo $1 + fi +} + +CASTESTQ3OUT=cassandraTestQ3.out + +function runTest { + totalG10=0 + totalG20=0 + totalG30=0 + totalG40=0 + totalG50=0 + totalG60=0 + totalG70=0 + totalG80=0 + totalG90=0 + totalG100=0 + for i in `seq 1 $NUM_LOOP`; do + printTo "loop i:$i, java -jar \ + $CASTEST_DIR/cassandratest/target/cassandratest-1.0-SNAPSHOT-jar-with-dependencies.jar \ + -conf $CASTEST_DIR/application.conf \ + -sql $CASTEST_DIR/q3.txt" + java -jar \ + $CASTEST_DIR/cassandratest/target/cassandratest-1.0-SNAPSHOT-jar-with-dependencies.jar \ + -conf $CASTEST_DIR/application.conf \ + -sql $CASTEST_DIR/q3.txt \ + > $CASTESTQ3OUT + G10=`grep "devgroup<10" $CASTESTQ3OUT| awk '{print $2}'` + totalG10=`echo "scale=4; $totalG10 + $G10" | bc` + G20=`grep "devgroup<20" $CASTESTQ3OUT| awk '{print $2}'` + totalG20=`echo "scale=4; $totalG20 + $G20" | bc` + G30=`grep "devgroup<30" $CASTESTQ3OUT| awk '{print $2}'` + totalG30=`echo "scale=4; $totalG30 + $G30" | bc` + G40=`grep "devgroup<40" $CASTESTQ3OUT| awk '{print $2}'` + totalG40=`echo "scale=4; $totalG40 + $G40" | bc` + G50=`grep "devgroup<50" $CASTESTQ3OUT| awk '{print $2}'` + totalG50=`echo "scale=4; $totalG50 + $G50" | bc` + G60=`grep "devgroup<60" $CASTESTQ3OUT| awk '{print $2}'` + totalG60=`echo "scale=4; $totalG60 + $G60" | bc` + G70=`grep "devgroup<70" $CASTESTQ3OUT| awk '{print $2}'` + totalG70=`echo "scale=4; $totalG70 + $G70" | bc` + G80=`grep "devgroup<80" $CASTESTQ3OUT| awk '{print $2}'` + totalG80=`echo "scale=4; $totalG80 + $G80" | bc` + G90=`grep "devgroup<90" $CASTESTQ3OUT| awk '{print $2}'` + totalG90=`echo "scale=4; $totalG90 + $G90" | bc` + G100=`grep "test group by " $CASTESTQ3OUT| awk '{print $2}'` + totalG100=`echo "scale=4; $totalG100 + $G100" | bc` + done + avgG10=`echo "scale=4; x = $totalG10 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG20=`echo "scale=4; x = $totalG20 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG30=`echo "scale=4; x = $totalG30 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG40=`echo "scale=4; x = $totalG40 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG50=`echo "scale=4; x = $totalG50 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG60=`echo "scale=4; x = $totalG60 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG70=`echo "scale=4; x = $totalG70 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG80=`echo "scale=4; x = $totalG80 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG90=`echo "scale=4; x = $totalG90 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG100=`echo "scale=4; x = $totalG100 / $NUM_LOOP; if(x<1) print 0; x" | bc` + echo "Latency, 10%, 20%, 30%, 40%, 50%, 60%, 70%, 80%, 90%, 100%" + echo "Cassandra, $avgG10, $avgG20, $avgG30, $avgG40, $avgG50, $avgG60, $avgG70, $avgG80, $avgG90, $avgG100" +} + +################ Main ################ + +verbose=false + +for arg in "$@" +do + case $arg in + -v) + verbose=true + shift ;; + + -c) + clients=$2 + shift 2;; + + -n) + NUM_LOOP=$2 + shift 2;; + + *) + ;; + esac +done + +WORK_DIR=/mnt/root/TDengine +CASTEST_DIR=$WORK_DIR/tests/comparisonTest/cassandra + +runTest + +printTo "Test done!" diff --git a/tests/perftest-scripts/cassandraTestQ4Loop.sh b/tests/perftest-scripts/cassandraTestQ4Loop.sh new file mode 100755 index 0000000000..85e45787b8 --- /dev/null +++ b/tests/perftest-scripts/cassandraTestQ4Loop.sh @@ -0,0 +1,123 @@ +#!/bin/bash + +DATA_DIR=/mnt/root/testdata +NUM_LOOP=5 + +function printTo { + if $verbose ; then + echo $1 + fi +} + +CASTESTQ4OUT=cassandraTestQ4.out + +function runTest { + totalG10=0 + totalG20=0 + totalG30=0 + totalG40=0 + totalG50=0 + totalG60=0 + totalG70=0 + totalG80=0 + totalG90=0 + totalG100=0 + for i in `seq 1 $NUM_LOOP`; do + if $regeneratedata ; then + printTo "java -jar $CASTEST_DIR/cassandratest-1.0-SNAPSHOT-jar-with-dependencies.jar \ + -datadir $DATA_DIR \ + -numofFiles 100 \ + -rowsperrequest 100 \ + -writeclients 4 \ + -conf $CASTEST_DIR/application.conf \ + -timetest" + java -jar $CASTEST_DIR/cassandratest-1.0-SNAPSHOT-jar-with-dependencies.jar \ + -datadir $DATA_DIR \ + -numofFiles 100 \ + -rowsperrequest 100 \ + -writeclients 4 \ + -conf $CASTEST_DIR/application.conf \ + -timetest + fi + + printTo "loop i:$i, java -jar \ + $CASTEST_DIR/cassandratest/target/cassandratest-1.0-SNAPSHOT-jar-with-dependencies.jar \ + -conf $CASTEST_DIR/application.conf \ + -sql $CASTEST_DIR/q4.txt" + java -jar \ + $CASTEST_DIR/cassandratest/target/cassandratest-1.0-SNAPSHOT-jar-with-dependencies.jar \ + -conf $CASTEST_DIR/application.conf \ + -sql $CASTEST_DIR/q4.txt \ + > $CASTESTQ4OUT + G10=`grep "devgroup<10" $CASTESTQ4OUT| awk '{print $2}'` + totalG10=`echo "scale=4; $totalG10 + $G10" | bc` + G20=`grep "devgroup<20" $CASTESTQ4OUT| awk '{print $2}'` + totalG20=`echo "scale=4; $totalG20 + $G20" | bc` + G30=`grep "devgroup<30" $CASTESTQ4OUT| awk '{print $2}'` + totalG30=`echo "scale=4; $totalG30 + $G30" | bc` + G40=`grep "devgroup<40" $CASTESTQ4OUT| awk '{print $2}'` + totalG40=`echo "scale=4; $totalG40 + $G40" | bc` + G50=`grep "devgroup<50" $CASTESTQ4OUT| awk '{print $2}'` + totalG50=`echo "scale=4; $totalG50 + $G50" | bc` + G60=`grep "devgroup<60" $CASTESTQ4OUT| awk '{print $2}'` + totalG60=`echo "scale=4; $totalG60 + $G60" | bc` + G70=`grep "devgroup<70" $CASTESTQ4OUT| awk '{print $2}'` + totalG70=`echo "scale=4; $totalG70 + $G70" | bc` + G80=`grep "devgroup<80" $CASTESTQ4OUT| awk '{print $2}'` + totalG80=`echo "scale=4; $totalG80 + $G80" | bc` + G90=`grep "devgroup<90" $CASTESTQ4OUT| awk '{print $2}'` + totalG90=`echo "scale=4; $totalG90 + $G90" | bc` + G100=`grep "test group by minute;" $CASTESTQ4OUT| awk '{print $2}'` + totalG100=`echo "scale=4; $totalG100 + $G100" | bc` + done + avgG10=`echo "scale=4; x = $totalG10 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG20=`echo "scale=4; x = $totalG20 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG30=`echo "scale=4; x = $totalG30 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG40=`echo "scale=4; x = $totalG40 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG50=`echo "scale=4; x = $totalG50 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG60=`echo "scale=4; x = $totalG60 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG70=`echo "scale=4; x = $totalG70 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG80=`echo "scale=4; x = $totalG80 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG90=`echo "scale=4; x = $totalG90 / $NUM_LOOP; if(x<1) print 0; x" | bc` + avgG100=`echo "scale=4; x = $totalG100 / $NUM_LOOP; if(x<1) print 0; x" | bc` + echo "Latency, 10%, 20%, 30%, 40%, 50%, 60%, 70%, 80%, 90%, 100%" + echo "Cassandra, $avgG10, $avgG20, $avgG30, $avgG40, $avgG50, $avgG60, $avgG70, $avgG80, $avgG90, $avgG100" +} + +################ Main ################ + +master=false +develop=true +verbose=false +regeneratedata=false + +for arg in "$@" +do + case $arg in + -v) + verbose=true + shift ;; + + -c) + clients=$2 + shift 2;; + + -r) + regeneratedata=true + ;; + + -n) + NUM_LOOP=$2 + shift 2;; + + *) + ;; + esac +done + +WORK_DIR=/mnt/root/TDengine +CASTEST_DIR=$WORK_DIR/tests/comparisonTest/cassandra + +runTest + +printTo "Test done!" diff --git a/tests/script/unique/clusterSimCase/client-01.sim b/tests/script/unique/clusterSimCase/client-01.sim new file mode 100644 index 0000000000..b35790d69e --- /dev/null +++ b/tests/script/unique/clusterSimCase/client-01.sim @@ -0,0 +1,69 @@ +# create database db1 +# create table tb () +# loop insert seq data +############################################################### +################ set script parameters ################ +$tblStart = 0 +$tblEnd = 10000 +$tsStart = 1325347200000 # 2012-01-01 00:00:00.000 +$preBinary = ' . client-01- +############################################################### + +sql connect + +print ================ create db +$db = db +sql create database if not exists $db replica 2 +sql use $db + +print ================ create table $tb (ts timestamp, c1 int, c2 binary(16)) + +$i = $tblStart +while $i < $tblEnd + $tb = tb . $i +# print create table if not exists $tb ( ts timestamp , c1 int , c2 binary(16) ) + sql create table if not exists $tb ( ts timestamp , c1 int , c2 binary(16) ) + $i = $i + 1 +endw + +$totalRows = 0 +$rowsPerTbl = 0 +$ts = $tsStart +$rowsPerLoop = 160 + +$loop_cnt = 0 +loop_run: +print ================ client-01 start loop insert data ( loop_cnt: $loop_cnt ) +$i = $tblStart +while $i < $tblEnd + $tb = tb . $i + + $c2Binary = $preBinary . $i + $c2Binary = $c2Binary . ' + + $x = 0 + while $x < $rowsPerLoop + sql insert into $tb values ( $ts + 0a , $x , $c2Binary ) ( $ts + 2a , $x , $c2Binary ) ( $ts + 4a , $x , $c2Binary ) ( $ts + 6a , $x , $c2Binary ) ( $ts + 8a , $x , $c2Binary ) ( $ts + 10a , $x , $c2Binary ) ( $ts + 12a , $x , $c2Binary ) ( $ts + 14a , $x , $c2Binary ) ( $ts + 16a , $x , $c2Binary ) ( $ts + 18a , $x , $c2Binary ) ( $ts + 20a , $x , $c2Binary ) ( $ts + 22a , $x , $c2Binary ) ( $ts + 24a , $x , $c2Binary ) ( $ts + 26a , $x , $c2Binary ) ( $ts + 28a , $x , $c2Binary ) ( $ts + 30a , $x , $c2Binary ) ( $ts + 32a , $x , $c2Binary ) ( $ts + 34a , $x , $c2Binary ) ( $ts + 36a , $x , $c2Binary ) ( $ts + 38a , $x , $c2Binary ) + $x = $x + 20 + $ts = $ts + 40a + endw + if $i == $tblStart then + $rowsPerTbl = $rowsPerTbl + $x + endi + + $totalRows = $totalRows + $x + $i = $i + 1 + +endw + +sql select count(*) from $tb +print data00 $data00 rowsPerTbl $rowsPerTbl +if $data00 != $rowsPerTbl then + print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + print ************ client-01 insert data into $tb error ***** ***** + print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + return -1 +endi + +$loop_cnt = $loop_cnt + 1 +goto loop_run diff --git a/tests/script/unique/clusterSimCase/client-02.sim b/tests/script/unique/clusterSimCase/client-02.sim new file mode 100644 index 0000000000..30a2ceb089 --- /dev/null +++ b/tests/script/unique/clusterSimCase/client-02.sim @@ -0,0 +1,70 @@ +# create database db1 +# create table tb () +# loop insert disorder data +############################################################### +################ set script parameters ################ +$tblStart = 0 +$tblEnd = 10000 +$tsStart = 1325347200000 # 2012-01-01 00:00:00.000 +$preBinary = ' . client-02- +############################################################### + +sql connect + +print ================ create db +$db = db +sql create database if not exists $db replica 2 +sql use $db + +print ================ create table $tb (ts timestamp, c1 int, c2 binary(16)) + +$i = $tblStart +while $i < $tblEnd + $tb = tb . $i +# print create table if not exists $tb ( ts timestamp , c1 int , c2 binary(16) ) + sql create table if not exists $tb ( ts timestamp , c1 int , c2 binary(16) ) + $i = $i + 1 +endw + +$totalRows = 0 +$rowsPerTbl = 0 +$ts = $tsStart +$rowsPerLoop = 160 + +$loop_cnt = 0 +loop_run: +print ================ client-02 start loop insert data ( loop_cnt: $loop_cnt ) +$i = $tblStart +while $i < $tblEnd + $tb = tb . $i + + $c2Binary = $preBinary . $i + $c2Binary = $c2Binary . ' + + $x = 0 + while $x < $rowsPerLoop + sql insert into $tb values ( $ts + 0a , $x , $c2Binary ) ( $ts + 2a , $x , $c2Binary ) ( $ts + 4a , $x , $c2Binary ) ( $ts + 6a , $x , $c2Binary ) ( $ts + 8a , $x , $c2Binary ) ( $ts + 10a , $x , $c2Binary ) ( $ts + 12a , $x , $c2Binary ) ( $ts + 14a , $x , $c2Binary ) ( $ts + 16a , $x , $c2Binary ) ( $ts + 18a , $x , $c2Binary ) ( $ts + 20a , $x , $c2Binary ) ( $ts + 22a , $x , $c2Binary ) ( $ts + 24a , $x , $c2Binary ) ( $ts + 26a , $x , $c2Binary ) ( $ts + 28a , $x , $c2Binary ) ( $ts + 30a , $x , $c2Binary ) ( $ts + 32a , $x , $c2Binary ) ( $ts + 34a , $x , $c2Binary ) ( $ts + 36a , $x , $c2Binary ) ( $ts + 38a , $x , $c2Binary ) + sql insert into $tb values ( $ts + 1a , $x , $c2Binary ) ( $ts + 3a , $x , $c2Binary ) ( $ts + 5a , $x , $c2Binary ) ( $ts + 7a , $x , $c2Binary ) ( $ts + 9a , $x , $c2Binary ) ( $ts + 11a , $x , $c2Binary ) ( $ts + 13a , $x , $c2Binary ) ( $ts + 15a , $x , $c2Binary ) ( $ts + 17a , $x , $c2Binary ) ( $ts + 19a , $x , $c2Binary ) ( $ts + 21a , $x , $c2Binary ) ( $ts + 23a , $x , $c2Binary ) ( $ts + 25a , $x , $c2Binary ) ( $ts + 27a , $x , $c2Binary ) ( $ts + 29a , $x , $c2Binary ) ( $ts + 31a , $x , $c2Binary ) ( $ts + 33a , $x , $c2Binary ) ( $ts + 35a , $x , $c2Binary ) ( $ts + 37a , $x , $c2Binary ) ( $ts + 39a , $x , $c2Binary ) + $x = $x + 40 + $ts = $ts + 40a + endw + if $i == $tblStart then + $rowsPerTbl = $rowsPerTbl + $x + endi + + $totalRows = $totalRows + $x + $i = $i + 1 + +endw + +sql select count(*) from $tb +print data00 $data00 rowsPerTbl $rowsPerTbl +if $data00 != $rowsPerTbl then + print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + print ************ client-02 insert data into $tb error ***** ***** + print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + return -1 +endi + +$loop_cnt = $loop_cnt + 1 +goto loop_run diff --git a/tests/script/unique/clusterSimCase/client-03.sim b/tests/script/unique/clusterSimCase/client-03.sim new file mode 100644 index 0000000000..78a25e26a5 --- /dev/null +++ b/tests/script/unique/clusterSimCase/client-03.sim @@ -0,0 +1,83 @@ +# create database db1 +# create table stb () +# loop dynamic create sub table and insert disorder data +############################################################### +################ set script parameters ################ +$tblStart = 0 +$tblEnd = 10000 +$tsStart = 1325347200000 # 2012-01-01 00:00:00.000 +$preBinary = ' . client-03- +############################################################### + +sql connect + +print ================ create db and super-table +$db = db +$stb = stb +sql create database if not exists $db replica 2 +sql use $db +sql create table if not exists $stb ( ts timestamp , c1 int , c2 binary(16) ) tags ( t1 int , t2 binary(16) ) + +#print ================ create table $tb (ts timestamp, c1 int, c2 binary(16)) +#$i = $tblStart +#while $i < $tblEnd +# $tb = tb . $i +## print create table if not exists $tb ( ts timestamp , c1 int , c2 binary(16) ) +# sql create table if not exists $tb ( ts timestamp , c1 int , c2 binary(16) ) +# $i = $i + 1 +#endw + +$totalRows = 0 +$rowsPerTbl = 0 +$ts = $tsStart +$rowsPerLoop = 160 + +$loop_cnt = 0 +loop_run: +print ================ client-03 start loop insert data ( loop_cnt: $loop_cnt ) +$i = $tblStart +while $i < $tblEnd + $tb = tb . $i + + $t2Binary = $preBinary . $i + $t2Binary = $t2Binary . ' + + $c2Binary = $preBinary . $i + $c2Binary = $c2Binary . ' + + $x = 0 + while $x < $rowsPerLoop + sql insert into $tb using $stb tags ( $i , $t2Binary ) values ( $ts + 0a , $x , $c2Binary ) ( $ts + 2a , $x , $c2Binary ) ( $ts + 4a , $x , $c2Binary ) ( $ts + 6a , $x , $c2Binary ) ( $ts + 8a , $x , $c2Binary ) ( $ts + 10a , $x , $c2Binary ) ( $ts + 12a , $x , $c2Binary ) ( $ts + 14a , $x , $c2Binary ) ( $ts + 16a , $x , $c2Binary ) ( $ts + 18a , $x , $c2Binary ) ( $ts + 20a , $x , $c2Binary ) ( $ts + 22a , $x , $c2Binary ) ( $ts + 24a , $x , $c2Binary ) ( $ts + 26a , $x , $c2Binary ) ( $ts + 28a , $x , $c2Binary ) ( $ts + 30a , $x , $c2Binary ) ( $ts + 32a , $x , $c2Binary ) ( $ts + 34a , $x , $c2Binary ) ( $ts + 36a , $x , $c2Binary ) ( $ts + 38a , $x , $c2Binary ) + sql insert into $tb using $stb tags ( $i , $t2Binary ) values ( $ts + 1a , $x , $c2Binary ) ( $ts + 3a , $x , $c2Binary ) ( $ts + 5a , $x , $c2Binary ) ( $ts + 7a , $x , $c2Binary ) ( $ts + 9a , $x , $c2Binary ) ( $ts + 11a , $x , $c2Binary ) ( $ts + 13a , $x , $c2Binary ) ( $ts + 15a , $x , $c2Binary ) ( $ts + 17a , $x , $c2Binary ) ( $ts + 19a , $x , $c2Binary ) ( $ts + 21a , $x , $c2Binary ) ( $ts + 23a , $x , $c2Binary ) ( $ts + 25a , $x , $c2Binary ) ( $ts + 27a , $x , $c2Binary ) ( $ts + 29a , $x , $c2Binary ) ( $ts + 31a , $x , $c2Binary ) ( $ts + 33a , $x , $c2Binary ) ( $ts + 35a , $x , $c2Binary ) ( $ts + 37a , $x , $c2Binary ) ( $ts + 39a , $x , $c2Binary ) + $x = $x + 40 + $ts = $ts + 40a + endw + if $i == $tblStart then + $rowsPerTbl = $rowsPerTbl + $x + endi + + $totalRows = $totalRows + $x + $i = $i + 1 + +endw + +sql select count(*) from $tb +print data00 $data00 rowsPerTbl $rowsPerTbl +if $data00 != $rowsPerTbl then + + print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + print ****** client-3 insert data into $tb error *** + print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + return -1 +endi + +sql select count(*) from $stb +print data00 $data00 totalRows $totalRows +if $data00 != $totalRows then + print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + print ***** client-3 insert data into $stbl error ******** + print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + return -1 +endi +$loop_cnt = $loop_cnt + 1 +goto loop_run diff --git a/tests/script/unique/clusterSimCase/client-04.sim b/tests/script/unique/clusterSimCase/client-04.sim new file mode 100644 index 0000000000..5248e9d7bb --- /dev/null +++ b/tests/script/unique/clusterSimCase/client-04.sim @@ -0,0 +1,81 @@ +# create database db1 +# create table tb () +# loop insert disorder data, drop database, sleep , +############################################################### +################ set script parameters ################ +$tblStart = 0 +$tblEnd = 10000 +$tsStart = 1325347200000 # 2012-01-01 00:00:00.000 +$preBinary = ' . client-04- +############################################################### + +$totalRows = 0 + +sql connect + +init_lable: +print ================ create db +$db = db +sql create database if not exists $db replica 2 +sql use $db + +print ================ create table $tb (ts timestamp, c1 int, c2 binary(16)) + +$i = $tblStart +while $i < $tblEnd + $tb = tb . $i +# print create table if not exists $tb ( ts timestamp , c1 int , c2 binary(16) ) + sql create table if not exists $tb ( ts timestamp , c1 int , c2 binary(16) ) + $i = $i + 1 +endw + +$rowsPerTbl = 0 +$ts = $tsStart +$rowsPerLoop = 160 + +$loopCnt = 0 + +loop_run: +print ================ client-04 start loop insert data ( loopCnt: $loopCnt ) +$i = $tblStart +while $i < $tblEnd + $tb = tb . $i + + $c2Binary = $preBinary . $i + $c2Binary = $c2Binary . ' + + $x = 0 + while $x < $rowsPerLoop + sql insert into $tb values ( $ts + 0a , $x , $c2Binary ) ( $ts + 2a , $x , $c2Binary ) ( $ts + 4a , $x , $c2Binary ) ( $ts + 6a , $x , $c2Binary ) ( $ts + 8a , $x , $c2Binary ) ( $ts + 10a , $x , $c2Binary ) ( $ts + 12a , $x , $c2Binary ) ( $ts + 14a , $x , $c2Binary ) ( $ts + 16a , $x , $c2Binary ) ( $ts + 18a , $x , $c2Binary ) ( $ts + 20a , $x , $c2Binary ) ( $ts + 22a , $x , $c2Binary ) ( $ts + 24a , $x , $c2Binary ) ( $ts + 26a , $x , $c2Binary ) ( $ts + 28a , $x , $c2Binary ) ( $ts + 30a , $x , $c2Binary ) ( $ts + 32a , $x , $c2Binary ) ( $ts + 34a , $x , $c2Binary ) ( $ts + 36a , $x , $c2Binary ) ( $ts + 38a , $x , $c2Binary ) + sql insert into $tb values ( $ts + 1a , $x , $c2Binary ) ( $ts + 3a , $x , $c2Binary ) ( $ts + 5a , $x , $c2Binary ) ( $ts + 7a , $x , $c2Binary ) ( $ts + 9a , $x , $c2Binary ) ( $ts + 11a , $x , $c2Binary ) ( $ts + 13a , $x , $c2Binary ) ( $ts + 15a , $x , $c2Binary ) ( $ts + 17a , $x , $c2Binary ) ( $ts + 19a , $x , $c2Binary ) ( $ts + 21a , $x , $c2Binary ) ( $ts + 23a , $x , $c2Binary ) ( $ts + 25a , $x , $c2Binary ) ( $ts + 27a , $x , $c2Binary ) ( $ts + 29a , $x , $c2Binary ) ( $ts + 31a , $x , $c2Binary ) ( $ts + 33a , $x , $c2Binary ) ( $ts + 35a , $x , $c2Binary ) ( $ts + 37a , $x , $c2Binary ) ( $ts + 39a , $x , $c2Binary ) + $x = $x + 40 + $ts = $ts + 40a + endw + if $i == $tblStart then + $rowsPerTbl = $rowsPerTbl + $x + endi + + $totalRows = $totalRows + $x + $i = $i + 1 + +endw + +sql select count(*) from $tb +print data00 $data00 rowsPerTbl $rowsPerTbl +if $data00 != $rowsPerTbl then + print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + print ************ client-04 insert data into $tb error ***** ***** + print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + return -1 +endi + +$loopCnt = $loopCnt + 1 + +if $loopCnt > 100 then + sql drop database if exists $db + sleep 20000 + $totalRows = 0 + goto init_lable +endi + +goto loop_run diff --git a/tests/script/unique/clusterSimCase/client-05.sim b/tests/script/unique/clusterSimCase/client-05.sim new file mode 100644 index 0000000000..a2deaba9b2 --- /dev/null +++ b/tests/script/unique/clusterSimCase/client-05.sim @@ -0,0 +1,86 @@ +# create database db1 +# loop create table , insert disorder data, drop table, sleep , +############################################################### +################ set script parameters ################ +$tblStart = 0 +$tblEnd = 10000 +$tsStart = 1325347200000 # 2012-01-01 00:00:00.000 +$preBinary = ' . client-05- +############################################################### + +$totalRows = 0 + +sql connect + +print ================ create db +$db = db +sql create database if not exists $db replica 2 +sql use $db + +init_lable: +print ================ create table $tb (ts timestamp, c1 int, c2 binary(16)) + +$i = $tblStart +while $i < $tblEnd + $tb = tb . $i +# print create table if not exists $tb ( ts timestamp , c1 int , c2 binary(16) ) + sql create table if not exists $tb ( ts timestamp , c1 int , c2 binary(16) ) + $i = $i + 1 +endw + +$rowsPerTbl = 0 +$ts = $tsStart +$rowsPerLoop = 160 + +$loopCnt = 0 + +loop_run: +print ================ client-05 start loop insert data ( loopCnt: $loopCnt ) +$i = $tblStart +while $i < $tblEnd + $tb = tb . $i + + $c2Binary = $preBinary . $i + $c2Binary = $c2Binary . ' + + $x = 0 + while $x < $rowsPerLoop + sql insert into $tb values ( $ts + 0a , $x , $c2Binary ) ( $ts + 2a , $x , $c2Binary ) ( $ts + 4a , $x , $c2Binary ) ( $ts + 6a , $x , $c2Binary ) ( $ts + 8a , $x , $c2Binary ) ( $ts + 10a , $x , $c2Binary ) ( $ts + 12a , $x , $c2Binary ) ( $ts + 14a , $x , $c2Binary ) ( $ts + 16a , $x , $c2Binary ) ( $ts + 18a , $x , $c2Binary ) ( $ts + 20a , $x , $c2Binary ) ( $ts + 22a , $x , $c2Binary ) ( $ts + 24a , $x , $c2Binary ) ( $ts + 26a , $x , $c2Binary ) ( $ts + 28a , $x , $c2Binary ) ( $ts + 30a , $x , $c2Binary ) ( $ts + 32a , $x , $c2Binary ) ( $ts + 34a , $x , $c2Binary ) ( $ts + 36a , $x , $c2Binary ) ( $ts + 38a , $x , $c2Binary ) + sql insert into $tb values ( $ts + 1a , $x , $c2Binary ) ( $ts + 3a , $x , $c2Binary ) ( $ts + 5a , $x , $c2Binary ) ( $ts + 7a , $x , $c2Binary ) ( $ts + 9a , $x , $c2Binary ) ( $ts + 11a , $x , $c2Binary ) ( $ts + 13a , $x , $c2Binary ) ( $ts + 15a , $x , $c2Binary ) ( $ts + 17a , $x , $c2Binary ) ( $ts + 19a , $x , $c2Binary ) ( $ts + 21a , $x , $c2Binary ) ( $ts + 23a , $x , $c2Binary ) ( $ts + 25a , $x , $c2Binary ) ( $ts + 27a , $x , $c2Binary ) ( $ts + 29a , $x , $c2Binary ) ( $ts + 31a , $x , $c2Binary ) ( $ts + 33a , $x , $c2Binary ) ( $ts + 35a , $x , $c2Binary ) ( $ts + 37a , $x , $c2Binary ) ( $ts + 39a , $x , $c2Binary ) + $x = $x + 40 + $ts = $ts + 40a + endw + if $i == $tblStart then + $rowsPerTbl = $rowsPerTbl + $x + endi + + $totalRows = $totalRows + $x + $i = $i + 1 + +endw + +sql select count(*) from $tb +print data00 $data00 rowsPerTbl $rowsPerTbl +if $data00 != $rowsPerTbl then + print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + print ************ client-05 insert data into $tb error ***** ***** + print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + return -1 +endi + +$loopCnt = $loopCnt + 1 + +if $loopCnt > 100 then + + $i = $tblStart + while $i < $tblEnd + $tb = tb . $i + sql drop table if exists $tb + $i = $i + 1 + $totalRows = $totalRows - $rowsPerTbl + endw + sleep 20000 + goto init_lable +endi + +goto loop_run diff --git a/tests/script/unique/clusterSimCase/client-06.sim b/tests/script/unique/clusterSimCase/client-06.sim new file mode 100644 index 0000000000..013eb1db85 --- /dev/null +++ b/tests/script/unique/clusterSimCase/client-06.sim @@ -0,0 +1,101 @@ +# create database db1 +# create table tb () +# loop insert disorder data, drop some tables and create new tables, sleep , +############################################################### +################ set script parameters ################ +$tblStart = 0 +$tblEnd = 10000 +$tsStart = 1325347200000 # 2012-01-01 00:00:00.000 +$preBinary = ' . client-06- +$numPerUpdateTbl = 2000 +############################################################### +$testCnt = 0 + +$totalRows = 0 +$dropTblStart = $tblStart +$dropTblEnd = $dropTblStart + +sql connect + +print ================ create db +$db = db +sql create database if not exists $db replica 2 +sql use $db + +init_lable: +print ================ create table $tb [ from $tblStart to $tblEnd ] (ts timestamp, c1 int, c2 binary(16)) + +$i = $tblStart +while $i < $tblEnd + $tb = tb . $i +# print create table if not exists $tb ( ts timestamp , c1 int , c2 binary(16) ) + sql create table if not exists $tb ( ts timestamp , c1 int , c2 binary(16) ) + $i = $i + 1 +endw + +$dropTblStart = $dropTblEnd +$dropTblEnd = $dropTblStart + $numPerUpdateTbl + +$tblStart = $tblEnd +$tblEnd = $tblEnd + $numPerUpdateTbl + +$rowsPerTbl = 0 +$ts = $tsStart +$rowsPerLoop = 160 + +$loopCnt = 0 + +loop_run: +print ================ client-06 start loop insert data from $dropTblStart to $tblStart ( loopCnt: $loopCnt ) +$i = $dropTblStart +while $i < $tblStart + $tb = tb . $i + + $c2Binary = $preBinary . $i + $c2Binary = $c2Binary . ' + + $x = 0 + while $x < $rowsPerLoop + sql insert into $tb values ( $ts + 0a , $x , $c2Binary ) ( $ts + 2a , $x , $c2Binary ) ( $ts + 4a , $x , $c2Binary ) ( $ts + 6a , $x , $c2Binary ) ( $ts + 8a , $x , $c2Binary ) ( $ts + 10a , $x , $c2Binary ) ( $ts + 12a , $x , $c2Binary ) ( $ts + 14a , $x , $c2Binary ) ( $ts + 16a , $x , $c2Binary ) ( $ts + 18a , $x , $c2Binary ) ( $ts + 20a , $x , $c2Binary ) ( $ts + 22a , $x , $c2Binary ) ( $ts + 24a , $x , $c2Binary ) ( $ts + 26a , $x , $c2Binary ) ( $ts + 28a , $x , $c2Binary ) ( $ts + 30a , $x , $c2Binary ) ( $ts + 32a , $x , $c2Binary ) ( $ts + 34a , $x , $c2Binary ) ( $ts + 36a , $x , $c2Binary ) ( $ts + 38a , $x , $c2Binary ) + sql insert into $tb values ( $ts + 1a , $x , $c2Binary ) ( $ts + 3a , $x , $c2Binary ) ( $ts + 5a , $x , $c2Binary ) ( $ts + 7a , $x , $c2Binary ) ( $ts + 9a , $x , $c2Binary ) ( $ts + 11a , $x , $c2Binary ) ( $ts + 13a , $x , $c2Binary ) ( $ts + 15a , $x , $c2Binary ) ( $ts + 17a , $x , $c2Binary ) ( $ts + 19a , $x , $c2Binary ) ( $ts + 21a , $x , $c2Binary ) ( $ts + 23a , $x , $c2Binary ) ( $ts + 25a , $x , $c2Binary ) ( $ts + 27a , $x , $c2Binary ) ( $ts + 29a , $x , $c2Binary ) ( $ts + 31a , $x , $c2Binary ) ( $ts + 33a , $x , $c2Binary ) ( $ts + 35a , $x , $c2Binary ) ( $ts + 37a , $x , $c2Binary ) ( $ts + 39a , $x , $c2Binary ) + $x = $x + 40 + $ts = $ts + 40a + endw + if $i == $dropTblStart then + $rowsPerTbl = $rowsPerTbl + $x + endi + + $totalRows = $totalRows + $x + $i = $i + 1 + +endw + +sql select count(*) from $tb +print data00 $data00 rowsPerTbl $rowsPerTbl +if $data00 != $rowsPerTbl then + print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + print ************ client-06 insert data into $tb error ***** ***** + print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + return -1 +endi + +$loopCnt = $loopCnt + 1 + +if $loopCnt > 2 then + print ================ client-06 drop table from $dropTblStart to $dropTblEnd + $i = $dropTblStart + while $i < $dropTblEnd + $tb = tb . $i + sql drop table if exists $tb + $i = $i + 1 + $totalRows = $totalRows - $rowsPerTbl + endw + sleep 20000 + $testCnt = $testCnt + 1 + goto init_lable +endi + +if $testCnt > 10 then + return 0 +endi +goto loop_run diff --git a/tests/script/unique/clusterSimCase/client-07.sim b/tests/script/unique/clusterSimCase/client-07.sim new file mode 100644 index 0000000000..0dde62248d --- /dev/null +++ b/tests/script/unique/clusterSimCase/client-07.sim @@ -0,0 +1,88 @@ +# create database db1 +# loop create table , insert disorder data, drop table, sleep , +############################################################### +################ set script parameters ################ +$tblStart = 0 +$tblEnd = 10000 +$tsStart = 1325347200000 # 2012-01-01 00:00:00.000 +$preBinary = ' . client-07- +############################################################### + +$totalRows = 0 + +sql connect + +print ================ create db +$db = db +sql create database if not exists $db replica 2 +sql use $db + +init_lable: +print ================ create table $tb (ts timestamp, c1 int, c2 binary(16)) + +$i = $tblStart +while $i < $tblEnd + $tb = tb . $i +# print create table if not exists $tb ( ts timestamp , c1 int , c2 binary(16) ) + sql create table if not exists $tb ( ts timestamp , c1 int , c2 binary(16) ) + $i = $i + 1 +endw + +$rowsPerTbl = 0 +$ts = $tsStart +$rowsPerLoop = 160 + +$loopCnt = 0 + +loop_run: +print ================ client-07 start loop insert data ( loopCnt: $loopCnt ) +$i = $tblStart +while $i < $tblEnd + $tb = tb . $i + + $c2Binary = $preBinary . $i + $c2Binary = $c2Binary . ' + + $x = 0 + while $x < $rowsPerLoop + sql insert into $tb values ( $ts + 0a , $x , $c2Binary ) ( $ts + 2a , $x , $c2Binary ) ( $ts + 4a , $x , $c2Binary ) ( $ts + 6a , $x , $c2Binary ) ( $ts + 8a , $x , $c2Binary ) ( $ts + 10a , $x , $c2Binary ) ( $ts + 12a , $x , $c2Binary ) ( $ts + 14a , $x , $c2Binary ) ( $ts + 16a , $x , $c2Binary ) ( $ts + 18a , $x , $c2Binary ) ( $ts + 20a , $x , $c2Binary ) ( $ts + 22a , $x , $c2Binary ) ( $ts + 24a , $x , $c2Binary ) ( $ts + 26a , $x , $c2Binary ) ( $ts + 28a , $x , $c2Binary ) ( $ts + 30a , $x , $c2Binary ) ( $ts + 32a , $x , $c2Binary ) ( $ts + 34a , $x , $c2Binary ) ( $ts + 36a , $x , $c2Binary ) ( $ts + 38a , $x , $c2Binary ) + sql insert into $tb values ( $ts + 1a , $x , $c2Binary ) ( $ts + 3a , $x , $c2Binary ) ( $ts + 5a , $x , $c2Binary ) ( $ts + 7a , $x , $c2Binary ) ( $ts + 9a , $x , $c2Binary ) ( $ts + 11a , $x , $c2Binary ) ( $ts + 13a , $x , $c2Binary ) ( $ts + 15a , $x , $c2Binary ) ( $ts + 17a , $x , $c2Binary ) ( $ts + 19a , $x , $c2Binary ) ( $ts + 21a , $x , $c2Binary ) ( $ts + 23a , $x , $c2Binary ) ( $ts + 25a , $x , $c2Binary ) ( $ts + 27a , $x , $c2Binary ) ( $ts + 29a , $x , $c2Binary ) ( $ts + 31a , $x , $c2Binary ) ( $ts + 33a , $x , $c2Binary ) ( $ts + 35a , $x , $c2Binary ) ( $ts + 37a , $x , $c2Binary ) ( $ts + 39a , $x , $c2Binary ) + $x = $x + 40 + $ts = $ts + 40a + endw + if $i == $tblStart then + $rowsPerTbl = $rowsPerTbl + $x + endi + + $totalRows = $totalRows + $x + $i = $i + 1 + +endw + +sql select count(*) from $tb +print data00 $data00 rowsPerTbl $rowsPerTbl +if $data00 != $rowsPerTbl then + print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + print ************ client-07 insert data into $tb error ***** ***** + print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + return -1 +endi + +$loopCnt = $loopCnt + 1 + +if $loopCnt > 100 then + + $i = $tblStart + while $i < $tblEnd + $tb = tb . $i + sql alter table $tb add column c3 double + sql alter table $tb add column c4 binary( 16 ) + sql alter table $tb drop column c1 + sql alter table $tb drop column c2 + $i = $i + 1 + endw + sleep 20000 + goto init_lable +endi + +goto loop_run diff --git a/tests/script/unique/clusterSimCase/client-08.sim b/tests/script/unique/clusterSimCase/client-08.sim new file mode 100644 index 0000000000..891b3991b6 --- /dev/null +++ b/tests/script/unique/clusterSimCase/client-08.sim @@ -0,0 +1,93 @@ +# create database db1 +# create table tb () +# loop insert disorder data, drop some tables and create new tables, sleep , +############################################################### +################ set script parameters ################ +$tblStart = 0 +$tblEnd = 10000 +$tsStart = 1325347200000 # 2012-01-01 00:00:00.000 +$preBinary = ' . client-08- +$numPerUpdateTbl = 2000 +############################################################### + +$totalRows = 0 + +sql connect + +print ================ create db +$db = db +sql create database if not exists $db replica 2 +sql use $db + +init_lable: +print ================ create table $tb (ts timestamp, c1 int, c2 binary(16)) + +$i = $tblStart +while $i < $tblEnd + $tb = tb . $i +# print create table if not exists $tb ( ts timestamp , c1 int , c2 binary(16) ) + sql create table if not exists $tb ( ts timestamp , c1 int , c2 binary(16) ) + $i = $i + 1 +endw + +$dropTblStart = $tblStart +$dropTblEnd = $tblStart + $numPerUpdateTbl + +$tblStart = $tblEnd +$tblEnd = $tblEnd + $numPerUpdateTbl + +$rowsPerTbl = 0 +$ts = $tsStart +$rowsPerLoop = 40 + +$loopCnt = 0 + +loop_run: +print ================ client-08 start loop insert data ( loopCnt: $loopCnt ) +$i = $dropTblStart +while $i < $tblEnd + $tb = tb . $i + + $c2Binary = $preBinary . $i + $c2Binary = $c2Binary . ' + + $x = 0 + while $x < $rowsPerLoop + sql insert into $tb values ( $ts + 0a , $x , $c2Binary ) ( $ts + 2a , $x , $c2Binary ) ( $ts + 4a , $x , $c2Binary ) ( $ts + 6a , $x , $c2Binary ) ( $ts + 8a , $x , $c2Binary ) ( $ts + 10a , $x , $c2Binary ) ( $ts + 12a , $x , $c2Binary ) ( $ts + 14a , $x , $c2Binary ) ( $ts + 16a , $x , $c2Binary ) ( $ts + 18a , $x , $c2Binary ) ( $ts + 20a , $x , $c2Binary ) ( $ts + 22a , $x , $c2Binary ) ( $ts + 24a , $x , $c2Binary ) ( $ts + 26a , $x , $c2Binary ) ( $ts + 28a , $x , $c2Binary ) ( $ts + 30a , $x , $c2Binary ) ( $ts + 32a , $x , $c2Binary ) ( $ts + 34a , $x , $c2Binary ) ( $ts + 36a , $x , $c2Binary ) ( $ts + 38a , $x , $c2Binary ) + sql insert into $tb values ( $ts + 1a , $x , $c2Binary ) ( $ts + 3a , $x , $c2Binary ) ( $ts + 5a , $x , $c2Binary ) ( $ts + 7a , $x , $c2Binary ) ( $ts + 9a , $x , $c2Binary ) ( $ts + 11a , $x , $c2Binary ) ( $ts + 13a , $x , $c2Binary ) ( $ts + 15a , $x , $c2Binary ) ( $ts + 17a , $x , $c2Binary ) ( $ts + 19a , $x , $c2Binary ) ( $ts + 21a , $x , $c2Binary ) ( $ts + 23a , $x , $c2Binary ) ( $ts + 25a , $x , $c2Binary ) ( $ts + 27a , $x , $c2Binary ) ( $ts + 29a , $x , $c2Binary ) ( $ts + 31a , $x , $c2Binary ) ( $ts + 33a , $x , $c2Binary ) ( $ts + 35a , $x , $c2Binary ) ( $ts + 37a , $x , $c2Binary ) ( $ts + 39a , $x , $c2Binary ) + $x = $x + 40 + $ts = $ts + 40a + endw + if $i == $tblStart then + $rowsPerTbl = $rowsPerTbl + $x + endi + + $totalRows = $totalRows + $x + $i = $i + 1 + +endw + +sql select count(*) from $tb +print data00 $data00 rowsPerTbl $rowsPerTbl +if $data00 != $rowsPerTbl then + print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + print ************ client-08 insert data into $tb error ***** ***** + print ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + return -1 +endi + +$loopCnt = $loopCnt + 1 + +#if $loopCnt > 100 then + $i = $dropTblStart + while $i < $dropTblEnd + $tb = tb . $i + sql drop table if exists $tb + $i = $i + 1 + $totalRows = $totalRows - $rowsPerTbl + endw + sleep 20000 + goto init_lable +#endi + +goto loop_run diff --git a/tests/script/unique/clusterSimCase/cluster_main.sim b/tests/script/unique/clusterSimCase/cluster_main.sim new file mode 100644 index 0000000000..a2ca34b189 --- /dev/null +++ b/tests/script/unique/clusterSimCase/cluster_main.sim @@ -0,0 +1,248 @@ +# scp -r root@114.116.252.220:/home/ubuntu/clusterSimCase unique/. + +#taos> show dnodes; +# id | end_point | vnodes | cores | status | role | create_time | +#================================================================================================================== +# 1 | ubuntu-OptiPlex-7060:7100 | 0 | 12 | ready | mnode | 2020-07-22 06:25:31.677 | +# 2 | ubuntu-OptiPlex-7060:7200 | 4 | 12 | ready | vnode | 2020-07-22 06:25:34.795 | +# 3 | ubuntu-OptiPlex-7060:7300 | 4 | 12 | ready | vnode | 2020-07-22 06:25:34.803 | +#Query OK, 3 row(s) in set (0.002448s) +#taos> show vgroups; +# vgId | tables | status | onlineVnodes | dnode | vstatus | dnode | vstatus | +#====================================================================================================== +# 2 | 1024 | ready | 2 | 3 | master | 2 | slave | +# 3 | 1024 | ready | 2 | 3 | master | 2 | slave | +# 4 | 1024 | ready | 2 | 3 | master | 2 | slave | +# 5 | 718 | ready | 2 | 3 | master | 2 | slave | +#Query OK, 4 row(s) in set (0.002749s) +# +#taos> show mnodes +# -> ; +# id | end_point | role | create_time | +#===================================================================================== +# 1 | ubuntu-OptiPlex-7060:7100 | master | 2020-07-22 06:25:31.677 | +#Query OK, 1 row(s) in set (0.002126s) + + +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 + +system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1 +system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1 +system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1 +system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1 + +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/cfg.sh -n dnode2 -c walLevel -v 1 +system sh/cfg.sh -n dnode3 -c walLevel -v 1 +system sh/cfg.sh -n dnode4 -c walLevel -v 1 + +system sh/cfg.sh -n dnode1 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode2 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode3 -c balanceInterval -v 10 +system sh/cfg.sh -n dnode4 -c balanceInterval -v 10 + +system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 256 +system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 256 +system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 256 +system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 256 + +#system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1 +#system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 1 +#system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 1 +#system sh/cfg.sh -n dnode4 -c maxVgroupsPerDb -v 1 + +#system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 256 +#system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 256 +#system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 256 +#system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 256 + +system sh/cfg.sh -n dnode1 -c role -v 1 +system sh/cfg.sh -n dnode2 -c role -v 2 +system sh/cfg.sh -n dnode3 -c role -v 2 +system sh/cfg.sh -n dnode4 -c role -v 2 + +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 5000 +system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 5000 +system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 5000 +system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 5000 + +system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator +system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator +system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator +system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator + +print ============== step0: start tarbitrator and dnode1 +system sh/exec_tarbitrator.sh -s start +system sh/exec.sh -n dnode1 -s start + +print ============== step2: start dnode2/dnode3 +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +sleep 1000 +sql connect + +print ============== step1: add dnode2/dnode3 into cluster +sql create dnode $hostname2 +sql create dnode $hostname3 +sleep 3000 +sleep 3000 +print ============== step3: start back client-01.sim +#run_back unique/clusterSimCase/client-01.sim +#run_back unique/clusterSimCase/client-02.sim +#run_back unique/clusterSimCase/client-03.sim +run_back unique/clusterSimCase/client-04.sim +#run_back unique/clusterSimCase/client-05.sim +#run_back unique/clusterSimCase/client-06.sim +#run_back unique/clusterSimCase/client-07.sim +#run_back unique/clusterSimCase/client-08.sim +#run_back unique/clusterSimCase/client-01.sim +#run_back unique/clusterSimCase/client-01.sim +sleep 20000 + +$db = db +sql use $db + +$loop_cnt = 0 +loop_cluster_do: +print **** **** **** START loop cluster do (loop_cnt: $loop_cnt )**** **** **** **** + +print ============== step4: stop dnode3 +system sh/exec.sh -n dnode3 -s stop -x SIGINT + +$checkCnt = 0 +wait_vgroup_chang_0: +$checkCnt = $checkCnt + 1 +if $loopCnt == 10 then + return -1 +endi +print show vgroups +sql show vgroups +#if $rows != 4 then +# print rows: $rows +# sleep 2000 +# goto wait_vgroup_chang_0 +#endi +print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 +print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 +print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 +$vg2Dnode3Status = $data5_2 +$vg2Dnode2Status = $data7_2 + +if $vg2Dnode3Status != offline then + sleep 2000 + goto wait_vgroup_chang_0 +endi +if $vg2Dnode2Status != master then + sleep 2000 + goto wait_vgroup_chang_0 +endi + + +sleep 3000 +print ============== step3: restart dnode3 +system sh/exec.sh -n dnode3 -s start + +$checkCnt = 0 +wait_vgroup_chang_1: +$checkCnt = $checkCnt + 1 +if $loopCnt == 10 then + return -1 +endi +print show vgroups +sql show vgroups +#if $rows != 4 then +# print rows: $rows +# sleep 2000 +# goto wait_vgroup_chang_1 +#endi +print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 +print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 +print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 +$vg2Dnode3Status = $data5_2 +$vg2Dnode2Status = $data7_2 + +if $vg2Dnode3Status != slave then + sleep 2000 + goto wait_vgroup_chang_1 +endi +if $vg2Dnode2Status != master then + sleep 2000 + goto wait_vgroup_chang_1 +endi + +sleep 3000 +print ============== step4: stop dnode2 +system sh/exec.sh -n dnode2 -s stop -x SIGINT + +$checkCnt = 0 +wait_vgroup_chang_2: +$checkCnt = $checkCnt + 1 +if $loopCnt == 10 then + return -1 +endi +print show vgroups +sql show vgroups +#if $rows != 4 then +# print rows: $rows +# sleep 2000 +# goto wait_vgroup_chang_2 +#endi +print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 +print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 +print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 +$vg2Dnode3Status = $data5_2 +$vg2Dnode2Status = $data7_2 + +if $vg2Dnode3Status != master then + sleep 2000 + goto wait_vgroup_chang_2 +endi +if $vg2Dnode2Status != offline then + sleep 2000 + goto wait_vgroup_chang_2 +endi + + +sleep 3000 +print ============== step5: restart dnode2 +system sh/exec.sh -n dnode2 -s start + +$checkCnt = 0 +wait_vgroup_chang_3: +$checkCnt = $checkCnt + 1 +if $loopCnt == 10 then + return -1 +endi +print show vgroups +sql show vgroups +#if $rows != 4 then +# print rows: $rows +# sleep 2000 +# goto wait_vgroup_chang_3 +#endi +print $data0_1 $data1_1 $data2_1 $data3_1 $data4_1 $data5_1 $data6_1 $data7_1 $data8_1 $data9_1 +print $data0_2 $data1_2 $data2_2 $data3_2 $data4_2 $data5_2 $data6_2 $data7_2 $data8_2 $data9_2 +print $data0_3 $data1_3 $data2_3 $data3_3 $data4_3 $data5_3 $data6_3 $data7_3 $data8_3 $data9_3 +$vg2Dnode3Status = $data5_2 +$vg2Dnode2Status = $data7_2 + +if $vg2Dnode2Status != slave then + sleep 2000 + goto wait_vgroup_chang_3 +endi +if $vg2Dnode3Status != master then + sleep 2000 + goto wait_vgroup_chang_3 +endi + +sleep 3000 +print **** **** **** (loop_cnt: $loop_cnt ) end, continue...... **** **** **** **** +$loop_cnt = $loop_cnt + 1 +if $loop_cnt == 50 then + return 0 +endi +goto loop_cluster_do