Merge branch '3.0' of https://github.com/taosdata/TDengine into refact/tsdb_last

cmake/cmake.define

@@ -81,7 +81,7 @@ ENDIF ()
 
 IF (TD_WINDOWS)
     MESSAGE("${Yellow} set compiler flag for Windows! ${ColourReset}")
-    SET(COMMON_FLAGS "/w /D_WIN32 /DWIN32 /Zi")
+    SET(COMMON_FLAGS "/w /D_WIN32 /DWIN32 /Zi /MTd")
     SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /MANIFEST:NO")
     # IF (MSVC AND (MSVC_VERSION GREATER_EQUAL 1900))
     #  SET(COMMON_FLAGS "${COMMON_FLAGS} /Wv:18")
@@ -92,6 +92,12 @@ IF (TD_WINDOWS)
     IF (CMAKE_DEPFILE_FLAGS_CXX)
         SET(CMAKE_DEPFILE_FLAGS_CXX "")
     ENDIF ()
+    IF (CMAKE_C_FLAGS_DEBUG)
+        SET(CMAKE_C_FLAGS_DEBUG "" CACHE STRING "" FORCE)
+    ENDIF ()
+    IF (CMAKE_CXX_FLAGS_DEBUG)
+        SET(CMAKE_CXX_FLAGS_DEBUG "" CACHE STRING "" FORCE)
+    ENDIF ()
 
     SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${COMMON_FLAGS}")
     SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COMMON_FLAGS}")

contrib/CMakeLists.txt

@@ -273,7 +273,7 @@ endif(${BUILD_WITH_NURAFT})
 
 # pthread
 if(${BUILD_PTHREAD})
-    set(CMAKE_BUILD_TYPE release)
+    set(CMAKE_BUILD_TYPE debug)
     add_definitions(-DPTW32_STATIC_LIB)
     add_subdirectory(pthread EXCLUDE_FROM_ALL)
     set_target_properties(libpthreadVC3 PROPERTIES OUTPUT_NAME pthread)

packaging/cfg/taos.cfg

@@ -5,137 +5,54 @@
 #                                                      #
 ########################################################
 
-# first fully qualified domain name (FQDN) for TDengine system
+# The end point of the first dnode in the cluster to be connected to when `taosd` or `taos` is started
 # firstEp                   hostname:6030
 
-# local fully qualified domain name (FQDN)
+# The end point of the second dnode to be connected to if the firstEp is not available when `taosd` or `taos` is started
+# secondEp
+
+# The FQDN of the host where `taosd` will be started. It can be IP address
 # fqdn                      hostname
 
-# first port number for the connection (12 continuous UDP/TCP port number are used) 
+# The port for external access after `taosd` is started     
 # serverPort                6030
 
-# log file's directory
+# The maximum number of connections a dnode can accept
+# maxShellConns         5000
+
+# The directory for writing log files
 # logDir                    /var/log/taos
 
-# data file's directory
+# All data files are stored in this directory
 # dataDir                   /var/lib/taos
 
 # temporary file's directory
 # tempDir                   /tmp/
 
-# the arbitrator's fully qualified domain name (FQDN) for TDengine system, for cluster only   
+# Switch for allowing TDengine to collect and report service usage information
-# arbitrator                arbitrator_hostname:6042     
-
-# number of threads per CPU core
-# numOfThreadsPerCore       1.0
-
-# number of threads to commit cache data
-# numOfCommitThreads        4
-
-# the proportion of total CPU cores available for query processing
-# 2.0: the query threads will be set to double of the CPU cores.
-# 1.0: all CPU cores are available for query processing [default].
-# 0.5: only half of the CPU cores are available for query.
-# 0.0: only one core available.
-# ratioOfQueryCores        1.0
-
-# the last_row/first/last aggregator will not change the original column name in the result fields
-keepColumnName            1
-
-# number of management nodes in the system
-# numOfMnodes               1
-
-# enable/disable backuping vnode directory when removing vnode
-# vnodeBak                  1
-
-# enable/disable installation / usage report
 # telemetryReporting        1
 
-# enable/disable load balancing
+# The maximum number of vnodes supported by dnode
-# balance                   1
+# supportVnodes             0
 
-# role for dnode. 0 - any, 1 - mnode, 2 - dnode
+# The interval of dnode reporting status to mnode
-# role                      0
-
-# max timer control blocks
-# maxTmrCtrl                512
-
-# time interval of system monitor, seconds 
-# monitorInterval           30
-
-# number of seconds allowed for a dnode to be offline, for cluster only 
-# offlineThreshold          864000
-
-# RPC re-try timer, millisecond
-# rpcTimer                  300
-
-# RPC maximum time for ack, seconds. 
-# rpcMaxTime                600
-
-# time interval of dnode status reporting to mnode, seconds, for cluster only 
 # statusInterval            1
 
-# time interval of heart beat from shell to dnode, seconds
+# The interval for taos shell to send heartbeat to mnode
 # shellActivityTimer        3
 
-# minimum sliding window time, milli-second
+# The minimum sliding window time, milli-second
 # minSlidingTime            10
 
-# minimum time window, milli-second
+# The minimum time window, milli-second
 # minIntervalTime           10
 
-# maximum delay before launching a stream computation, milli-second
+# The maximum allowed query buffer size in MB during query processing for each data node
-# maxStreamCompDelay        20000
+# -1 no limit (default)
+# 0  no query allowed, queries are disabled
+# queryBufferSize           -1
 
-# maximum delay before launching a stream computation for the first time, milli-second
+# The compressed rpc message, option:
-# maxFirstStreamCompDelay   10000
-
-# retry delay when a stream computation fails, milli-second
-# retryStreamCompDelay      10
-
-# the delayed time for launching a stream computation, from 0.1(default, 10% of whole computing time window) to 0.9
-# streamCompDelayRatio      0.1
-
-# max number of vgroups per db, 0 means configured automatically
-# maxVgroupsPerDb           0
-
-# max number of tables per vnode
-# maxTablesPerVnode         1000000
-
-# cache block size (Mbyte)
-# cache                     16
-
-# number of cache blocks per vnode
-# blocks                    6
-
-# number of days per DB file
-# days                  10
-
-# number of days to keep DB file
-# keep                  3650
-
-# minimum rows of records in file block
-# minRows               100
-
-# maximum rows of records in file block
-# maxRows               4096
-
-# the number of acknowledgments required for successful data writing
-# quorum                1     
-
-# enable/disable compression
-# comp                  2
-
-# write ahead log (WAL) level, 0: no wal; 1: write wal, but no fysnc; 2: write wal, and call fsync
-# walLevel              1
-
-# if walLevel is set to 2, the cycle of fsync being executed, if set to 0, fsync is called right away
-# fsync                 3000
-
-# number of replications, for cluster only 
-# replica               1
-
-# the compressed rpc message, option:
 #  -1 (no compression)
 #   0 (all message compressed),
 # > 0 (rpc message body which larger than this value will be compressed)
@@ -147,15 +64,6 @@ keepColumnName            1
 # > 0 (any retrieved column size greater than this value all data will be compressed.)
 # compressColData           -1
 
-# max length of an SQL
-# maxSQLLength          65480
-
-# max length of WildCards
-# maxWildCardsLength    100
-
-# the maximum number of records allowed for super table time sorting
-# maxNumOfOrderedRes    100000
-
 # system time zone
 # timezone              Asia/Shanghai (CST, +0800)
 # system time zone (for windows 10)
@@ -167,12 +75,6 @@ keepColumnName            1
 # default system charset
 # charset                   UTF-8
 
-# max number of connections allowed in dnode
-# maxShellConns         5000
-
-# max number of connections allowed in client
-# maxConnections        5000
-
 # stop writing logs when the disk size of the log folder is less than this value
 # minimalLogDirGB           1.0
 
@@ -182,30 +84,9 @@ keepColumnName            1
 # if disk free space is less than this value, taosd service exit directly within startup process
 # minimalDataDirGB          2.0
 
-# One mnode is equal to the number of vnode consumed
-# mnodeEqualVnodeNum    4
-
-# enbale/disable http service
-# http                  1
-
 # enable/disable system monitor 
 # monitor                   1
 
-# enable/disable recording the SQL statements via restful interface
-# httpEnableRecordSql   0
-
-# number of threads used to process http requests
-# httpMaxThreads        2
-
-# maximum number of rows returned by the restful interface
-# restfulRowLimit       10240
-
-# database name must be specified in restful interface if the following parameter is set, off by default
-# httpDbNameMandatory 1
-
-# http keep alive, default is 30 seconds
-# httpKeepAlive   30000
-
 # The following parameter is used to limit the maximum number of lines in log files.
 # max number of lines per log filters
 # numOfLogLines             10000000
@@ -216,7 +97,6 @@ keepColumnName            1
 # time of keeping log files, days
 # logKeepDays               0
 
-
 # The following parameters are used for debug purpose only.
 # debugFlag 8 bits mask: FILE-SCREEN-UNUSED-HeartBeat-DUMP-TRACE_WARN-ERROR
 # 131: output warning and error 
@@ -228,85 +108,62 @@ keepColumnName            1
 # debug flag for all log type, take effect when non-zero value
 # debugFlag                 0
 
-# debug flag for meta management messages
+# debug flag for timer
-# mDebugFlag            135
-
-# debug flag for dnode messages
-# dDebugFlag            135
-
-# debug flag for sync module
-# sDebugFlag            135
-
-# debug flag for WAL
-# wDebugFlag            135
-
-# debug flag for SDB
-# sdbDebugFlag          135
-
-# debug flag for RPC 
-# rpcDebugFlag          131
-
-# debug flag for TAOS TIMER
 # tmrDebugFlag              131
 
-# debug flag for TDengine client 
+# debug flag for util
-# cDebugFlag            131
-
-# debug flag for JNI
-# jniDebugFlag          131
-
-# debug flag for storage
 # uDebugFlag                131
 
-# debug flag for http server
+# debug flag for rpc 
-# httpDebugFlag         131
+# rpcDebugFlag              131
 
-# debug flag for monitor
+# debug flag for jni
-# monDebugFlag          131
+# jniDebugFlag              131
 
 # debug flag for query
 # qDebugFlag                131
 
+# debug flag for taosc driver 
+# cDebugFlag                131
+
+# debug flag for dnode messages
+# dDebugFlag                135
+
 # debug flag for vnode
 # vDebugFlag                131
 
-# debug flag for TSDB
+# debug flag for meta management messages
+# mDebugFlag                135
+
+# debug flag for wal
+# wDebugFlag                135
+
+# debug flag for sync module
+# sDebugFlag                135
+
+# debug flag for tsdb
 # tsdbDebugFlag             131
 
-# debug flag for continue query
+# debug flag for tq
-# cqDebugFlag           131
+# tqDebugFlag               131
 
-# enable/disable recording the SQL in taos client
+# debug flag for fs
-# enableRecordSql    0
+# fsDebugFlag               131
+
+# debug flag for udf
+# udfDebugFlag              131
+
+# debug flag for sma
+# smaDebugFlag              131
+
+# debug flag for index
+# idxDebugFlag              131
+
+# debug flag for tdb
+# tdbDebugFlag              131
+
+# debug flag for meta
+# metaDebugFlag             131
 
 # generate core file when service crash
 # enableCoreFile            1
-
-# maximum display width of binary and nchar fields in the shell. The parts exceeding this limit will be hidden
-# maxBinaryDisplayWidth 30
-
-# enable/disable stream (continuous query)
-# stream                1
-
-# in retrieve blocking model, only in 50% query threads will be used in query processing in dnode
-# retrieveBlockingModel    0
-
-# the maximum allowed query buffer size in MB during query processing for each data node
-# -1 no limit (default)
-# 0  no query allowed, queries are disabled
-# queryBufferSize         -1
-
-# percent of redundant data in tsdb meta will compact meta data,0 means donot compact
-# tsdbMetaCompactRatio    0
-
-# default string type used for storing JSON String, options can be binary/nchar, default is nchar
-# defaultJSONStrType      nchar
-
-# force TCP transmission 
-# rpcForceTcp        0
-
-# unit MB. Flush vnode wal file if walSize > walFlushSize and walSize > cache*0.5*blocks
-# walFlushSize         1024
-
-# unit Hour. Latency of data migration
-# keepTimeOffset     0

packaging/deb/DEBIAN/prerm

@@ -29,6 +29,7 @@ else
     # Remove all links
     ${csudo}rm -f ${bin_link_dir}/taos       || :
     ${csudo}rm -f ${bin_link_dir}/taosd      || :
+    ${csudo}rm -f ${bin_link_dir}/udfd       || :
     ${csudo}rm -f ${bin_link_dir}/taosadapter       || :
     ${csudo}rm -f ${bin_link_dir}/taosdemo   || :
     ${csudo}rm -f ${cfg_link_dir}/*          || :

packaging/deb/makedeb.sh

@@ -60,6 +60,7 @@ cp ${compile_dir}/../packaging/tools/set_core.sh    ${pkg_dir}${install_home_pat
 cp ${compile_dir}/../packaging/tools/taosd-dump-cfg.gdb    ${pkg_dir}${install_home_path}/bin
 
 cp ${compile_dir}/build/bin/taosd                   ${pkg_dir}${install_home_path}/bin
+cp ${compile_dir}/build/bin/udfd                   ${pkg_dir}${install_home_path}/bin
 cp ${compile_dir}/build/bin/taosBenchmark           ${pkg_dir}${install_home_path}/bin
 
 if [ -f "${compile_dir}/build/bin/taosadapter" ]; then

packaging/rpm/tdengine.spec

@@ -69,6 +69,7 @@ cp %{_compiledir}/../packaging/tools/set_core.sh    %{buildroot}%{homepath}/bin
 cp %{_compiledir}/../packaging/tools/taosd-dump-cfg.gdb    %{buildroot}%{homepath}/bin
 cp %{_compiledir}/build/bin/taos                    %{buildroot}%{homepath}/bin
 cp %{_compiledir}/build/bin/taosd                   %{buildroot}%{homepath}/bin
+cp %{_compiledir}/build/bin/udfd                    %{buildroot}%{homepath}/bin
 cp %{_compiledir}/build/bin/taosBenchmark           %{buildroot}%{homepath}/bin
 
 if [ -f %{_compiledir}/build/bin/taosadapter ]; then
@@ -204,6 +205,7 @@ if [ $1 -eq 0 ];then
     # Remove all links
     ${csudo}rm -f ${bin_link_dir}/taos       || :
     ${csudo}rm -f ${bin_link_dir}/taosd      || :
+    ${csudo}rm -f ${bin_link_dir}/udfd       || :
     ${csudo}rm -f ${bin_link_dir}/taosadapter       || :
     ${csudo}rm -f ${cfg_link_dir}/*          || :
     ${csudo}rm -f ${inc_link_dir}/taos.h     || :

packaging/tools/install.sh

@@ -18,6 +18,7 @@ script_dir=$(dirname $(readlink -f "$0"))
 
 clientName="taos"
 serverName="taosd"
+udfdName="udfd"
 configFile="taos.cfg"
 productName="TDengine"
 emailName="taosdata.com"
@@ -192,6 +193,7 @@ function install_bin() {
   # Remove links
   ${csudo}rm -f ${bin_link_dir}/${clientName} || :
   ${csudo}rm -f ${bin_link_dir}/${serverName} || :
+  ${csudo}rm -f ${bin_link_dir}/${udfdName} || :
   ${csudo}rm -f ${bin_link_dir}/${adapterName} || :
   ${csudo}rm -f ${bin_link_dir}/${uninstallScript} || :
   ${csudo}rm -f ${bin_link_dir}/${demoName} || :
@@ -205,6 +207,7 @@ function install_bin() {
   #Make link
   [ -x ${install_main_dir}/bin/${clientName} ] && ${csudo}ln -s ${install_main_dir}/bin/${clientName} ${bin_link_dir}/${clientName} || :
   [ -x ${install_main_dir}/bin/${serverName} ] && ${csudo}ln -s ${install_main_dir}/bin/${serverName} ${bin_link_dir}/${serverName} || :
+  [ -x ${install_main_dir}/bin/${udfdName} ] && ${csudo}ln -s ${install_main_dir}/bin/${udfdName} ${bin_link_dir}/${udfdName} || :
   [ -x ${install_main_dir}/bin/${adapterName} ] && ${csudo}ln -s ${install_main_dir}/bin/${adapterName} ${bin_link_dir}/${adapterName} || :
   [ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -s ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${demoName} || :
   [ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -s ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${benchmarkName} || :
@@ -742,7 +745,7 @@ function is_version_compatible() {
   fi
 
   exist_version=$(${installDir}/bin/${serverName} -V | head -1 | cut -d ' ' -f 3)
-  vercomp $exist_version "2.0.16.0"
+  vercomp $exist_version "3.0.0.0"
   case $? in
   2)
     prompt_force=1

packaging/tools/makepkg.sh

@@ -85,6 +85,7 @@ else
       ${build_dir}/bin/${clientName} \
       ${taostools_bin_files} \
       ${build_dir}/bin/taosadapter \
+      ${build_dir}/bin/udfd \
       ${script_dir}/remove.sh \
       ${script_dir}/set_core.sh \
       ${script_dir}/startPre.sh \
@@ -318,7 +319,7 @@ if [ "$verMode" == "cluster" ]; then
 fi
 
 # Copy release note
-cp ${script_dir}/release_note ${install_dir}
+# cp ${script_dir}/release_note ${install_dir}
 
 # exit 1
 

packaging/tools/post.sh

@@ -118,6 +118,7 @@ function install_bin() {
     # Remove links
     ${csudo}rm -f ${bin_link_dir}/taos     || :
     ${csudo}rm -f ${bin_link_dir}/taosd    || :
+    ${csudo}rm -f ${bin_link_dir}/udfd     || :
     ${csudo}rm -f ${bin_link_dir}/taosadapter     || :
     ${csudo}rm -f ${bin_link_dir}/taosBenchmark || :
     ${csudo}rm -f ${bin_link_dir}/taosdemo || :
@@ -130,6 +131,7 @@ function install_bin() {
     #Make link
     [ -x ${bin_dir}/taos ] && ${csudo}ln -s ${bin_dir}/taos ${bin_link_dir}/taos             || :
     [ -x ${bin_dir}/taosd ] && ${csudo}ln -s ${bin_dir}/taosd ${bin_link_dir}/taosd          || :
+    [ -x ${bin_dir}/udfd ] && ${csudo}ln -s ${bin_dir}/udfd ${bin_link_dir}/udfd             || :
     [ -x ${bin_dir}/taosadapter ] && ${csudo}ln -s ${bin_dir}/taosadapter ${bin_link_dir}/taosadapter             || :
     [ -x ${bin_dir}/taosBenchmark ] && ${csudo}ln -sf ${bin_dir}/taosBenchmark ${bin_link_dir}/taosdemo || :
     [ -x ${bin_dir}/taosBenchmark ] && ${csudo}ln -sf ${bin_dir}/taosBenchmark ${bin_link_dir}/taosBenchmark || :

packaging/tools/tdengine.iss

@@ -51,7 +51,7 @@ Source: taos.bat; DestDir: "{app}\include"; Flags: igNoreversion;
 ;Source: taosdemo.png; DestDir: "{app}\include"; Flags: igNoreversion;
 ;Source: taosShell.png; DestDir: "{app}\include"; Flags: igNoreversion;
 Source: favicon.ico; DestDir: "{app}\include"; Flags: igNoreversion;
-Source: {#MyAppSourceDir}{#MyAppDLLName}; DestDir: "{win}\System32"; Flags: igNoreversion;
+Source: {#MyAppSourceDir}{#MyAppDLLName}; DestDir: "{win}\System32"; Flags: 64bit;Check:IsWin64;
 Source: {#MyAppSourceDir}{#MyAppCfgName}; DestDir: "{app}\cfg"; Flags: igNoreversion recursesubdirs createallsubdirs onlyifdoesntexist uninsneveruninstall
 Source: {#MyAppSourceDir}{#MyAppDriverName}; DestDir: "{app}\driver"; Flags: igNoreversion recursesubdirs createallsubdirs
 ;Source: {#MyAppSourceDir}{#MyAppConnectorName}; DestDir: "{app}\connector"; Flags: igNoreversion recursesubdirs createallsubdirs

source/os/src/osRand.c

@@ -37,9 +37,13 @@ uint32_t taosRandR(uint32_t *pSeed) {
 
 uint32_t taosSafeRand(void) {
 #ifdef WINDOWS
-  uint32_t seed;
+  uint32_t seed = taosRand();
   HCRYPTPROV hCryptProv;
-  if (!CryptAcquireContext(&hCryptProv, NULL, NULL, PROV_RSA_FULL, 0)) return seed;
+  if (!CryptAcquireContext(&hCryptProv, NULL, NULL, PROV_RSA_FULL, 0)) {
+    if (!CryptAcquireContext(&hCryptProv, NULL, NULL, PROV_RSA_FULL, CRYPT_NEWKEYSET)) {
+      return seed;
+    }
+  }
   if (hCryptProv != NULL) {
     if (!CryptGenRandom(hCryptProv, 4, &seed)) return seed;
   }