diff --git a/.gitignore b/.gitignore index c5d90eea50..e6e327327c 100644 --- a/.gitignore +++ b/.gitignore @@ -66,6 +66,8 @@ CMakeError.log /test/cfg /src/.vs *.o +version.c +taos.rc src/connector/jdbc/.settings/ tests/comparisonTest/cassandra/cassandratest/.classpath tests/comparisonTest/cassandra/cassandratest/.project diff --git a/CMakeLists.txt b/CMakeLists.txt index bc6a888f9d..565ab32f00 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,6 +18,7 @@ SET(TD_COVER FALSE) SET(TD_MEM_CHECK FALSE) SET(TD_PAGMODE_LITE FALSE) +SET(TD_SOMODE_STATIC FALSE) SET(TD_GODLL FALSE) SET(TD_COMMUNITY_DIR ${PROJECT_SOURCE_DIR}) @@ -27,6 +28,7 @@ INCLUDE(cmake/input.inc) INCLUDE(cmake/platform.inc) INCLUDE(cmake/define.inc) INCLUDE(cmake/env.inc) +INCLUDE(cmake/version.inc) INCLUDE(cmake/install.inc) ADD_SUBDIRECTORY(deps) diff --git a/README.md b/README.md index 2d84389f78..8f29adf89b 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@ [![Build status](https://ci.appveyor.com/api/projects/status/kf3pwh2or5afsgl9/branch/master?svg=true)](https://ci.appveyor.com/project/sangshuduo/tdengine-2n8ge/branch/master) [![Coverage Status](https://coveralls.io/repos/github/taosdata/TDengine/badge.svg?branch=develop)](https://coveralls.io/github/taosdata/TDengine?branch=develop) [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/4201/badge)](https://bestpractices.coreinfrastructure.org/projects/4201) +[![Docker Pulls](https://img.shields.io/docker/pulls/tdengine/tdengine)](https://hub.docker.com/repository/docker/tdengine/tdengine) [![TDengine](TDenginelogo.png)](https://www.taosdata.com) diff --git a/cmake/define.inc b/cmake/define.inc index c72995159f..a63b9f1732 100755 --- a/cmake/define.inc +++ b/cmake/define.inc @@ -77,6 +77,11 @@ IF (TD_LINUX) ADD_DEFINITIONS(-D_LINUX) ADD_DEFINITIONS(-D_TD_LINUX) ADD_DEFINITIONS(-D_REENTRANT -D__USE_POSIX -D_LIBC_REENTRANT) + + IF (TD_NINGSI_60) + ADD_DEFINITIONS(-D_TD_NINGSI_60_) + MESSAGE(STATUS "set ningsi macro to true") + ENDIF () SET(DEBUG_FLAGS "-O0 -DDEBUG") SET(RELEASE_FLAGS "-O0") @@ -110,7 +115,7 @@ IF (TD_WINDOWS) ADD_DEFINITIONS(-D_MBCS -D_CRT_SECURE_NO_DEPRECATE -D_CRT_NONSTDC_NO_DEPRECATE) SET(CMAKE_GENERATOR "NMake Makefiles" CACHE INTERNAL "" FORCE) IF (NOT TD_GODLL) - SET(COMMON_FLAGS "/nologo /WX /Oi /Oy- /Gm- /EHsc /MT /GS /Gy /fp:precise /Zc:wchar_t /Zc:forScope /Gd /errorReport:prompt /analyze-") + SET(COMMON_FLAGS "/nologo /WX /wd4018 /wd2220 /Oi /Oy- /Gm- /EHsc /MT /GS /Gy /fp:precise /Zc:wchar_t /Zc:forScope /Gd /errorReport:prompt /analyze-") SET(DEBUG_FLAGS "/Zi /W3 /GL") SET(RELEASE_FLAGS "/W0 /GL") ENDIF () diff --git a/cmake/input.inc b/cmake/input.inc index e963e20240..0235ba42d3 100755 --- a/cmake/input.inc +++ b/cmake/input.inc @@ -22,6 +22,11 @@ IF (${PAGMODE} MATCHES "lite") MESSAGE(STATUS "Build with pagmode lite") ENDIF () +IF (${SOMODE} MATCHES "static") + SET(TD_SOMODE_STATIC TRUE) + MESSAGE(STATUS "Link so using static mode") +ENDIF () + IF (${DLLTYPE} MATCHES "go") SET(TD_GODLL TRUE) MESSAGE(STATUS "input dll type: " ${DLLTYPE}) diff --git a/cmake/platform.inc b/cmake/platform.inc index 71eee9d507..7834a35411 100755 --- a/cmake/platform.inc +++ b/cmake/platform.inc @@ -22,6 +22,9 @@ SET(TD_LINUX FALSE) SET(TD_MIPS_64 FALSE) SET(TD_MIPS_32 FALSE) SET(TD_APLHINE FALSE) + SET(TD_NINGSI FALSE) + SET(TD_NINGSI_60 FALSE) + SET(TD_NINGSI_80 FALSE) SET(TD_WINDOWS FALSE) SET(TD_WINDOWS_64 FALSE) SET(TD_WINDOWS_32 FALSE) @@ -99,3 +102,18 @@ ELSEIF (${CPUTYPE} MATCHES "x86") ELSE () MESSAGE(STATUS "input cpuType unknown " ${CPUTYPE}) ENDIF () + +# cmake -DOSTYPE=Ningsi +IF (${OSTYPE} MATCHES "Ningsi60") + SET(TD_NINGSI TRUE) + SET(TD_NINGSI_60 TRUE) + MESSAGE(STATUS "input osType: Ningsi60") +ELSEIF (${OSTYPE} MATCHES "Ningsi80") + SET(TD_NINGSI TRUE) + SET(TD_NINGSI_80 TRUE) + MESSAGE(STATUS "input osType: Ningsi80") +ELSEIF (${OSTYPE} MATCHES "Linux") + MESSAGE(STATUS "input osType: Linux") +ELSE () + MESSAGE(STATUS "input osType unknown: " ${OSTYPE}) +ENDIF () \ No newline at end of file diff --git a/cmake/version.inc b/cmake/version.inc new file mode 100644 index 0000000000..c620d753a6 --- /dev/null +++ b/cmake/version.inc @@ -0,0 +1,69 @@ +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +PROJECT(TDengine) + +IF (DEFINED VERNUMBER) + SET(TD_VER_NUMBER ${VERNUMBER}) +ELSE () + SET(TD_VER_NUMBER "2.0.2.0") +ENDIF () + +IF (DEFINED VERCOMPATIBLE) + SET(TD_VER_COMPATIBLE ${VERCOMPATIBLE}) +ELSE () + SET(TD_VER_COMPATIBLE "2.0.0.0") +ENDIF () + +IF (DEFINED GITINFO) + SET(TD_VER_GIT ${GITINFO}) +ELSE () + SET(TD_VER_GIT "community") +ENDIF () + +IF (DEFINED GITINFOI) + SET(TD_VER_GIT_INTERNAL ${GITINFOI}) +ELSE () + SET(TD_VER_GIT_INTERNAL "internal") +ENDIF () + +IF (DEFINED VERDATE) + SET(TD_VER_DATE ${VERDATE}) +ELSE () + STRING(TIMESTAMP TD_VER_DATE "%Y-%m-%d %H:%M:%S") +ENDIF () + +IF (DEFINED VERTYPE) + SET(TD_VER_VERTYPE ${VERTYPE}) +ELSE () + SET(TD_VER_VERTYPE "stable") +ENDIF () + +IF (DEFINED CPUTYPE) + SET(TD_VER_CPUTYPE ${CPUTYPE}) +ELSE () + IF (TD_WINDOWS_32) + SET(TD_VER_CPUTYPE "x86") + ELSE () + SET(TD_VER_CPUTYPE "x64") + ENDIF () +ENDIF () + +IF (DEFINED OSTYPE) + SET(TD_VER_OSTYPE ${OSTYPE}) +ELSE () + SET(TD_VER_OSTYPE "Linux") +ENDIF () + +MESSAGE(STATUS "============= compile version parameter information start ============= ") +MESSAGE(STATUS "ver number:" ${TD_VER_NUMBER}) +MESSAGE(STATUS "compatible ver number:" ${TD_VER_COMPATIBLE}) +MESSAGE(STATUS "communit commit id:" ${TD_VER_GIT}) +MESSAGE(STATUS "internal commit id:" ${TD_VER_GIT_INTERNAL}) +MESSAGE(STATUS "build date:" ${TD_VER_DATE}) +MESSAGE(STATUS "ver type:" ${TD_VER_VERTYPE}) +MESSAGE(STATUS "ver cpu:" ${TD_VER_CPUTYPE}) +MESSAGE(STATUS "os type:" ${TD_VER_OSTYPE}) +MESSAGE(STATUS "============= compile version parameter information end ============= ") + +STRING(REPLACE "." "_" TD_LIB_VER_NUMBER ${TD_VER_NUMBER}) + +CONFIGURE_FILE("${TD_COMMUNITY_DIR}/src/util/src/version.c.in" "${TD_COMMUNITY_DIR}/src/util/src/version.c") diff --git a/deps/MsvcLibX/CMakeLists.txt b/deps/MsvcLibX/CMakeLists.txt index fc77a3b447..4428579e1c 100644 --- a/deps/MsvcLibX/CMakeLists.txt +++ b/deps/MsvcLibX/CMakeLists.txt @@ -4,5 +4,5 @@ PROJECT(TDengine) IF (TD_WINDOWS) INCLUDE_DIRECTORIES(include) AUX_SOURCE_DIRECTORY(src SRC) - ADD_LIBRARY(MsvcLibXw64 ${SRC}) + ADD_LIBRARY(MsvcLibXw ${SRC}) ENDIF () diff --git a/deps/MsvcLibX/include/msvclibx.h b/deps/MsvcLibX/include/msvclibx.h index 4f5fc2733b..00953f13d0 100644 --- a/deps/MsvcLibX/include/msvclibx.h +++ b/deps/MsvcLibX/include/msvclibx.h @@ -59,7 +59,7 @@ /* Generate the OS-and-debug-mode-specific library name */ #define _MSVCLIBX_LIB "MsvcLibX" _MSVCLIBX_LIB_OS_SUFFIX _MSVCLIBX_LIB_DBG_SUFFIX ".lib" //#pragma message("Adding pragma comment(lib, \"" _MSVCLIBX_LIB "\")") -#pragma comment(lib, _MSVCLIBX_LIB) +//#pragma comment(lib, _MSVCLIBX_LIB) /* Library-specific routine used internally by many standard routines */ #if defined(_WIN32) diff --git a/documentation20/webdocs/markdowndocs/Model-ch.md b/documentation20/webdocs/markdowndocs/Model-ch.md index f5776ce6ce..31eb6b3744 100644 --- a/documentation20/webdocs/markdowndocs/Model-ch.md +++ b/documentation20/webdocs/markdowndocs/Model-ch.md @@ -9,9 +9,9 @@ TDengine采用关系型数据模型，需要建库、建表。因此对于一个 不同类型的数据采集点往往具有不同的数据特征，包括数据采集频率的高低，数据保留时间的长短，副本的数目，数据块的大小等等。为让各种场景下TDengine都能最大效率的工作，TDengine建议将不同数据特征的表创建在不同的库里，因为每个库可以配置不同的存储策略。创建一个库时，除SQL标准的选项外，应用还可以指定保留时长、副本数、内存块个数、时间精度、文件块里最大最小记录条数、是否压缩、一个数据文件覆盖的天数等多种参数。比如： ```cmd -CREATE DATABASE power KEEP 365 DAYS 10 REPLICA 3 BLOCKS 4; +CREATE DATABASE power KEEP 365 DAYS 10 BLOCKS 4; ``` -上述语句将创建一个名为power的库，这个库的数据将保留365天（超过365天将被自动删除），每10天一个数据文件，副本数为3, 内存块数为4。详细的语法及参数请见TAOS SQL +上述语句将创建一个名为power的库，这个库的数据将保留365天（超过365天将被自动删除），每10天一个数据文件，内存块数为4。详细的语法及参数请见TAOS SQL 创建库之后，需要使用SQL命令USE将当前库切换过来，例如： @@ -27,13 +27,15 @@ USE power; - 处于两个不同库的表是不能进行JOIN操作的。 ## 创建超级表 -一个物联网系统，往往存在多种类型的设备，比如对于电网，存在智能电表、变压器、母线、开关等等。为便于多表之间的聚合，使用TDengine, 需要对每个类型的设备创建一超级表。以表一中的智能电表为例，可以使用如下的SQL命令创建超级表： +一个物联网系统，往往存在多种类型的设备，比如对于电网，存在智能电表、变压器、母线、开关等等。为便于多表之间的聚合，使用TDengine, 需要对每个类型的数据采集点创建一超级表。以表一中的智能电表为例，可以使用如下的SQL命令创建超级表： ```cmd CREATE TABLE meters (ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupdId int); ``` -与创建普通表一样，创建表时，需要提供表名（示例中为meters），表结构Schema，即数据列的定义，为采集的物理量（示例中为ts, current, voltage, phase)，数据类型可以为整型、浮点型、字符串等。除此之外，还需要提供标签的schema (示例中为location, groupId)，标签的数据类型可以为整型、浮点型、字符串等。采集点的静态属性往往可以作为标签，比如采集点的地理位置、设备型号、设备组ID、管理员ID等等。标签的schema可以事后增加、删除、修改。具体定义以及细节请见 TAOS SQL 一节。 +与创建普通表一样，创建表时，需要提供表名（示例中为meters），表结构Schema，即数据列的定义。第一列必须为时间戳（示例中为ts)，其他列为采集的物理量（示例中为current, voltage, phase)，数据类型可以为整型、浮点型、字符串等。除此之外，还需要提供标签的schema (示例中为location, groupId)，标签的数据类型可以为整型、浮点型、字符串等。采集点的静态属性往往可以作为标签，比如采集点的地理位置、设备型号、设备组ID、管理员ID等等。标签的schema可以事后增加、删除、修改。具体定义以及细节请见 TAOS SQL 一节。 -每一种类型的数据采集点需要建立一个超级表，因此一个物联网系统，往往会有多个超级表。一个系统可以有多个DB，一个DB里可以有一到多个超级表。 +每一种类型的数据采集点需要建立一个超级表，因此一个物联网系统，往往会有多个超级表。对于电网，我们就需要对智能电表、变压器、母线、开关等都建立一个超级表。在物联网中，一个设备就可能有多个数据采集点（比如一台风力发电的风机，有的采集点采集电流、电压等电参数，有的采集点采集温度、湿度、风向等环境参数），这个时候，对这一类型的设备，需要建立多张超级表。一张超级表里包含的采集物理量必须是同时采集的（时间戳是一致的）。 + +一张超级表最多容许1024列，如果一个采集点采集的物理量个数超过1024，需要建多张超级表来处理。一个系统可以有多个DB，一个DB里可以有一到多个超级表。 ## 创建表 TDengine对每个数据采集点需要独立建表。与标准的关系型数据一样，一张表有表名，Schema，但除此之外，还可以带有一到多个标签。创建时，需要使用超级表做模板，同时指定标签的具体值。以表一中的智能电表为例，可以使用如下的SQL命令建表： @@ -51,5 +53,7 @@ INSERT INTO d1001 USING METERS TAGS ("Beijng.Chaoyang", 2) VALUES (now, 10.2, 21 ``` 上述SQL语句将记录(now, 10.2, 219, 0.32) 插入进表d1001。如果表d1001还未创建，则使用超级表meters做模板自动创建，同时打上标签值“Beijing.Chaoyang", 2。 -**多列模型**：TDengine支持多列模型，只要这些物理量是同时采集的，这些量就可以作为不同列放在同一张表里。有的数据采集点有多组采集量，每一组的数据采集时间是不一样的，这时需要对同一个采集点建多张表。但还有一种极限的设计，单列模型，无论是否同时采集，每个采集的物理量单独建表。TDengine建议，只要采集时间一致，就采用多列模型，因为插入效率以及存储效率更高。TDengine支持最大的列数为1024列。 +## 多列模型 vs 单列模型 +TDengine支持多列模型，只要物理量是一个数据采集点同时采集的（时间戳一致），这些量就可以作为不同列放在一张超级表里。但还有一种极限的设计，单列模型，每个采集的物理量都单独建表，因此每种类型的物理量都单独建立一超级表。比如电流、电压、相位，就建三张超级表。 +TDengine建议尽可能采用多列模型，因为插入效率以及存储效率更高。但对于有些场景，一个采集点的采集量的种类经常变化，这个时候，如果采用多列模型，就需要频繁修改超级表的结构定义，让应用变的复杂，这个时候，采用单列模型就会显得简单。 diff --git a/documentation20/webdocs/markdowndocs/administrator-ch.md b/documentation20/webdocs/markdowndocs/administrator-ch.md index 813d06a660..64cadf69cd 100644 --- a/documentation20/webdocs/markdowndocs/administrator-ch.md +++ b/documentation20/webdocs/markdowndocs/administrator-ch.md @@ -82,7 +82,7 @@ TDengine系统后台服务由taosd提供，可以在配置文件taos.cfg里修 - firstEp: taosd启动时，主动连接的集群中第一个dnode的end point, 默认值为localhost:6030。 - secondEp: taosd启动时，如果first连接不上，尝试连接集群中第二个dnode的end point, 默认值为空。 -- fqdn：数据节点的FQDN。如果为空，将自动获取操作系统配置的第一个, 默认值为空。 +- fqdn：数据节点的FQDN，缺省为操作系统配置的第一个hostname。如果习惯IP地址访问，可设置为该节点的IP地址。 - serverPort：taosd启动后，对外服务的端口号，默认值为6030。 - httpPort: RESTful服务使用的端口号，所有的HTTP请求（TCP）都需要向该接口发起查询/写入请求, 默认值为6041。 - dataDir: 数据文件目录，所有的数据文件都将写入该目录。默认值：/var/lib/taos。 @@ -94,7 +94,7 @@ TDengine系统后台服务由taosd提供，可以在配置文件taos.cfg里修 - maxSQLLength：单条SQL语句允许最长限制。默认值：65380字节。 - telemetryReporting: 是否允许 TDengine 采集和上报基本使用信息，0表示不允许，1表示允许。 默认值：1。 -**注意：**对于端口，TDengine会使用从serverPort起12个连续的TCP和UDP端口号，请务必在防火墙打开。因此如果是缺省配置，需要打开从6030都6041共12个端口，而且必须TCP和UDP都打开。 +**注意：**对于端口，TDengine会使用从serverPort起13个连续的TCP和UDP端口号，请务必在防火墙打开。因此如果是缺省配置，需要打开从6030都6042共13个端口，而且必须TCP和UDP都打开。 不同应用场景的数据往往具有不同的数据特征，比如保留天数、副本数、采集频次、记录大小、采集点的数量、压缩等都可完全不同。为获得在存储上的最高效率，TDengine提供如下存储相关的系统配置参数： @@ -153,10 +153,10 @@ TDengine系统的前台交互客户端应用程序为taos，它与taosd共享同 系统管理员可以在CLI界面里添加、删除用户，也可以修改密码。CLI里SQL语法如下： ``` -CREATE USER PASS <‘password’>; +CREATE USER PASS <'password'>; ``` -创建用户，并指定用户名和密码，密码需要用单引号引起来 +创建用户，并指定用户名和密码，密码需要用单引号引起来,单引号为英文半角 ``` DROP USER ; @@ -165,10 +165,10 @@ DROP USER ; 删除用户，限root用户使用 ``` -ALTER USER PASS <‘password’>; +ALTER USER PASS <'password'>; ``` -修改用户密码, 为避免被转换为小写，密码需要用单引号引用 +修改用户密码, 为避免被转换为小写，密码需要用单引号引用,单引号为英文半角 ``` SHOW USERS; diff --git a/documentation20/webdocs/markdowndocs/architecture-ch.md b/documentation20/webdocs/markdowndocs/architecture-ch.md index bfe3b55bd2..7ab4b5d096 100644 --- a/documentation20/webdocs/markdowndocs/architecture-ch.md +++ b/documentation20/webdocs/markdowndocs/architecture-ch.md @@ -82,7 +82,7 @@ TDengine 分布式架构的逻辑结构图如下： ### 节点之间的通讯 **通讯方式：**TDengine系统的各个节点之间的通讯是通过TCP/UDP进行的。因为考虑到物联网场景，数据写入的包一般不大，因此TDengine 除采用TCP做传输之外，还采用UDP方式，因为UDP 更加高效，而且不受连接数的限制。TDengine实现了自己的超时、重传、确认等机制，以确保UDP的可靠传输。对于数据量不到15K的数据包，采取UDP的方式进行传输，超过15K的，或者是查询类的操作，自动采取TCP的方式进行传输。同时，TDengine根据配置和数据包，会自动对数据进行压缩/解压缩，数字签名/认证等处理。对于数据节点之间的数据复制，只采用TCP方式进行数据传输。 -**FQDN配置**：一个数据节点有一个或多个FQDN，可以在系统配置文件taos.cfg通过选项“fqdn"进行指定，如果没有指定，系统将自动获取FQDN。如果节点没有配置FQDN，可以直接使用IP地址作为FQDN，但不建议使用，因为IP地址可变，一旦变化，将让集群无法正常工作。一个数据节点的EP(End Point)由FQDN + Port组成。 +**FQDN配置**：一个数据节点有一个或多个FQDN，可以在系统配置文件taos.cfg通过参数“fqdn"进行指定，如果没有指定，系统将自动获取FQDN。如果节点没有配置FQDN，可以直接将该节点的配置参数fqdn设置为它的IP地址。但不建议使用IP，因为IP地址可变，一旦变化，将让集群无法正常工作。一个数据节点的EP(End Point)由FQDN + Port组成。采用FQDN，需要保证DNS服务正常工作，或者在节点以及应用所在的节点配置好hosts文件。 **端口配置：**一个数据节点对外的端口由TDengine的系统配置参数serverPort决定，对集群内部通讯的端口是serverPort+5。集群内数据节点之间的数据复制操作还占有一个TCP端口，是serverPort+10. 为支持多线程高效的处理UDP数据，每个对内和对外的UDP链接，都需要占用5个连续的端口。因此一个数据节点总的端口范围为serverPort到serverPort + 10，总共11个TCP/UDP端口。使用时，需要确保防火墙将这些端口打开。每个数据节点可以配置不同的serverPort。 diff --git a/documentation20/webdocs/markdowndocs/cluster-ch.md b/documentation20/webdocs/markdowndocs/cluster-ch.md index afe0272387..097433a18a 100644 --- a/documentation20/webdocs/markdowndocs/cluster-ch.md +++ b/documentation20/webdocs/markdowndocs/cluster-ch.md @@ -1,16 +1,46 @@ -#TDengine 集群安装、管理 +# TDengine 集群安装、管理 -多个taosd的运行实例可以组成一个集群，以保证TDengine的高可靠运行，并提供水平扩展能力。要了解TDengine 2.0的集群管理，需要对集群的基本概念有所了解，请看TDengine 2.0整体架构一章。 +多个taosd的运行实例可以组成一个集群，以保证TDengine的高可靠运行，并提供水平扩展能力。要了解TDengine 2.0的集群管理，需要对集群的基本概念有所了解，请看TDengine 2.0整体架构一章。而且在安装集群之前，请按照[《立即开始》](https://www.taosdata.com/cn/getting-started20/)一章安装并体验过单节点功能。 -集群的每个节点是由End Point来唯一标识的，End Point是由FQDN(Fully Qualified Domain Name)外加Port组成，比如 h1.taosdata.com:6030。一般FQDN就是服务器的hostname，可通过Linux命令“hostname"获取。端口是这个节点对外服务的端口号，缺省是6030，但可以通过taos.cfg里配置参数serverPort进行修改。 +集群的每个节点是由End Point来唯一标识的，End Point是由FQDN(Fully Qualified Domain Name)外加Port组成，比如 h1.taosdata.com:6030。一般FQDN就是服务器的hostname，可通过Linux命令`hostname -f`获取。端口是这个节点对外服务的端口号，缺省是6030，但可以通过taos.cfg里配置参数serverPort进行修改。一个节点可能配置了多个hostname, TDengine会自动获取第一个，但也可以通过taos.cfg里配置参数fqdn进行指定。如果习惯IP地址直接访问，可以将参数fqdn设置为本节点的IP地址。 TDengine的集群管理极其简单，除添加和删除节点需要人工干预之外，其他全部是自动完成，最大程度的降低了运维的工作量。本章对集群管理的操作做详细的描述。 -##安装、创建第一个节点 +## 准备工作 -集群是由一个一个dnode组成的，是从一个dnode的创建开始的。创建第一个节点很简单，就按照["立即开始“](https://www.taosdata.com/cn/getting-started/)一章的方法进行安装、启动即可。 +**第一步**：如果搭建集群的节点中，存有之前的测试数据、装过1.X的版本，或者装过其他版本的TDengine，请先将其删除，并清空所有数据，具体步骤请参考博客[《TDengine多种安装包的安装和卸载》](https://www.taosdata.com/blog/2019/08/09/566.html ) -启动后，请执行taos, 启动taos shell，从shell里执行命令"show dnodes;"，如下所示： +**第二步**：建议关闭防火墙，至少保证端口：6030 - 6042的TCP和UDP端口都是开放的。**强烈建议**先关闭防火墙，集群搭建完毕之后，再来配置端口； + +**第三步**：在所有节点安装TDengine，且版本必须是一致的，**但不要启动taosd**； + +**第四步**：检查、配置所有节点的FQDN： + +1. 每个节点上执行命令`hostname -f`，查看和确认所有节点的hostname是不相同的； +2. 每个节点上执行`ping host`, 其中host是其他节点的hostname, 看能否ping通其它节点; 如果不能ping通，需要检查网络设置, 或/etc/hosts文件，或DNS的配置。如果无法ping通，是无法组成集群的。 +3. 每个节点的FQDN就是输出的hostname外加端口号，比如h1.taosdata.com:6030 + +**第五步**：修改TDengine的配置文件（所有节点的文件/etc/taos/taos.cfg都需要修改）。假设准备启动的第一个节点End Point为 h1.taosdata.com:6030, 那么以下几个参数与集群相关： + +``` +// firstEp 是每个节点启动后连接的第一个节点 +firstEp h1.taosdata.com:6030 + +// 配置本节点的FQDN，如果本机只有一个hostname, 无需配置 +fqdn h1.taosdata.com + +// 配置本节点的端口号，缺省是6030 +serverPort 6030 + +// 副本数为偶数的时候，需要配置，请参考《Arbitrator的使用》的部分 +arbitrator ha.taosdata.com:6042 +``` + +一定要修改的参数是firstEp, 其他参数可不做任何修改，除非你很清楚为什么要修改。 + +## 启动第一个节点 + +按照[《立即开始》](https://www.taosdata.com/cn/getting-started20/)里的指示，启动第一个节点h1.taosdata.com，然后执行taos, 启动taos shell，从shell里执行命令"show dnodes;"，如下所示： ``` Welcome to the TDengine shell from Linux, Client Version:2.0.0.0 Copyright (c) 2017 by TAOS Data, Inc. All rights reserved. @@ -25,71 +55,64 @@ taos> ``` 上述命令里，可以看到这个刚启动的这个节点的End Point是：h1.taos.com:6030 -## 安装、创建后续节点 +## 启动后续节点 -将新的节点添加到现有集群，具体有以下几步： +将后续的节点添加到现有集群，具体有以下几步： -1. 按照["立即开始“](https://www.taosdata.com/cn/getting-started/)一章的方法进行安装，**但不要启动taosd** +1. 按照["立即开始“](https://www.taosdata.com/cn/getting-started/)一章的方法在每个节点启动taosd。 -2. 如果是使用涛思数据的官方安装包进行安装，在安装结束时，会询问集群的End Port, 输入第一个节点的End Point即可。如果是源码安装，请编辑配置文件taos.cfg(缺省是在/etc/taos/目录)，增加一行： - - ``` - firstEp h1.taos.com:6030 - ``` - - 请注意将示例的“h1.taos.com:6030" 替换为你自己第一个节点的End Point - -3. 按照["立即开始“](https://www.taosdata.com/cn/getting-started/)一章的方法启动taosd - -4. 在Linux shell里执行命令"hostname"找出本机的FQDN, 假设为h2.taos.com。如果无法找到，可以查看taosd日志文件taosdlog.0里前面几行日志(一般在/var/log/taos目录)，fqdn以及port都会打印出来。 - -5. 在第一个节点，使用CLI程序taos, 登录进TDengine系统, 使用命令: +2. 在第一个节点，使用CLI程序taos, 登录进TDengine系统, 执行命令: ``` CREATE DNODE "h2.taos.com:6030"； ``` - 将新节点的End Point添加进集群的EP列表。**"fqdn:port"需要用双引号引起来**，否则出错。请注意将示例的“h2.taos.com:6030" 替换为你自己第一个节点的End Point + 将新节点的End Point (准备工作中第四步获知的) 添加进集群的EP列表。**"fqdn:port"需要用双引号引起来**，否则出错。请注意将示例的“h2.taos.com:6030" 替换为这个新节点的End Point。 -6. 使用命令 +3. 然后执行命令 ``` SHOW DNODES； ``` - 查看新节点是否被成功加入。 + 查看新节点是否被成功加入。如果该被加入的节点处于离线状态，请做两个检查 + + - 查看该节点的taosd是否正常工作，如果没有正常运行，需要先检查为什么 + - 查看该节点taosd日志文件taosdlog.0里前面几行日志(一般在/var/log/taos目录)，看日志里输出的该节点fqdn以及端口号是否为刚添加的End Point。如果不一致，需要将正确的End Point添加进去。 按照上述步骤可以源源不断的将新的节点加入到集群。 **提示：** -- firstEp, secondEp这两个参数仅仅在该节点第一次加入集群时有作用，加入集群后，该节点会保存最新的mnode的End Point列表，不再依赖这两个参数。 -- 两个没有配置firstEp, secondEp参数的dnode启动后，会独立运行起来。这个时候，无法将其中一个节点加入到另外一个节点，形成集群。**无法将两个独立的集群合并成为新的集群**。 +- firstEp这个参数仅仅在该节点第一次加入集群时有作用，加入集群后，该节点会保存最新的mnode的End Point列表，不再依赖这两个参数。 +- 两个没有配置firstEp参数的dnode启动后，会独立运行起来。这个时候，无法将其中一个节点加入到另外一个节点，形成集群。**无法将两个独立的集群合并成为新的集群**。 -##节点管理 +## 节点管理 -###添加节点 +### 添加节点 执行CLI程序taos, 使用root账号登录进系统, 执行: ``` CREATE DNODE "fqdn:port"； ``` 将新节点的End Point添加进集群的EP列表。**"fqdn:port"需要用双引号引起来**，否则出错。一个节点对外服务的fqdn和port可以通过配置文件taos.cfg进行配置，缺省是自动获取。 -###删除节点 +### 删除节点 执行CLI程序taos, 使用root账号登录进TDengine系统，执行： + ``` DROP DNODE "fqdn:port"; ``` 其中fqdn是被删除的节点的FQDN，port是其对外服务器的端口号 -###查看节点 +### 查看节点 执行CLI程序taos,使用root账号登录进TDengine系统，执行： + ``` SHOW DNODES; ``` 它将列出集群中所有的dnode,每个dnode的fqdn:port, 状态(ready, offline等），vnode数目，还未使用的vnode数目等信息。在添加或删除一个节点后，可以使用该命令查看。 -###查看虚拟节点组 +### 查看虚拟节点组 为充分利用多核技术，并提供scalability，数据需要分片处理。因此TDengine会将一个DB的数据切分成多份，存放在多个vnode里。这些vnode可能分布在多个dnode里，这样就实现了水平扩展。一个vnode仅仅属于一个DB，但一个DB可以有多个vnode。vnode的是mnode根据当前系统资源的情况，自动进行分配的，无需任何人工干预。 @@ -97,7 +120,7 @@ SHOW DNODES; ``` SHOW VGROUPS; ``` -##vnode的高可用性 +## vnode的高可用性 TDengine通过多副本的机制来提供系统的高可用性，包括vnode和mnode的高可用性。 vnode的副本数是与DB关联的，一个集群里可以有多个DB，根据运营的需求，每个DB可以配置不同的副本数。创建数据库时，通过参数replica 指定副本数（缺省为1）。如果副本数为1，系统的可靠性无法保证，只要数据所在的节点宕机，就将无法提供服务。集群的节点数必须大于等于副本数，否则创建表时将返回错误“more dnodes are needed"。比如下面的命令将创建副本数为3的数据库demo： @@ -111,7 +134,7 @@ CREATE DATABASE demo replica 3; 因为vnode的引入，无法简单的给出结论：“集群中过半dnode工作，集群就应该工作”。但是对于简单的情形，很好下结论。比如副本数为3，只有三个dnode，那如果仅有一个节点不工作，整个集群还是可以正常工作的，但如果有两个节点不工作，那整个集群就无法正常工作了。 -##Mnode的高可用性 +## Mnode的高可用性 TDengine集群是由mnode (taosd的一个模块，逻辑节点) 负责管理的，为保证mnode的高可用，可以配置多个mnode副本，副本数由系统配置参数numOfMnodes决定，有效范围为1-3。为保证元数据的强一致性，mnode副本之间是通过同步的方式进行数据复制的。 一个集群有多个dnode, 但一个dnode至多运行一个mnode实例。多个dnode情况下，哪个dnode可以作为mnode呢？这是完全由系统根据整个系统资源情况，自动指定的。用户可通过CLI程序taos，在TDengine的console里，执行如下命令： @@ -125,7 +148,7 @@ SHOW MNODES; **注意：**一个TDengine高可用系统，无论是vnode还是mnode, 都必须配置多个副本。 -##负载均衡 +## 负载均衡 有三种情况，将触发负载均衡，而且都无需人工干预。 @@ -142,8 +165,9 @@ SHOW MNODES; **注意：**如果一个虚拟节点组（包括mnode组）里每个节点都处于离线或unsynced状态，必须等该虚拟节点组里的所有节点都上线、都能交换状态信息后，才能选出Master，该虚拟节点组才能对外提供服务。比如整个集群有3个节点，副本数为3，如果3个节点都宕机，然后2个节点重启，是无法工作的，只有等3个节点都重启成功，才能对外服务。 -##Arbitrator的使用 +## Arbitrator的使用 如果副本数为偶数，当一个vnode group里一半或超过一半的vnode不工作时，是无法从中选出master的。同理，一半或超过一半的mnode不工作时，是无法选出mnode的master的，因为存在“split brain”问题。为解决这个问题，TDengine引入了arbitrator的概念。Arbitrator模拟一个vnode或mnode在工作，但只简单的负责网络连接，不处理任何数据插入或访问。只要包含arbitrator在内，超过半数的vnode或mnode工作，那么该vnode group或mnode组就可以正常的提供数据插入或查询服务。比如对于副本数为2的情形，如果一个节点A离线，但另外一个节点B正常，而且能连接到arbitrator, 那么节点B就能正常工作。 -TDengine安装包里带有一个执行程序tarbitrator, 找任何一台Linux服务器运行它即可。该程序对系统资源几乎没有要求，只需要保证有网络连接即可。该应用的命令行参数`-p`可以指定其对外服务的端口号，缺省是6030。配置每个taosd实例时，可以在配置文件taos.cfg里将参数arbitrator设置为arbitrator的End Point。如果该参数配置了，当副本数为偶数数，系统将自动连接配置的arbitrator。 +TDengine安装包里带有一个执行程序tarbitrator, 找任何一台Linux服务器运行它即可。该程序对系统资源几乎没有要求，只需要保证有网络连接即可。该应用的命令行参数`-p`可以指定其对外服务的端口号，缺省是6042。配置每个taosd实例时，可以在配置文件taos.cfg里将参数arbitrator设置为arbitrator的End Point。如果该参数配置了，当副本数为偶数数，系统将自动连接配置的arbitrator。 + diff --git a/documentation20/webdocs/markdowndocs/faq-ch.md b/documentation20/webdocs/markdowndocs/faq-ch.md index a4111e78fc..b760fe161a 100644 --- a/documentation20/webdocs/markdowndocs/faq-ch.md +++ b/documentation20/webdocs/markdowndocs/faq-ch.md @@ -1,6 +1,6 @@ -#常见问题 +# 常见问题 -#### 1. TDengine2.0之前的版本升级到2.0及以上的版本应该注意什么？☆☆☆ +## 1. TDengine2.0之前的版本升级到2.0及以上的版本应该注意什么？☆☆☆ 2.0版本在之前版本的基础上，进行了完全的重构，配置文件和数据文件是不兼容的。在升级之前务必进行如下操作： @@ -10,23 +10,23 @@ 4. 安装最新稳定版本的TDengine 5. 如果数据需要迁移数据或者数据文件损坏，请联系涛思数据官方技术支持团队，进行协助解决 -#### 2. Windows平台下JDBCDriver找不到动态链接库，怎么办？ +## 2. Windows平台下JDBCDriver找不到动态链接库，怎么办？ 请看为此问题撰写的技术博客 -#### 3. 创建数据表时提示more dnodes are needed +## 3. 创建数据表时提示more dnodes are needed 请看为此问题撰写的技术博客 -#### 4. 如何让TDengine crash时生成core文件？ +## 4. 如何让TDengine crash时生成core文件？ 请看为此问题撰写的技术博客 -#### 5. 遇到错误"Unable to establish connection", 我怎么办？ +## 5. 遇到错误"Unable to establish connection", 我怎么办？ 客户端遇到链接故障，请按照下面的步骤进行检查： 1. 确保客户端与服务端版本号是完全一致的，开源社区版和企业版也不能混用 2. 在服务器，执行 `systemctl status taosd` 检查*taosd*运行状态。如果没有运行，启动*taosd* -3. 确认客户端连接时指定了正确的服务器IP地址 -4. ping服务器IP，如果没有反应，请检查你的网络 +3. 确认客户端连接时指定了正确的服务器FQDN (Fully Qualified Domain Name(可在服务器上执行Linux命令hostname -f获得） +4. ping服务器FQDN，如果没有反应，请检查你的网络，DNS设置，或客户端所在计算机的系统hosts文件 5. 检查防火墙设置，确认TCP/UDP 端口6030-6039 是打开的 6. 对于Linux上的JDBC（ODBC, Python, Go等接口类似）连接, 确保*libtaos.so*在目录*/usr/local/lib/taos*里, 并且*/usr/local/lib/taos*在系统库函数搜索路径*LD_LIBRARY_PATH*里 7. 对于windows上的JDBC, ODBC, Python, Go等连接，确保*driver/c/taos.dll*在你的系统搜索目录里 (建议*taos.dll*放在目录 *C:\Windows\System32*) @@ -36,40 +36,61 @@ 检查客户端侧TCP端口链接是否工作：`nc {hostIP} {port}` -#### 6. 虽然语法正确，为什么我还是得到 "Invalid SQL" 错误 +## 6. 遇到错误“Unexpected generic error in RPC”， 我怎么办？ +产生这个错误，是由于客户端或数据节点无法解析FQDN(Fully Qualified Domain Name)导致。对于TAOS Shell或客户端应用，请做如下检查： + +1. 请检查连接的服务器的FQDN是否正确 +2. 如果网络配置有DNS server, 请检查是否正常工作 +3. 如果网络没有配置DNS server, 请检查客户端所在机器的hosts文件，查看该FQDN是否配置，并是否有正确的IP地址。 +4. 如果网络配置OK，从客户端所在机器，你需要能Ping该连接的FQDN，否则客户端是无法链接服务器的 + + +## 7. 虽然语法正确，为什么我还是得到 "Invalid SQL" 错误 如果你确认语法正确，2.0之前版本，请检查SQL语句长度是否超过64K。如果超过，也会返回这个错误。 -#### 7. 是否支持validation queries? +## 8. 是否支持validation queries? TDengine还没有一组专用的validation queries。然而建议你使用系统监测的数据库”log"来做。 -#### 8. 我可以删除或更新一条记录吗？ +## 9. 我可以删除或更新一条记录吗？ 不能。因为TDengine是为联网设备采集的数据设计的，不容许修改。但TDengine提供数据保留策略，只要数据记录超过保留时长，就会被自动删除。 -#### 10. 我怎么创建超过250列的表？ +## 10. 我怎么创建超过1024列的表？ 使用2.0及其以上版本，默认支持1024列；2.0之前的版本，TDengine最大允许创建250列的表。但是如果确实超过限值，建议按照数据特性，逻辑地将这个宽表分解成几个小表。 -#### 10. 最有效的写入数据的方法是什么？ +## 10. 最有效的写入数据的方法是什么？ -批量插入。每条写入语句可以一张表同时插入多条记录，也可以同时插入多张表的记录。 +批量插入。每条写入语句可以一张表同时插入多条记录，也可以同时插入多张表的多条记录。 -#### 11. 最有效的写入数据的方法是什么？windows系统下插入的nchar类数据中的汉字被解析成了乱码如何解决？ +## 11. 最有效的写入数据的方法是什么？windows系统下插入的nchar类数据中的汉字被解析成了乱码如何解决？ windows下插入nchar类的数据中如果有中文，请先确认系统的地区设置成了中国（在Control Panel里可以设置），这时cmd中的`taos`客户端应该已经可以正常工作了；如果是在IDE里开发Java应用，比如Eclipse， Intellij，请确认IDE里的文件编码为GBK（这是Java默认的编码类型），然后在生成Connection时，初始化客户端的配置，具体语句如下： - -​ Class.forName("com.taosdata.jdbc.TSDBDriver"); - -​ Properties properties = new Properties(); - -​ properties.setProperty(TSDBDriver.LOCALE_KEY, "UTF-8"); - -​ Connection = DriverManager.getConnection(url, properties); - -#### 12.TDengine GO windows驱动的如何编译？ +```JAVA +Class.forName("com.taosdata.jdbc.TSDBDriver"); +Properties properties = new Properties(); +properties.setProperty(TSDBDriver.LOCALE_KEY, "UTF-8"); +Connection = DriverManager.getConnection(url, properties); +``` +## 12.TDengine GO windows驱动的如何编译？ 请看为此问题撰写的技术博客 +## 13.JDBC报错： the excuted SQL is not a DML or a DDL？ +请更新至最新的JDBC驱动 +```JAVA + + com.taosdata.jdbc + taos-jdbcdriver + 2.0.4 + +``` +## 14. 怎么报告问题？ +如果 FAQ 中的信息不能够帮到您，需要 TDengine 技术团队的技术支持与协助，请将以下两个目录中内容打包: +1. /var/log/taos +2. /etc/taos +附上必要的问题描述，以及发生该问题的执行操作，出现问题的表征及大概的时间，在 GitHub提交Issue。 +为了保证有足够的debug信息，如果问题能够重复，请修改/etc/taos/taos.cfg文件，最后面添加一行“debugFlag 135"(不带引号本身），然后重启taosd, 重复问题，然后再递交。但系统正常运行时，请一定将debugFlag设置为131，否则会产生大量的日志信息，降低系统效率。 diff --git a/documentation20/webdocs/markdowndocs/insert-ch.md b/documentation20/webdocs/markdowndocs/insert-ch.md index 5eef99e891..5e4532dfd0 100644 --- a/documentation20/webdocs/markdowndocs/insert-ch.md +++ b/documentation20/webdocs/markdowndocs/insert-ch.md @@ -24,6 +24,7 @@ INSERT INTO d1001 VALUES (1538548685000, 10.3, 219, 0.31) (1538548695000, 12.6, - 要提高写入效率，需要批量写入。一批写入的记录条数越多，插入效率就越高。但一条记录不能超过16K，一条SQL语句总长度不能超过64K（可通过参数maxSQLLength配置，最大可配置为8M）。 - TDengine支持多线程同时写入，要进一步提高写入速度，一个客户端需要打开20个以上的线程同时写。但线程数达到一定数量后，无法再提高，甚至还会下降，因为线程切频繁切换，带来额外开销。 +- 对同一张表，如果新插入记录的时间戳已经存在，新记录将被直接抛弃，也就是说，在一张表里，时间戳必须是唯一的。如果应用自动生成记录，很有可能生成的时间戳是一样的，这样，成功插入的记录条数会小于应用插入的记录条数。 ## Prometheus直接写入 [Prometheus](https://www.prometheus.io/)作为Cloud Native Computing Fundation毕业的项目，在性能监控以及K8S性能监控领域有着非常广泛的应用。TDengine提供一个小工具[Bailongma](https://github.com/taosdata/Bailongma)，只需在Prometheus做简单配置，无需任何代码，就可将Prometheus采集的数据直接写入TDengine，并按规则在TDengine自动创建库和相关表项。博文[用Docker容器快速搭建一个Devops监控Demo](https://www.taosdata.com/blog/2020/02/03/1189.html)即是采用bailongma将Prometheus和Telegraf的数据写入TDengine中的示例，可以参考。 diff --git a/packaging/cfg/taos.cfg b/packaging/cfg/taos.cfg index 93a6936da3..a91dde8796 100644 --- a/packaging/cfg/taos.cfg +++ b/packaging/cfg/taos.cfg @@ -24,7 +24,7 @@ # dataDir /var/lib/taos # the arbitrator's fully qualified domain name (FQDN) for TDengine system, for cluster only -# arbitrator arbitrator_hostname:6030 +# arbitrator arbitrator_hostname:6042 # number of threads per CPU core # numOfThreadsPerCore 1.0 diff --git a/packaging/deb/tarbitratord b/packaging/deb/tarbitratord index bf4dba7258..3f97c3c0c2 100644 --- a/packaging/deb/tarbitratord +++ b/packaging/deb/tarbitratord @@ -7,19 +7,19 @@ # chkconfig: 2345 99 01 # ### BEGIN INIT INFO -# Provides: TDEngine +# Provides: taoscluster # Required-Start: $local_fs $network $syslog # Required-Stop: $local_fs $network $syslog # Default-Start: 2 3 4 5 # Default-Stop: 0 1 6 -# Short-Description: Starts TDEngine tarbitrator -# Description: Starts TDEngine tarbitrator, a arbitrator +# Short-Description: Starts taoscluster tarbitrator +# Description: Starts taoscluster tarbitrator, a arbitrator ### END INIT INFO set -e PATH="/bin:/usr/bin:/sbin:/usr/sbin" -NAME="tarbitrator" +NAME="taoscluster" USER="root" GROUP="root" DAEMON="/usr/local/taos/bin/tarbitrator" diff --git a/packaging/docker/Dockerfile b/packaging/docker/Dockerfile index 3fb34e8286..668d5a49eb 100644 --- a/packaging/docker/Dockerfile +++ b/packaging/docker/Dockerfile @@ -12,6 +12,6 @@ ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib" ENV LANG=en_US.UTF-8 ENV LANGUAGE=en_US:en ENV LC_ALL=en_US.UTF-8 -EXPOSE 6030 6031 6032 6033 6034 6035 6036 6037 6038 6039 6040 6041 +EXPOSE 6030 6031 6032 6033 6034 6035 6036 6037 6038 6039 6040 6041 6042 CMD ["taosd"] VOLUME [ "/var/lib/taos", "/var/log/taos","/etc/taos/" ] diff --git a/packaging/release.sh b/packaging/release.sh index 2302b45875..3c0378042d 100755 --- a/packaging/release.sh +++ b/packaging/release.sh @@ -7,20 +7,24 @@ set -e # releash.sh -v [cluster | edge] # -c [aarch32 | aarch64 | x64 | x86 | mips64 ...] -# -o [Linux | Kylin | Alpine | Raspberrypi | Darwin | Windows | ...] +# -o [Linux | Kylin | Alpine | Raspberrypi | Darwin | Windows | Ningsi60 | Ningsi80 |...] # -V [stable | beta] # -l [full | lite] +# -s [static | dynamic] # -n [2.0.0.3] +# -m [2.0.0.0] # set parameters by default value verMode=edge # [cluster, edge] verType=stable # [stable, beta] cpuType=x64 # [aarch32 | aarch64 | x64 | x86 | mips64 ...] -osType=Linux # [Linux | Kylin | Alpine | Raspberrypi | Darwin | Windows | ...] +osType=Linux # [Linux | Kylin | Alpine | Raspberrypi | Darwin | Windows | Ningsi60 | Ningsi80 |...] pagMode=full # [full | lite] +soMode=dynamic # [static | dynamic] verNumber="" +verNumberComp="2.0.0.0" -while getopts "hv:V:c:o:l:n:" arg +while getopts "hv:V:c:o:l:s:n:m:" arg do case $arg in v) @@ -39,10 +43,18 @@ do #echo "pagMode=$OPTARG" pagMode=$(echo $OPTARG) ;; + s) + #echo "soMode=$OPTARG" + soMode=$(echo $OPTARG) + ;; n) #echo "verNumber=$OPTARG" verNumber=$(echo $OPTARG) ;; + m) + #echo "verNumberComp=$OPTARG" + verNumberComp=$(echo $OPTARG) + ;; o) #echo "osType=$OPTARG" osType=$(echo $OPTARG) @@ -50,10 +62,12 @@ do h) echo "Usage: `basename $0` -v [cluster | edge] " echo " -c [aarch32 | aarch64 | x64 | x86 | mips64 ...] " - echo " -o [Linux | Kylin | Alpine | Raspberrypi | Darwin | Windows | ...] " + echo " -o [Linux | Kylin | Alpine | Raspberrypi | Darwin | Windows | Ningsi60 | Ningsi80 |...] " echo " -V [stable | beta] " echo " -l [full | lite] " + echo " -s [static | dynamic] " echo " -n [version number] " + echo " -m [compatible version number] " exit 0 ;; ?) #unknow option @@ -63,215 +77,142 @@ do esac done -echo "verMode=${verMode} verType=${verType} cpuType=${cpuType} osType=${osType} pagMode=${pagMode} verNumber=${verNumber}" +echo "verMode=${verMode} verType=${verType} cpuType=${cpuType} osType=${osType} pagMode=${pagMode} soMode=${soMode} verNumber=${verNumber} verNumberComp=${verNumberComp}" curr_dir=$(pwd) if [ "$osType" != "Darwin" ]; then - script_dir="$(dirname $(readlink -f $0))" - top_dir="$(readlink -f ${script_dir}/..)" + script_dir="$(dirname $(readlink -f $0))" + top_dir="$(readlink -f ${script_dir}/..)" else - script_dir=`dirname $0` - cd ${script_dir} - script_dir="$(pwd)" - top_dir=${script_dir}/.. + script_dir=`dirname $0` + cd ${script_dir} + script_dir="$(pwd)" + top_dir=${script_dir}/.. fi -versioninfo="${top_dir}/src/util/src/version.c" - csudo="" -if command -v sudo > /dev/null; then - csudo="sudo" -fi +#if command -v sudo > /dev/null; then +# csudo="sudo" +#fi function is_valid_version() { - [ -z $1 ] && return 1 || : + [ -z $1 ] && return 1 || : - rx='^([0-9]+\.){3}(\*|[0-9]+)$' - if [[ $1 =~ $rx ]]; then - return 0 - fi - return 1 + rx='^([0-9]+\.){3}(\*|[0-9]+)$' + if [[ $1 =~ $rx ]]; then + return 0 + fi + return 1 } function vercomp () { - if [[ $1 == $2 ]]; then - echo 0 - exit 0 - fi - - local IFS=. - local i ver1=($1) ver2=($2) - - # fill empty fields in ver1 with zeros - for ((i=${#ver1[@]}; i<${#ver2[@]}; i++)); do - ver1[i]=0 - done - - for ((i=0; i<${#ver1[@]}; i++)); do - if [[ -z ${ver2[i]} ]]; then - # fill empty fields in ver2 with zeros - ver2[i]=0 - fi - if ((10#${ver1[i]} > 10#${ver2[i]})); then - echo 1 - exit 0 - fi - if ((10#${ver1[i]} < 10#${ver2[i]})); then - echo 2 - exit 0 - fi - done + if [[ $1 == $2 ]]; then echo 0 -} + exit 0 + fi + + local IFS=. + local i ver1=($1) ver2=($2) -# 1. Read version information -version=$(cat ${versioninfo} | grep " version" | cut -d '"' -f2) -compatible_version=$(cat ${versioninfo} | grep " compatible_version" | cut -d '"' -f2) + # fill empty fields in ver1 with zeros + for ((i=${#ver1[@]}; i<${#ver2[@]}; i++)); do + ver1[i]=0 + done -if [ -z ${verNumber} ]; then - while true; do - read -p "Do you want to release a new version? [y/N]: " is_version_change - - if [[ ( "${is_version_change}" == "y") || ( "${is_version_change}" == "Y") ]]; then - read -p "Please enter the new version: " tversion - while true; do - if (! is_valid_version $tversion) || [ "$(vercomp $tversion $version)" = '2' ]; then - read -p "Please enter a correct version: " tversion - continue - fi - version=${tversion} - break - done - - echo - - read -p "Enter the oldest compatible version: " tversion - while true; do - - if [ -z $tversion ]; then - break - fi - - if (! is_valid_version $tversion) || [ "$(vercomp $version $tversion)" = '2' ]; then - read -p "enter correct compatible version: " tversion - else - compatible_version=$tversion - break - fi - done - - break - elif [[ ( "${is_version_change}" == "n") || ( "${is_version_change}" == "N") ]]; then - echo "Use old version: ${version} compatible version: ${compatible_version}." - break - else - continue + for ((i=0; i<${#ver1[@]}; i++)); do + if [[ -z ${ver2[i]} ]]; then + # fill empty fields in ver2 with zeros + ver2[i]=0 + fi + if ((10#${ver1[i]} > 10#${ver2[i]})); then + echo 1 + exit 0 + fi + if ((10#${ver1[i]} < 10#${ver2[i]})); then + echo 2 + exit 0 fi done -else - echo "old version: $version, new version: $verNumber" - #if ( ! is_valid_version $verNumber ) || [[ "$(vercomp $version $verNumber)" == '2' ]]; then - # echo "please enter correct version" - # exit 0 - #else - version=${verNumber} - #fi -fi + echo 0 +} -echo "=======================new version number: ${version}======================================" +# 1. check version information +if (( ! is_valid_version $verNumber ) || ( ! is_valid_version $verNumberComp ) || [[ "$(vercomp $verNumber $verNumberComp)" == '2' ]]); then + echo "please enter correct version" + exit 0 +fi + +echo "=======================new version number: ${verNumber}, compatible version: ${verNumberComp}======================================" -# output the version info to the buildinfo file. build_time=$(date +"%F %R") -echo "char version[12] = \"${version}\";" > ${versioninfo} -echo "char compatible_version[12] = \"${compatible_version}\";" >> ${versioninfo} -echo "char gitinfo[48] = \"$(git rev-parse --verify HEAD)\";" >> ${versioninfo} -if [ "$verMode" != "cluster" ]; then - echo "char gitinfoOfInternal[48] = \"\";" >> ${versioninfo} -else - enterprise_dir="${top_dir}/../enterprise" - cd ${enterprise_dir} - echo "char gitinfoOfInternal[48] = \"$(git rev-parse --verify HEAD)\";" >> ${versioninfo} - cd ${curr_dir} -fi -echo "char buildinfo[64] = \"Built by ${USER} at ${build_time}\";" >> ${versioninfo} -echo "" >> ${versioninfo} -tmp_version=$(echo $version | tr -s "." "_") -if [ "$verMode" == "cluster" ]; then - libtaos_info=${tmp_version}_${osType}_${cpuType} -else - libtaos_info=edge_${tmp_version}_${osType}_${cpuType} -fi -if [ "$verType" == "beta" ]; then - libtaos_info=${libtaos_info}_${verType} -fi -echo "void libtaos_${libtaos_info}() {};" >> ${versioninfo} + +# get commint id from git +gitinfo=$(git rev-parse --verify HEAD) +enterprise_dir="${top_dir}/../enterprise" +cd ${enterprise_dir} +gitinfoOfInternal=$(git rev-parse --verify HEAD) +cd ${curr_dir} # 2. cmake executable file compile_dir="${top_dir}/debug" if [ -d ${compile_dir} ]; then - ${csudo} rm -rf ${compile_dir} + ${csudo} rm -rf ${compile_dir} fi if [ "$osType" != "Darwin" ]; then - ${csudo} mkdir -p ${compile_dir} + ${csudo} mkdir -p ${compile_dir} else - mkdir -p ${compile_dir} + mkdir -p ${compile_dir} fi cd ${compile_dir} # check support cpu type if [[ "$cpuType" == "x64" ]] || [[ "$cpuType" == "aarch64" ]] || [[ "$cpuType" == "aarch32" ]] || [[ "$cpuType" == "mips64" ]] ; then - if [ "$verMode" != "cluster" ]; then - cmake ../ -DCPUTYPE=${cpuType} -DPAGMODE=${pagMode} - else - cmake ../../ -DCPUTYPE=${cpuType} - fi + if [ "$verMode" != "cluster" ]; then + cmake ../ -DCPUTYPE=${cpuType} -DOSTYPE=${osType} -DSOMODE=${soMode} -DVERTYPE=${verType} -DVERDATE="${build_time}" -DGITINFO=${gitinfo} -DGITINFOI=${gitinfoOfInternal} -DVERNUMBER=${verNumber} -DVERCOMPATIBLE=${verNumberComp} -DPAGMODE=${pagMode} + else + cmake ../../ -DCPUTYPE=${cpuType} -DOSTYPE=${osType} -DSOMODE=${soMode} -DVERTYPE=${verType} -DVERDATE="${build_time}" -DGITINFO=${gitinfo} -DGITINFOI=${gitinfoOfInternal} -DVERNUMBER=${verNumber} -DVERCOMPATIBLE=${verNumberComp} + fi else - echo "input cpuType=${cpuType} error!!!" - exit 1 + echo "input cpuType=${cpuType} error!!!" + exit 1 fi make cd ${curr_dir} -# 3. judge the operating system type, then Call the corresponding script for packaging -#osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release) -#osinfo=$(cat /etc/os-release | grep "NAME" | cut -d '"' -f2) -#echo "osinfo: ${osinfo}" - +# 3. Call the corresponding script for packaging if [ "$osType" != "Darwin" ]; then - if [[ "$verMode" != "cluster" ]] && [[ "$cpuType" == "x64" ]]; then - echo "====do deb package for the ubuntu system====" - output_dir="${top_dir}/debs" - if [ -d ${output_dir} ]; then - ${csudo} rm -rf ${output_dir} - fi - ${csudo} mkdir -p ${output_dir} - cd ${script_dir}/deb - ${csudo} ./makedeb.sh ${compile_dir} ${output_dir} ${version} ${cpuType} ${osType} ${verMode} ${verType} - - echo "====do rpm package for the centos system====" - output_dir="${top_dir}/rpms" - if [ -d ${output_dir} ]; then - ${csudo} rm -rf ${output_dir} - fi - ${csudo} mkdir -p ${output_dir} - cd ${script_dir}/rpm - ${csudo} ./makerpm.sh ${compile_dir} ${output_dir} ${version} ${cpuType} ${osType} ${verMode} ${verType} + if [[ "$verMode" != "cluster" ]] && [[ "$cpuType" == "x64" ]]; then + echo "====do deb package for the ubuntu system====" + output_dir="${top_dir}/debs" + if [ -d ${output_dir} ]; then + ${csudo} rm -rf ${output_dir} fi + ${csudo} mkdir -p ${output_dir} + cd ${script_dir}/deb + ${csudo} ./makedeb.sh ${compile_dir} ${output_dir} ${verNumber} ${cpuType} ${osType} ${verMode} ${verType} + + echo "====do rpm package for the centos system====" + output_dir="${top_dir}/rpms" + if [ -d ${output_dir} ]; then + ${csudo} rm -rf ${output_dir} + fi + ${csudo} mkdir -p ${output_dir} + cd ${script_dir}/rpm + ${csudo} ./makerpm.sh ${compile_dir} ${output_dir} ${verNumber} ${cpuType} ${osType} ${verMode} ${verType} + fi - echo "====do tar.gz package for all systems====" - cd ${script_dir}/tools - - ${csudo} ./makepkg.sh ${compile_dir} ${version} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} - ${csudo} ./makeclient.sh ${compile_dir} ${version} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} + echo "====do tar.gz package for all systems====" + cd ${script_dir}/tools + + ${csudo} ./makepkg.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} + ${csudo} ./makeclient.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} + ${csudo} ./makearbi.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} else - cd ${script_dir}/tools - ./makeclient.sh ${compile_dir} ${version} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} + cd ${script_dir}/tools + ./makeclient.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} fi -# 4. Clean up temporary compile directories -#${csudo} rm -rf ${compile_dir} - diff --git a/packaging/rpm/tarbitratord b/packaging/rpm/tarbitratord index 5d3f2b9c79..68138f5c1d 100644 --- a/packaging/rpm/tarbitratord +++ b/packaging/rpm/tarbitratord @@ -7,10 +7,10 @@ # # ### BEGIN INIT INFO -# Provides: TDEngine +# Provides: taoscluster # Required-Start: $network $local_fs $remote_fs # Required-Stop: $network $local_fs $remote_fs -# Short-Description: start and stop taosd +# Short-Description: start and stop tarbitrator # Description: tarbitrator is a arbitrator used in TDengine cluster. ### END INIT INFO diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh old mode 100644 new mode 100755 index 6f54993f91..1cbb3ead9c --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -76,7 +76,11 @@ fi # get the operating system type for using the corresponding init file # ubuntu/debian(deb), centos/fedora(rpm), others: opensuse, redhat, ..., no verification #osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release) -osinfo=$(cat /etc/os-release | grep "NAME" | cut -d '"' -f2) +if [[ -e /etc/os-release ]]; then + osinfo=$(cat /etc/os-release | grep "NAME" | cut -d '"' -f2) ||: +else + osinfo="" +fi #echo "osinfo: ${osinfo}" os_type=0 if echo $osinfo | grep -qwi "ubuntu" ; then @@ -95,8 +99,10 @@ elif echo $osinfo | grep -qwi "fedora" ; then # echo "This is fedora system" os_type=2 else - echo "${osinfo}: This is an officially unverified linux system, If there are any problems with the installation and operation, " - echo "please feel free to contact taosdata.com for support." + echo " osinfo: ${osinfo}" + echo " This is an officially unverified linux system," + echo " if there are any problems with the installation and operation, " + echo " please feel free to contact taosdata.com for support." os_type=1 fi @@ -192,13 +198,12 @@ function install_lib() { ${csudo} rm -f ${lib_link_dir}/libtaos.* || : ${csudo} rm -f ${lib64_link_dir}/libtaos.* || : #${csudo} rm -rf ${v15_java_app_dir} || : - ${csudo} cp -rf ${script_dir}/driver/* ${install_main_dir}/driver && ${csudo} chmod 777 ${install_main_dir}/driver/* ${csudo} ln -s ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1 ${csudo} ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so - if [ -d ${lib64_link_dir} ]; then + if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.so ]]; then ${csudo} ln -s ${install_main_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 || : ${csudo} ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || : fi @@ -306,14 +311,27 @@ function clean_service_on_sysvinit() { fi if ((${initd_mod}==1)); then + if [ -e ${service_config_dir}/taosd ]; then ${csudo} chkconfig --del taosd || : + fi + + if [ -e ${service_config_dir}/tarbitratord ]; then ${csudo} chkconfig --del tarbitratord || : + fi elif ((${initd_mod}==2)); then + if [ -e ${service_config_dir}/taosd ]; then ${csudo} insserv -r taosd || : + fi + if [ -e ${service_config_dir}/tarbitratord ]; then ${csudo} insserv -r tarbitratord || : + fi elif ((${initd_mod}==3)); then + if [ -e ${service_config_dir}/taosd ]; then ${csudo} update-rc.d -f taosd remove || : + fi + if [ -e ${service_config_dir}/tarbitratord ]; then ${csudo} update-rc.d -f tarbitratord remove || : + fi fi ${csudo} rm -f ${service_config_dir}/taosd || : @@ -326,7 +344,6 @@ function clean_service_on_sysvinit() { function install_service_on_sysvinit() { clean_service_on_sysvinit - sleep 1 # Install taosd service @@ -364,34 +381,29 @@ function install_service_on_sysvinit() { function clean_service_on_systemd() { taosd_service_config="${service_config_dir}/taosd.service" - if systemctl is-active --quiet taosd; then echo "TDengine is running, stopping it..." ${csudo} systemctl stop taosd &> /dev/null || echo &> /dev/null fi ${csudo} systemctl disable taosd &> /dev/null || echo &> /dev/null - ${csudo} rm -f ${taosd_service_config} - + + tarbitratord_service_config="${service_config_dir}/tarbitratord.service" + if systemctl is-active --quiet tarbitratord; then + echo "tarbitrator is running, stopping it..." + ${csudo} systemctl stop tarbitratord &> /dev/null || echo &> /dev/null + fi + ${csudo} systemctl disable tarbitratord &> /dev/null || echo &> /dev/null + ${csudo} rm -f ${tarbitratord_service_config} + if [ "$verMode" == "cluster" ]; then nginx_service_config="${service_config_dir}/nginxd.service" - if systemctl is-active --quiet nginxd; then echo "Nginx for TDengine is running, stopping it..." ${csudo} systemctl stop nginxd &> /dev/null || echo &> /dev/null fi ${csudo} systemctl disable nginxd &> /dev/null || echo &> /dev/null - ${csudo} rm -f ${nginx_service_config} - - tarbitratord_service_config="${service_config_dir}/tarbitratord.service" - if systemctl is-active --quiet tarbitratord; then - echo "tarbitrator is running, stopping it..." - ${csudo} systemctl stop tarbitratord &> /dev/null || echo &> /dev/null - fi - ${csudo} systemctl disable tarbitratord &> /dev/null || echo &> /dev/null - - ${csudo} rm -f ${tarbitratord_service_config} fi } @@ -401,7 +413,6 @@ function install_service_on_systemd() { clean_service_on_systemd taosd_service_config="${service_config_dir}/taosd.service" - ${csudo} bash -c "echo '[Unit]' >> ${taosd_service_config}" ${csudo} bash -c "echo 'Description=TDengine server service' >> ${taosd_service_config}" ${csudo} bash -c "echo 'After=network-online.target' >> ${taosd_service_config}" @@ -422,32 +433,30 @@ function install_service_on_systemd() { ${csudo} bash -c "echo '[Install]' >> ${taosd_service_config}" ${csudo} bash -c "echo 'WantedBy=multi-user.target' >> ${taosd_service_config}" ${csudo} systemctl enable taosd - + + tarbitratord_service_config="${service_config_dir}/tarbitratord.service" + ${csudo} bash -c "echo '[Unit]' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Description=TDengine arbitrator service' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'After=network-online.target' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Wants=network-online.target' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo '[Service]' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Type=simple' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'ExecStart=/usr/bin/tarbitrator' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'LimitNOFILE=infinity' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'LimitNPROC=infinity' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'LimitCORE=infinity' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'TimeoutStartSec=0' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'StandardOutput=null' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Restart=always' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'StartLimitBurst=3' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'StartLimitInterval=60s' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo '[Install]' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'WantedBy=multi-user.target' >> ${tarbitratord_service_config}" + #${csudo} systemctl enable tarbitratord + if [ "$verMode" == "cluster" ]; then - - tarbitratord_service_config="${service_config_dir}/tarbitratord.service" - - ${csudo} bash -c "echo '[Unit]' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo 'Description=TDengine arbitrator service' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo 'After=network-online.target' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo 'Wants=network-online.target' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo '[Service]' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo 'Type=simple' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo 'ExecStart=/usr/bin/tarbitrator' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo 'LimitNOFILE=infinity' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo 'LimitNPROC=infinity' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo 'LimitCORE=infinity' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo 'TimeoutStartSec=0' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo 'StandardOutput=null' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo 'Restart=always' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo 'StartLimitBurst=3' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo 'StartLimitInterval=60s' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo '[Install]' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo 'WantedBy=multi-user.target' >> ${tarbitratord_service_config}" - # ${csudo} systemctl enable tarbitratord - nginx_service_config="${service_config_dir}/nginxd.service" ${csudo} bash -c "echo '[Unit]' >> ${nginx_service_config}" ${csudo} bash -c "echo 'Description=Nginx For TDengine Service' >> ${nginx_service_config}" @@ -696,6 +705,7 @@ function install_TDengine() { echo echo -e "\033[44;32;1mTDengine client is installed successfully!${NC}" fi + touch ~/.taos_history rm -rf $(tar -tf taos.tar.gz) } diff --git a/packaging/tools/install_arbi.sh b/packaging/tools/install_arbi.sh new file mode 100755 index 0000000000..a89d2257dc --- /dev/null +++ b/packaging/tools/install_arbi.sh @@ -0,0 +1,297 @@ +#!/bin/bash +# +# This file is used to install database on linux systems. The operating system +# is required to use systemd to manage services at boot + +set -e +#set -x + +# -----------------------Variables definition--------------------- +script_dir=$(dirname $(readlink -f "$0")) + +bin_link_dir="/usr/bin" +#inc_link_dir="/usr/include" + +#install main path +install_main_dir="/usr/local/tarbitrator" + +# old bin dir +bin_dir="/usr/local/tarbitrator/bin" + +service_config_dir="/etc/systemd/system" + +# Color setting +RED='\033[0;31m' +GREEN='\033[1;32m' +GREEN_DARK='\033[0;32m' +GREEN_UNDERLINE='\033[4;32m' +NC='\033[0m' + +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo" +fi + +update_flag=0 + +initd_mod=0 +service_mod=2 +if pidof systemd &> /dev/null; then + service_mod=0 +elif $(which service &> /dev/null); then + service_mod=1 + service_config_dir="/etc/init.d" + if $(which chkconfig &> /dev/null); then + initd_mod=1 + elif $(which insserv &> /dev/null); then + initd_mod=2 + elif $(which update-rc.d &> /dev/null); then + initd_mod=3 + else + service_mod=2 + fi +else + service_mod=2 +fi + + +# get the operating system type for using the corresponding init file +# ubuntu/debian(deb), centos/fedora(rpm), others: opensuse, redhat, ..., no verification +#osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release) +if [[ -e /etc/os-release ]]; then + osinfo=$(cat /etc/os-release | grep "NAME" | cut -d '"' -f2) ||: +else + osinfo="" +fi +#echo "osinfo: ${osinfo}" +os_type=0 +if echo $osinfo | grep -qwi "ubuntu" ; then +# echo "This is ubuntu system" + os_type=1 +elif echo $osinfo | grep -qwi "debian" ; then +# echo "This is debian system" + os_type=1 +elif echo $osinfo | grep -qwi "Kylin" ; then +# echo "This is Kylin system" + os_type=1 +elif echo $osinfo | grep -qwi "centos" ; then +# echo "This is centos system" + os_type=2 +elif echo $osinfo | grep -qwi "fedora" ; then +# echo "This is fedora system" + os_type=2 +else + echo " osinfo: ${osinfo}" + echo " This is an officially unverified linux system," + echo " if there are any problems with the installation and operation, " + echo " please feel free to contact taosdata.com for support." + os_type=1 +fi + +function kill_tarbitrator() { + pid=$(ps -ef | grep "tarbitrator" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo} kill -9 $pid || : + fi +} + +function install_main_path() { + #create install main dir and all sub dir + ${csudo} rm -rf ${install_main_dir} || : + ${csudo} mkdir -p ${install_main_dir} + ${csudo} mkdir -p ${install_main_dir}/bin + #${csudo} mkdir -p ${install_main_dir}/include + ${csudo} mkdir -p ${install_main_dir}/init.d +} + +function install_bin() { + # Remove links + ${csudo} rm -f ${bin_link_dir}/rmtarbitrator || : + ${csudo} rm -f ${bin_link_dir}/tarbitrator || : + ${csudo} cp -r ${script_dir}/bin/* ${install_main_dir}/bin && ${csudo} chmod 0555 ${install_main_dir}/bin/* + + #Make link + [ -x ${install_main_dir}/bin/remove_arbi.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove_arbi.sh ${bin_link_dir}/rmtarbitrator || : + [ -x ${install_main_dir}/bin/tarbitrator ] && ${csudo} ln -s ${install_main_dir}/bin/tarbitrator ${bin_link_dir}/tarbitrator || : +} + +function install_header() { + ${csudo} rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taoserror.h || : + ${csudo} cp -f ${script_dir}/inc/* ${install_main_dir}/include && ${csudo} chmod 644 ${install_main_dir}/include/* + ${csudo} ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h + ${csudo} ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h +} + +function clean_service_on_sysvinit() { + #restart_config_str="taos:2345:respawn:${service_config_dir}/taosd start" + #${csudo} sed -i "\|${restart_config_str}|d" /etc/inittab || : + + if pidof tarbitrator &> /dev/null; then + ${csudo} service tarbitratord stop || : + fi + + if ((${initd_mod}==1)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} chkconfig --del tarbitratord || : + fi + elif ((${initd_mod}==2)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} insserv -r tarbitratord || : + fi + elif ((${initd_mod}==3)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} update-rc.d -f tarbitratord remove || : + fi + fi + + ${csudo} rm -f ${service_config_dir}/tarbitratord || : + + if $(which init &> /dev/null); then + ${csudo} init q || : + fi +} + +function install_service_on_sysvinit() { + clean_service_on_sysvinit + sleep 1 + + # Install taosd service + + if ((${os_type}==1)); then + ${csudo} cp -f ${script_dir}/init.d/tarbitratord.deb ${install_main_dir}/init.d/tarbitratord + ${csudo} cp ${script_dir}/init.d/tarbitratord.deb ${service_config_dir}/tarbitratord && ${csudo} chmod a+x ${service_config_dir}/tarbitratord + elif ((${os_type}==2)); then + ${csudo} cp -f ${script_dir}/init.d/tarbitratord.rpm ${install_main_dir}/init.d/tarbitratord + ${csudo} cp ${script_dir}/init.d/tarbitratord.rpm ${service_config_dir}/tarbitratord && ${csudo} chmod a+x ${service_config_dir}/tarbitratord + fi + + #restart_config_str="taos:2345:respawn:${service_config_dir}/taosd start" + #${csudo} grep -q -F "$restart_config_str" /etc/inittab || ${csudo} bash -c "echo '${restart_config_str}' >> /etc/inittab" + + if ((${initd_mod}==1)); then + ${csudo} chkconfig --add tarbitratord || : + ${csudo} chkconfig --level 2345 tarbitratord on || : + elif ((${initd_mod}==2)); then + ${csudo} insserv tarbitratord || : + ${csudo} insserv -d tarbitratord || : + elif ((${initd_mod}==3)); then + ${csudo} update-rc.d tarbitratord defaults || : + fi +} + +function clean_service_on_systemd() { + tarbitratord_service_config="${service_config_dir}/tarbitratord.service" + if systemctl is-active --quiet tarbitratord; then + echo "tarbitrator is running, stopping it..." + ${csudo} systemctl stop tarbitratord &> /dev/null || echo &> /dev/null + fi + ${csudo} systemctl disable tarbitratord &> /dev/null || echo &> /dev/null + + ${csudo} rm -f ${tarbitratord_service_config} +} + +# taos:2345:respawn:/etc/init.d/tarbitratord start + +function install_service_on_systemd() { + clean_service_on_systemd + + tarbitratord_service_config="${service_config_dir}/tarbitratord.service" + + ${csudo} bash -c "echo '[Unit]' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Description=TDengine arbitrator service' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'After=network-online.target' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Wants=network-online.target' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo '[Service]' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Type=simple' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'ExecStart=/usr/bin/tarbitrator' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'LimitNOFILE=infinity' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'LimitNPROC=infinity' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'LimitCORE=infinity' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'TimeoutStartSec=0' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'StandardOutput=null' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Restart=always' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'StartLimitBurst=3' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'StartLimitInterval=60s' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo '[Install]' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'WantedBy=multi-user.target' >> ${tarbitratord_service_config}" + ${csudo} systemctl enable tarbitratord +} + +function install_service() { + if ((${service_mod}==0)); then + install_service_on_systemd + elif ((${service_mod}==1)); then + install_service_on_sysvinit + else + # must manual stop taosd + kill_tarbitrator + fi +} + +function update_TDengine() { + # Start to update + echo -e "${GREEN}Start to update TDengine's arbitrator ...${NC}" + # Stop the service if running + if pidof tarbitrator &> /dev/null; then + if ((${service_mod}==0)); then + ${csudo} systemctl stop tarbitratord || : + elif ((${service_mod}==1)); then + ${csudo} service tarbitratord stop || : + else + kill_tarbitrator + fi + sleep 1 + fi + + install_main_path + #install_header + install_bin + install_service + + echo + #echo -e "${GREEN_DARK}To configure TDengine ${NC}: edit /etc/taos/taos.cfg" + if ((${service_mod}==0)); then + echo -e "${GREEN_DARK}To start arbitrator ${NC}: ${csudo} systemctl start tarbitratord${NC}" + elif ((${service_mod}==1)); then + echo -e "${GREEN_DARK}To start arbitrator ${NC}: ${csudo} service tarbitratord start${NC}" + else + echo -e "${GREEN_DARK}To start arbitrator ${NC}: ./tarbitrator${NC}" + fi + echo + echo -e "\033[44;32;1mTDengine's arbitrator is updated successfully!${NC}" +} + +function install_TDengine() { + # Start to install + echo -e "${GREEN}Start to install TDengine's arbitrator ...${NC}" + + install_main_path + #install_header + install_bin + install_service + echo + #echo -e "${GREEN_DARK}To configure TDengine ${NC}: edit /etc/taos/taos.cfg" + if ((${service_mod}==0)); then + echo -e "${GREEN_DARK}To start arbitrator ${NC}: ${csudo} systemctl start tarbitratord${NC}" + elif ((${service_mod}==1)); then + echo -e "${GREEN_DARK}To start arbitrator ${NC}: ${csudo} service tarbitratord start${NC}" + else + echo -e "${GREEN_DARK}To start arbitrator ${NC}: tarbitrator${NC}" + fi + + echo -e "\033[44;32;1mTDengine's arbitrator is installed successfully!${NC}" + echo +} + + +## ==============================Main program starts from here============================ +# Install server and client +if [ -x ${bin_dir}/tarbitrator ]; then + update_flag=1 + update_TDengine +else + install_TDengine +fi + diff --git a/packaging/tools/makearbi.sh b/packaging/tools/makearbi.sh new file mode 100755 index 0000000000..bc6179eff2 --- /dev/null +++ b/packaging/tools/makearbi.sh @@ -0,0 +1,75 @@ +#!/bin/bash +# +# Generate arbitrator's tar.gz setup package for all os system + +set -e +#set -x + +curr_dir=$(pwd) +compile_dir=$1 +version=$2 +build_time=$3 +cpuType=$4 +osType=$5 +verMode=$6 +verType=$7 +pagMode=$8 + +script_dir="$(dirname $(readlink -f $0))" +top_dir="$(readlink -f ${script_dir}/../..)" + +# create compressed install file. +build_dir="${compile_dir}/build" +code_dir="${top_dir}/src" +release_dir="${top_dir}/release" + +#package_name='linux' +if [ "$verMode" == "cluster" ]; then + install_dir="${release_dir}/TDengine-enterprise-arbitrator" +else + install_dir="${release_dir}/TDengine-arbitrator" +fi + +# Directories and files. +bin_files="${build_dir}/bin/tarbitrator ${script_dir}/remove_arbi.sh" +install_files="${script_dir}/install_arbi.sh" + +#header_files="${code_dir}/inc/taos.h ${code_dir}/inc/taoserror.h" +init_file_tarbitrator_deb=${script_dir}/../deb/tarbitratord +init_file_tarbitrator_rpm=${script_dir}/../rpm/tarbitratord + +# make directories. +mkdir -p ${install_dir} && cp ${install_files} ${install_dir} && chmod a+x ${install_dir}/install_arbi.sh || : +#mkdir -p ${install_dir}/inc && cp ${header_files} ${install_dir}/inc || : +mkdir -p ${install_dir}/bin && cp ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* || : +mkdir -p ${install_dir}/init.d && cp ${init_file_tarbitrator_deb} ${install_dir}/init.d/tarbitratord.deb || : +mkdir -p ${install_dir}/init.d && cp ${init_file_tarbitrator_rpm} ${install_dir}/init.d/tarbitratord.rpm || : + +cd ${release_dir} + +if [ "$verMode" == "cluster" ]; then + pkg_name=${install_dir}-${version}-${osType}-${cpuType} +elif [ "$verMode" == "edge" ]; then + pkg_name=${install_dir}-${version}-${osType}-${cpuType} +else + echo "unknow verMode, nor cluster or edge" + exit 1 +fi + +if [ "$verType" == "beta" ]; then + pkg_name=${pkg_name}-${verType} +elif [ "$verType" == "stable" ]; then + pkg_name=${pkg_name} +else + echo "unknow verType, nor stabel or beta" + exit 1 +fi + +tar -zcv -f "$(basename ${pkg_name}).tar.gz" $(basename ${install_dir}) --remove-files || : +exitcode=$? +if [ "$exitcode" != "0" ]; then + echo "tar ${pkg_name}.tar.gz error !!!" + exit $exitcode +fi + +cd ${curr_dir} diff --git a/packaging/tools/makeclient.sh b/packaging/tools/makeclient.sh index 5fe4cf50dd..8545a3e5e4 100755 --- a/packaging/tools/makeclient.sh +++ b/packaging/tools/makeclient.sh @@ -45,7 +45,7 @@ if [ "$osType" != "Darwin" ]; then strip ${build_dir}/bin/taos bin_files="${build_dir}/bin/taos ${script_dir}/remove_client.sh" else - bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdump ${script_dir}/remove_client.sh ${script_dir}/set_core.sh" + bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdemo ${script_dir}/remove_client.sh ${script_dir}/set_core.sh" fi lib_files="${build_dir}/lib/libtaos.so.${version}" else diff --git a/packaging/tools/remove.sh b/packaging/tools/remove.sh index 63e09dc568..dd6fe65eb7 100755 --- a/packaging/tools/remove.sh +++ b/packaging/tools/remove.sh @@ -69,11 +69,12 @@ function kill_tarbitrator() { } function clean_bin() { # Remove link - ${csudo} rm -f ${bin_link_dir}/taos || : - ${csudo} rm -f ${bin_link_dir}/taosd || : - ${csudo} rm -f ${bin_link_dir}/taosdemo || : - ${csudo} rm -f ${bin_link_dir}/taosdump || : - ${csudo} rm -f ${bin_link_dir}/rmtaos || : + ${csudo} rm -f ${bin_link_dir}/taos || : + ${csudo} rm -f ${bin_link_dir}/taosd || : + ${csudo} rm -f ${bin_link_dir}/taosdemo || : + ${csudo} rm -f ${bin_link_dir}/rmtaos || : + ${csudo} rm -f ${bin_link_dir}/tarbitrator || : + ${csudo} rm -f ${bin_link_dir}/set_core || : } function clean_lib() { @@ -86,7 +87,7 @@ function clean_lib() { function clean_header() { # Remove link ${csudo} rm -f ${inc_link_dir}/taos.h || : - ${csudo} rm -f ${inc_link_dir}/taoserror.h || : + ${csudo} rm -f ${inc_link_dir}/taoserror.h || : } function clean_config() { @@ -148,15 +149,27 @@ function clean_service_on_sysvinit() { ${csudo} service tarbitratord stop || : fi - if ((${initd_mod}==1)); then + if ((${initd_mod}==1)); then + if [ -e ${service_config_dir}/taosd ]; then ${csudo} chkconfig --del taosd || : + fi + if [ -e ${service_config_dir}/tarbitratord ]; then ${csudo} chkconfig --del tarbitratord || : - elif ((${initd_mod}==2)); then + fi + elif ((${initd_mod}==2)); then + if [ -e ${service_config_dir}/taosd ]; then ${csudo} insserv -r taosd || : + fi + if [ -e ${service_config_dir}/tarbitratord ]; then ${csudo} insserv -r tarbitratord || : - elif ((${initd_mod}==3)); then + fi + elif ((${initd_mod}==3)); then + if [ -e ${service_config_dir}/taosd ]; then ${csudo} update-rc.d -f taosd remove || : + fi + if [ -e ${service_config_dir}/tarbitratord ]; then ${csudo} update-rc.d -f tarbitratord remove || : + fi fi ${csudo} rm -f ${service_config_dir}/taosd || : @@ -196,13 +209,20 @@ ${csudo} rm -rf ${data_link_dir} || : ${csudo} rm -rf ${install_main_dir} ${csudo} rm -rf ${install_nginxd_dir} +if [[ -e /etc/os-release ]]; then + osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release) +else + osinfo="" +fi -osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release) if echo $osinfo | grep -qwi "ubuntu" ; then # echo "this is ubuntu system" ${csudo} rm -f /var/lib/dpkg/info/tdengine* || : +elif echo $osinfo | grep -qwi "debian" ; then +# echo "this is debian system" + ${csudo} rm -f /var/lib/dpkg/info/tdengine* || : elif echo $osinfo | grep -qwi "centos" ; then - echo "this is centos system" +# echo "this is centos system" ${csudo} rpm -e --noscripts tdengine || : fi diff --git a/packaging/tools/remove_arbi.sh b/packaging/tools/remove_arbi.sh new file mode 100755 index 0000000000..68fd9275fb --- /dev/null +++ b/packaging/tools/remove_arbi.sh @@ -0,0 +1,129 @@ +#!/bin/bash +# +# Script to stop the service and uninstall TDengine's arbitrator + +set -e +#set -x + +verMode=edge + +RED='\033[0;31m' +GREEN='\033[1;32m' +NC='\033[0m' + +#install main path +install_main_dir="/usr/local/tarbitrator" +bin_link_dir="/usr/bin" +#inc_link_dir="/usr/include" + +service_config_dir="/etc/systemd/system" +tarbitrator_service_name="tarbitratord" +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo" +fi + +initd_mod=0 +service_mod=2 +if pidof systemd &> /dev/null; then + service_mod=0 +elif $(which service &> /dev/null); then + service_mod=1 + service_config_dir="/etc/init.d" + if $(which chkconfig &> /dev/null); then + initd_mod=1 + elif $(which insserv &> /dev/null); then + initd_mod=2 + elif $(which update-rc.d &> /dev/null); then + initd_mod=3 + else + service_mod=2 + fi +else + service_mod=2 +fi + +function kill_tarbitrator() { + pid=$(ps -ef | grep "tarbitrator" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo} kill -9 $pid || : + fi +} +function clean_bin() { + # Remove link + ${csudo} rm -f ${bin_link_dir}/tarbitrator || : +} + +function clean_header() { + # Remove link + ${csudo} rm -f ${inc_link_dir}/taos.h || : + ${csudo} rm -f ${inc_link_dir}/taoserror.h || : +} + +function clean_log() { + # Remove link + ${csudo} rm -rf /arbitrator.log || : +} + +function clean_service_on_systemd() { + tarbitratord_service_config="${service_config_dir}/${tarbitrator_service_name}.service" + + if systemctl is-active --quiet ${tarbitrator_service_name}; then + echo "TDengine tarbitrator is running, stopping it..." + ${csudo} systemctl stop ${tarbitrator_service_name} &> /dev/null || echo &> /dev/null + fi + ${csudo} systemctl disable ${tarbitrator_service_name} &> /dev/null || echo &> /dev/null + + ${csudo} rm -f ${tarbitratord_service_config} +} + +function clean_service_on_sysvinit() { + if pidof tarbitrator &> /dev/null; then + echo "TDengine's tarbitrator is running, stopping it..." + ${csudo} service tarbitratord stop || : + fi + + if ((${initd_mod}==1)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} chkconfig --del tarbitratord || : + fi + elif ((${initd_mod}==2)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} insserv -r tarbitratord || : + fi + elif ((${initd_mod}==3)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} update-rc.d -f tarbitratord remove || : + fi + fi + + ${csudo} rm -f ${service_config_dir}/tarbitratord || : + + if $(which init &> /dev/null); then + ${csudo} init q || : + fi +} + +function clean_service() { + if ((${service_mod}==0)); then + clean_service_on_systemd + elif ((${service_mod}==1)); then + clean_service_on_sysvinit + else + # must manual stop + kill_tarbitrator + fi +} + +# Stop service and disable booting start. +clean_service +# Remove binary file and links +clean_bin +# Remove header file. +##clean_header +# Remove log file +clean_log + +${csudo} rm -rf ${install_main_dir} + +echo -e "${GREEN}TDengine's arbitrator is removed successfully!${NC}" diff --git a/packaging/tools/remove_client.sh b/packaging/tools/remove_client.sh index 4bc278fcf0..7cbf524d04 100755 --- a/packaging/tools/remove_client.sh +++ b/packaging/tools/remove_client.sh @@ -37,7 +37,7 @@ function kill_client() { function clean_bin() { # Remove link ${csudo} rm -f ${bin_link_dir}/taos || : - ${csudo} rm -f ${bin_link_dir}/taosump || : + ${csudo} rm -f ${bin_link_dir}/taosdemo || : ${csudo} rm -f ${bin_link_dir}/rmtaos || : } diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index d5cbf3cab6..5b5fb3435d 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -35,12 +35,14 @@ IF (TD_LINUX) ELSEIF (TD_WINDOWS) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/jni/windows) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/jni/windows/win32) - + + CONFIGURE_FILE("${TD_COMMUNITY_DIR}/src/client/src/taos.rc.in" "${TD_COMMUNITY_DIR}/src/client/src/taos.rc") + ADD_LIBRARY(taos_static STATIC ${SRC}) TARGET_LINK_LIBRARIES(taos_static trpc tutil query) # generate dynamic library (*.dll) - ADD_LIBRARY(taos SHARED ${SRC}) + ADD_LIBRARY(taos SHARED ${SRC} ${TD_COMMUNITY_DIR}/src/client/src/taos.rc) IF (NOT TD_GODLL) SET_TARGET_PROPERTIES(taos PROPERTIES LINK_FLAGS /DEF:${TD_COMMUNITY_DIR}/src/client/src/taos.def) ENDIF () diff --git a/src/client/inc/tscLocalMerge.h b/src/client/inc/tscLocalMerge.h index 397a60d140..4e579b0729 100644 --- a/src/client/inc/tscLocalMerge.h +++ b/src/client/inc/tscLocalMerge.h @@ -87,7 +87,6 @@ typedef struct SRetrieveSupport { SSqlObj * pParentSql; tFilePage * localBuffer; // temp buffer, there is a buffer for each vnode to uint32_t numOfRetry; // record the number of retry times - pthread_mutex_t queryMutex; } SRetrieveSupport; int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOrderDescriptor **pDesc, diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 2ca6ba6691..f77897a74b 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -108,7 +108,7 @@ int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOff void tscDestroyDataBlock(STableDataBlocks* pDataBlock); void tscSortRemoveDataBlockDupRows(STableDataBlocks* dataBuf); -SParamInfo* tscAddParamToDataBlock(STableDataBlocks* pDataBlock, char type, uint8_t timePrec, short bytes, +SParamInfo* tscAddParamToDataBlock(STableDataBlocks* pDataBlock, char type, uint8_t timePrec, int16_t bytes, uint32_t offset); void* tscDestroyBlockArrayList(SArray* pDataBlockList); @@ -138,10 +138,10 @@ bool tscIsProjectionQuery(SQueryInfo* pQueryInfo); bool tscIsTwoStageSTableQuery(SQueryInfo* pQueryInfo, int32_t tableIndex); bool tscQueryTags(SQueryInfo* pQueryInfo); -void tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId, SColumnIndex* pIndex, - SSchema* pColSchema, int16_t colType); +SSqlExpr* tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId, + SColumnIndex* pIndex, SSchema* pColSchema, int16_t colType); -int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SSQLToken* pzTableName, SSqlObj* pSql); +int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SStrToken* pzTableName, SSqlObj* pSql); void tscClearInterpInfo(SQueryInfo* pQueryInfo); bool tscIsInsertData(char* sqlstr); @@ -194,11 +194,11 @@ SColumn* tscColumnListInsert(SArray* pColList, SColumnIndex* colIndex); SArray* tscColumnListClone(const SArray* src, int16_t tableIndex); void tscColumnListDestroy(SArray* pColList); -int32_t tscValidateName(SSQLToken* pToken); +int32_t tscValidateName(SStrToken* pToken); void tscIncStreamExecutionCount(void* pStream); -bool tscValidateColumnId(STableMetaInfo* pTableMetaInfo, int32_t colId); +bool tscValidateColumnId(STableMetaInfo* pTableMetaInfo, int32_t colId, int32_t numOfParams); // get starter position of metric query condition (query on tags) in SSqlCmd.payload SCond* tsGetSTableQueryCond(STagCond* pCond, uint64_t uid); @@ -217,7 +217,7 @@ STableMetaInfo* tscGetTableMetaInfoFromCmd(SSqlCmd *pCmd, int32_t subClauseIndex STableMetaInfo* tscGetMetaInfo(SQueryInfo *pQueryInfo, int32_t tableIndex); SQueryInfo *tscGetQueryInfoDetail(SSqlCmd* pCmd, int32_t subClauseIndex); -int32_t tscGetQueryInfoDetailSafely(SSqlCmd *pCmd, int32_t subClauseIndex, SQueryInfo** pQueryInfo); +SQueryInfo *tscGetQueryInfoDetailSafely(SSqlCmd *pCmd, int32_t subClauseIndex); void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache); diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 6d02bc7fbd..d2c52e972a 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -35,6 +35,40 @@ extern "C" { #include "qTsbuf.h" #include "tcmdtype.h" +#if 0 +static UNUSED_FUNC void *u_malloc (size_t __size) { + uint32_t v = rand(); + + if (v % 5000 <= 0) { + return NULL; + } else { + return malloc(__size); + } +} + +static UNUSED_FUNC void* u_calloc(size_t num, size_t __size) { + uint32_t v = rand(); + if (v % 5000 <= 0) { + return NULL; + } else { + return calloc(num, __size); + } +} + +static UNUSED_FUNC void* u_realloc(void* p, size_t __size) { + uint32_t v = rand(); + if (v % 5000 <= 0) { + return NULL; + } else { + return realloc(p, __size); + } +} + +#define calloc u_calloc +#define malloc u_malloc +#define realloc u_realloc +#endif + // forward declaration struct SSqlInfo; struct SLocalReducer; @@ -195,9 +229,9 @@ typedef struct STableDataBlocks { typedef struct SQueryInfo { int16_t command; // the command may be different for each subclause, so keep it seperately. - uint32_t type; // query/insert/import type + uint32_t type; // query/insert type char slidingTimeUnit; - STimeWindow window; + STimeWindow window; // query time window int64_t intervalTime; // aggregation time interval int64_t slidingTime; // sliding window in mseconds SSqlGroupbyExpr groupbyExpr; // group by tags info @@ -216,6 +250,7 @@ typedef struct SQueryInfo { char * msg; // pointer to the pCmd->payload to keep error message temporarily int64_t clauseLimit; // limit for current sub clause int64_t prjOffset; // offset value in the original sql expression, only applied at client side + int32_t udColumnId; // current user-defined constant output field column id, monotonically decreases from TSDB_UD_COLUMN_INDEX } SQueryInfo; typedef struct { @@ -234,7 +269,7 @@ typedef struct { char * curSql; // current sql, resume position of sql after parsing paused int8_t parseFinished; - short numOfCols; + int16_t numOfCols; uint32_t allocSize; char * payload; int32_t payloadLen; @@ -431,31 +466,36 @@ static FORCE_INLINE void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pField int32_t bytes = pInfo->pSqlExpr->resBytes; char* pData = pRes->data + pInfo->pSqlExpr->offset * pRes->numOfRows + bytes * pRes->row; - if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BINARY) { - int32_t realLen = varDataLen(pData); - assert(realLen <= bytes - VARSTR_HEADER_SIZE); - if (isNull(pData, type)) { - pRes->tsrow[columnIndex] = NULL; + // user defined constant value output columns + if (pInfo->pSqlExpr->colInfo.flag == TSDB_COL_UDC) { + if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BINARY) { + pData = pInfo->pSqlExpr->param[1].pz; + pRes->length[columnIndex] = pInfo->pSqlExpr->param[1].nLen; + pRes->tsrow[columnIndex] = (pInfo->pSqlExpr->param[1].nType == TSDB_DATA_TYPE_NULL) ? NULL : pData; } else { - pRes->tsrow[columnIndex] = ((tstr*)pData)->data; - } + assert(bytes == tDataTypeDesc[type].nSize); - if (realLen < pInfo->pSqlExpr->resBytes - VARSTR_HEADER_SIZE) { // todo refactor - *(pData + realLen + VARSTR_HEADER_SIZE) = 0; + pRes->tsrow[columnIndex] = isNull(pData, type) ? NULL : &pInfo->pSqlExpr->param[1].i64Key; + pRes->length[columnIndex] = bytes; } - - pRes->length[columnIndex] = realLen; } else { - assert(bytes == tDataTypeDesc[type].nSize); + if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BINARY) { + int32_t realLen = varDataLen(pData); + assert(realLen <= bytes - VARSTR_HEADER_SIZE); - if (isNull(pData, type)) { - pRes->tsrow[columnIndex] = NULL; + pRes->tsrow[columnIndex] = (isNull(pData, type)) ? NULL : ((tstr *)pData)->data; + if (realLen < pInfo->pSqlExpr->resBytes - VARSTR_HEADER_SIZE) { // todo refactor + *(pData + realLen + VARSTR_HEADER_SIZE) = 0; + } + + pRes->length[columnIndex] = realLen; } else { - pRes->tsrow[columnIndex] = pData; - } + assert(bytes == tDataTypeDesc[type].nSize); - pRes->length[columnIndex] = bytes; + pRes->tsrow[columnIndex] = isNull(pData, type) ? NULL : pData; + pRes->length[columnIndex] = bytes; + } } } diff --git a/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h b/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h index d81bbd9b6d..eaea91d1bf 100644 --- a/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h +++ b/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h @@ -89,6 +89,14 @@ JNIEXPORT jstring JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getErrMsgImp JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getResultSetImp (JNIEnv *env, jobject jobj, jlong con, jlong tres); +/* + * Class: com_taosdata_jdbc_TSDBJNIConnector + * Method: isUpdateQueryImp + * Signature: (J)J + */ +JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_isUpdateQueryImp + (JNIEnv *env, jobject jobj, jlong con, jlong tres); + /* * Class: com_taosdata_jdbc_TSDBJNIConnector * Method: freeResultSetImp diff --git a/src/client/src/taos.rc.in b/src/client/src/taos.rc.in new file mode 100644 index 0000000000..751be85fd0 --- /dev/null +++ b/src/client/src/taos.rc.in @@ -0,0 +1,31 @@ +1 VERSIONINFO + FILEVERSION ${TD_VER_NUMBER} + PRODUCTVERSION ${TD_VER_NUMBER} + FILEFLAGSMASK 0x17L +#ifdef _DEBUG + FILEFLAGS 0x1L +#else + FILEFLAGS 0x0L +#endif + FILEOS 0x4L + FILETYPE 0x0L + FILESUBTYPE 0x0L +BEGIN + BLOCK "StringFileInfo" + BEGIN + BLOCK "040904b0" + BEGIN + VALUE "FileDescription", "Native C Driver for TDengine" + VALUE "FileVersion", "${TD_VER_NUMBER}" + VALUE "InternalName", "taos.dll(${TD_VER_CPUTYPE})" + VALUE "LegalCopyright", "Copyright (C) 2020 TAOS Data" + VALUE "OriginalFilename", "" + VALUE "ProductName", "taos.dll(${TD_VER_CPUTYPE})" + VALUE "ProductVersion", "${TD_VER_NUMBER}" + END + END + BLOCK "VarFileInfo" + BEGIN + VALUE "Translation", 0x409, 1200 + END +END \ No newline at end of file diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index b05aef76eb..4643d255dc 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -433,7 +433,8 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { tscError("%p get tableMeta failed, code:%s", pSql, tstrerror(code)); goto _error; } else { - tscDebug("%p get tableMeta successfully", pSql); + const char* msg = (pCmd->command == TSDB_SQL_STABLEVGROUP)? "vgroup-list":"table-meta"; + tscDebug("%p get %s successfully", pSql, msg); } if (pSql->pStream == NULL) { diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index 5c708dffee..e74fcba246 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -56,7 +56,8 @@ for (int32_t i = 0; i < (ctx)->tagInfo.numOfTagCols; ++i) { \ SQLFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[i]; \ if (__ctx->functionId == TSDB_FUNC_TS_DUMMY) { \ - __ctx->tag = (tVariant){.i64Key = (ts), .nType = TSDB_DATA_TYPE_BIGINT}; \ + __ctx->tag.i64Key = (ts); \ + __ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; \ } \ aAggs[TSDB_FUNC_TAG].xFunction(__ctx); \ } \ @@ -963,7 +964,8 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, for (int32_t i = 0; i < (pCtx)->tagInfo.numOfTagCols; ++i) { SQLFunctionCtx *__ctx = pCtx->tagInfo.pTagCtxList[i]; if (__ctx->functionId == TSDB_FUNC_TS_DUMMY) { - __ctx->tag = (tVariant){.i64Key = key, .nType = TSDB_DATA_TYPE_BIGINT}; + __ctx->tag.i64Key = key; + __ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; } aAggs[TSDB_FUNC_TAG].xFunction(__ctx); @@ -1811,23 +1813,19 @@ static void last_dist_func_second_merge(SQLFunctionCtx *pCtx) { * NOTE: last_row does not use the interResultBuf to keep the result */ static void last_row_function(SQLFunctionCtx *pCtx) { - assert(pCtx->size == 1); - + assert(pCtx->size >= 1); char *pData = GET_INPUT_CHAR(pCtx); - assignVal(pCtx->aOutputBuf, pData, pCtx->inputBytes, pCtx->inputType); + + // assign the last element in current data block + assignVal(pCtx->aOutputBuf, pData + (pCtx->size - 1) * pCtx->inputBytes, pCtx->inputBytes, pCtx->inputType); SResultInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; - SLastrowInfo *pInfo = (SLastrowInfo *)pResInfo->interResultBuf; - pInfo->ts = pCtx->ptsList[0]; - - pInfo->hasResult = DATA_SET_FLAG; - - // set the result to final result buffer + // set the result to final result buffer in case of super table query if (pResInfo->superTableQ) { SLastrowInfo *pInfo1 = (SLastrowInfo *)(pCtx->aOutputBuf + pCtx->inputBytes); - pInfo1->ts = pCtx->ptsList[0]; + pInfo1->ts = pCtx->ptsList[pCtx->size - 1]; pInfo1->hasResult = DATA_SET_FLAG; DO_UPDATE_TAG_COLUMNS(pCtx, pInfo1->ts); @@ -1867,7 +1865,8 @@ static void valuePairAssign(tValuePair *dst, int16_t type, const char *val, int6 for (int32_t i = 0; i < pTagInfo->numOfTagCols; ++i) { SQLFunctionCtx* ctx = pTagInfo->pTagCtxList[i]; if (ctx->functionId == TSDB_FUNC_TS_DUMMY) { - ctx->tag = (tVariant) {.nType = TSDB_DATA_TYPE_BIGINT, .i64Key = tsKey}; + ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; + ctx->tag.i64Key = tsKey; } tVariantDump(&ctx->tag, dst->pTags + size, ctx->tag.nType, true); @@ -2035,7 +2034,7 @@ static void copyTopBotRes(SQLFunctionCtx *pCtx, int32_t type) { tValuePair **tvp = pRes->res; int32_t step = QUERY_ASC_FORWARD_STEP; - int32_t len = GET_RES_INFO(pCtx)->numOfRes; + int32_t len = (int32_t)(GET_RES_INFO(pCtx)->numOfRes); switch (type) { case TSDB_DATA_TYPE_INT: { @@ -2409,10 +2408,10 @@ static void top_bottom_func_finalizer(SQLFunctionCtx *pCtx) { // user specify the order of output by sort the result according to timestamp if (pCtx->param[1].i64Key == PRIMARYKEY_TIMESTAMP_COL_INDEX) { __compar_fn_t comparator = (pCtx->param[2].i64Key == TSDB_ORDER_ASC) ? resAscComparFn : resDescComparFn; - qsort(tvp, pResInfo->numOfRes, POINTER_BYTES, comparator); + qsort(tvp, (size_t)pResInfo->numOfRes, POINTER_BYTES, comparator); } else if (pCtx->param[1].i64Key > PRIMARYKEY_TIMESTAMP_COL_INDEX) { __compar_fn_t comparator = (pCtx->param[2].i64Key == TSDB_ORDER_ASC) ? resDataAscComparFn : resDataDescComparFn; - qsort(tvp, pResInfo->numOfRes, POINTER_BYTES, comparator); + qsort(tvp, (size_t)pResInfo->numOfRes, POINTER_BYTES, comparator); } GET_TRUE_DATA_TYPE(); @@ -2906,33 +2905,41 @@ static FORCE_INLINE void date_col_output_function_f(SQLFunctionCtx *pCtx, int32_ } static void col_project_function(SQLFunctionCtx *pCtx) { + // the number of output rows should not affect the final number of rows, so set it to be 0 + if (pCtx->numOfParams == 2) { + return; + } + INC_INIT_VAL(pCtx, pCtx->size); - + char *pData = GET_INPUT_CHAR(pCtx); if (pCtx->order == TSDB_ORDER_ASC) { - memcpy(pCtx->aOutputBuf, pData, (size_t)pCtx->size * pCtx->inputBytes); + memcpy(pCtx->aOutputBuf, pData, (size_t) pCtx->size * pCtx->inputBytes); } else { for(int32_t i = 0; i < pCtx->size; ++i) { memcpy(pCtx->aOutputBuf + (pCtx->size - 1 - i) * pCtx->inputBytes, pData + i * pCtx->inputBytes, pCtx->inputBytes); } } - + pCtx->aOutputBuf += pCtx->size * pCtx->outputBytes; } static void col_project_function_f(SQLFunctionCtx *pCtx, int32_t index) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); - + if (pCtx->numOfParams == 2) { // the number of output rows should not affect the final number of rows, so set it to be 0 + return; + } + // only one output if (pCtx->param[0].i64Key == 1 && pResInfo->numOfRes >= 1) { return; } - + INC_INIT_VAL(pCtx, 1); char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); - + pCtx->aOutputBuf += pCtx->inputBytes; } @@ -3900,11 +3907,11 @@ static void ts_comp_function(SQLFunctionCtx *pCtx) { // primary ts must be existed, so no need to check its existance if (pCtx->order == TSDB_ORDER_ASC) { - tsBufAppend(pTSbuf, 0, pCtx->tag.i64Key, input, pCtx->size * TSDB_KEYSIZE); + tsBufAppend(pTSbuf, 0, &pCtx->tag, input, pCtx->size * TSDB_KEYSIZE); } else { for (int32_t i = pCtx->size - 1; i >= 0; --i) { char *d = GET_INPUT_CHAR_INDEX(pCtx, i); - tsBufAppend(pTSbuf, 0, pCtx->tag.i64Key, d, TSDB_KEYSIZE); + tsBufAppend(pTSbuf, 0, &pCtx->tag, d, TSDB_KEYSIZE); } } @@ -3923,7 +3930,7 @@ static void ts_comp_function_f(SQLFunctionCtx *pCtx, int32_t index) { STSBuf *pTSbuf = pInfo->pTSBuf; - tsBufAppend(pTSbuf, 0, pCtx->tag.i64Key, pData, TSDB_KEYSIZE); + tsBufAppend(pTSbuf, 0, &pCtx->tag, pData, TSDB_KEYSIZE); SET_VAL(pCtx, pCtx->size, 1); pResInfo->hasResult = DATA_SET_FLAG; diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c index 99c3bc4fb3..563c9fa84e 100644 --- a/src/client/src/tscLocalMerge.c +++ b/src/client/src/tscLocalMerge.c @@ -370,7 +370,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); - TSKEY stime = MIN(pQueryInfo->window.skey, pQueryInfo->window.ekey); + TSKEY stime = (pQueryInfo->order.order == TSDB_ORDER_ASC)? pQueryInfo->window.skey : pQueryInfo->window.ekey; int64_t revisedSTime = taosGetIntervalStartTimestamp(stime, pQueryInfo->slidingTime, pQueryInfo->intervalTime, pQueryInfo->slidingTimeUnit, tinfo.precision); @@ -843,28 +843,6 @@ void savePrevRecordAndSetupFillInfo(SLocalReducer *pLocalReducer, SQueryInfo *pQ tColModelAppend(pModel, pLocalReducer->discardData, pLocalReducer->prevRowOfInput, 0, 1, 1); } -static void reversedCopyFromInterpolationToDstBuf(SQueryInfo *pQueryInfo, SSqlRes *pRes, tFilePage **pResPages, - SLocalReducer *pLocalReducer) { - assert(0); - size_t size = tscSqlExprNumOfExprs(pQueryInfo); - - for (int32_t i = 0; i < size; ++i) { - TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); - - int32_t offset = tscFieldInfoGetOffset(pQueryInfo, i); - assert(offset == getColumnModelOffset(pLocalReducer->resColModel, i)); - - char *src = pResPages[i]->data + (pRes->numOfRows - 1) * pField->bytes; - char *dst = pRes->data + pRes->numOfRows * offset; - - for (int32_t j = 0; j < pRes->numOfRows; ++j) { - memcpy(dst, src, (size_t)pField->bytes); - dst += pField->bytes; - src -= pField->bytes; - } - } -} - static void genFinalResWithoutFill(SSqlRes* pRes, SLocalReducer *pLocalReducer, SQueryInfo* pQueryInfo) { assert(pQueryInfo->intervalTime == 0 || pQueryInfo->fillType == TSDB_FILL_NONE); @@ -907,7 +885,7 @@ static void genFinalResWithoutFill(SSqlRes* pRes, SLocalReducer *pLocalReducer, savePrevRecordAndSetupFillInfo(pLocalReducer, pQueryInfo, pLocalReducer->pFillInfo); } - memcpy(pRes->data, pBeforeFillData->data, pRes->numOfRows * pLocalReducer->finalRowSize); + memcpy(pRes->data, pBeforeFillData->data, (size_t)(pRes->numOfRows * pLocalReducer->finalRowSize)); pRes->numOfClauseTotal += pRes->numOfRows; pBeforeFillData->num = 0; @@ -925,7 +903,8 @@ static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneO SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); SFillInfo *pFillInfo = pLocalReducer->pFillInfo; - int64_t actualETime = MAX(pQueryInfo->window.skey, pQueryInfo->window.ekey); + // todo extract function + int64_t actualETime = (pQueryInfo->order.order == TSDB_ORDER_ASC)? pQueryInfo->window.ekey: pQueryInfo->window.skey; tFilePage **pResPages = malloc(POINTER_BYTES * pQueryInfo->fieldsInfo.numOfOutput); for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { @@ -943,7 +922,7 @@ static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneO for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); memmove(pResPages[i]->data, pResPages[i]->data + pField->bytes * pQueryInfo->limit.offset, - newRows * pField->bytes); + (size_t)(newRows * pField->bytes)); } } @@ -984,14 +963,10 @@ static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneO savePrevRecordAndSetupFillInfo(pLocalReducer, pQueryInfo, pFillInfo); } - if (pQueryInfo->order.order == TSDB_ORDER_ASC) { - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { - TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); - int16_t offset = getColumnModelOffset(pLocalReducer->resColModel, i); - memcpy(pRes->data + offset * pRes->numOfRows, pResPages[i]->data, pField->bytes * pRes->numOfRows); - } - } else { // todo bug?? - reversedCopyFromInterpolationToDstBuf(pQueryInfo, pRes, pResPages, pLocalReducer); + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); + int16_t offset = getColumnModelOffset(pLocalReducer->resColModel, i); + memcpy(pRes->data + offset * pRes->numOfRows, pResPages[i]->data, (size_t)(pField->bytes * pRes->numOfRows)); } pRes->numOfRowsGroup += pRes->numOfRows; @@ -1248,8 +1223,6 @@ bool genFinalResults(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool noMoreCur printf("final result before interpo:\n"); // tColModelDisplay(pLocalReducer->resColModel, pLocalReducer->pBufForInterpo, pResBuf->num, pResBuf->num); #endif - - // no interval query, no fill operation if (pQueryInfo->intervalTime == 0 || pQueryInfo->fillType == TSDB_FILL_NONE) { @@ -1257,7 +1230,9 @@ bool genFinalResults(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool noMoreCur } else { SFillInfo* pFillInfo = pLocalReducer->pFillInfo; if (pFillInfo != NULL) { - taosFillSetStartInfo(pFillInfo, (int32_t)pResBuf->num, pQueryInfo->window.ekey); + TSKEY ekey = (pQueryInfo->order.order == TSDB_ORDER_ASC)? pQueryInfo->window.ekey: pQueryInfo->window.skey; + + taosFillSetStartInfo(pFillInfo, (int32_t)pResBuf->num, ekey); taosFillCopyInputDataFromOneFilePage(pFillInfo, pResBuf); } @@ -1292,7 +1267,7 @@ static void resetEnvForNewResultset(SSqlRes *pRes, SSqlCmd *pCmd, SLocalReducer // for group result interpolation, do not return if not data is generated if (pQueryInfo->fillType != TSDB_FILL_NONE) { - TSKEY skey = MIN(pQueryInfo->window.skey, pQueryInfo->window.ekey); + TSKEY skey = (pQueryInfo->order.order == TSDB_ORDER_ASC)? pQueryInfo->window.skey:pQueryInfo->window.ekey;//MIN(pQueryInfo->window.skey, pQueryInfo->window.ekey); int64_t newTime = taosGetIntervalStartTimestamp(skey, pQueryInfo->slidingTime, pQueryInfo->intervalTime, pQueryInfo->slidingTimeUnit, precision); taosResetFillInfo(pLocalReducer->pFillInfo, newTime); @@ -1345,7 +1320,7 @@ static bool doHandleLastRemainData(SSqlObj *pSql) { // if fillType == TSDB_FILL_NONE, return directly if (pQueryInfo->fillType != TSDB_FILL_NONE && ((pRes->numOfRowsGroup < pQueryInfo->limit.limit && pQueryInfo->limit.limit > 0) || (pQueryInfo->limit.limit < 0))) { - int64_t etime = (pQueryInfo->window.skey < pQueryInfo->window.ekey) ? pQueryInfo->window.ekey : pQueryInfo->window.skey; + int64_t etime = (pQueryInfo->order.order == TSDB_ORDER_ASC)? pQueryInfo->window.ekey : pQueryInfo->window.skey; int32_t rows = (int32_t)getFilledNumOfRes(pFillInfo, etime, pLocalReducer->resColModel->capacity); if (rows > 0) { @@ -1402,13 +1377,12 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) { tscResetForNextRetrieve(pRes); if (pSql->signature != pSql || pRes == NULL || pRes->pLocalReducer == NULL) { // all data has been processed - tscDebug("%p %s call the drop local reducer", pSql, __FUNCTION__); - tscDestroyLocalReducer(pSql); - return 0; + tscError("%p local merge abort due to error occurs, code:%s", pSql, tstrerror(pRes->code)); + return pRes->code; } SLocalReducer *pLocalReducer = pRes->pLocalReducer; - SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); + SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); // set the data merge in progress int32_t prevStatus = @@ -1503,8 +1477,8 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) { * so the processing of previous group is completed. */ int32_t numOfRes = finalizeRes(pQueryInfo, pLocalReducer); + bool sameGroup = isSameGroup(pCmd, pLocalReducer, pLocalReducer->prevRowOfInput, tmpBuffer); - bool sameGroup = isSameGroup(pCmd, pLocalReducer, pLocalReducer->prevRowOfInput, tmpBuffer); tFilePage *pResBuf = pLocalReducer->pResultBuf; /* diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index cb49bd80b7..47bfe0fcdc 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -40,7 +40,7 @@ enum { static int32_t tscAllocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t * numOfRows); -static int32_t tscToInteger(SSQLToken *pToken, int64_t *value, char **endPtr) { +static int32_t tscToInteger(SStrToken *pToken, int64_t *value, char **endPtr) { if (pToken->n == 0) { return TK_ILLEGAL; } @@ -73,7 +73,7 @@ static int32_t tscToInteger(SSQLToken *pToken, int64_t *value, char **endPtr) { return pToken->type; } -static int32_t tscToDouble(SSQLToken *pToken, double *value, char **endPtr) { +static int32_t tscToDouble(SStrToken *pToken, double *value, char **endPtr) { if (pToken->n == 0) { return TK_ILLEGAL; } @@ -89,9 +89,9 @@ static int32_t tscToDouble(SSQLToken *pToken, double *value, char **endPtr) { return pToken->type; } -int tsParseTime(SSQLToken *pToken, int64_t *time, char **next, char *error, int16_t timePrec) { +int tsParseTime(SStrToken *pToken, int64_t *time, char **next, char *error, int16_t timePrec) { int32_t index = 0; - SSQLToken sToken; + SStrToken sToken; int64_t interval; int64_t useconds = 0; char * pTokenEnd = *next; @@ -128,7 +128,7 @@ int tsParseTime(SSQLToken *pToken, int64_t *time, char **next, char *error, int1 * time expression: * e.g., now+12a, now-5h */ - SSQLToken valueToken; + SStrToken valueToken; index = 0; sToken = tStrGetToken(pTokenEnd, &index, false, 0, NULL); pTokenEnd += index; @@ -163,7 +163,7 @@ int tsParseTime(SSQLToken *pToken, int64_t *time, char **next, char *error, int1 return TSDB_CODE_SUCCESS; } -int32_t tsParseOneColumnData(SSchema *pSchema, SSQLToken *pToken, char *payload, char *msg, char **str, bool primaryKey, +int32_t tsParseOneColumnData(SSchema *pSchema, SStrToken *pToken, char *payload, char *msg, char **str, bool primaryKey, int16_t timePrec) { int64_t iv; int32_t numType; @@ -409,7 +409,7 @@ static int32_t tsCheckTimestamp(STableDataBlocks *pDataBlocks, const char *start int tsParseOneRowData(char **str, STableDataBlocks *pDataBlocks, SSchema schema[], SParsedDataColInfo *spd, char *error, int16_t timePrec, int32_t *code, char *tmpTokenBuf) { int32_t index = 0; - SSQLToken sToken = {0}; + SStrToken sToken = {0}; char * payload = pDataBlocks->pData + pDataBlocks->size; // 1. set the parsed value from sql string @@ -524,7 +524,7 @@ static int32_t rowDataCompar(const void *lhs, const void *rhs) { int tsParseValues(char **str, STableDataBlocks *pDataBlock, STableMeta *pTableMeta, int maxRows, SParsedDataColInfo *spd, char *error, int32_t *code, char *tmpTokenBuf) { int32_t index = 0; - SSQLToken sToken; + SStrToken sToken; int16_t numOfRows = 0; @@ -734,8 +734,8 @@ static int32_t doParseInsertStatement(SSqlObj *pSql, void *pTableList, char **st static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { int32_t index = 0; - SSQLToken sToken = {0}; - SSQLToken tableToken = {0}; + SStrToken sToken = {0}; + SStrToken tableToken = {0}; int32_t code = TSDB_CODE_SUCCESS; const int32_t TABLE_INDEX = 0; @@ -993,7 +993,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { return code; } -int validateTableName(char *tblName, int len, SSQLToken* psTblToken) { +int validateTableName(char *tblName, int len, SStrToken* psTblToken) { tstrncpy(psTblToken->z, tblName, TSDB_TABLE_FNAME_LEN); psTblToken->n = len; @@ -1031,11 +1031,11 @@ int tsParseInsertSql(SSqlObj *pSql) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); assert(pQueryInfo != NULL); - STableMetaInfo *pTableMetaInfo = NULL; - if (pQueryInfo->numOfTables == 0) { - pTableMetaInfo = tscAddEmptyMetaInfo(pQueryInfo); - } else { - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + STableMetaInfo *pTableMetaInfo = (pQueryInfo->numOfTables == 0)? tscAddEmptyMetaInfo(pQueryInfo):tscGetMetaInfo(pQueryInfo, 0); + if (pTableMetaInfo == NULL) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + code = terrno; + return code; } if ((code = tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE)) != TSDB_CODE_SUCCESS) { @@ -1057,7 +1057,7 @@ int tsParseInsertSql(SSqlObj *pSql) { while (1) { int32_t index = 0; - SSQLToken sToken = tStrGetToken(str, &index, false, 0, NULL); + SStrToken sToken = tStrGetToken(str, &index, false, 0, NULL); // no data in the sql string anymore. if (sToken.n == 0) { @@ -1083,7 +1083,7 @@ int tsParseInsertSql(SSqlObj *pSql) { pCmd->curSql = sToken.z; char buf[TSDB_TABLE_FNAME_LEN]; - SSQLToken sTblToken; + SStrToken sTblToken; sTblToken.z = buf; // Check if the table name available or not if (validateTableName(sToken.z, sToken.n, &sTblToken) != TSDB_CODE_SUCCESS) { @@ -1285,15 +1285,14 @@ int tsInsertInitialCheck(SSqlObj *pSql) { int32_t index = 0; SSqlCmd *pCmd = &pSql->cmd; - SSQLToken sToken = tStrGetToken(pSql->sqlstr, &index, false, 0, NULL); + SStrToken sToken = tStrGetToken(pSql->sqlstr, &index, false, 0, NULL); assert(sToken.type == TK_INSERT || sToken.type == TK_IMPORT); pCmd->count = 0; pCmd->command = TSDB_SQL_INSERT; pSql->res.numOfRows = 0; - SQueryInfo *pQueryInfo = NULL; - tscGetQueryInfoDetailSafely(pCmd, pCmd->clauseIndex, &pQueryInfo); + SQueryInfo *pQueryInfo = tscGetQueryInfoDetailSafely(pCmd, pCmd->clauseIndex); TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_INSERT | pCmd->insertType); diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index b996dd958a..620e8ea57a 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -155,7 +155,7 @@ static int normalStmtPrepare(STscStmt* stmt) { uint32_t i = 0, start = 0; while (sql[i] != 0) { - SSQLToken token = {0}; + SStrToken token = {0}; token.n = tSQLGetToken(sql + i, &token.type); if (token.type == TK_QUESTION) { diff --git a/src/client/src/tscProfile.c b/src/client/src/tscProfile.c index f48e7b7691..6ff97e9d00 100644 --- a/src/client/src/tscProfile.c +++ b/src/client/src/tscProfile.c @@ -285,9 +285,9 @@ void tscKillConnection(STscObj *pObj) { SSqlObj *pSql = pObj->sqlList; while (pSql) { - //taosStopRpcConn(pSql->thandle); pSql = pSql->next; } + SSqlStream *pStream = pObj->streamList; while (pStream) { diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index d108349085..5ce4c7125f 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -33,8 +33,8 @@ #define DEFAULT_PRIMARY_TIMESTAMP_COL_NAME "_c0" -// -1 is tbname column index, so here use the -2 as the initial value -#define COLUMN_INDEX_INITIAL_VAL (-2) +// -1 is tbname column index, so here use the -3 as the initial value +#define COLUMN_INDEX_INITIAL_VAL (-3) #define COLUMN_INDEX_INITIALIZER \ { COLUMN_INDEX_INITIAL_VAL, COLUMN_INDEX_INITIAL_VAL } #define COLUMN_INDEX_VALIDE(index) (((index).tableIndex >= 0) && ((index).columnIndex >= TSDB_TBNAME_COLUMN_INDEX)) @@ -51,12 +51,12 @@ static int32_t setShowInfo(SSqlObj* pSql, SSqlInfo* pInfo); static char* getAccountId(SSqlObj* pSql); static bool has(tFieldList* pFieldList, int32_t startIdx, const char* name); -static void getCurrentDBName(SSqlObj* pSql, SSQLToken* pDBToken); -static bool hasSpecifyDB(SSQLToken* pTableName); +static void getCurrentDBName(SSqlObj* pSql, SStrToken* pDBToken); +static bool hasSpecifyDB(SStrToken* pTableName); static bool validateTableColumnInfo(tFieldList* pFieldList, SSqlCmd* pCmd); static bool validateTagParams(tFieldList* pTagsList, tFieldList* pFieldList, SSqlCmd* pCmd); -static int32_t setObjFullName(char* fullName, const char* account, SSQLToken* pDB, SSQLToken* tableName, int32_t* len); +static int32_t setObjFullName(char* fullName, const char* account, SStrToken* pDB, SStrToken* tableName, int32_t* len); static void getColumnName(tSQLExprItem* pItem, char* resultFieldName, int32_t nameLength); static void getRevisedName(char* resultFieldName, int32_t functionId, int32_t maxLen, char* columnName); @@ -65,11 +65,11 @@ static int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int3 static int32_t insertResultField(SQueryInfo* pQueryInfo, int32_t outputIndex, SColumnList* pIdList, int16_t bytes, int8_t type, char* fieldName, SSqlExpr* pSqlExpr); static int32_t changeFunctionID(int32_t optr, int16_t* functionId); -static int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSelection, bool isSTable); +static int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSelection, bool isSTable, bool joinQuery); static bool validateIpAddress(const char* ip, size_t size); static bool hasUnsupportFunctionsForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo); -static bool functionCompatibleCheck(SQueryInfo* pQueryInfo); +static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery); static void setColumnOffsetValueInResultset(SQueryInfo* pQueryInfo); static int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* pCmd); @@ -100,11 +100,11 @@ static bool hasTimestampForPointInterpQuery(SQueryInfo* pQueryInfo); static int32_t parseLimitClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t index, SQuerySQL* pQuerySql, SSqlObj* pSql); static int32_t parseCreateDBOptions(SSqlCmd* pCmd, SCreateDBInfo* pCreateDbSql); -static int32_t getColumnIndexByName(SSqlCmd* pCmd, const SSQLToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex); -static int32_t getTableIndexByName(SSQLToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex); +static int32_t getColumnIndexByName(SSqlCmd* pCmd, const SStrToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex); +static int32_t getTableIndexByName(SStrToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex); static int32_t optrToString(tSQLExpr* pExpr, char** exprString); -static int32_t getTableIndexImpl(SSQLToken* pTableToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex); +static int32_t getTableIndexImpl(SStrToken* pTableToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex); static int32_t doFunctionsCompatibleCheck(SSqlCmd* pCmd, SQueryInfo* pQueryInfo); static int32_t doLocalQueryProcess(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql); static int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCMCreateDbMsg* pCreate); @@ -151,7 +151,7 @@ static int setColumnFilterInfoForTimestamp(SSqlCmd* pCmd, SQueryInfo* pQueryInfo return TSDB_CODE_SUCCESS; } -static int32_t handlePassword(SSqlCmd* pCmd, SSQLToken* pPwd) { +static int32_t handlePassword(SSqlCmd* pCmd, SStrToken* pPwd) { const char* msg1 = "password can not be empty"; const char* msg2 = "name or password too long"; const char* msg3 = "password needs single quote marks enclosed"; @@ -179,20 +179,24 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { return TSDB_CODE_TSC_APP_ERROR; } - SSqlCmd* pCmd = &(pSql->cmd); - SQueryInfo* pQueryInfo = NULL; + SSqlCmd* pCmd = &pSql->cmd; + SSqlRes* pRes = &pSql->res; + int32_t code = TSDB_CODE_SUCCESS; if (!pInfo->valid) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), pInfo->pzErrMsg); } - int32_t code = tscGetQueryInfoDetailSafely(pCmd, pCmd->clauseIndex, &pQueryInfo); + SQueryInfo* pQueryInfo = tscGetQueryInfoDetailSafely(pCmd, pCmd->clauseIndex); + if (pQueryInfo == NULL) { + pRes->code = terrno; + return pRes->code; + } - STableMetaInfo* pTableMetaInfo = NULL; - if (pQueryInfo->numOfTables == 0) { - pTableMetaInfo = tscAddEmptyMetaInfo(pQueryInfo); - } else { - pTableMetaInfo = pQueryInfo->pTableMetaInfo[0]; + STableMetaInfo* pTableMetaInfo = (pQueryInfo->numOfTables == 0)? tscAddEmptyMetaInfo(pQueryInfo) : pQueryInfo->pTableMetaInfo[0]; + if (pTableMetaInfo == NULL) { + pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; + return pRes->code; } pCmd->command = pInfo->type; @@ -206,7 +210,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { const char* msg2 = "invalid name"; const char* msg3 = "param name too long"; - SSQLToken* pzName = &pInfo->pDCLInfo->a[0]; + SStrToken* pzName = &pInfo->pDCLInfo->a[0]; if ((pInfo->type != TSDB_SQL_DROP_DNODE) && (tscValidateName(pzName) != TSDB_CODE_SUCCESS)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } @@ -241,7 +245,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { case TSDB_SQL_USE_DB: { const char* msg = "invalid db name"; - SSQLToken* pToken = &pInfo->pDCLInfo->a[0]; + SStrToken* pToken = &pInfo->pDCLInfo->a[0]; if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); @@ -296,7 +300,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); } - SSQLToken* pIpAddr = &pInfo->pDCLInfo->a[0]; + SStrToken* pIpAddr = &pInfo->pDCLInfo->a[0]; pIpAddr->n = strdequote(pIpAddr->z); break; } @@ -307,8 +311,8 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { const char* msg2 = "invalid user/account name"; const char* msg3 = "name too long"; - SSQLToken* pName = &pInfo->pDCLInfo->user.user; - SSQLToken* pPwd = &pInfo->pDCLInfo->user.passwd; + SStrToken* pName = &pInfo->pDCLInfo->user.user; + SStrToken* pPwd = &pInfo->pDCLInfo->user.passwd; if (handlePassword(pCmd, pPwd) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; @@ -337,7 +341,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { } case TSDB_SQL_DESCRIBE_TABLE: { - SSQLToken* pToken = &pInfo->pDCLInfo->a[0]; + SStrToken* pToken = &pInfo->pDCLInfo->a[0]; const char* msg2 = "table name is too long"; const char* msg1 = "invalid table name"; @@ -400,8 +404,8 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { // tDCLSQL* pDCL = pInfo->pDCLInfo; SUserInfo* pUser = &pInfo->pDCLInfo->user; - SSQLToken* pName = &pUser->user; - SSQLToken* pPwd = &pUser->passwd; + SStrToken* pName = &pUser->user; + SStrToken* pPwd = &pUser->passwd; if (pName->n >= TSDB_USER_LEN) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); @@ -423,7 +427,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { } else if (pUser->type == TSDB_ALTER_USER_PRIVILEGES) { assert(pPwd->type == TSDB_DATA_TYPE_NULL); - SSQLToken* pPrivilege = &pUser->privilege; + SStrToken* pPrivilege = &pUser->privilege; if (strncasecmp(pPrivilege->z, "super", 5) == 0 && pPrivilege->n == 5) { pCmd->count = 1; @@ -487,9 +491,10 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { const char* msg1 = "columns in select clause not identical"; for (int32_t i = pCmd->numOfClause; i < pInfo->subclauseInfo.numOfClause; ++i) { - SQueryInfo* pqi = NULL; - if ((code = tscGetQueryInfoDetailSafely(pCmd, i, &pqi)) != TSDB_CODE_SUCCESS) { - return code; + SQueryInfo* pqi = tscGetQueryInfoDetailSafely(pCmd, i); + if (pqi == NULL) { + pRes->code = terrno; + return pRes->code; } } @@ -581,7 +586,7 @@ int32_t parseIntervalClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQ } // interval is not null - SSQLToken* t = &pQuerySql->interval; + SStrToken* t = &pQuerySql->interval; if (getTimestampInUsFromStr(t->z, t->n, &pQueryInfo->intervalTime) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -667,7 +672,7 @@ int32_t parseSlidingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); - SSQLToken* pSliding = &pQuerySql->sliding; + SStrToken* pSliding = &pQuerySql->sliding; if (pSliding->n != 0) { getTimestampInUsFromStr(pSliding->z, pSliding->n, &pQueryInfo->slidingTime); if (tinfo.precision == TSDB_TIME_PRECISION_MILLI) { @@ -692,7 +697,7 @@ int32_t parseSlidingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu return TSDB_CODE_SUCCESS; } -int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SSQLToken* pzTableName, SSqlObj* pSql) { +int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SStrToken* pzTableName, SSqlObj* pSql) { const char* msg = "name too long"; SSqlCmd* pCmd = &pSql->cmd; @@ -709,7 +714,7 @@ int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SSQLToken* pzTableNa // db has been specified in sql string so we ignore current db path code = setObjFullName(pTableMetaInfo->name, getAccountId(pSql), NULL, pzTableName, NULL); } else { // get current DB name first, then set it into path - SSQLToken t = {0}; + SStrToken t = {0}; getCurrentDBName(pSql, &t); code = setObjFullName(pTableMetaInfo->name, NULL, &t, pzTableName, NULL); @@ -1027,13 +1032,13 @@ static bool has(tFieldList* pFieldList, int32_t startIdx, const char* name) { static char* getAccountId(SSqlObj* pSql) { return pSql->pTscObj->acctId; } -static void getCurrentDBName(SSqlObj* pSql, SSQLToken* pDBToken) { +static void getCurrentDBName(SSqlObj* pSql, SStrToken* pDBToken) { pDBToken->z = pSql->pTscObj->db; pDBToken->n = (uint32_t)strlen(pSql->pTscObj->db); } /* length limitation, strstr cannot be applied */ -static bool hasSpecifyDB(SSQLToken* pTableName) { +static bool hasSpecifyDB(SStrToken* pTableName) { for (uint32_t i = 0; i < pTableName->n; ++i) { if (pTableName->z[i] == TS_PATH_DELIMITER[0]) { return true; @@ -1043,7 +1048,7 @@ static bool hasSpecifyDB(SSQLToken* pTableName) { return false; } -int32_t setObjFullName(char* fullName, const char* account, SSQLToken* pDB, SSQLToken* tableName, int32_t* xlen) { +int32_t setObjFullName(char* fullName, const char* account, SStrToken* pDB, SStrToken* tableName, int32_t* xlen) { int32_t totalLen = 0; if (account != NULL) { @@ -1094,18 +1099,6 @@ int32_t setObjFullName(char* fullName, const char* account, SSQLToken* pDB, SSQL return (totalLen < TSDB_TABLE_FNAME_LEN) ? TSDB_CODE_SUCCESS : TSDB_CODE_TSC_INVALID_SQL; } -static void extractColumnNameFromString(tSQLExprItem* pItem) { - if (pItem->pNode->nSQLOptr == TK_STRING) { - pItem->pNode->val.nLen = strdequote(pItem->pNode->val.pz); - pItem->pNode->nSQLOptr = TK_ID; - - SSQLToken* pIdToken = &pItem->pNode->colInfo; - pIdToken->type = TK_ID; - pIdToken->z = pItem->pNode->val.pz; - pIdToken->n = pItem->pNode->val.nLen; - } -} - static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t exprIndex, tSQLExprItem* pItem) { const char* msg1 = "invalid column name, or illegal column type"; const char* msg2 = "invalid arithmetic expression in select clause"; @@ -1123,42 +1116,31 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t } int32_t tableIndex = columnList.ids[0].tableIndex; - - // todo potential data overflow - char* arithmeticExprStr = malloc(1024*1024); - char* p = arithmeticExprStr; - if (arithmeticType == NORMAL_ARITHMETIC) { pQueryInfo->type |= TSDB_QUERY_TYPE_PROJECTION_QUERY; // all columns in arithmetic expression must belong to the same table for (int32_t f = 1; f < columnList.num; ++f) { if (columnList.ids[f].tableIndex != tableIndex) { - taosTFree(arithmeticExprStr); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); } } - if (arithmeticExprToString(pItem->pNode, &p) != TSDB_CODE_SUCCESS) { - taosTFree(arithmeticExprStr); - return TSDB_CODE_TSC_INVALID_SQL; - } - // expr string is set as the parameter of function SColumnIndex index = {.tableIndex = tableIndex}; SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_ARITHM, &index, TSDB_DATA_TYPE_DOUBLE, sizeof(double), sizeof(double), false); - char* name = (pItem->aliasName != NULL)? pItem->aliasName:arithmeticExprStr; - tstrncpy(pExpr->aliasName, name, sizeof(pExpr->aliasName)); + char* name = (pItem->aliasName != NULL)? pItem->aliasName:pItem->pNode->token.z; + size_t len = MIN(sizeof(pExpr->aliasName), pItem->pNode->token.n + 1); + tstrncpy(pExpr->aliasName, name, len); tExprNode* pNode = NULL; SArray* colList = taosArrayInit(10, sizeof(SColIndex)); int32_t ret = exprTreeFromSqlExpr(pCmd, &pNode, pItem->pNode, pQueryInfo->exprList, pQueryInfo, colList); if (ret != TSDB_CODE_SUCCESS) { - taosTFree(arithmeticExprStr); taosArrayDestroy(colList); tExprTreeDestroy(&pNode, NULL); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); @@ -1167,8 +1149,8 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t size_t numOfNode = taosArrayGetSize(colList); for(int32_t k = 0; k < numOfNode; ++k) { SColIndex* pIndex = taosArrayGet(colList, k); - if (pIndex->flag == 1) { - taosTFree(arithmeticExprStr); + if (TSDB_COL_IS_TAG(pIndex->flag)) { + tExprTreeDestroy(&pNode, NULL); taosArrayDestroy(colList); tExprTreeDestroy(&pNode, NULL); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); @@ -1185,7 +1167,7 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t // TODO: other error handling } END_TRY - size_t len = tbufTell(&bw); + len = tbufTell(&bw); char* c = tbufGetData(&bw, true); // set the serialized binary string as the parameter of arithmetic expression @@ -1196,16 +1178,18 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t taosArrayDestroy(colList); tExprTreeDestroy(&pNode, NULL); } else { - if (arithmeticExprToString(pItem->pNode, &p) != TSDB_CODE_SUCCESS) { - taosTFree(arithmeticExprStr); - return TSDB_CODE_TSC_INVALID_SQL; - } - columnList.num = 0; columnList.ids[0] = (SColumnIndex) {0, 0}; - char* name = (pItem->aliasName != NULL)? pItem->aliasName:arithmeticExprStr; - insertResultField(pQueryInfo, exprIndex, &columnList, sizeof(double), TSDB_DATA_TYPE_DOUBLE, name, NULL); + char aliasName[TSDB_COL_NAME_LEN] = {0}; + if (pItem->aliasName != NULL) { + tstrncpy(aliasName, pItem->aliasName, TSDB_COL_NAME_LEN); + } else { + int32_t nameLen = MIN(TSDB_COL_NAME_LEN, pItem->pNode->token.n + 1); + tstrncpy(aliasName, pItem->pNode->token.z, nameLen); + } + + insertResultField(pQueryInfo, exprIndex, &columnList, sizeof(double), TSDB_DATA_TYPE_DOUBLE, aliasName, NULL); int32_t slot = tscNumOfFields(pQueryInfo) - 1; SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, slot); @@ -1221,7 +1205,6 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t int32_t ret = exprTreeFromSqlExpr(pCmd, &pArithExprInfo->pExpr, pItem->pNode, pQueryInfo->exprList, pQueryInfo, NULL); if (ret != TSDB_CODE_SUCCESS) { tExprTreeDestroy(&pArithExprInfo->pExpr, NULL); - taosTFree(arithmeticExprStr); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "invalid expression in select clause"); } @@ -1229,11 +1212,38 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t } } - taosTFree(arithmeticExprStr); return TSDB_CODE_SUCCESS; } -int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSelection, bool isSTable) { +static void tscInsertPrimaryTSSourceColumn(SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { + SColumnIndex tsCol = {.tableIndex = pIndex->tableIndex, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX}; + tscColumnListInsert(pQueryInfo->colList, &tsCol); +} + +static void addProjectQueryCol(SQueryInfo* pQueryInfo, int32_t startPos, SColumnIndex* pIndex, tSQLExprItem* pItem) { + SSqlExpr* pExpr = doAddProjectCol(pQueryInfo, startPos, pIndex->columnIndex, pIndex->tableIndex); + + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pIndex->tableIndex); + STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; + + SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, pIndex->columnIndex); + + char* colName = (pItem->aliasName == NULL) ? pSchema->name : pItem->aliasName; + tstrncpy(pExpr->aliasName, colName, sizeof(pExpr->aliasName)); + + SColumnList ids = {0}; + ids.num = 1; + ids.ids[0] = *pIndex; + + if (pIndex->columnIndex == TSDB_TBNAME_COLUMN_INDEX || pIndex->columnIndex == TSDB_UD_COLUMN_INDEX || + pIndex->columnIndex >= tscGetNumOfColumns(pTableMeta)) { + ids.num = 0; + } + + insertResultField(pQueryInfo, startPos, &ids, pExpr->resBytes, (int8_t)pExpr->resType, pExpr->aliasName, pExpr); +} + +int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSelection, bool isSTable, bool joinQuery) { assert(pSelection != NULL && pCmd != NULL); const char* msg2 = "functions can not be mixed up"; @@ -1251,16 +1261,14 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel tSQLExprItem* pItem = &pSelection->a[i]; // project on all fields - if (pItem->pNode->nSQLOptr == TK_ALL || pItem->pNode->nSQLOptr == TK_ID || pItem->pNode->nSQLOptr == TK_STRING) { + int32_t optr = pItem->pNode->nSQLOptr; + + if (optr == TK_ALL || optr == TK_ID || optr == TK_STRING || optr == TK_INTEGER || optr == TK_FLOAT) { // it is actually a function, but the function name is invalid if (pItem->pNode->nSQLOptr == TK_ID && (pItem->pNode->colInfo.z == NULL && pItem->pNode->colInfo.n == 0)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); } - // if the name of column is quoted, remove it and set the right information for later process - extractColumnNameFromString(pItem); - TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_PROJECTION_QUERY); - // select table_name1.field_name1, table_name2.field_name2 from table_name1, table_name2 if (addProjectionExprAndResultField(pCmd, pQueryInfo, pItem) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; @@ -1276,12 +1284,7 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel if (code != TSDB_CODE_SUCCESS) { return code; } - } else { - /* - * not support such expression - * e.g., select 12+5 from table_name - */ return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } @@ -1290,7 +1293,26 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel } } - if (!functionCompatibleCheck(pQueryInfo)) { + // there is only one user-defined column in the final result field, add the timestamp column. + size_t numOfSrcCols = taosArrayGetSize(pQueryInfo->colList); + if (numOfSrcCols <= 0 && !tscQueryTags(pQueryInfo)) { + SColumnIndex index = {0}; + + // set the constant column value always attached to first table. + STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, clauseIndex, 0); + SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, PRIMARYKEY_TIMESTAMP_COL_INDEX); + + // add the timestamp column into the output columns + int32_t numOfCols = (int32_t)tscSqlExprNumOfExprs(pQueryInfo); + tscAddSpecialColumnForSelect(pQueryInfo, numOfCols, TSDB_FUNC_PRJ, &index, pSchema, TSDB_COL_NORMAL); + + SFieldSupInfo* pSupInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, numOfCols); + pSupInfo->visible = false; + + pQueryInfo->type |= TSDB_QUERY_TYPE_PROJECTION_QUERY; + } + + if (!functionCompatibleCheck(pQueryInfo, joinQuery)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } @@ -1353,32 +1375,10 @@ SSqlExpr* doAddProjectCol(SQueryInfo* pQueryInfo, int32_t outputIndex, int32_t c pSchema->bytes, functionId == TSDB_FUNC_TAGPRJ); } -static void addProjectQueryCol(SQueryInfo* pQueryInfo, int32_t startPos, SColumnIndex* pIndex, tSQLExprItem* pItem) { - SSqlExpr* pExpr = doAddProjectCol(pQueryInfo, startPos, pIndex->columnIndex, pIndex->tableIndex); - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pIndex->tableIndex); - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - - SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, pIndex->columnIndex); - - char* colName = (pItem->aliasName == NULL) ? pSchema->name : pItem->aliasName; - tstrncpy(pExpr->aliasName, colName, sizeof(pExpr->aliasName)); - - SColumnList ids = {0}; - ids.num = 1; - ids.ids[0] = *pIndex; - - if (pIndex->columnIndex >= tscGetNumOfColumns(pTableMeta) || pIndex->columnIndex == TSDB_TBNAME_COLUMN_INDEX) { - ids.num = 0; - } - - insertResultField(pQueryInfo, startPos, &ids, pExpr->resBytes, (int8_t)pExpr->resType, pExpr->aliasName, pExpr); -} - -void tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId, +SSqlExpr* tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId, SColumnIndex* pIndex, SSchema* pColSchema, int16_t flag) { SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, outputColIndex, functionId, pIndex, pColSchema->type, - pColSchema->bytes, pColSchema->bytes, flag); + pColSchema->bytes, pColSchema->bytes, TSDB_COL_IS_TAG(flag)); tstrncpy(pExpr->aliasName, pColSchema->name, sizeof(pExpr->aliasName)); SColumnList ids = getColumnList(1, pIndex->tableIndex, pIndex->columnIndex); @@ -1394,6 +1394,8 @@ void tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex if (TSDB_COL_IS_TAG(flag)) { tscColumnListInsert(pTableMetaInfo->tagColList, pIndex); } + + return pExpr; } static int32_t doAddProjectionExprAndResultFields(SQueryInfo* pQueryInfo, SColumnIndex* pIndex, int32_t startPos) { @@ -1426,18 +1428,16 @@ static int32_t doAddProjectionExprAndResultFields(SQueryInfo* pQueryInfo, SColum return numOfTotalColumns; } -static void tscInsertPrimaryTSSourceColumn(SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { - SColumnIndex tsCol = {.tableIndex = pIndex->tableIndex, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX}; - tscColumnListInsert(pQueryInfo->colList, &tsCol); -} - int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExprItem* pItem) { const char* msg0 = "invalid column name"; const char* msg1 = "tag for normal table query is not allowed"; - - int32_t startPos = (int32_t)tscSqlExprNumOfExprs(pQueryInfo); - if (pItem->pNode->nSQLOptr == TK_ALL) { // project on all fields + int32_t startPos = (int32_t)tscSqlExprNumOfExprs(pQueryInfo); + int32_t optr = pItem->pNode->nSQLOptr; + + if (optr == TK_ALL) { // project on all fields + TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_PROJECTION_QUERY); + SColumnIndex index = COLUMN_INDEX_INITIALIZER; if (getTableIndexByName(&pItem->pNode->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg0); @@ -1453,7 +1453,24 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t } else { doAddProjectionExprAndResultFields(pQueryInfo, &index, startPos); } - } else if (pItem->pNode->nSQLOptr == TK_ID) { // simple column projection query + + // add the primary timestamp column even though it is not required by user + tscInsertPrimaryTSSourceColumn(pQueryInfo, &index); + } else if (optr == TK_STRING || optr == TK_INTEGER || optr == TK_FLOAT) { // simple column projection query + SColumnIndex index = COLUMN_INDEX_INITIALIZER; + + // user-specified constant value as a new result column + index.columnIndex = (pQueryInfo->udColumnId--); + index.tableIndex = 0; + + SSchema colSchema = tGetUserSpecifiedColumnSchema(&pItem->pNode->val, &pItem->pNode->token, pItem->aliasName); + SSqlExpr* pExpr = + tscAddSpecialColumnForSelect(pQueryInfo, startPos, TSDB_FUNC_PRJ, &index, &colSchema, TSDB_COL_UDC); + + // NOTE: the first parameter is reserved for the tag column id during join query process. + pExpr->numOfParams = 2; + tVariantAssign(&pExpr->param[1], &pItem->pNode->val); + } else if (optr == TK_ID) { SColumnIndex index = COLUMN_INDEX_INITIALIZER; if (getColumnIndexByName(pCmd, &pItem->pNode->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { @@ -1472,8 +1489,10 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t } addProjectQueryCol(pQueryInfo, startPos, &index, pItem); + pQueryInfo->type |= TSDB_QUERY_TYPE_PROJECTION_QUERY; } + // add the primary timestamp column even though it is not required by user tscInsertPrimaryTSSourceColumn(pQueryInfo, &index); } else { return TSDB_CODE_TSC_INVALID_SQL; @@ -1561,7 +1580,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col SColumnIndex index = COLUMN_INDEX_INITIALIZER; if (pItem->pNode->pParam != NULL) { - SSQLToken* pToken = &pItem->pNode->pParam->a[0].pNode->colInfo; + SStrToken* pToken = &pItem->pNode->pParam->a[0].pNode->colInfo; if (pToken->z == NULL || pToken->n == 0) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } @@ -1570,7 +1589,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col if (pParamElem->pNode->nSQLOptr == TK_ALL) { // select table.* // check if the table name is valid or not - SSQLToken tmpToken = pParamElem->pNode->colInfo; + SStrToken tmpToken = pParamElem->pNode->colInfo; if (getTableIndexByName(&tmpToken, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); @@ -1768,7 +1787,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col if (pParamElem->pNode->nSQLOptr == TK_ALL) { // select table.* - SSQLToken tmpToken = pParamElem->pNode->colInfo; + SStrToken tmpToken = pParamElem->pNode->colInfo; if (getTableIndexByName(&tmpToken, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); @@ -1834,10 +1853,11 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col for (int32_t i = 0; i < tscGetNumOfColumns(pTableMetaInfo->pTableMeta); ++i) { SColumnIndex index = {.tableIndex = j, .columnIndex = i}; - if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, functionID, pItem->aliasName, colIndex + i + j, &index) != - 0) { + if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, functionID, pItem->aliasName, colIndex, &index) != 0) { return TSDB_CODE_TSC_INVALID_SQL; } + + colIndex++; } numOfFields += tscGetNumOfColumns(pTableMetaInfo->pTableMeta); @@ -2045,7 +2065,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col // todo refactor static SColumnList getColumnList(int32_t num, int16_t tableIndex, int32_t columnIndex) { - assert(num == 1 && columnIndex >= -1 && tableIndex >= 0); + assert(num == 1 && tableIndex >= 0); SColumnList columnList = {0}; columnList.num = num; @@ -2070,16 +2090,16 @@ void getRevisedName(char* resultFieldName, int32_t functionId, int32_t maxLen, c snprintf(resultFieldName, maxLen, "%s(%s)", aAggs[functionId].aName, columnName); } -static bool isTablenameToken(SSQLToken* token) { - SSQLToken tmpToken = *token; - SSQLToken tableToken = {0}; +static bool isTablenameToken(SStrToken* token) { + SStrToken tmpToken = *token; + SStrToken tableToken = {0}; extractTableNameFromToken(&tmpToken, &tableToken); return (strncasecmp(TSQL_TBNAME_L, tmpToken.z, tmpToken.n) == 0 && tmpToken.n == strlen(TSQL_TBNAME_L)); } -static int16_t doGetColumnIndex(SQueryInfo* pQueryInfo, int32_t index, SSQLToken* pToken) { +static int16_t doGetColumnIndex(SQueryInfo* pQueryInfo, int32_t index, SStrToken* pToken) { STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, index)->pTableMeta; int32_t numOfCols = tscGetNumOfColumns(pTableMeta) + tscGetNumOfTags(pTableMeta); @@ -2101,7 +2121,7 @@ static int16_t doGetColumnIndex(SQueryInfo* pQueryInfo, int32_t index, SSQLToken return columnIndex; } -int32_t doGetColumnIndexByName(SSqlCmd* pCmd, SSQLToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { +int32_t doGetColumnIndexByName(SSqlCmd* pCmd, SStrToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { const char* msg0 = "ambiguous column name"; const char* msg1 = "invalid column name"; @@ -2143,7 +2163,7 @@ int32_t doGetColumnIndexByName(SSqlCmd* pCmd, SSQLToken* pToken, SQueryInfo* pQu } } -int32_t getTableIndexImpl(SSQLToken* pTableToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { +int32_t getTableIndexImpl(SStrToken* pTableToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { if (pTableToken->n == 0) { // only one table and no table name prefix in column name if (pQueryInfo->numOfTables == 1) { pIndex->tableIndex = 0; @@ -2169,8 +2189,8 @@ int32_t getTableIndexImpl(SSQLToken* pTableToken, SQueryInfo* pQueryInfo, SColum return TSDB_CODE_SUCCESS; } -int32_t getTableIndexByName(SSQLToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { - SSQLToken tableToken = {0}; +int32_t getTableIndexByName(SStrToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { + SStrToken tableToken = {0}; extractTableNameFromToken(pToken, &tableToken); if (getTableIndexImpl(&tableToken, pQueryInfo, pIndex) != TSDB_CODE_SUCCESS) { @@ -2180,12 +2200,12 @@ int32_t getTableIndexByName(SSQLToken* pToken, SQueryInfo* pQueryInfo, SColumnIn return TSDB_CODE_SUCCESS; } -int32_t getColumnIndexByName(SSqlCmd* pCmd, const SSQLToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { +int32_t getColumnIndexByName(SSqlCmd* pCmd, const SStrToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { if (pQueryInfo->pTableMetaInfo == NULL || pQueryInfo->numOfTables == 0) { return TSDB_CODE_TSC_INVALID_SQL; } - SSQLToken tmpToken = *pToken; + SStrToken tmpToken = *pToken; if (getTableIndexByName(&tmpToken, pQueryInfo, pIndex) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; @@ -2297,7 +2317,7 @@ int32_t setShowInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { int16_t showType = pShowInfo->showType; if (showType == TSDB_MGMT_TABLE_TABLE || showType == TSDB_MGMT_TABLE_METRIC || showType == TSDB_MGMT_TABLE_VGROUP) { // db prefix in tagCond, show table conds in payload - SSQLToken* pDbPrefixToken = &pShowInfo->prefix; + SStrToken* pDbPrefixToken = &pShowInfo->prefix; if (pDbPrefixToken->type != 0) { if (pDbPrefixToken->n >= TSDB_DB_NAME_LEN) { // db name is too long @@ -2319,7 +2339,7 @@ int32_t setShowInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { } // show table/stable like 'xxxx', set the like pattern for show tables - SSQLToken* pPattern = &pShowInfo->pattern; + SStrToken* pPattern = &pShowInfo->pattern; if (pPattern->type != 0) { pPattern->n = strdequote(pPattern->z); @@ -2337,7 +2357,7 @@ int32_t setShowInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { } // show vnodes may be ip addr of dnode in payload - SSQLToken* pDnodeIp = &pShowInfo->prefix; + SStrToken* pDnodeIp = &pShowInfo->prefix; if (pDnodeIp->n >= TSDB_IPv4ADDR_LEN) { // ip addr is too long return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } @@ -2358,7 +2378,7 @@ int32_t setKillInfo(SSqlObj* pSql, struct SSqlInfo* pInfo, int32_t killType) { SSqlCmd* pCmd = &pSql->cmd; pCmd->command = pInfo->type; - SSQLToken* idStr = &(pInfo->pDCLInfo->ip); + SStrToken* idStr = &(pInfo->pDCLInfo->ip); if (idStr->n > TSDB_KILL_MSG_LEN) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -2510,7 +2530,7 @@ bool hasUnsupportFunctionsForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) return false; } -static bool functionCompatibleCheck(SQueryInfo* pQueryInfo) { +static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery) { int32_t startIdx = 0; SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, startIdx); @@ -2542,6 +2562,10 @@ static bool functionCompatibleCheck(SQueryInfo* pQueryInfo) { if (functionCompatList[functionId] != factor) { return false; } + + if (functionId == TSDB_FUNC_LAST_ROW && joinQuery) { + return false; + } } return true; @@ -2579,7 +2603,7 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* for (int32_t i = 0; i < pList->nExpr; ++i) { tVariant* pVar = &pList->a[i].pVar; - SSQLToken token = {pVar->nLen, pVar->nType, pVar->pz}; + SStrToken token = {pVar->nLen, pVar->nType, pVar->pz}; SColumnIndex index = COLUMN_INDEX_INITIALIZER; if (getColumnIndexByName(pCmd, &token, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { @@ -2659,9 +2683,12 @@ static SColumnFilterInfo* addColumnFilterInfo(SColumn* pColumn) { } int32_t size = pColumn->numOfFilters + 1; - char* tmp = (char*)realloc((void*)(pColumn->filterInfo), sizeof(SColumnFilterInfo) * (size)); + + char* tmp = (char*) realloc((void*)(pColumn->filterInfo), sizeof(SColumnFilterInfo) * (size)); if (tmp != NULL) { pColumn->filterInfo = (SColumnFilterInfo*)tmp; + } else { + return NULL; } pColumn->numOfFilters++; @@ -2945,9 +2972,16 @@ static int32_t extractColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SC } else { // update the existed column filter information, find the filter info here pColFilter = &pColumn->filterInfo[0]; } + + if (pColFilter == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } } else if (sqlOptr == TK_OR) { // TODO fixme: failed to invalid the filter expression: "col1 = 1 OR col2 = 2" pColFilter = addColumnFilterInfo(pColumn); + if (pColFilter == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } } else { // error; return TSDB_CODE_TSC_INVALID_SQL; } @@ -3072,7 +3106,6 @@ static int32_t getColumnQueryCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQ static int32_t getJoinCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr* pExpr) { const char* msg1 = "invalid join query condition"; - const char* msg2 = "join on binary/nchar not supported"; const char* msg3 = "type of join columns must be identical"; const char* msg4 = "invalid column name in join condition"; @@ -3116,10 +3149,6 @@ static int32_t getJoinCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr* return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } - if (pTagSchema1->type == TSDB_DATA_TYPE_BINARY || pTagSchema1->type == TSDB_DATA_TYPE_NCHAR) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - pTagCond->joinInfo.hasJoin = true; return TSDB_CODE_SUCCESS; } @@ -3156,7 +3185,7 @@ int32_t doArithmeticExprToString(tSQLExpr* pExpr, char** exprString) { return TSDB_CODE_SUCCESS; } -static int32_t arithmeticExprToString(tSQLExpr* pExpr, char** str) { +static UNUSED_FUNC int32_t arithmeticExprToString(tSQLExpr* pExpr, char** str) { char* start = *str; int32_t code = doArithmeticExprToString(pExpr, str); @@ -3610,7 +3639,7 @@ static void doExtractExprForSTable(SSqlCmd* pCmd, tSQLExpr** pExpr, SQueryInfo* return; } - SSQLToken t = {0}; + SStrToken t = {0}; extractTableNameFromToken(&pLeft->colInfo, &t); *pOut = *pExpr; @@ -3690,7 +3719,7 @@ static int32_t setTableCondForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, num = j; char* name = extractDBName(pTableMetaInfo->name, db); - SSQLToken dbToken = { .type = TK_STRING, .z = name, .n = (uint32_t)strlen(name) }; + SStrToken dbToken = { .type = TK_STRING, .z = name, .n = (uint32_t)strlen(name) }; for (int32_t i = 0; i < num; ++i) { if (i >= 1) { @@ -3699,7 +3728,7 @@ static int32_t setTableCondForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, char idBuf[TSDB_TABLE_FNAME_LEN] = {0}; int32_t xlen = (int32_t)strlen(segments[i]); - SSQLToken t = {.z = segments[i], .n = xlen, .type = TK_STRING}; + SStrToken t = {.z = segments[i], .n = xlen, .type = TK_STRING}; int32_t ret = setObjFullName(idBuf, account, &dbToken, &t, &xlen); if (ret != TSDB_CODE_SUCCESS) { @@ -4020,7 +4049,7 @@ int32_t getTimeRange(STimeWindow* win, tSQLExpr* pRight, int32_t optr, int16_t t return TSDB_CODE_TSC_INVALID_SQL; } } else { - SSQLToken token = {.z = pRight->val.pz, .n = pRight->val.nLen, .type = TK_ID}; + SStrToken token = {.z = pRight->val.pz, .n = pRight->val.nLen, .type = TK_ID}; int32_t len = tSQLGetToken(pRight->val.pz, &token.type); if ((token.type != TK_INTEGER && token.type != TK_FLOAT) || len != pRight->val.nLen) { @@ -4266,7 +4295,7 @@ int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu return TSDB_CODE_SUCCESS; } - SSQLToken columnName = {pVar->nLen, pVar->nType, pVar->pz}; + SStrToken columnName = {pVar->nLen, pVar->nType, pVar->pz}; SColumnIndex index = {0}; if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { // super table query @@ -4335,7 +4364,7 @@ int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu } tVariant* pVar2 = &pSortorder->a[1].pVar; - SSQLToken cname = {pVar2->nLen, pVar2->nType, pVar2->pz}; + SStrToken cname = {pVar2->nLen, pVar2->nType, pVar2->pz}; if (getColumnIndexByName(pCmd, &cname, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } @@ -4464,7 +4493,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { } SColumnIndex index = COLUMN_INDEX_INITIALIZER; - SSQLToken name = {.z = pItem->pVar.pz, .n = pItem->pVar.nLen, .type = TK_STRING}; + SStrToken name = {.z = pItem->pVar.pz, .n = pItem->pVar.nLen, .type = TK_STRING}; if (getColumnIndexByName(pCmd, &name, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; @@ -4502,12 +4531,12 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { SColumnIndex srcIndex = COLUMN_INDEX_INITIALIZER; SColumnIndex destIndex = COLUMN_INDEX_INITIALIZER; - SSQLToken srcToken = {.z = pSrcItem->pVar.pz, .n = pSrcItem->pVar.nLen, .type = TK_STRING}; + SStrToken srcToken = {.z = pSrcItem->pVar.pz, .n = pSrcItem->pVar.nLen, .type = TK_STRING}; if (getColumnIndexByName(pCmd, &srcToken, pQueryInfo, &srcIndex) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg17); } - SSQLToken destToken = {.z = pDstItem->pVar.pz, .n = pDstItem->pVar.nLen, .type = TK_STRING}; + SStrToken destToken = {.z = pDstItem->pVar.pz, .n = pDstItem->pVar.nLen, .type = TK_STRING}; if (getColumnIndexByName(pCmd, &destToken, pQueryInfo, &destIndex) == TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg19); } @@ -4531,7 +4560,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { int16_t numOfTags = tscGetNumOfTags(pTableMeta); SColumnIndex columnIndex = COLUMN_INDEX_INITIALIZER; - SSQLToken name = {.type = TK_STRING, .z = pTagName->pz, .n = pTagName->nLen}; + SStrToken name = {.type = TK_STRING, .z = pTagName->pz, .n = pTagName->nLen}; if (getColumnIndexByName(pCmd, &name, pQueryInfo, &columnIndex) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -4624,7 +4653,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { tVariantListItem* pItem = &pAlterSQL->varList->a[0]; SColumnIndex columnIndex = COLUMN_INDEX_INITIALIZER; - SSQLToken name = {.type = TK_STRING, .z = pItem->pVar.pz, .n = pItem->pVar.nLen}; + SStrToken name = {.type = TK_STRING, .z = pItem->pVar.pz, .n = pItem->pVar.nLen}; if (getColumnIndexByName(pCmd, &name, pQueryInfo, &columnIndex) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg17); } @@ -4687,7 +4716,7 @@ int32_t validateFunctionsInIntervalOrGroupbyQuery(SSqlCmd* pCmd, SQueryInfo* pQu } } - if (pExpr->functionId == TSDB_FUNC_PRJ || pExpr->functionId == TSDB_FUNC_DIFF || + if ((pExpr->functionId == TSDB_FUNC_PRJ && pExpr->numOfParams == 0) || pExpr->functionId == TSDB_FUNC_DIFF || pExpr->functionId == TSDB_FUNC_ARITHM) { isProjectionFunction = true; } @@ -4744,7 +4773,7 @@ int32_t validateDNodeConfig(tDCLSQL* pOptions) { {"dDebugFlag", 10}, {"mqttDebugFlag", 13}, {"wDebugFlag", 10}, {"tmrDebugFlag", 12}, }; - SSQLToken* pOptionToken = &pOptions->a[1]; + SStrToken* pOptionToken = &pOptions->a[1]; if (pOptions->nTokens == 2) { // reset log and reset query cache does not need value @@ -4756,7 +4785,7 @@ int32_t validateDNodeConfig(tDCLSQL* pOptions) { } } else if ((strncasecmp(cfgOptions[tokenBalance].name, pOptionToken->z, pOptionToken->n) == 0) && (cfgOptions[tokenBalance].len == pOptionToken->n)) { - SSQLToken* pValToken = &pOptions->a[2]; + SStrToken* pValToken = &pOptions->a[2]; int32_t vnodeId = 0; int32_t dnodeId = 0; strdequote(pValToken->z); @@ -4767,14 +4796,14 @@ int32_t validateDNodeConfig(tDCLSQL* pOptions) { return TSDB_CODE_SUCCESS; } else if ((strncasecmp(cfgOptions[tokenMonitor].name, pOptionToken->z, pOptionToken->n) == 0) && (cfgOptions[tokenMonitor].len == pOptionToken->n)) { - SSQLToken* pValToken = &pOptions->a[2]; + SStrToken* pValToken = &pOptions->a[2]; int32_t val = strtol(pValToken->z, NULL, 10); if (val != 0 && val != 1) { return TSDB_CODE_TSC_INVALID_SQL; // options value is invalid } return TSDB_CODE_SUCCESS; } else { - SSQLToken* pValToken = &pOptions->a[2]; + SStrToken* pValToken = &pOptions->a[2]; int32_t val = strtol(pValToken->z, NULL, 10); if (val < 0 || val > 256) { @@ -4803,7 +4832,7 @@ int32_t validateLocalConfig(tDCLSQL* pOptions) { SDNodeDynConfOption LOCAL_DYNAMIC_CFG_OPTIONS[6] = {{"resetLog", 8}, {"rpcDebugFlag", 12}, {"tmrDebugFlag", 12}, {"cDebugFlag", 10}, {"uDebugFlag", 10}, {"debugFlag", 9}}; - SSQLToken* pOptionToken = &pOptions->a[0]; + SStrToken* pOptionToken = &pOptions->a[0]; if (pOptions->nTokens == 1) { // reset log does not need value @@ -4814,7 +4843,7 @@ int32_t validateLocalConfig(tDCLSQL* pOptions) { } } } else { - SSQLToken* pValToken = &pOptions->a[1]; + SStrToken* pValToken = &pOptions->a[1]; int32_t val = strtol(pValToken->z, NULL, 10); if (val < 131 || val > 199) { @@ -4839,7 +4868,7 @@ int32_t validateColumnName(char* name) { return TSDB_CODE_TSC_INVALID_SQL; } - SSQLToken token = {.z = name}; + SStrToken token = {.z = name}; token.n = tSQLGetToken(name, &token.type); if (token.type != TK_STRING && token.type != TK_ID) { @@ -5018,7 +5047,7 @@ static int32_t setTimePrecision(SSqlCmd* pCmd, SCMCreateDbMsg* pMsg, SCreateDBIn pMsg->precision = TSDB_TIME_PRECISION_MILLI; // millisecond by default - SSQLToken* pToken = &pCreateDbInfo->precision; + SStrToken* pToken = &pCreateDbInfo->precision; if (pToken->n > 0) { pToken->n = strdequote(pToken->z); @@ -5689,7 +5718,7 @@ int32_t doCheckForCreateTable(SSqlObj* pSql, int32_t subClauseIndex, SSqlInfo* p assert(pFieldList != NULL); // if sql specifies db, use it, otherwise use default db - SSQLToken* pzTableName = &(pCreateTable->name); + SStrToken* pzTableName = &(pCreateTable->name); if (tscValidateName(pzTableName) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); @@ -5744,7 +5773,7 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { STableMetaInfo* pStableMeterMetaInfo = tscGetMetaInfo(pQueryInfo, STABLE_INDEX); // super table name, create table by using dst - SSQLToken* pToken = &(pCreateTable->usingInfo.stableName); + SStrToken* pToken = &(pCreateTable->usingInfo.stableName); if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); @@ -5837,7 +5866,7 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); // if sql specifies db, use it, otherwise use default db - SSQLToken* pzTableName = &(pCreateTable->name); + SStrToken* pzTableName = &(pCreateTable->name); SQuerySQL* pQuerySql = pCreateTable->pSelect; if (tscValidateName(pzTableName) != TSDB_CODE_SUCCESS) { @@ -5850,7 +5879,7 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { } tVariant* pVar = &pSrcMeterName->a[0].pVar; - SSQLToken srcToken = {.z = pVar->pz, .n = pVar->nLen, .type = TK_STRING}; + SStrToken srcToken = {.z = pVar->pz, .n = pVar->nLen, .type = TK_STRING}; if (tscValidateName(&srcToken) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } @@ -5865,7 +5894,7 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { } bool isSTable = UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo); - if (parseSelectClause(&pSql->cmd, 0, pQuerySql->pSelection, isSTable) != TSDB_CODE_SUCCESS) { + if (parseSelectClause(&pSql->cmd, 0, pQuerySql->pSelection, isSTable, false) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -5991,7 +6020,7 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { pTableItem->nLen = strdequote(pTableItem->pz); - SSQLToken tableName = {.z = pTableItem->pz, .n = pTableItem->nLen, .type = TK_STRING}; + SStrToken tableName = {.z = pTableItem->pz, .n = pTableItem->nLen, .type = TK_STRING}; if (tscValidateName(&tableName) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg0); } @@ -6002,13 +6031,13 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { STableMetaInfo* pTableMetaInfo1 = tscGetMetaInfo(pQueryInfo, i/2); - SSQLToken t = {.type = TSDB_DATA_TYPE_BINARY, .n = pTableItem->nLen, .z = pTableItem->pz}; + SStrToken t = {.type = TSDB_DATA_TYPE_BINARY, .n = pTableItem->nLen, .z = pTableItem->pz}; if (tscSetTableFullName(pTableMetaInfo1, &t, pSql) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } tVariant* pTableItem1 = &pQuerySql->from->a[i + 1].pVar; - SSQLToken aliasName = {.z = pTableItem1->pz, .n = pTableItem1->nLen, .type = TK_STRING}; + SStrToken aliasName = {.z = pTableItem1->pz, .n = pTableItem1->nLen, .type = TK_STRING}; if (tscValidateName(&aliasName) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg11); } @@ -6048,7 +6077,8 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { return TSDB_CODE_TSC_INVALID_SQL; } - if (parseSelectClause(pCmd, index, pQuerySql->pSelection, isSTable) != TSDB_CODE_SUCCESS) { + int32_t joinQuery = (pQuerySql->from != NULL && pQuerySql->from->nExpr > 2); + if (parseSelectClause(pCmd, index, pQuerySql->pSelection, isSTable, joinQuery) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -6082,6 +6112,9 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { } } else { // set the time rang pQueryInfo->window = TSWINDOW_INITIALIZER; + if (pQuerySql->from->nExpr > 2) { // it is a join query, no wher clause is not allowed. + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "condition missing for join query "); + } } // user does not specified the query time window, twa is not allowed in such case. @@ -6131,7 +6164,7 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { if (pQueryInfo->intervalTime > 0) { int64_t timeRange = ABS(pQueryInfo->window.skey - pQueryInfo->window.ekey); // number of result is not greater than 10,000,000 - if ((timeRange == 0) || (timeRange / pQueryInfo->intervalTime) > MAX_RETRIEVE_ROWS_IN_INTERVAL_QUERY) { + if ((timeRange == 0) || (timeRange / pQueryInfo->intervalTime) > MAX_INTERVAL_TIME_WINDOW) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6); } } @@ -6230,7 +6263,7 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pS (*pExpr)->_node.pLeft = pLeft; (*pExpr)->_node.pRight = pRight; - SSQLToken t = {.type = pSqlExpr->nSQLOptr}; + SStrToken t = {.type = pSqlExpr->nSQLOptr}; (*pExpr)->_node.optr = getBinaryExprOptr(&t); assert((*pExpr)->_node.optr != 0); diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 0785dd8b53..ecb85472fc 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -43,6 +43,14 @@ void tscUpdateSubscriptionProgress(void* sub, int64_t uid, TSKEY ts); void tscSaveSubscriptionProgress(void* sub); static int32_t minMsgSize() { return tsRpcHeadSize + 100; } +static int32_t getWaitingTimeInterval(int32_t count) { + int32_t initial = 100; // 100 ms by default + if (count <= 1) { + return 0; + } + + return initial * (2<<(count - 2)); +} static void tscSetDnodeEpSet(SSqlObj* pSql, SCMVgroupInfo* pVgroupInfo) { assert(pSql != NULL && pVgroupInfo != NULL && pVgroupInfo->numOfEps > 0); @@ -117,7 +125,7 @@ static void tscUpdateVgroupInfo(SSqlObj *pObj, SRpcEpSet *pEpSet) { pVgroupInfo->inUse = pEpSet->inUse; pVgroupInfo->numOfEps = pEpSet->numOfEps; for (int32_t i = 0; i < pVgroupInfo->numOfEps; i++) { - tstrncpy(pVgroupInfo->epAddr[i].fqdn, pEpSet->fqdn[i], sizeof(pEpSet->fqdn[i])); + tstrncpy(pVgroupInfo->epAddr[i].fqdn, pEpSet->fqdn[i], TSDB_FQDN_LEN); pVgroupInfo->epAddr[i].port = pEpSet->port[i]; } tscDebug("after: EndPoint in use: %d", pVgroupInfo->inUse); @@ -218,13 +226,17 @@ int tscSendMsgToServer(SSqlObj *pSql) { .handle = &pSql->pRpcCtx, .code = 0 }; - // NOTE: the rpc context should be acquired before sending data to server. // Otherwise, the pSql object may have been released already during the response function, which is // processMsgFromServer function. In the meanwhile, the assignment of the rpc context to sql object will absolutely // cause crash. - rpcSendRequest(pObj->pDnodeConn, &pSql->epSet, &rpcMsg); - return TSDB_CODE_SUCCESS; + if (pObj != NULL && pObj->signature == pObj) { + rpcSendRequest(pObj->pDnodeConn, &pSql->epSet, &rpcMsg); + return TSDB_CODE_SUCCESS; + } else { + //pObj->signature has been reset by other thread, ignore concurrency problem + return TSDB_CODE_TSC_CONN_KILLED; + } } void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { @@ -275,6 +287,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { (rpcMsg->code == TSDB_CODE_TDB_INVALID_TABLE_ID || rpcMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID || rpcMsg->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || + rpcMsg->code == TSDB_CODE_APP_NOT_READY || rpcMsg->code == TSDB_CODE_TDB_TABLE_RECONFIGURE)) { tscWarn("%p it shall renew table meta, code:%s, retry:%d", pSql, tstrerror(rpcMsg->code), ++pSql->retry); @@ -287,6 +300,12 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { if (pSql->retry > pSql->maxRetry) { tscError("%p max retry %d reached, give up", pSql, pSql->maxRetry); } else { + // wait for a little bit moment and then retry + if (rpcMsg->code == TSDB_CODE_APP_NOT_READY || rpcMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID) { + int32_t duration = getWaitingTimeInterval(pSql->retry); + taosMsleep(duration); + } + rpcMsg->code = tscRenewTableMeta(pSql, pTableMetaInfo->name); // if there is an error occurring, proceed to the following error handling procedure. @@ -299,10 +318,10 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { pRes->rspLen = 0; - if (pRes->code != TSDB_CODE_TSC_QUERY_CANCELLED) { - pRes->code = (rpcMsg->code != TSDB_CODE_SUCCESS) ? rpcMsg->code : TSDB_CODE_RPC_NETWORK_UNAVAIL; - } else { + if (pRes->code == TSDB_CODE_TSC_QUERY_CANCELLED) { tscDebug("%p query is cancelled, code:%s", pSql, tstrerror(pRes->code)); + } else { + pRes->code = rpcMsg->code; } if (pRes->code == TSDB_CODE_SUCCESS) { @@ -439,35 +458,21 @@ void tscKillSTableQuery(SSqlObj *pSql) { return; } + pSql->res.code = TSDB_CODE_TSC_QUERY_CANCELLED; + for (int i = 0; i < pSql->numOfSubs; ++i) { + // NOTE: pSub may have been released already here SSqlObj *pSub = pSql->pSubs[i]; if (pSub == NULL) { continue; } - /* - * here, we cannot set the command = TSDB_SQL_KILL_QUERY. Otherwise, it may cause - * sub-queries not correctly released and master sql object of super table query reaches an abnormal state. - */ - rpcCancelRequest(pSub->pRpcCtx); pSub->res.code = TSDB_CODE_TSC_QUERY_CANCELLED; - tscQueueAsyncRes(pSub); - } - - /* - * 1. if the subqueries are not launched or partially launched, we need to waiting the launched - * query return to successfully free allocated resources. - * 2. if no any subqueries are launched yet, which means the super table query only in parse sql stage, - * set the res.code, and return. - */ - const int64_t MAX_WAITING_TIME = 10000; // 10 Sec. - int64_t stime = taosGetTimestampMs(); - - while (pCmd->command != TSDB_SQL_RETRIEVE_LOCALMERGE && pCmd->command != TSDB_SQL_RETRIEVE_EMPTY_RESULT) { - taosMsleep(100); - if (taosGetTimestampMs() - stime > MAX_WAITING_TIME) { - break; + if (pSub->pRpcCtx != NULL) { + rpcCancelRequest(pSub->pRpcCtx); } + + tscQueueAsyncRes(pSub); // async res? not other functions? } tscDebug("%p super table query cancelled", pSql); @@ -623,26 +628,29 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { tscError("%p failed to malloc for query msg", pSql); - return -1; // todo add test for this + return TSDB_CODE_TSC_INVALID_SQL; // todo add test for this } SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; - - if (taosArrayGetSize(pQueryInfo->colList) <= 0 && !tscQueryTags(pQueryInfo)) { - tscError("%p illegal value of numOfCols in query msg: %d", pSql, tscGetNumOfColumns(pTableMeta)); - return -1; + + size_t numOfSrcCols = taosArrayGetSize(pQueryInfo->colList); + if (numOfSrcCols <= 0 && !tscQueryTags(pQueryInfo)) { + tscError("%p illegal value of numOfCols in query msg: %"PRIu64", table cols:%d", pSql, numOfSrcCols, + tscGetNumOfColumns(pTableMeta)); + + return TSDB_CODE_TSC_INVALID_SQL; } if (pQueryInfo->intervalTime < 0) { tscError("%p illegal value of aggregation time interval in query msg: %ld", pSql, pQueryInfo->intervalTime); - return -1; + return TSDB_CODE_TSC_INVALID_SQL; } if (pQueryInfo->groupbyExpr.numOfGroupCols < 0) { tscError("%p illegal value of numOfGroupCols in query msg: %d", pSql, pQueryInfo->groupbyExpr.numOfGroupCols); - return -1; + return TSDB_CODE_TSC_INVALID_SQL; } SQueryTableMsg *pQueryMsg = (SQueryTableMsg *)pCmd->payload; @@ -708,7 +716,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { if (pColFilter->filterstr) { pFilterMsg->len = htobe64(pColFilter->len); - memcpy(pMsg, (void *)pColFilter->pz, pColFilter->len + 1); + memcpy(pMsg, (void *)pColFilter->pz, (size_t)(pColFilter->len + 1)); pMsg += (pColFilter->len + 1); // append the additional filter binary info } else { pFilterMsg->lowerBndi = htobe64(pColFilter->lowerBndi); @@ -720,7 +728,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { if (pColFilter->lowerRelOptr == TSDB_RELATION_INVALID && pColFilter->upperRelOptr == TSDB_RELATION_INVALID) { tscError("invalid filter info"); - return -1; + return TSDB_CODE_TSC_INVALID_SQL; } } } @@ -729,10 +737,10 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { for (int32_t i = 0; i < tscSqlExprNumOfExprs(pQueryInfo); ++i) { SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, i); - if (!tscValidateColumnId(pTableMetaInfo, pExpr->colInfo.colId)) { + if (!tscValidateColumnId(pTableMetaInfo, pExpr->colInfo.colId, pExpr->numOfParams)) { /* column id is not valid according to the cached table meta, the table meta is expired */ tscError("%p table schema is not matched with parsed sql", pSql); - return -1; + return TSDB_CODE_TSC_INVALID_SQL; } pSqlFuncExpr->colInfo.colId = htons(pExpr->colInfo.colId); @@ -929,8 +937,8 @@ int32_t tscBuildAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SCMCreateAcctMsg *pAlterMsg = (SCMCreateAcctMsg *)pCmd->payload; - SSQLToken *pName = &pInfo->pDCLInfo->user.user; - SSQLToken *pPwd = &pInfo->pDCLInfo->user.passwd; + SStrToken *pName = &pInfo->pDCLInfo->user.user; + SStrToken *pPwd = &pInfo->pDCLInfo->user.passwd; strncpy(pAlterMsg->user, pName->z, pName->n); strncpy(pAlterMsg->pass, pPwd->z, pPwd->n); @@ -1132,13 +1140,13 @@ int32_t tscBuildShowMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pShowMsg->type = pShowInfo->showType; if (pShowInfo->showType != TSDB_MGMT_TABLE_VNODES) { - SSQLToken *pPattern = &pShowInfo->pattern; + SStrToken *pPattern = &pShowInfo->pattern; if (pPattern->type > 0) { // only show tables support wildcard query strncpy(pShowMsg->payload, pPattern->z, pPattern->n); pShowMsg->payloadLen = htons(pPattern->n); } } else { - SSQLToken *pEpAddr = &pShowInfo->prefix; + SStrToken *pEpAddr = &pShowInfo->prefix; assert(pEpAddr->n > 0 && pEpAddr->type > 0); strncpy(pShowMsg->payload, pEpAddr->z, pEpAddr->n); @@ -1280,7 +1288,7 @@ int tscBuildAlterTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { int size = tscEstimateAlterTableMsgLength(pCmd); if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { tscError("%p failed to malloc for alter table msg", pSql); - return -1; + return TSDB_CODE_TSC_OUT_OF_MEMORY; } SCMAlterTableMsg *pAlterTableMsg = (SCMAlterTableMsg *)pCmd->payload; @@ -1428,6 +1436,12 @@ int tscProcessRetrieveLocalMergeRsp(SSqlObj *pSql) { SSqlRes *pRes = &pSql->res; SSqlCmd *pCmd = &pSql->cmd; + int32_t code = pRes->code; + if (pRes->code != TSDB_CODE_SUCCESS) { + tscQueueAsyncRes(pSql); + return code; + } + pRes->code = tscDoLocalMerge(pSql); SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); @@ -1438,7 +1452,7 @@ int tscProcessRetrieveLocalMergeRsp(SSqlObj *pSql) { pRes->row = 0; pRes->completed = (pRes->numOfRows == 0); - int32_t code = pRes->code; + code = pRes->code; if (pRes->code == TSDB_CODE_SUCCESS) { (*pSql->fp)(pSql->param, pSql, pRes->numOfRows); } else { @@ -1632,7 +1646,7 @@ int tscBuildHeartBeatMsg(SSqlObj *pSql, SSqlInfo *pInfo) { if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { pthread_mutex_unlock(&pObj->mutex); tscError("%p failed to malloc for heartbeat msg", pSql); - return -1; + return TSDB_CODE_TSC_OUT_OF_MEMORY; } SCMHeartBeatMsg *pHeartbeat = (SCMHeartBeatMsg *)pCmd->payload; @@ -1702,7 +1716,7 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { assert(pTableMetaInfo->pTableMeta == NULL); pTableMetaInfo->pTableMeta = (STableMeta *) taosCachePut(tscCacheHandle, pTableMetaInfo->name, - strlen(pTableMetaInfo->name), pTableMeta, size, tsTableMetaKeepTimer); + strlen(pTableMetaInfo->name), pTableMeta, size, tsTableMetaKeepTimer * 1000); // todo handle out of memory case if (pTableMetaInfo->pTableMeta == NULL) { @@ -1906,7 +1920,7 @@ int tscProcessShowRsp(SSqlObj *pSql) { STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg, &size); pTableMetaInfo->pTableMeta = taosCachePut(tscCacheHandle, key, strlen(key), (char *)pTableMeta, size, - tsTableMetaKeepTimer); + tsTableMetaKeepTimer * 1000); SSchema *pTableSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); if (pQueryInfo->colList == NULL) { @@ -1946,8 +1960,12 @@ static void createHBObj(STscObj* pObj) { pSql->fp = tscProcessHeartBeatRsp; - SQueryInfo *pQueryInfo = NULL; - tscGetQueryInfoDetailSafely(&pSql->cmd, 0, &pQueryInfo); + SQueryInfo *pQueryInfo = tscGetQueryInfoDetailSafely(&pSql->cmd, 0); + if (pQueryInfo == NULL) { + pSql->res.code = terrno; + return; + } + pQueryInfo->command = TSDB_SQL_HB; pSql->cmd.command = pQueryInfo->command; @@ -2132,8 +2150,7 @@ static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInf tscAddSubqueryInfo(&pNew->cmd); - SQueryInfo *pNewQueryInfo = NULL; - tscGetQueryInfoDetailSafely(&pNew->cmd, 0, &pNewQueryInfo); + SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetailSafely(&pNew->cmd, 0); pNew->cmd.autoCreated = pSql->cmd.autoCreated; // create table if not exists if (TSDB_CODE_SUCCESS != tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE + pSql->cmd.payloadLen)) { @@ -2236,8 +2253,8 @@ int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) { pNew->cmd.command = TSDB_SQL_STABLEVGROUP; - SQueryInfo *pNewQueryInfo = NULL; - if ((code = tscGetQueryInfoDetailSafely(&pNew->cmd, 0, &pNewQueryInfo)) != TSDB_CODE_SUCCESS) { + SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetailSafely(&pNew->cmd, 0); + if (pNewQueryInfo == NULL) { tscFreeSqlObj(pNew); return code; } diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 1bd885466c..f63923e046 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -201,7 +201,7 @@ TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, } TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port) { - tscDebug("try to create a connection to %s:%u, user:%s db:%s", ip, port, user, db); + tscDebug("try to create a connection to %s:%u, user:%s db:%s", ip, port != 0 ? port : tsServerPort , user, db); if (user == NULL) user = TSDB_DEFAULT_USER; if (pass == NULL) pass = TSDB_DEFAULT_PASS; @@ -655,27 +655,30 @@ int* taos_fetch_lengths(TAOS_RES *res) { char *taos_get_client_info() { return version; } void taos_stop_query(TAOS_RES *res) { - if (res == NULL) { + SSqlObj *pSql = (SSqlObj *)res; + if (pSql == NULL || pSql->signature != pSql) { return; } - SSqlObj *pSql = (SSqlObj *)res; + tscDebug("%p start to cancel query", res); SSqlCmd *pCmd = &pSql->cmd; - if (pSql->signature != pSql) return; - tscDebug("%p start to cancel query", res); - - + // TODO there are multi-thread problem. + // It may have been released by the other thread already. + // The ref count may fix this problem. SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) { - tscKillSTableQuery(pSql); - } - if (pSql->cmd.command < TSDB_SQL_LOCAL) { - rpcCancelRequest(pSql->pRpcCtx); - } + // set the error code for master pSqlObj firstly pSql->res.code = TSDB_CODE_TSC_QUERY_CANCELLED; - tscQueueAsyncRes(pSql); + + if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) { + assert(pSql->pRpcCtx == NULL); + tscKillSTableQuery(pSql); + } else { + if (pSql->cmd.command < TSDB_SQL_LOCAL) { + rpcCancelRequest(pSql->pRpcCtx); + } + } tscDebug("%p query is cancelled", res); } @@ -824,8 +827,11 @@ static int tscParseTblNameList(SSqlObj *pSql, const char *tblNameList, int32_t t int code = TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; char *str = (char *)tblNameList; - SQueryInfo *pQueryInfo = NULL; - tscGetQueryInfoDetailSafely(pCmd, pCmd->clauseIndex, &pQueryInfo); + SQueryInfo *pQueryInfo = tscGetQueryInfoDetailSafely(pCmd, pCmd->clauseIndex); + if (pQueryInfo == NULL) { + pSql->res.code = terrno; + return terrno; + } STableMetaInfo *pTableMetaInfo = tscAddEmptyMetaInfo(pQueryInfo); @@ -850,7 +856,7 @@ static int tscParseTblNameList(SSqlObj *pSql, const char *tblNameList, int32_t t str = nextStr + 1; len = (int32_t)strtrim(tblName); - SSQLToken sToken = {.n = len, .type = TK_ID, .z = tblName}; + SStrToken sToken = {.n = len, .type = TK_ID, .z = tblName}; tSQLGetToken(tblName, &sToken.type); // Check if the table name available or not diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index d2b5439f8d..2fb264c756 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -12,6 +12,8 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ +#define _GNU_SOURCE + #include "os.h" #include "qAst.h" @@ -93,13 +95,14 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, SJ tscInfo("%" PRId64 ", tags:%d \t %" PRId64 ", tags:%d", elem1.ts, elem1.tag, elem2.ts, elem2.tag); #endif - if (elem1.tag < elem2.tag || (elem1.tag == elem2.tag && tsCompare(order, elem1.ts, elem2.ts))) { + int32_t res = tVariantCompare(&elem1.tag, &elem2.tag); + if (res == -1 || (res == 0 && tsCompare(order, elem1.ts, elem2.ts))) { if (!tsBufNextPos(pSupporter1->pTSBuf)) { break; } numOfInput1++; - } else if (elem1.tag > elem2.tag || (elem1.tag == elem2.tag && tsCompare(order, elem2.ts, elem1.ts))) { + } else if ((res > 0) || (res == 0 && tsCompare(order, elem2.ts, elem1.ts))) { if (!tsBufNextPos(pSupporter2->pTSBuf)) { break; } @@ -119,8 +122,8 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, SJ win->ekey = elem1.ts; } - tsBufAppend(output1, elem1.vnode, elem1.tag, (const char*)&elem1.ts, sizeof(elem1.ts)); - tsBufAppend(output2, elem2.vnode, elem2.tag, (const char*)&elem2.ts, sizeof(elem2.ts)); + tsBufAppend(output1, elem1.vnode, &elem1.tag, (const char*)&elem1.ts, sizeof(elem1.ts)); + tsBufAppend(output2, elem2.vnode, &elem2.tag, (const char*)&elem2.ts, sizeof(elem2.ts)); } else { pLimit->offset -= 1; } @@ -352,11 +355,12 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) { pExpr = tscSqlExprGet(pQueryInfo, 0); } - // set the join condition tag column info, to do extract method + // set the join condition tag column info, todo extract method if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { assert(pQueryInfo->tagCond.joinInfo.hasJoin); int16_t colId = tscGetJoinTagColIdByUid(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->id.uid); + // set the tag column id for executor to extract correct tag value pExpr->param[0].i64Key = colId; pExpr->numOfParams = 1; } @@ -433,6 +437,7 @@ int32_t tscCompareTidTags(const void* p1, const void* p2) { if (t1->vgId != t2->vgId) { return (t1->vgId > t2->vgId) ? 1 : -1; } + if (t1->tid != t2->tid) { return (t1->tid > t2->tid) ? 1 : -1; } @@ -539,6 +544,7 @@ static bool checkForDuplicateTagVal(SQueryInfo* pQueryInfo, SJoinSupporter* p1, for(int32_t i = 1; i < p1->num; ++i) { STidTags* prev = (STidTags*) varDataVal(p1->pIdTagList + (i - 1) * p1->tagSize); STidTags* p = (STidTags*) varDataVal(p1->pIdTagList + i * p1->tagSize); + assert(prev->vgId >= 1 && p->vgId >= 1); if (doCompare(prev->tag, p->tag, pColSchema->type, pColSchema->bytes) == 0) { tscError("%p join tags have same value for different table, free all sub SqlObj and quit", pPSqlObj); @@ -575,6 +581,7 @@ static int32_t getIntersectionOfTableTuple(SQueryInfo* pQueryInfo, SSqlObj* pPar while(i < p1->num && j < p2->num) { STidTags* pp1 = (STidTags*) varDataVal(p1->pIdTagList + i * p1->tagSize); STidTags* pp2 = (STidTags*) varDataVal(p2->pIdTagList + j * p2->tagSize); + assert(pp1->tid != 0 && pp2->tid != 0); int32_t ret = doCompare(pp1->tag, pp2->tag, pColSchema->type, pColSchema->bytes); if (ret == 0) { @@ -623,7 +630,7 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow // keep the results in memory if (numOfRows > 0) { - size_t validLen = pSupporter->tagSize * pRes->numOfRows; + size_t validLen = (size_t)(pSupporter->tagSize * pRes->numOfRows); size_t length = pSupporter->totalLen + validLen; // todo handle memory error @@ -684,6 +691,7 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow freeJoinSubqueryObj(pParentSql); pParentSql->res.code = code; tscQueueAsyncRes(pParentSql); + taosArrayDestroy(s1); taosArrayDestroy(s2); return; @@ -748,7 +756,7 @@ static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow } if (numOfRows > 0) { // write the compressed timestamp to disk file - fwrite(pRes->data, pRes->numOfRows, 1, pSupporter->f); + fwrite(pRes->data, (size_t)pRes->numOfRows, 1, pSupporter->f); fclose(pSupporter->f); pSupporter->f = NULL; @@ -1143,7 +1151,7 @@ void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) { ///////////////////////////////////////////////////////////////////////////////////////// static void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code); -static SSqlObj *tscCreateSqlObjForSubquery(SSqlObj *pSql, SRetrieveSupport *trsupport, SSqlObj *prevSqlObj); +static SSqlObj *tscCreateSTableSubquery(SSqlObj *pSql, SRetrieveSupport *trsupport, SSqlObj *prevSqlObj); int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter *pSupporter) { SSqlCmd * pCmd = &pSql->cmd; @@ -1216,6 +1224,16 @@ int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter int32_t tagColId = tscGetJoinTagColIdByUid(pTagCond, pTableMetaInfo->pTableMeta->id.uid); SSchema* s = tscGetTableColumnSchemaById(pTableMetaInfo->pTableMeta, tagColId); + // get the tag colId column index + int32_t numOfTags = tscGetNumOfTags(pTableMetaInfo->pTableMeta); + SSchema* pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); + for(int32_t i = 0; i < numOfTags; ++i) { + if (pSchema[i].colId == tagColId) { + index.columnIndex = i; + break; + } + } + int16_t bytes = 0; int16_t type = 0; int32_t inter = 0; @@ -1285,8 +1303,14 @@ int32_t tscHandleMasterJoinQuery(SSqlObj* pSql) { SSqlCmd* pCmd = &pSql->cmd; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); assert((pQueryInfo->type & TSDB_QUERY_TYPE_SUBQUERY) == 0); - + + // todo add test SSubqueryState *pState = calloc(1, sizeof(SSubqueryState)); + if (pState == NULL) { + pSql->res.code = TSDB_CODE_TSC_OUT_OF_MEMORY; + return pSql->res.code; + } + pState->numOfTotal = pQueryInfo->numOfTables; pState->numOfRemain = pState->numOfTotal; @@ -1300,7 +1324,7 @@ int32_t tscHandleMasterJoinQuery(SSqlObj* pSql) { pSql->res.code = TSDB_CODE_TSC_OUT_OF_MEMORY; if (0 == i) { taosTFree(pState); - } + } return pSql->res.code; } @@ -1330,10 +1354,6 @@ static void doCleanupSubqueries(SSqlObj *pSql, int32_t numOfSubs, SSubqueryState SRetrieveSupport* pSupport = pSub->param; taosTFree(pSupport->localBuffer); - - pthread_mutex_unlock(&pSupport->queryMutex); - pthread_mutex_destroy(&pSupport->queryMutex); - taosTFree(pSupport); tscFreeSqlObj(pSub); @@ -1406,14 +1426,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { trs->pParentSql = pSql; trs->pFinalColModel = pModel; - pthread_mutexattr_t mutexattr; - memset(&mutexattr, 0, sizeof(pthread_mutexattr_t)); - - pthread_mutexattr_settype(&mutexattr, PTHREAD_MUTEX_RECURSIVE_NP); - pthread_mutex_init(&trs->queryMutex, &mutexattr); - pthread_mutexattr_destroy(&mutexattr); - - SSqlObj *pNew = tscCreateSqlObjForSubquery(pSql, trs, NULL); + SSqlObj *pNew = tscCreateSTableSubquery(pSql, trs, NULL); if (pNew == NULL) { tscError("%p failed to malloc buffer for subObj, orderOfSub:%d, reason:%s", pSql, i, strerror(errno)); taosTFree(trs->localBuffer); @@ -1458,15 +1471,16 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { } static void tscFreeSubSqlObj(SRetrieveSupport *trsupport, SSqlObj *pSql) { - tscDebug("%p start to free subquery result", pSql); - + tscDebug("%p start to free subquery obj", pSql); + + int32_t index = trsupport->subqueryIndex; + SSqlObj *pParentSql = trsupport->pParentSql; + + assert(pSql == pParentSql->pSubs[index]); + pParentSql->pSubs[index] = NULL; + taos_free_result(pSql); - taosTFree(trsupport->localBuffer); - - pthread_mutex_unlock(&trsupport->queryMutex); - pthread_mutex_destroy(&trsupport->queryMutex); - taosTFree(trsupport); } @@ -1475,23 +1489,11 @@ static void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, i static void tscAbortFurtherRetryRetrieval(SRetrieveSupport *trsupport, TAOS_RES *tres, int32_t code) { // set no disk space error info -#ifdef WINDOWS - LPVOID lpMsgBuf; - FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, NULL, - GetLastError(), MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), // Default language - (LPTSTR)&lpMsgBuf, 0, NULL); - tscError("sub:%p failed to flush data to disk:reason:%s", tres, lpMsgBuf); - LocalFree(lpMsgBuf); -#else tscError("sub:%p failed to flush data to disk, reason:%s", tres, tstrerror(code)); -#endif - SSqlObj* pParentSql = trsupport->pParentSql; pParentSql->res.code = code; trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; - - pthread_mutex_unlock(&trsupport->queryMutex); tscHandleSubqueryError(trsupport, tres, pParentSql->res.code); } @@ -1510,13 +1512,10 @@ static int32_t tscReissueSubquery(SRetrieveSupport *trsupport, SSqlObj *pSql, in // clear local saved number of results trsupport->localBuffer->num = 0; - pthread_mutex_unlock(&trsupport->queryMutex); - - tscTrace("%p sub:%p retrieve failed, code:%s, orderOfSub:%d, retry:%d", trsupport->pParentSql, pSql, + tscError("%p sub:%p retrieve/query failed, code:%s, orderOfSub:%d, retry:%d", trsupport->pParentSql, pSql, tstrerror(code), subqueryIndex, trsupport->numOfRetry); - SSqlObj *pNew = tscCreateSqlObjForSubquery(trsupport->pParentSql, trsupport, pSql); - + SSqlObj *pNew = tscCreateSTableSubquery(trsupport->pParentSql, trsupport, pSql); if (pNew == NULL) { tscError("%p sub:%p failed to create new subquery due to error:%s, abort retry, vgId:%d, orderOfSub:%d", trsupport->pParentSql, pSql, tstrerror(terrno), pVgroup->vgId, trsupport->subqueryIndex); @@ -1527,8 +1526,15 @@ static int32_t tscReissueSubquery(SRetrieveSupport *trsupport, SSqlObj *pSql, in return pParentSql->res.code; } - taos_free_result(pSql); - return tscProcessSql(pNew); + int32_t ret = tscProcessSql(pNew); + + // if failed to process sql, let following code handle the pSql + if (ret == TSDB_CODE_SUCCESS) { + taos_free_result(pSql); + return ret; + } else { + return ret; + } } void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numOfRows) { @@ -1548,14 +1554,14 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO */ pSql->res.numOfRows = 0; trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; // disable retry efforts - tscDebug("%p query is cancelled, sub:%p, orderOfSub:%d abort retrieve, code:%d", pParentSql, pSql, - subqueryIndex, pParentSql->res.code); + tscDebug("%p query is cancelled, sub:%p, orderOfSub:%d abort retrieve, code:%s", pParentSql, pSql, + subqueryIndex, tstrerror(pParentSql->res.code)); } if (numOfRows >= 0) { // current query is successful, but other sub query failed, still abort current query. tscDebug("%p sub:%p retrieve numOfRows:%d,orderOfSub:%d", pParentSql, pSql, numOfRows, subqueryIndex); - tscError("%p sub:%p abort further retrieval due to other queries failure,orderOfSub:%d,code:%d", pParentSql, pSql, - subqueryIndex, pParentSql->res.code); + tscError("%p sub:%p abort further retrieval due to other queries failure,orderOfSub:%d,code:%s", pParentSql, pSql, + subqueryIndex, tstrerror(pParentSql->res.code)); } else { if (trsupport->numOfRetry++ < MAX_NUM_OF_SUBQUERY_RETRY && pParentSql->res.code == TSDB_CODE_SUCCESS) { if (tscReissueSubquery(trsupport, pSql, numOfRows) == TSDB_CODE_SUCCESS) { @@ -1587,10 +1593,10 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO taosTFree(trsupport->pState); tscFreeSubSqlObj(trsupport, pSql); - + // in case of second stage join subquery, invoke its callback function instead of regular QueueAsyncRes SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pParentSql->cmd, 0); - + if (!TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_SEC_STAGE)) { (*pParentSql->fp)(pParentSql->param, pParentSql, pParentSql->res.code); } else { // regular super table query @@ -1669,7 +1675,7 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p // only free once taosTFree(trsupport->pState); tscFreeSubSqlObj(trsupport, pSql); - + // set the command flag must be after the semaphore been correctly set. pParentSql->cmd.command = TSDB_SQL_RETRIEVE_LOCALMERGE; if (pParentSql->res.code == TSDB_CODE_SUCCESS) { @@ -1685,24 +1691,22 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR int32_t idx = trsupport->subqueryIndex; SSqlObj * pParentSql = trsupport->pParentSql; + assert(tres != NULL); SSqlObj *pSql = (SSqlObj *)tres; - if (pSql == NULL) { // sql object has been released in error process, return immediately - tscDebug("%p subquery has been released, idx:%d, abort", pParentSql, idx); - return; - } - +// if (pSql == NULL) { // sql object has been released in error process, return immediately +// tscDebug("%p subquery has been released, idx:%d, abort", pParentSql, idx); +// return; +// } + SSubqueryState* pState = trsupport->pState; assert(pState->numOfRemain <= pState->numOfTotal && pState->numOfRemain >= 0 && pParentSql->numOfSubs == pState->numOfTotal); - // query process and cancel query process may execute at the same time - pthread_mutex_lock(&trsupport->queryMutex); - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); - SCMVgroupInfo* pVgroup = &pTableMetaInfo->vgroupList->vgroups[0]; + SCMVgroupInfo *pVgroup = &pTableMetaInfo->vgroupList->vgroups[0]; if (pParentSql->res.code != TSDB_CODE_SUCCESS) { trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; - tscTrace("%p query cancelled or failed, sub:%p, vgId:%d, orderOfSub:%d, code:%s, global code:%s", + tscDebug("%p query cancelled/failed, sub:%p, vgId:%d, orderOfSub:%d, code:%s, global code:%s", pParentSql, pSql, pVgroup->vgId, trsupport->subqueryIndex, tstrerror(numOfRows), tstrerror(pParentSql->res.code)); tscHandleSubqueryError(param, tres, numOfRows); @@ -1713,13 +1717,13 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR assert(numOfRows == taos_errno(pSql)); if (trsupport->numOfRetry++ < MAX_NUM_OF_SUBQUERY_RETRY) { - tscTrace("%p sub:%p failed code:%s, retry:%d", pParentSql, pSql, tstrerror(numOfRows), trsupport->numOfRetry); + tscError("%p sub:%p failed code:%s, retry:%d", pParentSql, pSql, tstrerror(numOfRows), trsupport->numOfRetry); if (tscReissueSubquery(trsupport, pSql, numOfRows) == TSDB_CODE_SUCCESS) { return; } } else { - tscTrace("%p sub:%p reach the max retry times, set global code:%s", pParentSql, pSql, tstrerror(numOfRows)); + tscDebug("%p sub:%p reach the max retry times, set global code:%s", pParentSql, pSql, tstrerror(numOfRows)); atomic_val_compare_exchange_32(&pParentSql->res.code, TSDB_CODE_SUCCESS, numOfRows); // set global code and abort } @@ -1764,13 +1768,9 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR (int32_t)pRes->numOfRows, pQueryInfo->groupbyExpr.orderType); if (ret != 0) { // set no disk space error info, and abort retry tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_TSC_NO_DISKSPACE); - } else if (pRes->completed) { tscAllDataRetrievedFromDnode(trsupport, pSql); - return; - } else { // continue fetch data from dnode - pthread_mutex_unlock(&trsupport->queryMutex); taos_fetch_rows_a(tres, tscRetrieveFromDnodeCallBack, param); } @@ -1779,15 +1779,15 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR } } -static SSqlObj *tscCreateSqlObjForSubquery(SSqlObj *pSql, SRetrieveSupport *trsupport, SSqlObj *prevSqlObj) { +static SSqlObj *tscCreateSTableSubquery(SSqlObj *pSql, SRetrieveSupport *trsupport, SSqlObj *prevSqlObj) { const int32_t table_index = 0; SSqlObj *pNew = createSubqueryObj(pSql, table_index, tscRetrieveDataRes, trsupport, TSDB_SQL_SELECT, prevSqlObj); if (pNew != NULL) { // the sub query of two-stage super table query SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0); + pQueryInfo->type |= TSDB_QUERY_TYPE_STABLE_SUBQUERY; - - assert(pQueryInfo->numOfTables == 1 && pNew->cmd.numOfClause == 1); + assert(pQueryInfo->numOfTables == 1 && pNew->cmd.numOfClause == 1 && trsupport->subqueryIndex < pSql->numOfSubs); // launch subquery for each vnode, so the subquery index equals to the vgroupIndex. STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, table_index); @@ -1804,7 +1804,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { SSqlObj* pParentSql = trsupport->pParentSql; SSqlObj* pSql = (SSqlObj *) tres; - + SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); assert(pSql->cmd.numOfClause == 1 && pQueryInfo->numOfTables == 1); @@ -1814,7 +1814,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { // stable query killed or other subquery failed, all query stopped if (pParentSql->res.code != TSDB_CODE_SUCCESS) { trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; - tscTrace("%p query cancelled or failed, sub:%p, vgId:%d, orderOfSub:%d, code:%s, global code:%s", + tscError("%p query cancelled or failed, sub:%p, vgId:%d, orderOfSub:%d, code:%s, global code:%s", pParentSql, pSql, pVgroup->vgId, trsupport->subqueryIndex, tstrerror(code), tstrerror(pParentSql->res.code)); tscHandleSubqueryError(param, tres, code); @@ -1832,12 +1832,12 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { assert(code == taos_errno(pSql)); if (trsupport->numOfRetry++ < MAX_NUM_OF_SUBQUERY_RETRY) { - tscTrace("%p sub:%p failed code:%s, retry:%d", pParentSql, pSql, tstrerror(code), trsupport->numOfRetry); + tscError("%p sub:%p failed code:%s, retry:%d", pParentSql, pSql, tstrerror(code), trsupport->numOfRetry); if (tscReissueSubquery(trsupport, pSql, code) == TSDB_CODE_SUCCESS) { return; } } else { - tscTrace("%p sub:%p reach the max retry times, set global code:%s", pParentSql, pSql, tstrerror(code)); + tscError("%p sub:%p reach the max retry times, set global code:%s", pParentSql, pSql, tstrerror(code)); atomic_val_compare_exchange_32(&pParentSql->res.code, TSDB_CODE_SUCCESS, code); // set global code and abort } @@ -1845,7 +1845,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { return; } - tscTrace("%p sub:%p query complete, ep:%s, vgId:%d, orderOfSub:%d, retrieve data", trsupport->pParentSql, pSql, + tscDebug("%p sub:%p query complete, ep:%s, vgId:%d, orderOfSub:%d, retrieve data", trsupport->pParentSql, pSql, pVgroup->epAddr[0].fqdn, pVgroup->vgId, trsupport->subqueryIndex); if (pSql->res.qhandle == 0) { // qhandle is NULL, code is TSDB_CODE_SUCCESS means no results generated from this vnode @@ -1924,8 +1924,14 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) { size_t size = taosArrayGetSize(pCmd->pDataBlocks); assert(size > 0); - pSql->pSubs = calloc(size, POINTER_BYTES); + // the number of already initialized subqueries + int32_t numOfSub = 0; + pSql->numOfSubs = (uint16_t)size; + pSql->pSubs = calloc(size, POINTER_BYTES); + if (pSql->pSubs == NULL) { + goto _error; + } tscDebug("%p submit data to %" PRIzu " vnode(s)", pSql, size); @@ -1934,10 +1940,13 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) { pState->numOfRemain = pSql->numOfSubs; pRes->code = TSDB_CODE_SUCCESS; - int32_t numOfSub = 0; while(numOfSub < pSql->numOfSubs) { SInsertSupporter* pSupporter = calloc(1, sizeof(SInsertSupporter)); + if (pSupporter == NULL) { + goto _error; + } + pSupporter->pSql = pSql; pSupporter->pState = pState; pSupporter->index = numOfSub; @@ -1970,7 +1979,7 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) { if (numOfSub < pSql->numOfSubs) { tscError("%p failed to prepare subObj structure and launch sub-insertion", pSql); pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; - return pRes->code; // free all allocated resource + goto _error; } pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); @@ -2068,46 +2077,8 @@ void tscBuildResFromSubqueries(SSqlObj *pSql) { doBuildResFromSubqueries(pSql); tsem_post(&pSql->rspSem); - return; - - // continue retrieve data from vnode -// if (!tscHasRemainDataInSubqueryResultSet(pSql)) { -// tscDebug("%p at least one subquery exhausted, free all other %d subqueries", pSql, pSql->numOfSubs - 1); -// SSubqueryState* pState = NULL; -// -// // free all sub sqlobj -// for (int32_t i = 0; i < pSql->numOfSubs; ++i) { -// SSqlObj* pChildObj = pSql->pSubs[i]; -// if (pChildObj == NULL) { -// continue; -// } -// -// SJoinSupporter* pSupporter = (SJoinSupporter*)pChildObj->param; -// pState = pSupporter->pState; -// -// tscDestroyJoinSupporter(pChildObj->param); -// taos_free_result(pChildObj); -// } -// -// free(pState); -// -// pRes->completed = true; // set query completed -// tsem_post(&pSql->rspSem); -// return; -// } - -// tscFetchDatablockFromSubquery(pSql); -// if (pRes->code != TSDB_CODE_SUCCESS) { -// return; -// } } - -// if (pSql->res.code == TSDB_CODE_SUCCESS) { -// (*pSql->fp)(pSql->param, pSql, pRes->numOfRows); -// } else { -// tscQueueAsyncRes(pSql); -// } } static void transferNcharData(SSqlObj *pSql, int32_t columnIndex, TAOS_FIELD *pField) { @@ -2157,7 +2128,6 @@ void **doSetResultRowData(SSqlObj *pSql, bool finalResult) { SSqlRes *pRes = &pSql->res; assert(pRes->row >= 0 && pRes->row <= pRes->numOfRows); - if (pRes->row >= pRes->numOfRows) { // all the results has returned to invoker taosTFree(pRes->tsrow); return pRes->tsrow; diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 64a871ff74..b61fd7e8c9 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -373,7 +373,7 @@ void tscFreeSqlObj(SSqlObj* pSql) { if (pSql == NULL || pSql->signature != pSql) { return; } - + tscDebug("%p start to free sql object", pSql); tscPartiallyFreeSqlObj(pSql); @@ -388,6 +388,7 @@ void tscFreeSqlObj(SSqlObj* pSql) { taosTFree(pSql->sqlstr); tsem_destroy(&pSql->rspSem); + free(pSql); } @@ -404,7 +405,7 @@ void tscDestroyDataBlock(STableDataBlocks* pDataBlock) { taosTFree(pDataBlock); } -SParamInfo* tscAddParamToDataBlock(STableDataBlocks* pDataBlock, char type, uint8_t timePrec, short bytes, +SParamInfo* tscAddParamToDataBlock(STableDataBlocks* pDataBlock, char type, uint8_t timePrec, int16_t bytes, uint32_t offset) { uint32_t needed = pDataBlock->numOfParams + 1; if (needed > pDataBlock->numOfAllocedParams) { @@ -485,15 +486,6 @@ int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock) { return TSDB_CODE_SUCCESS; } -//void tscFreeUnusedDataBlocks(SDataBlockList* pList) { -// /* release additional memory consumption */ -// for (int32_t i = 0; i < pList->nSize; ++i) { -// STableDataBlocks* pDataBlock = pList->pData[i]; -// pDataBlock->pData = realloc(pDataBlock->pData, pDataBlock->size); -// pDataBlock->nAllocSize = (uint32_t)pDataBlock->size; -// } -//} - /** * create the in-memory buffer for each table to keep the submitted data block * @param initialSize @@ -518,6 +510,11 @@ int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOff } dataBuf->pData = calloc(1, dataBuf->nAllocSize); + if (dataBuf->pData == NULL) { + tscError("failed to allocated memory, reason:%s", strerror(errno)); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + dataBuf->ordered = true; dataBuf->prevTS = INT64_MIN; @@ -742,7 +739,7 @@ bool tscIsInsertData(char* sqlstr) { int32_t index = 0; do { - SSQLToken t0 = tStrGetToken(sqlstr, &index, false, 0, NULL); + SStrToken t0 = tStrGetToken(sqlstr, &index, false, 0, NULL); if (t0.type != TK_LP) { return t0.type == TK_INSERT || t0.type == TK_IMPORT; } @@ -926,17 +923,23 @@ void tscFieldInfoClear(SFieldInfo* pFieldInfo) { } static SSqlExpr* doBuildSqlExpr(SQueryInfo* pQueryInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type, - int16_t size, int16_t interSize, bool isTagCol) { + int16_t size, int16_t interSize, int32_t colType) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pColIndex->tableIndex); SSqlExpr* pExpr = calloc(1, sizeof(SSqlExpr)); + if (pExpr == NULL) { + return NULL; + } + pExpr->functionId = functionId; - + // set the correct columnIndex index if (pColIndex->columnIndex == TSDB_TBNAME_COLUMN_INDEX) { pExpr->colInfo.colId = TSDB_TBNAME_COLUMN_INDEX; + } else if (pColIndex->columnIndex <= TSDB_UD_COLUMN_INDEX) { + pExpr->colInfo.colId = pColIndex->columnIndex; } else { - if (isTagCol) { + if (TSDB_COL_IS_TAG(colType)) { SSchema* pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); pExpr->colInfo.colId = pSchema[pColIndex->columnIndex].colId; tstrncpy(pExpr->colInfo.name, pSchema[pColIndex->columnIndex].name, sizeof(pExpr->colInfo.name)); @@ -948,9 +951,9 @@ static SSqlExpr* doBuildSqlExpr(SQueryInfo* pQueryInfo, int16_t functionId, SCol } } - pExpr->colInfo.flag = isTagCol? TSDB_COL_TAG:TSDB_COL_NORMAL; - + pExpr->colInfo.flag = colType; pExpr->colInfo.colIndex = pColIndex->columnIndex; + pExpr->resType = type; pExpr->resBytes = size; pExpr->interBytes = interSize; @@ -1060,8 +1063,11 @@ void tscSqlExprCopy(SArray* dst, const SArray* src, uint64_t uid, bool deepcopy) if (deepcopy) { SSqlExpr* p1 = calloc(1, sizeof(SSqlExpr)); + if (p1 == NULL) { + assert(0); + } + *p1 = *pExpr; - for (int32_t j = 0; j < pExpr->numOfParams; ++j) { tVariantAssign(&p1->param[j], &pExpr->param[j]); } @@ -1097,16 +1103,22 @@ SColumn* tscColumnListInsert(SArray* pColumnList, SColumnIndex* pColIndex) { if (i >= numOfCols || numOfCols == 0) { SColumn* b = calloc(1, sizeof(SColumn)); + if (b == NULL) { + return NULL; + } + b->colIndex = *pColIndex; - taosArrayInsert(pColumnList, i, &b); } else { SColumn* pCol = taosArrayGetP(pColumnList, i); if (i < numOfCols && (pCol->colIndex.columnIndex > col || pCol->colIndex.tableIndex != pColIndex->tableIndex)) { SColumn* b = calloc(1, sizeof(SColumn)); + if (b == NULL) { + return NULL; + } + b->colIndex = *pColIndex; - taosArrayInsert(pColumnList, i, &b); } } @@ -1128,7 +1140,10 @@ SColumn* tscColumnClone(const SColumn* src) { assert(src != NULL); SColumn* dst = calloc(1, sizeof(SColumn)); - + if (dst == NULL) { + return NULL; + } + dst->colIndex = src->colIndex; dst->numOfFilters = src->numOfFilters; dst->filterInfo = tscFilterInfoClone(src->filterInfo, src->numOfFilters); @@ -1183,7 +1198,7 @@ void tscColumnListDestroy(SArray* pColumnList) { * 'first_part.second_part' * */ -static int32_t validateQuoteToken(SSQLToken* pToken) { +static int32_t validateQuoteToken(SStrToken* pToken) { strdequote(pToken->z); pToken->n = (uint32_t)strtrim(pToken->z); @@ -1199,7 +1214,7 @@ static int32_t validateQuoteToken(SSQLToken* pToken) { return TSDB_CODE_SUCCESS; } -int32_t tscValidateName(SSQLToken* pToken) { +int32_t tscValidateName(SStrToken* pToken) { if (pToken->type != TK_STRING && pToken->type != TK_ID) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -1286,12 +1301,12 @@ void tscIncStreamExecutionCount(void* pStream) { ps->num += 1; } -bool tscValidateColumnId(STableMetaInfo* pTableMetaInfo, int32_t colId) { +bool tscValidateColumnId(STableMetaInfo* pTableMetaInfo, int32_t colId, int32_t numOfParams) { if (pTableMetaInfo->pTableMeta == NULL) { return false; } - if (colId == TSDB_TBNAME_COLUMN_INDEX) { + if (colId == TSDB_TBNAME_COLUMN_INDEX || (colId <= TSDB_UD_COLUMN_INDEX && numOfParams == 2)) { return true; } @@ -1338,6 +1353,10 @@ void tscTagCondCopy(STagCond* dest, const STagCond* src) { if (pCond->len > 0) { assert(pCond->cond != NULL); c.cond = malloc(c.len); + if (c.cond == NULL) { + assert(0); + } + memcpy(c.cond, pCond->cond, c.len); } @@ -1463,20 +1482,20 @@ STableMetaInfo* tscGetMetaInfo(SQueryInfo* pQueryInfo, int32_t tableIndex) { return pQueryInfo->pTableMetaInfo[tableIndex]; } -int32_t tscGetQueryInfoDetailSafely(SSqlCmd* pCmd, int32_t subClauseIndex, SQueryInfo** pQueryInfo) { +SQueryInfo* tscGetQueryInfoDetailSafely(SSqlCmd* pCmd, int32_t subClauseIndex) { + SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, subClauseIndex); int32_t ret = TSDB_CODE_SUCCESS; - *pQueryInfo = tscGetQueryInfoDetail(pCmd, subClauseIndex); - - while ((*pQueryInfo) == NULL) { + while ((pQueryInfo) == NULL) { if ((ret = tscAddSubqueryInfo(pCmd)) != TSDB_CODE_SUCCESS) { - return ret; + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + return NULL; } - (*pQueryInfo) = tscGetQueryInfoDetail(pCmd, subClauseIndex); + pQueryInfo = tscGetQueryInfoDetail(pCmd, subClauseIndex); } - return TSDB_CODE_SUCCESS; + return pQueryInfo; } STableMetaInfo* tscGetTableMetaInfoByUid(SQueryInfo* pQueryInfo, uint64_t uid, int32_t* index) { @@ -1507,6 +1526,7 @@ void tscInitQueryInfo(SQueryInfo* pQueryInfo) { assert(pQueryInfo->exprList == NULL); pQueryInfo->exprList = taosArrayInit(4, POINTER_BYTES); pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); + pQueryInfo->udColumnId = TSDB_UD_COLUMN_INDEX; } int32_t tscAddSubqueryInfo(SSqlCmd* pCmd) { @@ -1522,8 +1542,11 @@ int32_t tscAddSubqueryInfo(SSqlCmd* pCmd) { pCmd->pQueryInfo = (SQueryInfo**)tmp; SQueryInfo* pQueryInfo = calloc(1, sizeof(SQueryInfo)); + if (pQueryInfo == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + tscInitQueryInfo(pQueryInfo); - pQueryInfo->msg = pCmd->payload; // pointer to the parent error message buffer pCmd->pQueryInfo[pCmd->numOfClause++] = pQueryInfo; @@ -1584,14 +1607,18 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST SVgroupsInfo* vgroupList, SArray* pTagCols) { void* pAlloc = realloc(pQueryInfo->pTableMetaInfo, (pQueryInfo->numOfTables + 1) * POINTER_BYTES); if (pAlloc == NULL) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; return NULL; } pQueryInfo->pTableMetaInfo = pAlloc; - pQueryInfo->pTableMetaInfo[pQueryInfo->numOfTables] = calloc(1, sizeof(STableMetaInfo)); + STableMetaInfo* pTableMetaInfo = calloc(1, sizeof(STableMetaInfo)); + if (pTableMetaInfo == NULL) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + return NULL; + } - STableMetaInfo* pTableMetaInfo = pQueryInfo->pTableMetaInfo[pQueryInfo->numOfTables]; - assert(pTableMetaInfo != NULL); + pQueryInfo->pTableMetaInfo[pQueryInfo->numOfTables] = pTableMetaInfo; if (name != NULL) { tstrncpy(pTableMetaInfo->name, name, sizeof(pTableMetaInfo->name)); @@ -1602,10 +1629,18 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST if (vgroupList != NULL) { size_t size = sizeof(SVgroupsInfo) + sizeof(SCMVgroupInfo) * vgroupList->numOfVgroups; pTableMetaInfo->vgroupList = malloc(size); + if (pTableMetaInfo->vgroupList == NULL) { + return NULL; + } + memcpy(pTableMetaInfo->vgroupList, vgroupList, size); } pTableMetaInfo->tagColList = taosArrayInit(4, POINTER_BYTES); + if (pTableMetaInfo->tagColList == NULL) { + return NULL; + } + if (pTagCols != NULL) { tscColumnListCopy(pTableMetaInfo->tagColList, pTagCols, -1); } @@ -1671,8 +1706,7 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cm return NULL; } - SQueryInfo* pQueryInfo = NULL; - tscGetQueryInfoDetailSafely(pCmd, 0, &pQueryInfo); + SQueryInfo* pQueryInfo = tscGetQueryInfoDetailSafely(pCmd, 0); assert(pSql->cmd.clauseIndex == 0); STableMetaInfo* pMasterTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, pSql->cmd.clauseIndex, 0); @@ -1754,6 +1788,7 @@ static void doSetSqlExprAndResultFieldInfo(SQueryInfo* pQueryInfo, SQueryInfo* p SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void* param, int32_t cmd, SSqlObj* pPrevSql) { SSqlCmd* pCmd = &pSql->cmd; + SSqlObj* pNew = (SSqlObj*)calloc(1, sizeof(SSqlObj)); if (pNew == NULL) { tscError("%p new subquery failed, tableIndex:%d", pSql, tableIndex); @@ -1769,10 +1804,8 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void pNew->sqlstr = strdup(pSql->sqlstr); if (pNew->sqlstr == NULL) { tscError("%p new subquery failed, tableIndex:%d, vgroupIndex:%d", pSql, tableIndex, pTableMetaInfo->vgroupIndex); - - free(pNew); terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - return NULL; + goto _error; } SSqlCmd* pnCmd = &pNew->cmd; @@ -1789,9 +1822,8 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void pnCmd->parseFinished = 1; if (tscAddSubqueryInfo(pnCmd) != TSDB_CODE_SUCCESS) { - tscFreeSqlObj(pNew); terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - return NULL; + goto _error; } SQueryInfo* pNewQueryInfo = tscGetQueryInfoDetail(pnCmd, 0); @@ -1816,20 +1848,28 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void pNewQueryInfo->groupbyExpr = pQueryInfo->groupbyExpr; if (pQueryInfo->groupbyExpr.columnInfo != NULL) { pNewQueryInfo->groupbyExpr.columnInfo = taosArrayClone(pQueryInfo->groupbyExpr.columnInfo); + if (pNewQueryInfo->groupbyExpr.columnInfo == NULL) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto _error; + } } tscTagCondCopy(&pNewQueryInfo->tagCond, &pQueryInfo->tagCond); if (pQueryInfo->fillType != TSDB_FILL_NONE) { pNewQueryInfo->fillVal = malloc(pQueryInfo->fieldsInfo.numOfOutput * sizeof(int64_t)); + if (pNewQueryInfo->fillVal == NULL) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto _error; + } + memcpy(pNewQueryInfo->fillVal, pQueryInfo->fillVal, pQueryInfo->fieldsInfo.numOfOutput * sizeof(int64_t)); } if (tscAllocPayload(pnCmd, TSDB_DEFAULT_PAYLOAD_SIZE) != TSDB_CODE_SUCCESS) { tscError("%p new subquery failed, tableIndex:%d, vgroupIndex:%d", pSql, tableIndex, pTableMetaInfo->vgroupIndex); - tscFreeSqlObj(pNew); terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - return NULL; + goto _error; } tscColumnListCopy(pNewQueryInfo->colList, pQueryInfo->colList, (int16_t)tableIndex); @@ -1872,16 +1912,15 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void if (pFinalInfo->pTableMeta == NULL) { tscError("%p new subquery failed since no tableMeta in cache, name:%s", pSql, name); - tscFreeSqlObj(pNew); - if (pPrevSql != NULL) { + if (pPrevSql != NULL) { // pass the previous error to client assert(pPrevSql->res.code != TSDB_CODE_SUCCESS); terrno = pPrevSql->res.code; } else { terrno = TSDB_CODE_TSC_APP_ERROR; } - return NULL; + goto _error; } assert(pNewQueryInfo->numOfTables == 1); @@ -1906,6 +1945,10 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void } return pNew; + +_error: + tscFreeSqlObj(pNew); + return NULL; } /** diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index 4ffc631566..ef0713c415 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -29,6 +29,7 @@ extern uint16_t tsServerPort; extern uint16_t tsDnodeShellPort; extern uint16_t tsDnodeDnodePort; extern uint16_t tsSyncPort; +extern uint16_t tsArbitratorPort; extern int32_t tsStatusInterval; extern int32_t tsNumOfMnodes; extern int32_t tsEnableVnodeBak; diff --git a/src/common/inc/tname.h b/src/common/inc/tname.h index 2263a5dae1..2a4ac3fc40 100644 --- a/src/common/inc/tname.h +++ b/src/common/inc/tname.h @@ -4,6 +4,7 @@ #include "os.h" #include "taosmsg.h" #include "tstoken.h" +#include "tvariant.h" typedef struct SDataStatis { int16_t colId; @@ -24,10 +25,12 @@ void extractTableName(const char *tableId, char *name); char* extractDBName(const char *tableId, char *name); -void extractTableNameFromToken(SSQLToken *pToken, SSQLToken* pTable); +void extractTableNameFromToken(SStrToken *pToken, SStrToken* pTable); SSchema tGetTableNameColumnSchema(); +SSchema tGetUserSpecifiedColumnSchema(tVariant* pVal, SStrToken* exprStr, const char* name); + bool tscValidateTableNameLength(size_t len); SColumnFilterInfo* tscFilterInfoClone(const SColumnFilterInfo* src, int32_t numOfFilters); diff --git a/src/common/inc/tvariant.h b/src/common/inc/tvariant.h index 4fd6ea5541..e9973bcb95 100644 --- a/src/common/inc/tvariant.h +++ b/src/common/inc/tvariant.h @@ -36,7 +36,7 @@ typedef struct tVariant { }; } tVariant; -void tVariantCreate(tVariant *pVar, SSQLToken *token); +void tVariantCreate(tVariant *pVar, SStrToken *token); void tVariantCreateFromString(tVariant *pVar, char *pz, uint32_t len, uint32_t type); @@ -46,6 +46,8 @@ void tVariantDestroy(tVariant *pV); void tVariantAssign(tVariant *pDst, const tVariant *pSrc); +int32_t tVariantCompare(const tVariant* p1, const tVariant* p2); + int32_t tVariantToString(tVariant *pVar, char *dst); int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool includeLengthPrefix); diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index e9d7a71477..795585e5c9 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -37,6 +37,7 @@ uint16_t tsServerPort = 6030; uint16_t tsDnodeShellPort = 6030; // udp[6035-6039] tcp[6035] uint16_t tsDnodeDnodePort = 6035; // udp/tcp uint16_t tsSyncPort = 6040; +uint16_t tsArbitratorPort = 6042; int32_t tsStatusInterval = 1; // second int32_t tsNumOfMnodes = 3; int32_t tsEnableVnodeBak = 1; @@ -54,7 +55,7 @@ int8_t tsDaylight = 0; char tsTimezone[TSDB_TIMEZONE_LEN] = {0}; char tsLocale[TSDB_LOCALE_LEN] = {0}; char tsCharset[TSDB_LOCALE_LEN] = {0}; // default encode string -int32_t tsEnableCoreFile = 1; +int32_t tsEnableCoreFile = 0; int32_t tsMaxBinaryDisplayWidth = 30; /* @@ -1331,7 +1332,10 @@ int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port) { *port = atoi(temp+1); } - if (*port == 0) *port = tsServerPort; + if (*port == 0) { + *port = tsServerPort; + return -1; + } return 0; } diff --git a/src/common/src/tname.c b/src/common/src/tname.c index d80ddb0ee2..01945dbb00 100644 --- a/src/common/src/tname.c +++ b/src/common/src/tname.c @@ -4,6 +4,7 @@ #include "tname.h" #include "tstoken.h" #include "ttokendef.h" +#include "tvariant.h" // todo refactor UNUSED_FUNC static FORCE_INLINE const char* skipSegments(const char* input, char delim, int32_t num) { @@ -43,7 +44,30 @@ SSchema tGetTableNameColumnSchema() { s.bytes = TSDB_TABLE_NAME_LEN - 1 + VARSTR_HEADER_SIZE; s.type = TSDB_DATA_TYPE_BINARY; s.colId = TSDB_TBNAME_COLUMN_INDEX; - strncpy(s.name, TSQL_TBNAME_L, TSDB_COL_NAME_LEN); + tstrncpy(s.name, TSQL_TBNAME_L, TSDB_COL_NAME_LEN); + return s; +} + +SSchema tGetUserSpecifiedColumnSchema(tVariant* pVal, SStrToken* exprStr, const char* name) { + SSchema s = {0}; + + s.type = pVal->nType; + if (s.type == TSDB_DATA_TYPE_BINARY || s.type == TSDB_DATA_TYPE_NCHAR) { + s.bytes = (int16_t)(pVal->nLen + VARSTR_HEADER_SIZE); + } else { + s.bytes = tDataTypeDesc[pVal->nType].nSize; + } + + s.colId = TSDB_UD_COLUMN_INDEX; + if (name != NULL) { + tstrncpy(s.name, name, sizeof(s.name)); + } else { + size_t len = strdequote(exprStr->z); + size_t tlen = MIN(sizeof(s.name), len + 1); + + tstrncpy(s.name, exprStr->z, tlen); + } + return s; } @@ -110,7 +134,7 @@ int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, in * tablePrefix.columnName * extract table name and save it in pTable, with only column name in pToken */ -void extractTableNameFromToken(SSQLToken* pToken, SSQLToken* pTable) { +void extractTableNameFromToken(SStrToken* pToken, SStrToken* pTable) { const char sep = TS_PATH_DELIMITER[0]; if (pToken == pTable || pToken == NULL || pTable == NULL) { diff --git a/src/common/src/tvariant.c b/src/common/src/tvariant.c index 5460d21252..069d8eb251 100644 --- a/src/common/src/tvariant.c +++ b/src/common/src/tvariant.c @@ -12,12 +12,10 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ +#include "os.h" #include "tvariant.h" #include "hash.h" -#include "hashfunc.h" -#include "os.h" -#include "hash.h" #include "taos.h" #include "taosdef.h" #include "tstoken.h" @@ -25,7 +23,7 @@ #include "tutil.h" // todo support scientific expression number and oct number -void tVariantCreate(tVariant *pVar, SSQLToken *token) { tVariantCreateFromString(pVar, token->z, token->n, token->type); } +void tVariantCreate(tVariant *pVar, SStrToken *token) { tVariantCreateFromString(pVar, token->z, token->n, token->type); } void tVariantCreateFromString(tVariant *pVar, char *pz, uint32_t len, uint32_t type) { memset(pVar, 0, sizeof(tVariant)); @@ -102,10 +100,9 @@ void tVariantCreateFromBinary(tVariant *pVar, const char *pz, size_t len, uint32 } case TSDB_DATA_TYPE_NCHAR: { // here we get the nchar length from raw binary bits length size_t lenInwchar = len / TSDB_NCHAR_SIZE; + pVar->wpz = calloc(1, (lenInwchar + 1) * TSDB_NCHAR_SIZE); - - wcsncpy(pVar->wpz, (wchar_t *)pz, lenInwchar); - pVar->wpz[lenInwchar] = 0; + memcpy(pVar->wpz, pz, lenInwchar * TSDB_NCHAR_SIZE); pVar->nLen = (int32_t)len; break; @@ -169,6 +166,50 @@ void tVariantAssign(tVariant *pDst, const tVariant *pSrc) { char* n = strdup(p); taosArrayPush(pDst->arr, &n); } + + return; + } + + pDst->nLen = tDataTypeDesc[pDst->nType].nSize; +} + +int32_t tVariantCompare(const tVariant* p1, const tVariant* p2) { + if (p1->nType == TSDB_DATA_TYPE_NULL && p2->nType == TSDB_DATA_TYPE_NULL) { + return 0; + } + + if (p1->nType == TSDB_DATA_TYPE_NULL) { + return -1; + } + + if (p2->nType == TSDB_DATA_TYPE_NULL) { + return 1; + } + + switch (p1->nType) { + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: { + if (p1->nLen == p2->nLen) { + return memcmp(p1->pz, p2->pz, p1->nLen); + } else { + return p1->nLen > p2->nLen? 1:-1; + } + }; + + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_DOUBLE: + if (p1->dKey == p2->dKey) { + return 0; + } else { + return p1->dKey > p2->dKey? 1:-1; + } + + default: + if (p1->i64Key == p2->i64Key) { + return 0; + } else { + return p1->i64Key > p2->i64Key? 1:-1; + } } } @@ -228,7 +269,7 @@ static int32_t doConvertToInteger(tVariant *pVariant, char *pDest, int32_t type, errno = 0; char *endPtr = NULL; - SSQLToken token = {0}; + SStrToken token = {0}; token.n = tSQLGetToken(pVariant->pz, &token.type); if (token.type == TK_MINUS || token.type == TK_PLUS) { @@ -277,7 +318,7 @@ static int32_t doConvertToInteger(tVariant *pVariant, char *pDest, int32_t type, errno = 0; wchar_t *endPtr = NULL; - SSQLToken token = {0}; + SStrToken token = {0}; token.n = tSQLGetToken(pVariant->pz, &token.type); if (token.type == TK_MINUS || token.type == TK_PLUS) { @@ -436,7 +477,7 @@ static int32_t toNchar(tVariant *pVariant, char **pDest, int32_t *pDestSize) { } static FORCE_INLINE int32_t convertToDouble(char *pStr, int32_t len, double *value) { - SSQLToken stoken = {.z = pStr, .n = len}; + SStrToken stoken = {.z = pStr, .n = len}; if (TK_ILLEGAL == isValidNumber(&stoken)) { return -1; @@ -462,7 +503,7 @@ static FORCE_INLINE int32_t convertToInteger(tVariant *pVariant, int64_t *result errno = 0; char *endPtr = NULL; - SSQLToken token = {0}; + SStrToken token = {0}; token.n = tSQLGetToken(pVariant->pz, &token.type); if (token.type == TK_MINUS || token.type == TK_PLUS) { @@ -479,7 +520,7 @@ static FORCE_INLINE int32_t convertToInteger(tVariant *pVariant, int64_t *result return 0; } - SSQLToken sToken = {.z = pVariant->pz, .n = pVariant->nLen}; + SStrToken sToken = {.z = pVariant->pz, .n = pVariant->nLen}; if (TK_ILLEGAL == isValidNumber(&sToken)) { return -1; } @@ -515,7 +556,7 @@ static FORCE_INLINE int32_t convertToInteger(tVariant *pVariant, int64_t *result errno = 0; wchar_t *endPtr = NULL; - SSQLToken token = {0}; + SStrToken token = {0}; token.n = tSQLGetToken(pVariant->pz, &token.type); if (token.type == TK_MINUS || token.type == TK_PLUS) { diff --git a/src/connector/jdbc/.classpath b/src/connector/jdbc/.classpath index 39abf1c5e9..a5d95095cc 100644 --- a/src/connector/jdbc/.classpath +++ b/src/connector/jdbc/.classpath @@ -28,22 +28,5 @@ - - - - - - - - - - - - - - - - - diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java index 2ce39b7ee4..86d179eae4 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConnection.java @@ -14,6 +14,7 @@ *****************************************************************************/ package com.taosdata.jdbc; +import java.io.*; import java.sql.Array; import java.sql.Blob; import java.sql.CallableStatement; @@ -30,336 +31,392 @@ import java.sql.SQLXML; import java.sql.Savepoint; import java.sql.Statement; import java.sql.Struct; -import java.util.Enumeration; -import java.util.Map; -import java.util.Properties; +import java.util.*; import java.util.concurrent.Executor; public class TSDBConnection implements Connection { - private TSDBJNIConnector connector = null; - - protected Properties props = null; - - private String catalog = null; - - private TSDBDatabaseMetaData dbMetaData = null; - - private Properties clientInfoProps = new Properties(); - - private int timeoutMilliseconds = 0; - - private String tsCharSet = ""; - - public TSDBConnection(Properties info, TSDBDatabaseMetaData meta) throws SQLException { - this.dbMetaData = meta; - connect(info.getProperty(TSDBDriver.PROPERTY_KEY_HOST), - Integer.parseInt(info.getProperty(TSDBDriver.PROPERTY_KEY_PORT, "0")), - info.getProperty(TSDBDriver.PROPERTY_KEY_DBNAME), info.getProperty(TSDBDriver.PROPERTY_KEY_USER), - info.getProperty(TSDBDriver.PROPERTY_KEY_PASSWORD)); - } - - private void connect(String host, int port, String dbName, String user, String password) throws SQLException { - this.connector = new TSDBJNIConnector(); - this.connector.connect(host, port, dbName, user, password); - - try { - this.setCatalog(dbName); - } catch (SQLException e) { - e.printStackTrace(); - } - - this.dbMetaData.setConnection(this); - } - - public TSDBJNIConnector getConnection() { - return this.connector; - } - - public Statement createStatement() throws SQLException { - if (!this.connector.isClosed()) { - return new TSDBStatement(this.connector); - } else { - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); - } - } - - public TSDBSubscribe subscribe(String topic, String sql, boolean restart) throws SQLException { - if (this.connector.isClosed()) { - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); - } - - long id = this.connector.subscribe(topic, sql, restart, 0); - if (id == 0) { - throw new SQLException(TSDBConstants.WrapErrMsg("failed to create subscription")); - } - - return new TSDBSubscribe(this.connector, id); - } - - public PreparedStatement prepareStatement(String sql) throws SQLException { - if (!this.connector.isClosed()) { - return new TSDBPreparedStatement(this.connector, sql); - } else { - throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); - } - } - - public CallableStatement prepareCall(String sql) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public String nativeSQL(String sql) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void setAutoCommit(boolean autoCommit) throws SQLException { - } - - public boolean getAutoCommit() throws SQLException { - return true; - } - - public void commit() throws SQLException { - } - - public void rollback() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void close() throws SQLException { - if (this.connector != null && !this.connector.isClosed()) { - this.connector.closeConnection(); - } else { - throw new SQLException(TSDBConstants.WrapErrMsg("connection is already closed!")); - } - } - - public boolean isClosed() throws SQLException { - return this.connector.isClosed(); - } - - /** - * A connection's database is able to provide information describing its tables, - * its supported SQL grammar, its stored procedures, the capabilities of this - * connection, etc. This information is made available through a - * DatabaseMetaData object. - * - * @return a DatabaseMetaData object for this connection - * @exception SQLException - * if a database access error occurs - */ - public DatabaseMetaData getMetaData() throws SQLException { - return this.dbMetaData; - } - - /** - * This readOnly option is not supported by TDengine. However, the method is intentionally left blank here to - * support HikariCP connection. - * @param readOnly - * @throws SQLException - */ - public void setReadOnly(boolean readOnly) throws SQLException { - } - - public boolean isReadOnly() throws SQLException { - return true; - } - - public void setCatalog(String catalog) throws SQLException { - this.catalog = catalog; - } - - public String getCatalog() throws SQLException { - return this.catalog; - } - - /** - * The transaction isolation level option is not supported by TDengine. - * This method is intentionally left empty to support HikariCP connection. - * @param level - * @throws SQLException - */ - public void setTransactionIsolation(int level) throws SQLException { - } - - /** - * The transaction isolation level option is not supported by TDengine. - * @return - * @throws SQLException - */ - public int getTransactionIsolation() throws SQLException { - return Connection.TRANSACTION_NONE; - } - - public SQLWarning getWarnings() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void clearWarnings() throws SQLException { - // left blank to support HikariCP connection - //todo: implement getWarnings according to the warning messages returned from TDengine - } - - public Statement createStatement(int resultSetType, int resultSetConcurrency) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public PreparedStatement prepareStatement(String sql, int resultSetType, int resultSetConcurrency) - throws SQLException { - // This method is implemented in the current way to support Spark - if (resultSetType != ResultSet.TYPE_FORWARD_ONLY) { - throw new SQLException(TSDBConstants.INVALID_VARIABLES); - } - - if (resultSetConcurrency != ResultSet.CONCUR_READ_ONLY) { - throw new SQLException(TSDBConstants.INVALID_VARIABLES); - } - - return this.prepareStatement(sql); - } - - public CallableStatement prepareCall(String sql, int resultSetType, int resultSetConcurrency) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Map> getTypeMap() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void setTypeMap(Map> map) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void setHoldability(int holdability) throws SQLException { - // intentionally left empty to support druid connection pool. - } - - /** - * the transaction is not supported by TDengine, so the opened ResultSet Objects will remain open - * @return - * @throws SQLException - */ - public int getHoldability() throws SQLException { - //intentionally left empty to support HikariCP connection. - return ResultSet.HOLD_CURSORS_OVER_COMMIT; - } - - public Savepoint setSavepoint() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Savepoint setSavepoint(String name) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void rollback(Savepoint savepoint) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void releaseSavepoint(Savepoint savepoint) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Statement createStatement(int resultSetType, int resultSetConcurrency, int resultSetHoldability) - throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public PreparedStatement prepareStatement(String sql, int resultSetType, int resultSetConcurrency, - int resultSetHoldability) throws SQLException { - return this.prepareStatement(sql, resultSetType, resultSetConcurrency); - } - - public CallableStatement prepareCall(String sql, int resultSetType, int resultSetConcurrency, - int resultSetHoldability) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public PreparedStatement prepareStatement(String sql, int autoGeneratedKeys) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public PreparedStatement prepareStatement(String sql, int[] columnIndexes) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public PreparedStatement prepareStatement(String sql, String[] columnNames) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Clob createClob() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Blob createBlob() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public NClob createNClob() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public SQLXML createSQLXML() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean isValid(int timeout) throws SQLException { - return !this.isClosed(); - } - - public void setClientInfo(String name, String value) throws SQLClientInfoException { - clientInfoProps.setProperty(name, value); - } - - public void setClientInfo(Properties properties) throws SQLClientInfoException { - for (Enumeration enumer = properties.keys(); enumer.hasMoreElements();) { - String name = (String) enumer.nextElement(); - clientInfoProps.put(name, properties.getProperty(name)); - } - } - - public String getClientInfo(String name) throws SQLException { - return clientInfoProps.getProperty(name); - } - - public Properties getClientInfo() throws SQLException { - return clientInfoProps; - } - - public Array createArrayOf(String typeName, Object[] elements) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public Struct createStruct(String typeName, Object[] attributes) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void setSchema(String schema) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public String getSchema() throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void abort(Executor executor) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public void setNetworkTimeout(Executor executor, int milliseconds) throws SQLException { - this.timeoutMilliseconds = milliseconds; - } - - public int getNetworkTimeout() throws SQLException { - return this.timeoutMilliseconds; - } - - public T unwrap(Class iface) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } - - public boolean isWrapperFor(Class iface) throws SQLException { - throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); - } + private TSDBJNIConnector connector = null; + + protected Properties props = null; + + private String catalog = null; + + private TSDBDatabaseMetaData dbMetaData = null; + + private Properties clientInfoProps = new Properties(); + + private int timeoutMilliseconds = 0; + + private String tsCharSet = ""; + + public TSDBConnection(Properties info, TSDBDatabaseMetaData meta) throws SQLException { + this.dbMetaData = meta; + + //load taos.cfg start + File cfgDir = loadConfigDir(info.getProperty(TSDBDriver.PROPERTY_KEY_CONFIG_DIR)); + File cfgFile = cfgDir.listFiles((dir, name) -> "taos.cfg".equalsIgnoreCase(name))[0]; + List endpoints = loadConfigEndpoints(cfgFile); + if (!endpoints.isEmpty()){ + info.setProperty(TSDBDriver.PROPERTY_KEY_HOST,endpoints.get(0).split(":")[0]); + info.setProperty(TSDBDriver.PROPERTY_KEY_PORT,endpoints.get(0).split(":")[1]); + } + //load taos.cfg end + + connect(info.getProperty(TSDBDriver.PROPERTY_KEY_HOST), + Integer.parseInt(info.getProperty(TSDBDriver.PROPERTY_KEY_PORT, "0")), + info.getProperty(TSDBDriver.PROPERTY_KEY_DBNAME), info.getProperty(TSDBDriver.PROPERTY_KEY_USER), + info.getProperty(TSDBDriver.PROPERTY_KEY_PASSWORD)); + } + + private List loadConfigEndpoints(File cfgFile){ + List endpoints = new ArrayList<>(); + try(BufferedReader reader = new BufferedReader(new FileReader(cfgFile))) { + String line = null; + while ((line = reader.readLine())!=null){ + if (line.trim().startsWith("firstEp") || line.trim().startsWith("secondEp")){ + endpoints.add(line.substring(line.indexOf('p')+1).trim()); + } + if (endpoints.size()>1) + break; + } + } catch (FileNotFoundException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } + return endpoints; + } + + /** + * @param cfgDirPath + * @return return the config dir + * **/ + private File loadConfigDir(String cfgDirPath) { + if (cfgDirPath == null) + return loadDefaultConfigDir(); + File cfgDir = new File(cfgDirPath); + if (!cfgDir.exists()) + return loadDefaultConfigDir(); + return cfgDir; + } + + /** + * @return search the default config dir, if the config dir is not exist will return null + * */ + private File loadDefaultConfigDir(){ + File cfgDir; + File cfgDir_linux = new File("/etc/taos"); + cfgDir = cfgDir_linux.exists() ? cfgDir_linux : null; + File cfgDir_windows = new File("C:\\TDengine\\cfg"); + cfgDir = (cfgDir == null && cfgDir_windows.exists()) ? cfgDir_windows : cfgDir; + return cfgDir; + } + + private void connect(String host, int port, String dbName, String user, String password) throws SQLException { + this.connector = new TSDBJNIConnector(); + this.connector.connect(host, port, dbName, user, password); + + try { + this.setCatalog(dbName); + } catch (SQLException e) { + e.printStackTrace(); + } + + this.dbMetaData.setConnection(this); + } + + public TSDBJNIConnector getConnection() { + return this.connector; + } + + public Statement createStatement() throws SQLException { + if (!this.connector.isClosed()) { + return new TSDBStatement(this.connector); + } else { + throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); + } + } + + public TSDBSubscribe subscribe(String topic, String sql, boolean restart) throws SQLException { + if (this.connector.isClosed()) { + throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); + } + + long id = this.connector.subscribe(topic, sql, restart, 0); + if (id == 0) { + throw new SQLException(TSDBConstants.WrapErrMsg("failed to create subscription")); + } + + return new TSDBSubscribe(this.connector, id); + } + + public PreparedStatement prepareStatement(String sql) throws SQLException { + if (!this.connector.isClosed()) { + return new TSDBPreparedStatement(this.connector, sql); + } else { + throw new SQLException(TSDBConstants.FixErrMsg(TSDBConstants.JNI_CONNECTION_NULL)); + } + } + + public CallableStatement prepareCall(String sql) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public String nativeSQL(String sql) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void setAutoCommit(boolean autoCommit) throws SQLException { + } + + public boolean getAutoCommit() throws SQLException { + return true; + } + + public void commit() throws SQLException { + } + + public void rollback() throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void close() throws SQLException { + if (this.connector != null && !this.connector.isClosed()) { + this.connector.closeConnection(); + } else { + throw new SQLException(TSDBConstants.WrapErrMsg("connection is already closed!")); + } + } + + public boolean isClosed() throws SQLException { + return this.connector.isClosed(); + } + + /** + * A connection's database is able to provide information describing its tables, + * its supported SQL grammar, its stored procedures, the capabilities of this + * connection, etc. This information is made available through a + * DatabaseMetaData object. + * + * @return a DatabaseMetaData object for this connection + * @throws SQLException if a database access error occurs + */ + public DatabaseMetaData getMetaData() throws SQLException { + return this.dbMetaData; + } + + /** + * This readOnly option is not supported by TDengine. However, the method is intentionally left blank here to + * support HikariCP connection. + * + * @param readOnly + * @throws SQLException + */ + public void setReadOnly(boolean readOnly) throws SQLException { + } + + public boolean isReadOnly() throws SQLException { + return true; + } + + public void setCatalog(String catalog) throws SQLException { + this.catalog = catalog; + } + + public String getCatalog() throws SQLException { + return this.catalog; + } + + /** + * The transaction isolation level option is not supported by TDengine. + * This method is intentionally left empty to support HikariCP connection. + * + * @param level + * @throws SQLException + */ + public void setTransactionIsolation(int level) throws SQLException { + } + + /** + * The transaction isolation level option is not supported by TDengine. + * + * @return + * @throws SQLException + */ + public int getTransactionIsolation() throws SQLException { + return Connection.TRANSACTION_NONE; + } + + public SQLWarning getWarnings() throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void clearWarnings() throws SQLException { + // left blank to support HikariCP connection + //todo: implement getWarnings according to the warning messages returned from TDengine + } + + public Statement createStatement(int resultSetType, int resultSetConcurrency) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public PreparedStatement prepareStatement(String sql, int resultSetType, int resultSetConcurrency) + throws SQLException { + // This method is implemented in the current way to support Spark + if (resultSetType != ResultSet.TYPE_FORWARD_ONLY) { + throw new SQLException(TSDBConstants.INVALID_VARIABLES); + } + + if (resultSetConcurrency != ResultSet.CONCUR_READ_ONLY) { + throw new SQLException(TSDBConstants.INVALID_VARIABLES); + } + + return this.prepareStatement(sql); + } + + public CallableStatement prepareCall(String sql, int resultSetType, int resultSetConcurrency) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public Map> getTypeMap() throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void setTypeMap(Map> map) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void setHoldability(int holdability) throws SQLException { + // intentionally left empty to support druid connection pool. + } + + /** + * the transaction is not supported by TDengine, so the opened ResultSet Objects will remain open + * + * @return + * @throws SQLException + */ + public int getHoldability() throws SQLException { + //intentionally left empty to support HikariCP connection. + return ResultSet.HOLD_CURSORS_OVER_COMMIT; + } + + public Savepoint setSavepoint() throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public Savepoint setSavepoint(String name) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void rollback(Savepoint savepoint) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void releaseSavepoint(Savepoint savepoint) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public Statement createStatement(int resultSetType, int resultSetConcurrency, int resultSetHoldability) + throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public PreparedStatement prepareStatement(String sql, int resultSetType, int resultSetConcurrency, + int resultSetHoldability) throws SQLException { + return this.prepareStatement(sql, resultSetType, resultSetConcurrency); + } + + public CallableStatement prepareCall(String sql, int resultSetType, int resultSetConcurrency, + int resultSetHoldability) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public PreparedStatement prepareStatement(String sql, int autoGeneratedKeys) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public PreparedStatement prepareStatement(String sql, int[] columnIndexes) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public PreparedStatement prepareStatement(String sql, String[] columnNames) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public Clob createClob() throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public Blob createBlob() throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public NClob createNClob() throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public SQLXML createSQLXML() throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public boolean isValid(int timeout) throws SQLException { + return !this.isClosed(); + } + + public void setClientInfo(String name, String value) throws SQLClientInfoException { + clientInfoProps.setProperty(name, value); + } + + public void setClientInfo(Properties properties) throws SQLClientInfoException { + for (Enumeration enumer = properties.keys(); enumer.hasMoreElements(); ) { + String name = (String) enumer.nextElement(); + clientInfoProps.put(name, properties.getProperty(name)); + } + } + + public String getClientInfo(String name) throws SQLException { + return clientInfoProps.getProperty(name); + } + + public Properties getClientInfo() throws SQLException { + return clientInfoProps; + } + + public Array createArrayOf(String typeName, Object[] elements) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public Struct createStruct(String typeName, Object[] attributes) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void setSchema(String schema) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public String getSchema() throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void abort(Executor executor) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public void setNetworkTimeout(Executor executor, int milliseconds) throws SQLException { + this.timeoutMilliseconds = milliseconds; + } + + public int getNetworkTimeout() throws SQLException { + return this.timeoutMilliseconds; + } + + public T unwrap(Class iface) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } + + public boolean isWrapperFor(Class iface) throws SQLException { + throw new SQLException(TSDBConstants.UNSUPPORT_METHOD_EXCEPTIONZ_MSG); + } } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java index a36f7c67a0..06ae449596 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java @@ -52,7 +52,7 @@ public class TSDBStatement implements Statement { if (isClosed) { throw new SQLException("Invalid method call on a closed statement."); } - + // TODO make sure it is not a update query pSql = this.connecter.executeQuery(sql); @@ -68,21 +68,21 @@ public class TSDBStatement implements Statement { this.connecter.freeResultSet(pSql); return null; } - + if (!this.connecter.isUpdateQuery(pSql)) { return new TSDBResultSet(this.connecter, resultSetPointer); } else { this.connecter.freeResultSet(pSql); return null; } - + } public int executeUpdate(String sql) throws SQLException { if (isClosed) { throw new SQLException("Invalid method call on a closed statement."); } - + // TODO check if current query is update query pSql = this.connecter.executeQuery(sql); long resultSetPointer = this.connecter.getResultSet(); @@ -94,7 +94,7 @@ public class TSDBStatement implements Statement { this.affectedRows = this.connecter.getAffectedRows(pSql); this.connecter.freeResultSet(pSql); - + return this.affectedRows; } @@ -192,7 +192,7 @@ public class TSDBStatement implements Statement { if (isClosed) { throw new SQLException("Invalid method call on a closed statement."); } - + return this.affectedRows; } diff --git a/src/connector/jdbc/src/test/java/TestPreparedStatement.java b/src/connector/jdbc/src/test/java/TestPreparedStatement.java index 2e387206a4..1edb957493 100644 --- a/src/connector/jdbc/src/test/java/TestPreparedStatement.java +++ b/src/connector/jdbc/src/test/java/TestPreparedStatement.java @@ -12,7 +12,7 @@ public class TestPreparedStatement { Class.forName("com.taosdata.jdbc.TSDBDriver"); Properties properties = new Properties(); properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, "localhost"); - connection = DriverManager.getConnection("jdbc:TAOS://localhost:0/?user=root&password=taosdata", properties); + connection = DriverManager.getConnection("jdbc:TAOS://localhost:0/", properties); String rawSql = "select * from test.log0601"; // String[] params = new String[]{"ts", "c1"}; PreparedStatement pstmt = (TSDBPreparedStatement) connection.prepareStatement(rawSql); diff --git a/src/connector/jdbc/src/test/java/TestTSDBDatabaseMetaData.java b/src/connector/jdbc/src/test/java/TestTSDBDatabaseMetaData.java index f7e0e78441..39a08f0fe9 100644 --- a/src/connector/jdbc/src/test/java/TestTSDBDatabaseMetaData.java +++ b/src/connector/jdbc/src/test/java/TestTSDBDatabaseMetaData.java @@ -13,7 +13,7 @@ public class TestTSDBDatabaseMetaData { Class.forName("com.taosdata.jdbc.TSDBDriver"); Properties properties = new Properties(); properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, "localhost"); - connection = DriverManager.getConnection("jdbc:TAOS://localhost:0/?user=root&password=taosdata", properties); + connection = DriverManager.getConnection("jdbc:TAOS://localhost:0/", properties); dbMetaData = connection.getMetaData(); resSet = dbMetaData.getCatalogs(); while(resSet.next()) { diff --git a/src/connector/jdbc/src/test/java/TestTSDBSubscribe.java b/src/connector/jdbc/src/test/java/TestTSDBSubscribe.java index df730efa69..47acb20064 100644 --- a/src/connector/jdbc/src/test/java/TestTSDBSubscribe.java +++ b/src/connector/jdbc/src/test/java/TestTSDBSubscribe.java @@ -3,7 +3,6 @@ import com.taosdata.jdbc.TSDBDriver; import com.taosdata.jdbc.TSDBResultSet; import com.taosdata.jdbc.TSDBSubscribe; -import java.sql.Connection; import java.sql.DriverManager; import java.util.Properties; @@ -17,12 +16,10 @@ public class TestTSDBSubscribe { properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - String cs = String.format("jdbc:TAOS://%s:0/%s?user=root&password=taosdata", host, database); + String cs = String.format("jdbc:TAOS://%s:0/%s", host, database); return (TSDBConnection)DriverManager.getConnection(cs, properties); } - - public static void main(String[] args) throws Exception { String usage = "java -Djava.ext.dirs=../ TestTSDBSubscribe [-host host] <-db database> <-topic topic> <-sql sql>"; if (args.length < 2) { diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/BatchInsertTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/BatchInsertTest.java index cb78a5ca0e..c49293c96b 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/BatchInsertTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/BatchInsertTest.java @@ -40,8 +40,7 @@ public class BatchInsertTest extends BaseTest { properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + "?user=root&password=taosdata" - , properties); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); statement = connection.createStatement(); statement.executeUpdate("drop database if exists " + dbName); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ConnectionTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ConnectionTest.java index 3fc8913ca3..a54ece4ead 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ConnectionTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ConnectionTest.java @@ -29,16 +29,14 @@ public class ConnectionTest extends BaseTest { properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + "?user=root&password=taosdata" - , properties); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); assertTrue(null != connection); statement = connection.createStatement(); assertTrue(null != statement); // try reconnect - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + "?user=root&password=taosdata" - , properties); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); try { statement.execute("create database if not exists " + dbName); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/DatabaseMetaDataTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/DatabaseMetaDataTest.java index 4c4d56020d..284af3dfe7 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/DatabaseMetaDataTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/DatabaseMetaDataTest.java @@ -26,8 +26,7 @@ public class DatabaseMetaDataTest extends BaseTest { properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + "?user=root&password=taosdata", - properties); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); String sql = "drop database if exists " + dbName; statement = (TSDBPreparedStatement) connection.prepareStatement(sql); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ImportTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ImportTest.java index 016eff1a35..dbe16d9fea 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ImportTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ImportTest.java @@ -28,8 +28,7 @@ public class ImportTest extends BaseTest { properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + "?user=root&password=taosdata" - , properties); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); statement = connection.createStatement(); statement.executeUpdate("drop database if exists " + dbName); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/PreparedStatementTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/PreparedStatementTest.java index d391771269..2cf88f6630 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/PreparedStatementTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/PreparedStatementTest.java @@ -33,8 +33,7 @@ public class PreparedStatementTest extends BaseTest { properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + "?user=root&password=taosdata", - properties); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/",properties); String sql = "drop database if exists " + dbName; statement = (TSDBPreparedStatement) connection.prepareStatement(sql); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ResultSetTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ResultSetTest.java index a0b9c051c6..8067c547df 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ResultSetTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ResultSetTest.java @@ -35,8 +35,7 @@ public class ResultSetTest extends BaseTest { properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + "?user=root&password=taosdata" - , properties); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); statement = connection.createStatement(); statement.executeUpdate("drop database if exists " + dbName); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SelectTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SelectTest.java index de21cc6195..1844a92b47 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SelectTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SelectTest.java @@ -28,8 +28,7 @@ public class SelectTest extends BaseTest { properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + "?user=root&password=taosdata" - , properties); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); statement = connection.createStatement(); statement.executeUpdate("drop database if exists " + dbName); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StableTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StableTest.java index c1321d08fc..6e01fb7c34 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StableTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StableTest.java @@ -31,8 +31,7 @@ public class StableTest extends BaseTest { properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + "?user=root&password=taosdata" - , properties); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); statement = connection.createStatement(); statement.executeUpdate("create database if not exists " + dbName); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StatementTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StatementTest.java index 4fe5c39486..db7b8c8cb1 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StatementTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StatementTest.java @@ -30,8 +30,7 @@ public class StatementTest extends BaseTest { properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + "?user=root&password=taosdata" - , properties); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); statement = connection.createStatement(); statement.executeUpdate("drop database if exists " + dbName); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java index 2dc27adab7..07b43d1227 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java @@ -32,8 +32,7 @@ public class SubscribeTest extends BaseTest { properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + "?user=root&password=taosdata" - , properties); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); statement = connection.createStatement(); statement.executeUpdate("create database if not exists " + dbName); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java index 9ea5a431a5..a0981063a5 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java @@ -12,7 +12,7 @@ public class TSDBDriverTest { @Test public void urlParserTest() throws SQLException { TSDBDriver driver = new TSDBDriver(); - String url = "jdbc:TSDB://127.0.0.1:0/db?user=root&password=your_password"; + String url = "jdbc:TSDB://127.0.0.1:0/db"; Properties properties = new Properties(); driver.parseURL(url, properties); diff --git a/src/cq/CMakeLists.txt b/src/cq/CMakeLists.txt index 4e63cd4953..db366639ef 100644 --- a/src/cq/CMakeLists.txt +++ b/src/cq/CMakeLists.txt @@ -6,6 +6,10 @@ AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR}/src SRC) IF (TD_LINUX) ADD_LIBRARY(tcq ${SRC}) - TARGET_LINK_LIBRARIES(tcq tutil common taos) + IF (TD_SOMODE_STATIC) + TARGET_LINK_LIBRARIES(tcq tutil common taos_static) + ELSE () + TARGET_LINK_LIBRARIES(tcq tutil common taos) + ENDIF () ADD_SUBDIRECTORY(test) ENDIF () diff --git a/src/dnode/CMakeLists.txt b/src/dnode/CMakeLists.txt index 032e12624f..24a109dd29 100644 --- a/src/dnode/CMakeLists.txt +++ b/src/dnode/CMakeLists.txt @@ -11,7 +11,11 @@ AUX_SOURCE_DIRECTORY(src SRC) IF (TD_LINUX) ADD_EXECUTABLE(taosd ${SRC}) - TARGET_LINK_LIBRARIES(taosd mnode taos monitor http mqtt tsdb twal vnode cJson lz4 balance sync) + IF (TD_SOMODE_STATIC) + TARGET_LINK_LIBRARIES(taosd mnode taos_static monitor http mqtt tsdb twal vnode cJson lz4 balance sync) + ELSE () + TARGET_LINK_LIBRARIES(taosd mnode taos monitor http mqtt tsdb twal vnode cJson lz4 balance sync) + ENDIF () IF (TD_ACCOUNT) TARGET_LINK_LIBRARIES(taosd account) @@ -35,4 +39,4 @@ IF (TD_LINUX) COMMAND ${CMAKE_COMMAND} -E echo charset UTF-8 >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg COMMENT "prepare taosd environment") ADD_CUSTOM_TARGET(${PREPARE_ENV_TARGET} ALL WORKING_DIRECTORY ${TD_EXECUTABLE_OUTPUT_PATH} DEPENDS ${PREPARE_ENV_CMD}) -ENDIF () \ No newline at end of file +ENDIF () diff --git a/src/dnode/src/dnodeMgmt.c b/src/dnode/src/dnodeMgmt.c index 4b06626753..1f41bc23eb 100644 --- a/src/dnode/src/dnodeMgmt.c +++ b/src/dnode/src/dnodeMgmt.c @@ -165,6 +165,13 @@ int32_t dnodeInitMgmtTimer() { return TSDB_CODE_SUCCESS; } +void dnodeSendStatusMsgToMnode() { + if (tsDnodeTmr != NULL && tsStatusTimer != NULL) { + dInfo("force send status msg to mnode"); + taosTmrReset(dnodeSendStatusMsg, 3, NULL, tsDnodeTmr, &tsStatusTimer); + } +} + void dnodeCleanupMgmtTimer() { if (tsStatusTimer != NULL) { taosTmrStopA(&tsStatusTimer); @@ -717,6 +724,9 @@ static void dnodeSendStatusMsg(void *handle, void *tmrId) { pStatus->clusterCfg.maxVgroupsPerDb = htonl(tsMaxVgroupsPerDb); tstrncpy(pStatus->clusterCfg.arbitrator, tsArbitrator, TSDB_EP_LEN); tstrncpy(pStatus->clusterCfg.timezone, tsTimezone, 64); + pStatus->clusterCfg.checkTime = 0; + char timestr[32] = "1970-01-01 00:00:00.00"; + (void)taosParseTime(timestr, &pStatus->clusterCfg.checkTime, strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); tstrncpy(pStatus->clusterCfg.locale, tsLocale, TSDB_LOCALE_LEN); tstrncpy(pStatus->clusterCfg.charset, tsCharset, TSDB_LOCALE_LEN); diff --git a/src/dnode/src/dnodePeer.c b/src/dnode/src/dnodePeer.c index b27f56a871..c09d742239 100644 --- a/src/dnode/src/dnodePeer.c +++ b/src/dnode/src/dnodePeer.c @@ -93,7 +93,7 @@ static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { if (pMsg->pCont == NULL) return; if (dnodeGetRunStatus() != TSDB_DNODE_RUN_STATUS_RUNING) { - rspMsg.code = TSDB_CODE_RPC_NOT_READY; + rspMsg.code = TSDB_CODE_APP_NOT_READY; rpcSendResponse(&rspMsg); rpcFreeCont(pMsg->pCont); dDebug("RPC %p, msg:%s is ignored since dnode not running", pMsg->handle, taosMsg[pMsg->msgType]); diff --git a/src/dnode/src/dnodeShell.c b/src/dnode/src/dnodeShell.c index f9d137bb99..5daf616706 100644 --- a/src/dnode/src/dnodeShell.c +++ b/src/dnode/src/dnodeShell.c @@ -119,7 +119,7 @@ void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { if (dnodeGetRunStatus() != TSDB_DNODE_RUN_STATUS_RUNING) { dError("RPC %p, shell msg:%s is ignored since dnode not running", pMsg->handle, taosMsg[pMsg->msgType]); - rpcMsg.code = TSDB_CODE_RPC_NOT_READY; + rpcMsg.code = TSDB_CODE_APP_NOT_READY; rpcSendResponse(&rpcMsg); rpcFreeCont(pMsg->pCont); return; @@ -144,7 +144,7 @@ void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char *secret, char *ckey) { int code = mnodeRetriveAuth(user, spi, encrypt, secret, ckey); - if (code != TSDB_CODE_RPC_NOT_READY) return code; + if (code != TSDB_CODE_APP_NOT_READY) return code; SDMAuthMsg *pMsg = rpcMallocCont(sizeof(SDMAuthMsg)); tstrncpy(pMsg->user, user, sizeof(pMsg->user)); diff --git a/src/dnode/src/dnodeSystem.c b/src/dnode/src/dnodeSystem.c index 6f32bc0f7a..56316e9619 100644 --- a/src/dnode/src/dnodeSystem.c +++ b/src/dnode/src/dnodeSystem.c @@ -119,11 +119,8 @@ int32_t main(int32_t argc, char *argv[]) { syslog(LOG_INFO, "Started TDengine service successfully."); - for (int res = tsem_wait(&exitSem); res != 0; res = tsem_wait(&exitSem)) { - if (res != EINTR) { - syslog(LOG_ERR, "failed to wait exit semphore: %d", res); - break; - } + if (tsem_wait(&exitSem) != 0) { + syslog(LOG_ERR, "failed to wait exit semphore: %s", strerror(errno)); } dnodeCleanUpSystem(); diff --git a/src/inc/dnode.h b/src/inc/dnode.h index 5a059c93a6..093ce93205 100644 --- a/src/inc/dnode.h +++ b/src/inc/dnode.h @@ -65,6 +65,8 @@ void dnodeSendRpcMnodeWriteRsp(void *pMsg, int32_t code); void dnodeReprocessMnodeWriteMsg(void *pMsg); void dnodeDelayReprocessMnodeWriteMsg(void *pMsg); +void dnodeSendStatusMsgToMnode(); + #ifdef __cplusplus } #endif diff --git a/src/inc/taos.h b/src/inc/taos.h index f3cc9bb4d7..7e8f174b7c 100644 --- a/src/inc/taos.h +++ b/src/inc/taos.h @@ -93,7 +93,7 @@ DLL_EXPORT int taos_result_precision(TAOS_RES *res); // get the time precision DLL_EXPORT void taos_free_result(TAOS_RES *res); DLL_EXPORT int taos_field_count(TAOS_RES *tres); DLL_EXPORT int taos_num_fields(TAOS_RES *res); -DLL_EXPORT int taos_affected_rows(TAOS_RES *taos); +DLL_EXPORT int taos_affected_rows(TAOS_RES *res); DLL_EXPORT TAOS_FIELD *taos_fetch_fields(TAOS_RES *res); DLL_EXPORT int taos_select_db(TAOS *taos, const char *db); DLL_EXPORT int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields); diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 3dea8da18a..b61890347f 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -63,7 +63,7 @@ typedef struct tstr { extern const int32_t TYPE_BYTES[11]; // TODO: replace and remove code below #define CHAR_BYTES sizeof(char) -#define SHORT_BYTES sizeof(short) +#define SHORT_BYTES sizeof(int16_t) #define INT_BYTES sizeof(int) #define LONG_BYTES sizeof(int64_t) #define FLOAT_BYTES sizeof(float) @@ -286,7 +286,8 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_MAX_REPLICA 5 -#define TSDB_TBNAME_COLUMN_INDEX (-1) +#define TSDB_TBNAME_COLUMN_INDEX (-1) +#define TSDB_UD_COLUMN_INDEX (-100) #define TSDB_MULTI_METERMETA_MAX_NUM 100000 // maximum batch size allowed to load metermeta #define TSDB_MIN_CACHE_BLOCK_SIZE 1 @@ -395,6 +396,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_PORT_DNODEDNODE 5 #define TSDB_PORT_SYNC 10 #define TSDB_PORT_HTTP 11 +#define TSDB_PORT_ARBITRATOR 12 #define TAOS_QTYPE_RPC 0 #define TAOS_QTYPE_FWD 1 diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 9af4cee28a..951c511022 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -65,6 +65,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_SESSION_ID, 0, 0x0010, "Invalid se TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_MSG_TYPE, 0, 0x0011, "Invalid message type") TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_RESPONSE_TYPE, 0, 0x0012, "Invalid response type") TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_TIME_STAMP, 0, 0x0013, "Invalid timestamp") +TAOS_DEFINE_ERROR(TSDB_CODE_APP_NOT_READY, 0, 0x0014, "Database not ready") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_FQDN_ERROR, 0, 0x0015, "Unable to resolve FQDN") //common & util TAOS_DEFINE_ERROR(TSDB_CODE_COM_OPS_NOT_SUPPORT, 0, 0x0100, "Operation not supported") @@ -95,6 +97,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_APP_ERROR, 0, 0x0211, "Applicatio TAOS_DEFINE_ERROR(TSDB_CODE_TSC_ACTION_IN_PROGRESS, 0, 0x0212, "Action in progress") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_DISCONNECTED, 0, 0x0213, "Disconnected from service") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_NO_WRITE_AUTH, 0, 0x0214, "No write permission") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_CONN_KILLED, 0, 0x0215, "Connection killed") // mnode TAOS_DEFINE_ERROR(TSDB_CODE_MND_MSG_NOT_PROCESSED, 0, 0x0300, "Message not processed") @@ -137,6 +140,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_NOT_READY, 0, 0x033C, "Cluster no TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACCT_ALREADY_EXIST, 0, 0x0340, "Account already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_ACCT, 0, 0x0341, "Invalid account") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_ACCT_OPTION, 0, 0x0342, "Invalid account options") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACCT_EXPIRED, 0, 0x0343, "Account authorization has expired") TAOS_DEFINE_ERROR(TSDB_CODE_MND_USER_ALREADY_EXIST, 0, 0x0350, "User already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_USER, 0, 0x0351, "Invalid user") @@ -184,7 +188,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_DISK_PERMISSIONS, 0, 0x0506, "No write p TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_SUCH_FILE_OR_DIR, 0, 0x0507, "Missing data file") TAOS_DEFINE_ERROR(TSDB_CODE_VND_OUT_OF_MEMORY, 0, 0x0508, "Out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_VND_APP_ERROR, 0, 0x0509, "Unexpected generic error in vnode") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_STATUS, 0, 0x0510, "Database not ready") TAOS_DEFINE_ERROR(TSDB_CODE_VND_NOT_SYNCED, 0, 0x0511, "Database suspended") TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_WRITE_AUTH, 0, 0x0512, "Write operation denied") @@ -220,6 +223,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_EXCEED_TAGS_LIMIT, 0, 0x0706, "Tag condit TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_READY, 0, 0x0707, "Query not ready") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_HAS_RSP, 0, 0x0708, "Query should response") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_IN_EXEC, 0, 0x0709, "Multiple retrieval of this query") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW, 0, 0x070A, "Too many time window in query") // grant TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_EXPIRED, 0, 0x0800, "License expired") diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 265bf47d6d..761a267ce5 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -133,7 +133,7 @@ enum _mgmt_table { TSDB_MGMT_TABLE_MODULE, TSDB_MGMT_TABLE_QUERIES, TSDB_MGMT_TABLE_STREAMS, - TSDB_MGMT_TABLE_CONFIGS, + TSDB_MGMT_TABLE_VARIABLES, TSDB_MGMT_TABLE_CONNS, TSDB_MGMT_TABLE_SCORES, TSDB_MGMT_TABLE_GRANTS, @@ -167,9 +167,9 @@ enum _mgmt_table { #define TSDB_VN_WRITE_ACCCESS ((char)0x2) #define TSDB_VN_ALL_ACCCESS (TSDB_VN_READ_ACCCESS | TSDB_VN_WRITE_ACCCESS) -#define TSDB_COL_NORMAL 0x0u -#define TSDB_COL_TAG 0x1u -#define TSDB_COL_JOIN 0x2u +#define TSDB_COL_NORMAL 0x0u // the normal column of the table +#define TSDB_COL_TAG 0x1u // the tag column type +#define TSDB_COL_UDC 0x2u // the user specified normal string column, it is a dummy column extern char *taosMsg[]; @@ -424,7 +424,10 @@ typedef struct SColumnInfo { int16_t type; int16_t bytes; int16_t numOfFilters; - SColumnFilterInfo *filters; + union{ + int64_t placeholder; + SColumnFilterInfo *filters; + }; } SColumnInfo; typedef struct STableIdInfo { @@ -575,6 +578,7 @@ typedef struct { int32_t maxVgroupsPerDb; char arbitrator[TSDB_EP_LEN]; // tsArbitrator char timezone[64]; // tsTimezone + int64_t checkTime; // 1970-01-01 00:00:00.000 char locale[TSDB_LOCALE_LEN]; // tsLocale char charset[TSDB_LOCALE_LEN]; // tsCharset } SClusterCfg; diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index 4776d1cda7..be73d8d383 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -115,7 +115,7 @@ int tsdbDropTable(TSDB_REPO_T *pRepo, STableId tableId); int tsdbUpdateTableTagValue(TSDB_REPO_T *repo, SUpdateTableTagValMsg *pMsg); TSKEY tsdbGetTableLastKey(TSDB_REPO_T *repo, uint64_t uid); -uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_t eindex, int32_t *size); +uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_t eindex, int64_t *size); // the TSDB repository info typedef struct STsdbRepoInfo { diff --git a/src/inc/tsync.h b/src/inc/tsync.h index 2dfac7ec32..ff9c9901bd 100644 --- a/src/inc/tsync.h +++ b/src/inc/tsync.h @@ -64,7 +64,7 @@ typedef struct { if name is provided(name[0] is not zero), get the named file at the specified index. If not there, return zero. If it is there, set the size to file size, and return file magic number. Index shall not be updated. */ -typedef uint32_t (*FGetFileInfo)(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int32_t *size, uint64_t *fversion); +typedef uint32_t (*FGetFileInfo)(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion); // get the wal file from index or after // return value, -1: error, 1:more wal files, 0:last WAL. if name[0]==0, no WAL file diff --git a/src/inc/ttokendef.h b/src/inc/ttokendef.h index e77f5741d3..c1579fe9e7 100644 --- a/src/inc/ttokendef.h +++ b/src/inc/ttokendef.h @@ -69,7 +69,7 @@ #define TK_QUERIES 51 #define TK_CONNECTIONS 52 #define TK_STREAMS 53 -#define TK_CONFIGS 54 +#define TK_VARIABLES 54 #define TK_SCORES 55 #define TK_GRANTS 56 #define TK_VNODES 57 diff --git a/src/kit/shell/CMakeLists.txt b/src/kit/shell/CMakeLists.txt index 76af19f481..0305d9f1cc 100644 --- a/src/kit/shell/CMakeLists.txt +++ b/src/kit/shell/CMakeLists.txt @@ -11,11 +11,11 @@ IF (TD_LINUX) LIST(REMOVE_ITEM SRC ./src/shellDarwin.c) ADD_EXECUTABLE(shell ${SRC}) -# IF (TD_PAGMODE_LITE) + IF (TD_SOMODE_STATIC) + TARGET_LINK_LIBRARIES(shell taos_static) + ELSE () TARGET_LINK_LIBRARIES(shell taos) -# ELSE () -# TARGET_LINK_LIBRARIES(shell taos_static) -# ENDIF () + ENDIF () SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME taos) ELSEIF (TD_WINDOWS) diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c index d82ddf8871..8f7f4d9324 100644 --- a/src/kit/shell/src/shellEngine.c +++ b/src/kit/shell/src/shellEngine.c @@ -25,6 +25,8 @@ #include "taosdef.h" #include "taoserror.h" #include "tglobal.h" +#include "tsclient.h" + #include /**************** Global variables ****************/ @@ -64,11 +66,6 @@ TAOS *shellInit(SShellArguments *args) { } taos_init(); - /* - * set tsTableMetaKeepTimer = 3000ms - * means not save cache in shell - */ - tsTableMetaKeepTimer = 3000; // Connect to the database. TAOS *con = NULL; @@ -287,7 +284,7 @@ void shellRunCommandOnServer(TAOS *con, char command[]) { st = taosGetTimestampUs(); TAOS_RES* pSql = taos_query(con, command); - result = pSql; // set it into the global variable + atomic_store_ptr(&result, pSql); // set the global TAOS_RES pointer if (taos_errno(pSql)) { taos_error(pSql); @@ -298,17 +295,16 @@ void shellRunCommandOnServer(TAOS *con, char command[]) { fprintf(stdout, "Database changed.\n\n"); fflush(stdout); - result = NULL; + atomic_store_ptr(&result, 0); taos_free_result(pSql); return; } - int num_fields = taos_field_count(pSql); - if (num_fields != 0) { // select and show kinds of commands + if (!tscIsUpdateQuery(pSql)) { // select and show kinds of commands int error_no = 0; int numOfRows = shellDumpResult(pSql, fname, &error_no, printMode); if (numOfRows < 0) { - result = NULL; + atomic_store_ptr(&result, 0); taos_free_result(pSql); return; } @@ -331,7 +327,7 @@ void shellRunCommandOnServer(TAOS *con, char command[]) { wordfree(&full_path); } - result = NULL; + atomic_store_ptr(&result, 0); taos_free_result(pSql); } @@ -497,7 +493,6 @@ static int dumpResultToFile(const char* fname, TAOS_RES* tres) { } while( row != NULL); result = NULL; - //taos_free_result(tres); fclose(fp); return numOfRows; @@ -802,8 +797,8 @@ void write_history() { } void taos_error(TAOS_RES *tres) { + atomic_store_ptr(&result, 0); fprintf(stderr, "\nDB error: %s\n", taos_errstr(tres)); - result = NULL; taos_free_result(tres); } diff --git a/src/kit/shell/src/shellMain.c b/src/kit/shell/src/shellMain.c index 8481f498dd..44de6641f6 100644 --- a/src/kit/shell/src/shellMain.c +++ b/src/kit/shell/src/shellMain.c @@ -18,11 +18,10 @@ pthread_t pid; -// TODO: IMPLEMENT INTERRUPT HANDLER. -void interruptHandler(int signum) { +void shellQueryInterruptHandler(int signum) { #ifdef LINUX - taos_stop_query(result); - result = NULL; + void* pResHandle = atomic_val_compare_exchange_64(&result, result, 0); + taos_stop_query(pResHandle); #else printf("\nReceive ctrl+c or other signal, quit shell.\n"); exit(0); @@ -86,7 +85,7 @@ int main(int argc, char* argv[]) { struct sigaction act; memset(&act, 0, sizeof(struct sigaction)); - act.sa_handler = interruptHandler; + act.sa_handler = shellQueryInterruptHandler; sigaction(SIGTERM, &act, NULL); sigaction(SIGINT, &act, NULL); diff --git a/src/kit/taosdemo/CMakeLists.txt b/src/kit/taosdemo/CMakeLists.txt index 1698c85915..ab02bdb64d 100644 --- a/src/kit/taosdemo/CMakeLists.txt +++ b/src/kit/taosdemo/CMakeLists.txt @@ -8,11 +8,11 @@ IF (TD_LINUX) AUX_SOURCE_DIRECTORY(. SRC) ADD_EXECUTABLE(taosdemo ${SRC}) -# IF (TD_PAGMODE_LITE) + IF (TD_SOMODE_STATIC) + TARGET_LINK_LIBRARIES(taosdemo taos_static) + ELSE () TARGET_LINK_LIBRARIES(taosdemo taos) -# ELSE () -# TARGET_LINK_LIBRARIES(taosdemo taos_static) -# ENDIF () + ENDIF () ELSEIF (TD_WINDOWS) AUX_SOURCE_DIRECTORY(. SRC) ADD_EXECUTABLE(taosdemo ${SRC}) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 74bdf6f074..6469990c78 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -57,29 +57,30 @@ extern char configDir[]; /* Used by main to communicate with parse_opt. */ typedef struct DemoArguments { - char *host; - uint16_t port; - char *user; - char *password; - char *database; - char *tb_prefix; - char *sqlFile; - bool use_metric; - bool insert_only; - char *output_file; - int mode; - char *datatype[MAX_NUM_DATATYPE+1]; - int len_of_binary; - int num_of_CPR; - int num_of_threads; - int num_of_RPR; - int num_of_tables; - int num_of_DPT; - int abort; - int order; - int rate; - int method_of_delete; - char **arg_list; + char * host; + uint16_t port; + char * user; + char * password; + char * database; + int replica; + char * tb_prefix; + char * sqlFile; + bool use_metric; + bool insert_only; + char * output_file; + int mode; + char * datatype[MAX_NUM_DATATYPE + 1]; + int len_of_binary; + int num_of_CPR; + int num_of_threads; + int num_of_RPR; + int num_of_tables; + int num_of_DPT; + int abort; + int order; + int rate; + int method_of_delete; + char ** arg_list; } SDemoArguments; #ifdef LINUX @@ -90,6 +91,7 @@ typedef struct DemoArguments { {0, 'u', "user", 0, "The TDengine user name to use when connecting to the server. Default is 'root'.", 2}, {0, 'P', "password", 0, "The password to use when connecting to the server. Default is 'taosdata'.", 3}, {0, 'd', "database", 0, "Destination database. Default is 'test'.", 3}, + {0, 'a', "replica", 0, "Set the replica parameters of the database, Default 1, min: 1, max: 3.", 3}, {0, 'm', "table_prefix", 0, "Table prefix name. Default is 't'.", 3}, {0, 's', "sql file", 0, "The select sql file.", 3}, {0, 'M', 0, 0, "Use metric flag.", 13}, @@ -225,6 +227,13 @@ typedef struct DemoArguments { arguments->rate = 10; } break; + case 'a': + arguments->replica = atoi(arg); + if (arguments->replica > 3 || arguments->replica < 1) + { + arguments->replica = 1; + } + break; case 'D': arguments->method_of_delete = atoi(arg); if (arguments->method_of_delete < 0 || arguments->method_of_delete > 3) @@ -273,6 +282,8 @@ typedef struct DemoArguments { printf("%s%s%s\n", indent, indent, "password, The password to use when connecting to the server. Default is 'taosdata'."); printf("%s%s\n", indent, "-d"); printf("%s%s%s\n", indent, indent, "database, Destination database. Default is 'test'."); + printf("%s%s\n", indent, "-a"); + printf("%s%s%s\n", indent, indent, "replica, Set the replica parameters of the database, Default 1, min: 1, max: 3."); printf("%s%s\n", indent, "-m"); printf("%s%s%s\n", indent, indent, "table_prefix, Table prefix name. Default is 't'."); printf("%s%s\n", indent, "-s"); @@ -396,6 +407,11 @@ typedef struct DemoArguments { if (arguments->order == 1 && (arguments->rate > 50 || arguments->rate <= 0)) { arguments->rate = 10; } + } else if (strcmp(argv[i], "-a") == 0) { + arguments->replica = atoi(argv[++i]); + if (arguments->rate > 3 || arguments->rate < 1) { + arguments->rate = 1; + } } else if (strcmp(argv[i], "-D") == 0) { arguments->method_of_delete = atoi(argv[++i]); if (arguments->method_of_delete < 0 || arguments->method_of_delete > 3) { @@ -499,6 +515,7 @@ int main(int argc, char *argv[]) { "root", // user "taosdata", // password "test", // database + 1, // replica "t", // tb_prefix NULL, false, // use_metric @@ -564,6 +581,7 @@ int main(int argc, char *argv[]) { int count_data_type = 0; char dataString[STRING_LEN]; bool do_aggreFunc = true; + int replica = arguments.replica; if (NULL != arguments.sqlFile) { TAOS* qtaos = taos_connect(ip_addr, user, pass, db_name, port); @@ -661,7 +679,7 @@ int main(int argc, char *argv[]) { TAOS_RES* res = taos_query(taos, command); taos_free_result(res); - sprintf(command, "create database %s;", db_name); + sprintf(command, "create database %s replica %d;", db_name, replica); res = taos_query(taos, command); taos_free_result(res); diff --git a/src/kit/taosnetwork/client.c b/src/kit/taosnetwork/client.c index 102f9b9d89..b7db2ba0a2 100644 --- a/src/kit/taosnetwork/client.c +++ b/src/kit/taosnetwork/client.c @@ -27,35 +27,46 @@ #include #include #include +#include -#define MAX_PKG_LEN (64*1000) -#define BUFFER_SIZE (MAX_PKG_LEN + 1024) +#define MAX_PKG_LEN (64*1000) +#define BUFFER_SIZE (MAX_PKG_LEN + 1024) +#define TEST_FQDN_LEN 128 +#define TEST_IPv4ADDR_LEN 16 typedef struct { - int port; - char *host; + uint16_t port; + uint32_t hostIp; + char fqdn[TEST_FQDN_LEN]; uint16_t pktLen; } info_s; typedef struct Arguments { - char * host; + char host[TEST_IPv4ADDR_LEN]; + char fqdn[TEST_FQDN_LEN]; uint16_t port; uint16_t max_port; uint16_t pktLen; } SArguments; static struct argp_option options[] = { - {0, 'h', "host", 0, "The host to connect to TDEngine. Default is localhost.", 0}, + {0, 'h', "host ip", 0, "The host ip to connect to TDEngine. Default is localhost.", 0}, {0, 'p', "port", 0, "The TCP or UDP port number to use for the connection. Default is 6030.", 1}, - {0, 'm', "max port", 0, "The max TCP or UDP port number to use for the connection. Default is 6060.", 2}, + {0, 'm', "max port", 0, "The max TCP or UDP port number to use for the connection. Default is 6042.", 2}, + {0, 'f', "host fqdn", 0, "The host fqdn to connect to TDEngine.", 3}, {0, 'l', "test pkg len", 0, "The len of pkg for test. Default is 1000 Bytes, max not greater than 64k Bytes.\nNotes: This parameter must be consistent between the client and the server.", 3}}; static error_t parse_opt(int key, char *arg, struct argp_state *state) { - + wordexp_t full_path; SArguments *arguments = state->input; switch (key) { case 'h': - arguments->host = arg; + if (wordexp(arg, &full_path, 0) != 0) { + fprintf(stderr, "Invalid host ip %s\n", arg); + return -1; + } + strcpy(arguments->host, full_path.we_wordv[0]); + wordfree(&full_path); break; case 'p': arguments->port = atoi(arg); @@ -66,6 +77,14 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { case 'l': arguments->pktLen = atoi(arg); break; + case 'f': + if (wordexp(arg, &full_path, 0) != 0) { + fprintf(stderr, "Invalid host fqdn %s\n", arg); + return -1; + } + strcpy(arguments->fqdn, full_path.we_wordv[0]); + wordfree(&full_path); + break; default: return ARGP_ERR_UNKNOWN; @@ -76,8 +95,6 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { static struct argp argp = {options, parse_opt, 0, 0}; int checkTcpPort(info_s *info) { - int port = info->port; - char *host = info->host; int clientSocket; struct sockaddr_in serverAddr; @@ -88,21 +105,35 @@ int checkTcpPort(info_s *info) { printf("socket() fail: %s\n", strerror(errno)); return -1; } - serverAddr.sin_family = AF_INET; - serverAddr.sin_port = htons(port); - serverAddr.sin_addr.s_addr = inet_addr(host); + // set send and recv overtime + struct timeval timeout; + timeout.tv_sec = 2; //s + timeout.tv_usec = 0; //us + if (setsockopt(clientSocket, SOL_SOCKET,SO_SNDTIMEO, (char *)&timeout, sizeof(struct timeval)) == -1) { + perror("setsockopt send timer failed:"); + } + if (setsockopt(clientSocket, SOL_SOCKET,SO_RCVTIMEO, (char *)&timeout, sizeof(struct timeval)) == -1) { + perror("setsockopt recv timer failed:"); + } + + serverAddr.sin_family = AF_INET; + serverAddr.sin_port = htons(info->port); + + serverAddr.sin_addr.s_addr = info->hostIp; //printf("=================================\n"); if (connect(clientSocket, (struct sockaddr *)&serverAddr, sizeof(serverAddr)) < 0) { - printf("connect() fail: %s\n", strerror(errno)); + printf("connect() fail: %s\t", strerror(errno)); return -1; } //printf("Connect to: %s:%d...success\n", host, port); memset(sendbuf, 0, BUFFER_SIZE); memset(recvbuf, 0, BUFFER_SIZE); - sprintf(sendbuf, "client send tcp pkg to %s:%d, content: 1122334455", host, port); + struct in_addr ipStr; + memcpy(&ipStr, &info->hostIp, 4); + sprintf(sendbuf, "client send tcp pkg to %s:%d, content: 1122334455", inet_ntoa(ipStr), info->port); sprintf(sendbuf + info->pktLen - 16, "1122334455667788"); send(clientSocket, sendbuf, info->pktLen, 0); @@ -120,7 +151,7 @@ int checkTcpPort(info_s *info) { if (errno == EINTR) { continue; } else { - printf("recv ack pkg from TCP port: %d fail:%s.\n", port, strerror(errno)); + printf("recv ack pkg from TCP port: %d fail:%s.\n", info->port, strerror(errno)); close(clientSocket); return -1; } @@ -132,7 +163,7 @@ int checkTcpPort(info_s *info) { } if (iDataNum < info->pktLen) { - printf("recv ack pkg len: %d, less than req pkg len: %d from tcp port: %d\n", iDataNum, info->pktLen, port); + printf("recv ack pkg len: %d, less than req pkg len: %d from tcp port: %d\n", iDataNum, info->pktLen, info->port); return -1; } //printf("Read ack pkg len:%d from tcp port: %d, buffer: %s %s\n", info->pktLen, port, recvbuf, recvbuf+iDataNum-8); @@ -142,8 +173,6 @@ int checkTcpPort(info_s *info) { } int checkUdpPort(info_s *info) { - int port = info->port; - char *host = info->host; int clientSocket; struct sockaddr_in serverAddr; @@ -154,15 +183,28 @@ int checkUdpPort(info_s *info) { perror("socket"); return -1; } + + // set overtime + struct timeval timeout; + timeout.tv_sec = 2; //s + timeout.tv_usec = 0; //us + if (setsockopt(clientSocket, SOL_SOCKET,SO_SNDTIMEO, (char *)&timeout, sizeof(struct timeval)) == -1) { + perror("setsockopt send timer failed:"); + } + if (setsockopt(clientSocket, SOL_SOCKET,SO_RCVTIMEO, (char *)&timeout, sizeof(struct timeval)) == -1) { + perror("setsockopt recv timer failed:"); + } serverAddr.sin_family = AF_INET; - serverAddr.sin_port = htons(port); - serverAddr.sin_addr.s_addr = inet_addr(host); + serverAddr.sin_port = htons(info->port); + serverAddr.sin_addr.s_addr = info->hostIp; memset(sendbuf, 0, BUFFER_SIZE); memset(recvbuf, 0, BUFFER_SIZE); - sprintf(sendbuf, "client send udp pkg to %s:%d, content: 1122334455", host, port); + struct in_addr ipStr; + memcpy(&ipStr, &info->hostIp, 4); + sprintf(sendbuf, "client send udp pkg to %s:%d, content: 1122334455", inet_ntoa(ipStr), info->port); sprintf(sendbuf + info->pktLen - 16, "1122334455667788"); socklen_t sin_size = sizeof(*(struct sockaddr *)&serverAddr); @@ -176,7 +218,7 @@ int checkUdpPort(info_s *info) { iDataNum = recvfrom(clientSocket, recvbuf, BUFFER_SIZE, 0, (struct sockaddr *)&serverAddr, &sin_size); if (iDataNum < info->pktLen) { - printf("Read ack pkg len: %d, less than req pkg len: %d from udp port: %d\n", iDataNum, info->pktLen, port); + printf("Read ack pkg len: %d, less than req pkg len: %d from udp port: %d\t\t", iDataNum, info->pktLen, info->port); return -1; } @@ -185,10 +227,61 @@ int checkUdpPort(info_s *info) { return 0; } -int main(int argc, char *argv[]) { - SArguments arguments = {"127.0.0.1", 6030, 6060, 1000}; - info_s info; +int32_t getIpFromFqdn(const char *fqdn, uint32_t* ip) { + struct addrinfo hints = {0}; + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + + struct addrinfo *result = NULL; + + int32_t ret = getaddrinfo(fqdn, NULL, &hints, &result); + if (result) { + struct sockaddr *sa = result->ai_addr; + struct sockaddr_in *si = (struct sockaddr_in*)sa; + struct in_addr ia = si->sin_addr; + *ip = ia.s_addr; + freeaddrinfo(result); + return 0; + } else { + printf("Failed get the ip address from fqdn:%s, code:%d, reason:%s", fqdn, ret, gai_strerror(ret)); + return -1; + } +} + +void checkPort(uint32_t hostIp, uint16_t startPort, uint16_t maxPort, uint16_t pktLen) { int ret; + info_s info; + memset(&info, 0, sizeof(info_s)); + info.hostIp = hostIp; + info.pktLen = pktLen; + + for (uint16_t port = startPort; port <= maxPort; port++) { + //printf("test: %s:%d\n", info.host, port); + printf("\n"); + + info.port = port; + ret = checkTcpPort(&info); + if (ret != 0) { + printf("tcp port:%d test fail.\t\n", port); + } else { + printf("tcp port:%d test ok.\t\t", port); + } + + ret = checkUdpPort(&info); + if (ret != 0) { + printf("udp port:%d test fail.\t\n", port); + } else { + printf("udp port:%d test ok.\t\t", port); + } + } + + printf("\n"); + return ; +} + +int main(int argc, char *argv[]) { + SArguments arguments = {"127.0.0.1", "", 6030, 6042, 1000}; + int ret; argp_parse(&argp, argc, argv, 0, 0, &arguments); if (arguments.pktLen > MAX_PKG_LEN) { @@ -196,32 +289,25 @@ int main(int argc, char *argv[]) { exit(0); } - printf("host: %s\tport: %d\tmax_port: %d\tpkgLen: %d\n", arguments.host, arguments.port, arguments.max_port, arguments.pktLen); + printf("host ip: %s\thost fqdn: %s\tport: %d\tmax_port: %d\tpkgLen: %d\n", arguments.host, arguments.fqdn, arguments.port, arguments.max_port, arguments.pktLen); - int port = arguments.port; - - info.host = arguments.host; - info.pktLen = arguments.pktLen; - - for (; port <= arguments.max_port; port++) { - //printf("test: %s:%d\n", info.host, port); + if (arguments.host[0] != 0) { + printf("\nstart connect to %s test:\n", arguments.host); + checkPort(inet_addr(arguments.host), arguments.port, arguments.max_port, arguments.pktLen); printf("\n"); - - info.port = port; - ret = checkTcpPort(&info); - if (ret != 0) { - printf("tcp port:%d test fail.\t\t", port); - } else { - printf("tcp port:%d test ok.\t\t", port); - } - - ret = checkUdpPort(&info); - if (ret != 0) { - printf("udp port:%d test fail.\t\t", port); - } else { - printf("udp port:%d test ok.\t\t", port); - } } - printf("\n"); + + if (arguments.fqdn[0] != 0) { + uint32_t hostIp = 0; + ret = getIpFromFqdn(arguments.fqdn, &hostIp); + if (ret) { + printf("\n"); + return 0; + } + printf("\nstart connetc to %s test:\n", arguments.fqdn); + checkPort(hostIp, arguments.port, arguments.max_port, arguments.pktLen); + printf("\n"); + } + return 0; } diff --git a/src/kit/taosnetwork/server.c b/src/kit/taosnetwork/server.c index 1c3bc6fa09..97be1d3b63 100644 --- a/src/kit/taosnetwork/server.c +++ b/src/kit/taosnetwork/server.c @@ -142,9 +142,9 @@ static void *bindTcpPort(void *sarg) { printf("recv Client: %s pkg from TCP port: %d, pkg len: %d\n", inet_ntoa(clientAddr.sin_addr), port, iDataNum); if (iDataNum > 0) { send(client, buffer, iDataNum, 0); - break; } } + close(serverSocket); return NULL; } @@ -201,7 +201,7 @@ static void *bindUdpPort(void *sarg) { int main(int argc, char *argv[]) { - SArguments arguments = {"127.0.0.1", 6030, 6060, 1000}; + SArguments arguments = {"127.0.0.1", 6030, 6042, 1000}; argp_parse(&argp, argc, argv, 0, 0, &arguments); if (arguments.pktLen > MAX_PKG_LEN) { printf("test pkg len overflow: %d, max len not greater than %d bytes\n", arguments.pktLen, MAX_PKG_LEN); diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index 78c022ce46..cdea9eda60 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -41,7 +41,7 @@ static void * tsDbSdb = NULL; static int32_t tsDbUpdateSize; -static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate, void *pMsg); +static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate, SMnodeMsg *pMsg); static int32_t mnodeDropDb(SMnodeMsg *newMsg); static int32_t mnodeSetDbDropping(SDbObj *pDb); static int32_t mnodeGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); @@ -343,7 +343,7 @@ static int32_t mnodeCreateDbCb(SMnodeMsg *pMsg, int32_t code) { return code; } -static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate, void *pMsg) { +static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate, SMnodeMsg *pMsg) { int32_t code = acctCheck(pAcct, ACCT_GRANT_DB); if (code != 0) return code; @@ -354,7 +354,7 @@ static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate, void *pMs mDebug("db:%s, already exist, ignore exist is set", pCreate->db); return TSDB_CODE_SUCCESS; } else { - mError("db:%s, is already exist, ignore exist not set", pCreate->db); + mError("db:%s, already exist, ignore exist not set", pCreate->db); return TSDB_CODE_MND_DB_ALREADY_EXIST; } } @@ -393,6 +393,9 @@ static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate, void *pMs return code; } + pMsg->pDb = pDb; + mnodeIncDbRef(pDb); + SSdbOper oper = { .type = SDB_OPER_GLOBAL, .table = tsDbSdb, @@ -405,6 +408,7 @@ static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate, void *pMs code = sdbInsertRow(&oper); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("db:%s, failed to create, reason:%s", pDb->name, tstrerror(code)); + pMsg->pDb = NULL; mnodeDestroyDb(pDb); } @@ -1055,10 +1059,12 @@ static int32_t mnodeProcessDropDbMsg(SMnodeMsg *pMsg) { } } +#if 0 if (mnodeCheckIsMonitorDB(pMsg->pDb->name, tsMonitorDbName)) { mError("db:%s, can't drop monitor database", pDrop->db); return TSDB_CODE_MND_MONITOR_DB_FORBIDDEN; } +#endif int32_t code = mnodeSetDbDropping(pMsg->pDb); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index 261f63d5cf..d5b9d18e37 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -49,7 +49,7 @@ static int32_t mnodeProcessCreateDnodeMsg(SMnodeMsg *pMsg); static int32_t mnodeProcessDropDnodeMsg(SMnodeMsg *pMsg); static int32_t mnodeProcessCfgDnodeMsg(SMnodeMsg *pMsg); static void mnodeProcessCfgDnodeMsgRsp(SRpcMsg *rpcMsg) ; -static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *rpcMsg); +static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg); static int32_t mnodeGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mnodeRetrieveModules(SShowObj *pShow, char *data, int32_t rows, void *pConn); static int32_t mnodeGetConfigMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); @@ -161,8 +161,8 @@ int32_t mnodeInitDnodes() { mnodeAddPeerMsgHandle(TSDB_MSG_TYPE_DM_STATUS, mnodeProcessDnodeStatusMsg); mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_MODULE, mnodeGetModuleMeta); mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_MODULE, mnodeRetrieveModules); - mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_CONFIGS, mnodeGetConfigMeta); - mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_CONFIGS, mnodeRetrieveConfigs); + mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_VARIABLES, mnodeGetConfigMeta); + mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_VARIABLES, mnodeRetrieveConfigs); mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_VNODES, mnodeGetVnodeMeta); mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_VNODES, mnodeRetrieveVnodes); mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_DNODE, mnodeGetDnodeMeta); @@ -363,10 +363,15 @@ static bool mnodeCheckClusterCfgPara(const SClusterCfg *clusterCfg) { mError("\"arbitrator\"[%s - %s] cfg parameters inconsistent", clusterCfg->arbitrator, tsArbitrator); return false; } - if (0 != strncasecmp(clusterCfg->timezone, tsTimezone, strlen(tsTimezone))) { - mError("\"timezone\"[%s - %s] cfg parameters inconsistent", clusterCfg->timezone, tsTimezone); + + int64_t checkTime = 0; + char timestr[32] = "1970-01-01 00:00:00.00"; + (void)taosParseTime(timestr, &checkTime, strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); + if ((0 != strncasecmp(clusterCfg->timezone, tsTimezone, strlen(tsTimezone))) && (checkTime != clusterCfg->checkTime)) { + mError("\"timezone\"[%s - %s] [%" PRId64 " - %" PRId64"] cfg parameters inconsistent", clusterCfg->timezone, tsTimezone, clusterCfg->checkTime, checkTime); return false; } + if (0 != strncasecmp(clusterCfg->locale, tsLocale, strlen(tsLocale))) { mError("\"locale\"[%s - %s] cfg parameters inconsistent", clusterCfg->locale, tsLocale); return false; @@ -513,7 +518,7 @@ static int32_t mnodeCreateDnode(char *ep, SMnodeMsg *pMsg) { SDnodeObj *pDnode = mnodeGetDnodeByEp(ep); if (pDnode != NULL) { mnodeDecDnodeRef(pDnode); - mError("dnode:%d is already exist, %s:%d", pDnode->dnodeId, pDnode->dnodeFqdn, pDnode->dnodePort); + mError("dnode:%d, already exist, %s:%d", pDnode->dnodeId, pDnode->dnodeFqdn, pDnode->dnodePort); return TSDB_CODE_MND_DNODE_ALREADY_EXIST; } @@ -728,7 +733,7 @@ static int32_t mnodeRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, vo } static bool mnodeCheckModuleInDnode(SDnodeObj *pDnode, int32_t moduleType) { - uint32_t status = pDnode->moduleStatus & (1 << moduleType); + uint32_t status = pDnode->moduleStatus & (1u << moduleType); return status > 0; } @@ -753,7 +758,7 @@ static int32_t mnodeGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *p pShow->bytes[cols] = 40 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "end point"); + strcpy(pSchema[cols].name, "end_point"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -787,7 +792,9 @@ static int32_t mnodeGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *p int32_t mnodeRetrieveModules(SShowObj *pShow, char *data, int32_t rows, void *pConn) { int32_t numOfRows = 0; - char * pWrite; + + char* pWrite; + char* moduleName[5] = { "MNODE", "HTTP", "MONITOR", "MQTT", "UNKNOWN" }; while (numOfRows < rows) { SDnodeObj *pDnode = NULL; @@ -802,28 +809,18 @@ int32_t mnodeRetrieveModules(SShowObj *pShow, char *data, int32_t rows, void *pC cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strncpy(pWrite, pDnode->dnodeEp, pShow->bytes[cols]-1); + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pDnode->dnodeEp, pShow->bytes[cols] - 1); cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - switch (moduleType) { - case TSDB_MOD_MNODE: - strcpy(pWrite, "mnode"); - break; - case TSDB_MOD_HTTP: - strcpy(pWrite, "http"); - break; - case TSDB_MOD_MONITOR: - strcpy(pWrite, "monitor"); - break; - default: - strcpy(pWrite, "unknown"); - } + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, moduleName[moduleType], pShow->bytes[cols]); cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; bool enable = mnodeCheckModuleInDnode(pDnode, moduleType); - strcpy(pWrite, enable ? "enable" : "disable"); + + char* v = enable? "enable":"disable"; + STR_TO_VARSTR(pWrite, v); cols++; numOfRows++; @@ -857,13 +854,13 @@ static int32_t mnodeGetConfigMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *p pShow->bytes[cols] = TSDB_CFG_OPTION_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "config name"); + tstrncpy(pSchema[cols].name, "name", sizeof(pSchema[cols].name)); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; pShow->bytes[cols] = TSDB_CFG_VALUE_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "config value"); + tstrncpy(pSchema[cols].name, "value", sizeof(pSchema[cols].name)); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -898,27 +895,32 @@ static int32_t mnodeRetrieveConfigs(SShowObj *pShow, char *data, int32_t rows, v int32_t cols = 0; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - snprintf(pWrite, TSDB_CFG_OPTION_LEN, "%s", cfg->option); + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, cfg->option, TSDB_CFG_OPTION_LEN); + cols++; + int32_t t = 0; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; switch (cfg->valType) { case TAOS_CFG_VTYPE_INT16: - snprintf(pWrite, TSDB_CFG_VALUE_LEN, "%d", *((int16_t *)cfg->ptr)); + t = snprintf(varDataVal(pWrite), TSDB_CFG_VALUE_LEN, "%d", *((int16_t *)cfg->ptr)); + varDataSetLen(pWrite, t); numOfRows++; break; case TAOS_CFG_VTYPE_INT32: - snprintf(pWrite, TSDB_CFG_VALUE_LEN, "%d", *((int32_t *)cfg->ptr)); + t = snprintf(varDataVal(pWrite), TSDB_CFG_VALUE_LEN, "%d", *((int32_t *)cfg->ptr)); + varDataSetLen(pWrite, t); numOfRows++; break; case TAOS_CFG_VTYPE_FLOAT: - snprintf(pWrite, TSDB_CFG_VALUE_LEN, "%f", *((float *)cfg->ptr)); + t = snprintf(varDataVal(pWrite), TSDB_CFG_VALUE_LEN, "%f", *((float *)cfg->ptr)); + varDataSetLen(pWrite, t); numOfRows++; break; case TAOS_CFG_VTYPE_STRING: case TAOS_CFG_VTYPE_IPSTR: case TAOS_CFG_VTYPE_DIRECTORY: - snprintf(pWrite, TSDB_CFG_VALUE_LEN, "%s", (char *)cfg->ptr); + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, cfg->ptr, TSDB_CFG_VALUE_LEN); numOfRows++; break; default: diff --git a/src/mnode/src/mnodeMain.c b/src/mnode/src/mnodeMain.c index 53f3474c3c..d63a575868 100644 --- a/src/mnode/src/mnodeMain.c +++ b/src/mnode/src/mnodeMain.c @@ -21,6 +21,7 @@ #include "tgrant.h" #include "ttimer.h" #include "tglobal.h" +#include "mnode.h" #include "dnode.h" #include "mnodeDef.h" #include "mnodeInt.h" @@ -107,13 +108,18 @@ int32_t mnodeStartSystem() { tsMgmtIsRunning = true; mInfo("mnode is initialized successfully"); + + sdbUpdateSync(); + return 0; } int32_t mnodeInitSystem() { mnodeInitTimer(); - if (!mnodeNeedStart()) return 0; - return mnodeStartSystem(); + if (mnodeNeedStart()) { + return mnodeStartSystem(); + } + return 0; } void mnodeCleanupSystem() { @@ -159,14 +165,19 @@ static void mnodeCleanupTimer() { static bool mnodeNeedStart() { struct stat dirstat; - bool fileExist = (stat(tsMnodeDir, &dirstat) == 0); + char mnodeFileName[TSDB_FILENAME_LEN * 2] = {0}; + sprintf(mnodeFileName, "%s/wal/wal0", tsMnodeDir); + + bool fileExist = (stat(mnodeFileName, &dirstat) == 0); bool asMaster = (strcmp(tsFirst, tsLocalEp) == 0); if (asMaster || fileExist) { + mDebug("mnode module start, asMaster:%d fileExist:%d", asMaster, fileExist); return true; + } else { + mDebug("mnode module won't start, asMaster:%d fileExist:%d", asMaster, fileExist); + return false; } - - return false; } bool mnodeIsRunning() { diff --git a/src/mnode/src/mnodeProfile.c b/src/mnode/src/mnodeProfile.c index 01a824baa7..06f992c26a 100644 --- a/src/mnode/src/mnodeProfile.c +++ b/src/mnode/src/mnodeProfile.c @@ -35,8 +35,8 @@ #include "mnodeVgroup.h" #include "mnodeWrite.h" -#define CONN_KEEP_TIME (tsShellActivityTimer * 3000) -#define CONN_CHECK_TIME (tsShellActivityTimer * 2000) +#define CONN_KEEP_TIME (tsShellActivityTimer * 3) +#define CONN_CHECK_TIME (tsShellActivityTimer * 2) #define QUERY_ID_SIZE 20 #define QUERY_STREAM_SAVE_SIZE 20 @@ -100,7 +100,7 @@ SConnObj *mnodeCreateConn(char *user, uint32_t ip, uint16_t port) { }; tstrncpy(connObj.user, user, sizeof(connObj.user)); - SConnObj *pConn = taosCachePut(tsMnodeConnCache, &connId, sizeof(int32_t), &connObj, sizeof(connObj), CONN_KEEP_TIME); + SConnObj *pConn = taosCachePut(tsMnodeConnCache, &connId, sizeof(int32_t), &connObj, sizeof(connObj), CONN_KEEP_TIME * 1000); mDebug("connId:%d, is created, user:%s ip:%s:%u", connId, user, taosIpStr(ip), port); return pConn; diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 3651aa8aad..e3ed7daf8c 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -224,7 +224,7 @@ void sdbUpdateMnodeRoles() { mnodeUpdateMnodeEpSet(); } -static uint32_t sdbGetFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int32_t *size, uint64_t *fversion) { +static uint32_t sdbGetFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion) { sdbUpdateMnodeRoles(); return 0; } @@ -291,6 +291,11 @@ static void sdbConfirmForward(void *ahandle, void *param, int32_t code) { } void sdbUpdateSync() { + if (!mnodeIsRunning()) { + mDebug("mnode not start yet, update sync info later"); + return; + } + SSyncCfg syncCfg = {0}; int32_t index = 0; diff --git a/src/mnode/src/mnodeShow.c b/src/mnode/src/mnodeShow.c index 8a84b66a33..f66ef6b7a3 100644 --- a/src/mnode/src/mnodeShow.c +++ b/src/mnode/src/mnodeShow.c @@ -98,7 +98,7 @@ static char *mnodeGetShowType(int32_t showType) { case TSDB_MGMT_TABLE_MODULE: return "show modules"; case TSDB_MGMT_TABLE_QUERIES: return "show queries"; case TSDB_MGMT_TABLE_STREAMS: return "show streams"; - case TSDB_MGMT_TABLE_CONFIGS: return "show configs"; + case TSDB_MGMT_TABLE_VARIABLES: return "show configs"; case TSDB_MGMT_TABLE_CONNS: return "show connections"; case TSDB_MGMT_TABLE_SCORES: return "show scores"; case TSDB_MGMT_TABLE_GRANTS: return "show grants"; @@ -313,6 +313,7 @@ static int32_t mnodeProcessConnectMsg(SMnodeMsg *pMsg) { if (pDb->status != TSDB_DB_STATUS_READY) { mError("db:%s, status:%d, in dropping", pDb->name, pDb->status); code = TSDB_CODE_MND_DB_IN_DROPPING; + mnodeDecDbRef(pDb); goto connect_over; } mnodeDecDbRef(pDb); diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index dbc1bffa5c..03b1399ea7 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -294,6 +294,7 @@ static int32_t mnodeChildTableActionRestored() { SSdbOper desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .table = tsChildTableSdb}; sdbDeleteRow(&desc); mnodeDecTableRef(pTable); + mnodeDecDbRef(pDb); continue; } mnodeDecDbRef(pDb); @@ -1259,6 +1260,7 @@ static int32_t mnodeGetShowSuperTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, if (pDb->status != TSDB_DB_STATUS_READY) { mError("db:%s, status:%d, in dropping", pDb->name, pDb->status); + mnodeDecDbRef(pDb); return TSDB_CODE_MND_DB_IN_DROPPING; } @@ -1323,6 +1325,7 @@ int32_t mnodeRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows, if (pDb->status != TSDB_DB_STATUS_READY) { mError("db:%s, status:%d, in dropping", pDb->name, pDb->status); + mnodeDecDbRef(pDb); return 0; } @@ -1714,7 +1717,8 @@ static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) { (sdbGetVersion() & ((1ul << 16) - 1ul)); pTable->superTable = pMsg->pSTable; } else { - pTable->uid = (((uint64_t) pTable->createdTime) << 16) + (sdbGetVersion() & ((1ul << 16) - 1ul)); + pTable->uid = (((uint64_t)pTable->vgId) << 40) + ((((uint64_t)pTable->sid) & ((1ul << 24) - 1ul)) << 16) + + (sdbGetVersion() & ((1ul << 16) - 1ul)); pTable->sversion = 0; pTable->numOfColumns = htons(pCreate->numOfColumns); pTable->sqlLen = htons(pCreate->sqlLen); @@ -2494,6 +2498,7 @@ static int32_t mnodeGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void if (pDb->status != TSDB_DB_STATUS_READY) { mError("db:%s, status:%d, in dropping", pDb->name, pDb->status); + mnodeDecDbRef(pDb); return TSDB_CODE_MND_DB_IN_DROPPING; } @@ -2547,6 +2552,7 @@ static int32_t mnodeRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows if (pDb->status != TSDB_DB_STATUS_READY) { mError("db:%s, status:%d, in dropping", pDb->name, pDb->status); + mnodeDecDbRef(pDb); return 0; } @@ -2715,6 +2721,7 @@ static int32_t mnodeGetStreamTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, vo if (pDb->status != TSDB_DB_STATUS_READY) { mError("db:%s, status:%d, in dropping", pDb->name, pDb->status); + mnodeDecDbRef(pDb); return TSDB_CODE_MND_DB_IN_DROPPING; } @@ -2767,6 +2774,7 @@ static int32_t mnodeRetrieveStreamTables(SShowObj *pShow, char *data, int32_t ro if (pDb->status != TSDB_DB_STATUS_READY) { mError("db:%s, status:%d, in dropping", pDb->name, pDb->status); + mnodeDecDbRef(pDb); return 0; } diff --git a/src/mnode/src/mnodeUser.c b/src/mnode/src/mnodeUser.c index 6e63a8f2de..f4cb1a9ef3 100644 --- a/src/mnode/src/mnodeUser.c +++ b/src/mnode/src/mnodeUser.c @@ -581,8 +581,8 @@ void mnodeDropAllUsers(SAcctObj *pAcct) { int32_t mnodeRetriveAuth(char *user, char *spi, char *encrypt, char *secret, char *ckey) { if (!sdbIsMaster()) { *secret = 0; - mDebug("user:%s, failed to auth user, reason:%s", user, tstrerror(TSDB_CODE_RPC_NOT_READY)); - return TSDB_CODE_RPC_NOT_READY; + mDebug("user:%s, failed to auth user, reason:%s", user, tstrerror(TSDB_CODE_APP_NOT_READY)); + return TSDB_CODE_APP_NOT_READY; } SUserObj *pUser = mnodeGetUser(user); diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index ff253c1935..aa6631ff83 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -89,6 +89,7 @@ static int32_t mnodeVgroupActionInsert(SSdbOper *pOper) { if (pDb->status != TSDB_DB_STATUS_READY) { mError("vgId:%d, db:%s status:%d, in dropping", pVgroup->vgId, pDb->name, pDb->status); + mnodeDecDbRef(pDb); return TSDB_CODE_MND_DB_IN_DROPPING; } @@ -617,6 +618,7 @@ static int32_t mnodeGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *p if (pDb->status != TSDB_DB_STATUS_READY) { mError("db:%s, status:%d, in dropping", pDb->name, pDb->status); + mnodeDecDbRef(pDb); return TSDB_CODE_MND_DB_IN_DROPPING; } @@ -708,6 +710,7 @@ static int32_t mnodeRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, v if (pDb->status != TSDB_DB_STATUS_READY) { mError("db:%s, status:%d, in dropping", pDb->name, pDb->status); + mnodeDecDbRef(pDb); return 0; } @@ -784,7 +787,10 @@ void mnodeAddTableIntoVgroup(SVgObj *pVgroup, SChildTableObj *pTable) { if (pTable->sid >= 1) { taosIdPoolMarkStatus(pVgroup->idPool, pTable->sid); pVgroup->numOfTables++; - mnodeIncVgroupRef(pVgroup); + // The create vgroup message may be received later than the create table message + // and the writing order in sdb is therefore uncertain + // which will cause the reference count of the vgroup to be incorrect when restarting + // mnodeIncVgroupRef(pVgroup); } } @@ -792,7 +798,10 @@ void mnodeRemoveTableFromVgroup(SVgObj *pVgroup, SChildTableObj *pTable) { if (pTable->sid >= 1) { taosFreeId(pVgroup->idPool, pTable->sid); pVgroup->numOfTables--; - mnodeDecVgroupRef(pVgroup); + // The create vgroup message may be received later than the create table message + // and the writing order in sdb is therefore uncertain + // which will cause the reference count of the vgroup to be incorrect when restarting + // mnodeDecVgroupRef(pVgroup); } } diff --git a/src/os/inc/os.h b/src/os/inc/os.h index 4953416bde..df4b847fbb 100644 --- a/src/os/inc/os.h +++ b/src/os/inc/os.h @@ -40,6 +40,10 @@ extern "C" { #include "osAlpine.h" #endif +#ifdef _TD_NINGSI_60_ +#include "osNingsi.h" +#endif + #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) #include "osWindows.h" #endif diff --git a/src/os/inc/osDir.h b/src/os/inc/osDir.h index 73e4b216e6..e7dc04fd15 100644 --- a/src/os/inc/osDir.h +++ b/src/os/inc/osDir.h @@ -23,7 +23,7 @@ extern "C" { // TAOS_OS_FUNC_DIR void taosRemoveDir(char *rootDir); int taosMkDir(const char *pathname, mode_t mode); -void taosMvDir(char* destDir, char *srcDir); +void taosRename(char* oldName, char *newName); #ifdef __cplusplus } diff --git a/src/os/inc/osNingsi.h b/src/os/inc/osNingsi.h new file mode 100644 index 0000000000..da7f796b59 --- /dev/null +++ b/src/os/inc/osNingsi.h @@ -0,0 +1,138 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_OS_NINGSI_H +#define TDENGINE_OS_NINGSI_H + +#ifdef __cplusplus +extern "C" { +#endif + +#define TAOS_OS_FUNC_ATOMIC +/* + * type __sync_fetch_and_add (type *ptr, type value); + * type __sync_fetch_and_sub (type *ptr, type value); + * type __sync_fetch_and_or (type *ptr, type value); + * type __sync_fetch_and_and (type *ptr, type value); + * type __sync_fetch_and_xor (type *ptr, type value); + * type __sync_fetch_and_nand (type *ptr, type value); + * type __sync_add_and_fetch (type *ptr, type value); + * type __sync_sub_and_fetch (type *ptr, type value); + * type __sync_or_and_fetch (type *ptr, type value); + * type __sync_and_and_fetch (type *ptr, type value); + * type __sync_xor_and_fetch (type *ptr, type value); + * type __sync_nand_and_fetch (type *ptr, type value); + * + * bool __sync_bool_compare_and_swap (type*ptr, type oldval, type newval, ...) + * type __sync_val_compare_and_swap (type *ptr, type oldval, ?type newval, ...) + * */ + +#define atomic_load_8(ptr) __sync_fetch_and_add((ptr), 0) +#define atomic_load_16(ptr) __sync_fetch_and_add((ptr), 0) +#define atomic_load_32(ptr) __sync_fetch_and_add((ptr), 0) +#define atomic_load_64(ptr) __sync_fetch_and_add((ptr), 0) +#define atomic_load_ptr(ptr) __sync_fetch_and_add((ptr), 0) + +#define atomic_store_8(ptr, val) (*(ptr)=(val)) +#define atomic_store_16(ptr, val) (*(ptr)=(val)) +#define atomic_store_32(ptr, val) (*(ptr)=(val)) +#define atomic_store_64(ptr, val) (*(ptr)=(val)) +#define atomic_store_ptr(ptr, val) (*(ptr)=(val)) + +int8_t atomic_exchange_8_impl(int8_t* ptr, int8_t val ); +int16_t atomic_exchange_16_impl(int16_t* ptr, int16_t val ); +int32_t atomic_exchange_32_impl(int32_t* ptr, int32_t val ); +int64_t atomic_exchange_64_impl(int64_t* ptr, int64_t val ); +void* atomic_exchange_ptr_impl( void **ptr, void *val ); + +#define atomic_exchange_8(ptr, val) atomic_exchange_8_impl((int8_t*)ptr, (int8_t)val) +#define atomic_exchange_16(ptr, val) atomic_exchange_16_impl((int16_t*)ptr, (int16_t)val) +#define atomic_exchange_32(ptr, val) atomic_exchange_32_impl((int32_t*)ptr, (int32_t)val) +#define atomic_exchange_64(ptr, val) atomic_exchange_64_impl((int64_t*)ptr, (int64_t)val) +#define atomic_exchange_ptr(ptr, val) atomic_exchange_ptr_impl((void **)ptr, (void*)val) + +#define atomic_val_compare_exchange_8 __sync_val_compare_and_swap +#define atomic_val_compare_exchange_16 __sync_val_compare_and_swap +#define atomic_val_compare_exchange_32 __sync_val_compare_and_swap +#define atomic_val_compare_exchange_64 __sync_val_compare_and_swap +#define atomic_val_compare_exchange_ptr __sync_val_compare_and_swap + +#define atomic_add_fetch_8(ptr, val) __sync_add_and_fetch((ptr), (val)) +#define atomic_add_fetch_16(ptr, val) __sync_add_and_fetch((ptr), (val)) +#define atomic_add_fetch_32(ptr, val) __sync_add_and_fetch((ptr), (val)) +#define atomic_add_fetch_64(ptr, val) __sync_add_and_fetch((ptr), (val)) +#define atomic_add_fetch_ptr(ptr, val) __sync_add_and_fetch((ptr), (val)) + +#define atomic_fetch_add_8(ptr, val) __sync_fetch_and_add((ptr), (val)) +#define atomic_fetch_add_16(ptr, val) __sync_fetch_and_add((ptr), (val)) +#define atomic_fetch_add_32(ptr, val) __sync_fetch_and_add((ptr), (val)) +#define atomic_fetch_add_64(ptr, val) __sync_fetch_and_add((ptr), (val)) +#define atomic_fetch_add_ptr(ptr, val) __sync_fetch_and_add((ptr), (val)) + +#define atomic_sub_fetch_8(ptr, val) __sync_sub_and_fetch((ptr), (val)) +#define atomic_sub_fetch_16(ptr, val) __sync_sub_and_fetch((ptr), (val)) +#define atomic_sub_fetch_32(ptr, val) __sync_sub_and_fetch((ptr), (val)) +#define atomic_sub_fetch_64(ptr, val) __sync_sub_and_fetch((ptr), (val)) +#define atomic_sub_fetch_ptr(ptr, val) __sync_sub_and_fetch((ptr), (val)) + +#define atomic_fetch_sub_8(ptr, val) __sync_fetch_and_sub((ptr), (val)) +#define atomic_fetch_sub_16(ptr, val) __sync_fetch_and_sub((ptr), (val)) +#define atomic_fetch_sub_32(ptr, val) __sync_fetch_and_sub((ptr), (val)) +#define atomic_fetch_sub_64(ptr, val) __sync_fetch_and_sub((ptr), (val)) +#define atomic_fetch_sub_ptr(ptr, val) __sync_fetch_and_sub((ptr), (val)) + +#define atomic_and_fetch_8(ptr, val) __sync_and_and_fetch((ptr), (val)) +#define atomic_and_fetch_16(ptr, val) __sync_and_and_fetch((ptr), (val)) +#define atomic_and_fetch_32(ptr, val) __sync_and_and_fetch((ptr), (val)) +#define atomic_and_fetch_64(ptr, val) __sync_and_and_fetch((ptr), (val)) +#define atomic_and_fetch_ptr(ptr, val) __sync_and_and_fetch((ptr), (val)) + +#define atomic_fetch_and_8(ptr, val) __sync_fetch_and_and((ptr), (val)) +#define atomic_fetch_and_16(ptr, val) __sync_fetch_and_and((ptr), (val)) +#define atomic_fetch_and_32(ptr, val) __sync_fetch_and_and((ptr), (val)) +#define atomic_fetch_and_64(ptr, val) __sync_fetch_and_and((ptr), (val)) +#define atomic_fetch_and_ptr(ptr, val) __sync_fetch_and_and((ptr), (val)) + +#define atomic_or_fetch_8(ptr, val) __sync_or_and_fetch((ptr), (val)) +#define atomic_or_fetch_16(ptr, val) __sync_or_and_fetch((ptr), (val)) +#define atomic_or_fetch_32(ptr, val) __sync_or_and_fetch((ptr), (val)) +#define atomic_or_fetch_64(ptr, val) __sync_or_and_fetch((ptr), (val)) +#define atomic_or_fetch_ptr(ptr, val) __sync_or_and_fetch((ptr), (val)) + +#define atomic_fetch_or_8(ptr, val) __sync_fetch_and_or((ptr), (val)) +#define atomic_fetch_or_16(ptr, val) __sync_fetch_and_or((ptr), (val)) +#define atomic_fetch_or_32(ptr, val) __sync_fetch_and_or((ptr), (val)) +#define atomic_fetch_or_64(ptr, val) __sync_fetch_and_or((ptr), (val)) +#define atomic_fetch_or_ptr(ptr, val) __sync_fetch_and_or((ptr), (val)) + +#define atomic_xor_fetch_8(ptr, val) __sync_xor_and_fetch((ptr), (val)) +#define atomic_xor_fetch_16(ptr, val) __sync_xor_and_fetch((ptr), (val)) +#define atomic_xor_fetch_32(ptr, val) __sync_xor_and_fetch((ptr), (val)) +#define atomic_xor_fetch_64(ptr, val) __sync_xor_and_fetch((ptr), (val)) +#define atomic_xor_fetch_ptr(ptr, val) __sync_xor_and_fetch((ptr), (val)) + +#define atomic_fetch_xor_8(ptr, val) __sync_fetch_and_xor((ptr), (val)) +#define atomic_fetch_xor_16(ptr, val) __sync_fetch_and_xor((ptr), (val)) +#define atomic_fetch_xor_32(ptr, val) __sync_fetch_and_xor((ptr), (val)) +#define atomic_fetch_xor_64(ptr, val) __sync_fetch_and_xor((ptr), (val)) +#define atomic_fetch_xor_ptr(ptr, val) __sync_fetch_and_xor((ptr), (val)) + + + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/os/inc/osSemphone.h b/src/os/inc/osSemphone.h index fd88d2d798..4280b458a6 100644 --- a/src/os/inc/osSemphone.h +++ b/src/os/inc/osSemphone.h @@ -23,7 +23,7 @@ extern "C" { #ifndef TAOS_OS_FUNC_SEMPHONE #define tsem_t sem_t #define tsem_init sem_init - #define tsem_wait sem_wait + int tsem_wait(tsem_t* sem); #define tsem_post sem_post #define tsem_destroy sem_destroy #endif diff --git a/src/os/inc/osWindows.h b/src/os/inc/osWindows.h index 348f50891e..0dbc3aac9c 100644 --- a/src/os/inc/osWindows.h +++ b/src/os/inc/osWindows.h @@ -230,8 +230,12 @@ void wordfree(wordexp_t *pwordexp); #define atomic_exchange_16(ptr, val) _InterlockedExchange16((short volatile*)(ptr), (short)(val)) #define atomic_exchange_32(ptr, val) _InterlockedExchange((long volatile*)(ptr), (long)(val)) #define atomic_exchange_64(ptr, val) _InterlockedExchange64((__int64 volatile*)(ptr), (__int64)(val)) - #define atomic_exchange_ptr(ptr, val) _InterlockedExchangePointer((void* volatile*)(ptr), (void*)(val)) - + #ifdef _WIN64 + #define atomic_exchange_ptr(ptr, val) _InterlockedExchangePointer((void* volatile*)(ptr), (void*)(val)) + #else + #define atomic_exchange_ptr(ptr, val) _InlineInterlockedExchangePointer((void* volatile*)(ptr), (void*)(val)) + #endif + #ifdef _TD_GO_DLL_ #define atomic_val_compare_exchange_8 __sync_val_compare_and_swap #else @@ -247,162 +251,104 @@ void wordfree(wordexp_t *pwordexp); long interlocked_add_fetch_32(long volatile *ptr, long val); __int64 interlocked_add_fetch_64(__int64 volatile *ptr, __int64 val); + char interlocked_and_fetch_8(char volatile* ptr, char val); + short interlocked_and_fetch_16(short volatile* ptr, short val); + long interlocked_and_fetch_32(long volatile* ptr, long val); + __int64 interlocked_and_fetch_64(__int64 volatile* ptr, __int64 val); + + __int64 interlocked_fetch_and_64(__int64 volatile* ptr, __int64 val); + + char interlocked_or_fetch_8(char volatile* ptr, char val); + short interlocked_or_fetch_16(short volatile* ptr, short val); + long interlocked_or_fetch_32(long volatile* ptr, long val); + __int64 interlocked_or_fetch_64(__int64 volatile* ptr, __int64 val); + + char interlocked_xor_fetch_8(char volatile* ptr, char val); + short interlocked_xor_fetch_16(short volatile* ptr, short val); + long interlocked_xor_fetch_32(long volatile* ptr, long val); + __int64 interlocked_xor_fetch_64(__int64 volatile* ptr, __int64 val); + + __int64 interlocked_fetch_xor_64(__int64 volatile* ptr, __int64 val); + #define atomic_add_fetch_8(ptr, val) interlocked_add_fetch_8((char volatile*)(ptr), (char)(val)) #define atomic_add_fetch_16(ptr, val) interlocked_add_fetch_16((short volatile*)(ptr), (short)(val)) #define atomic_add_fetch_32(ptr, val) interlocked_add_fetch_32((long volatile*)(ptr), (long)(val)) #define atomic_add_fetch_64(ptr, val) interlocked_add_fetch_64((__int64 volatile*)(ptr), (__int64)(val)) - #ifdef _WIN64 - #define atomic_add_fetch_ptr atomic_add_fetch_64 - #else - #define atomic_add_fetch_ptr atomic_add_fetch_32 - #endif - #ifdef _TD_GO_DLL_ #define atomic_fetch_add_8 __sync_fetch_and_ad #define atomic_fetch_add_16 __sync_fetch_and_add #else #define atomic_fetch_add_8(ptr, val) _InterlockedExchangeAdd8((char volatile*)(ptr), (char)(val)) #define atomic_fetch_add_16(ptr, val) _InterlockedExchangeAdd16((short volatile*)(ptr), (short)(val)) - #endif + #endif + #define atomic_fetch_add_8(ptr, val) _InterlockedExchangeAdd8((char volatile*)(ptr), (char)(val)) + #define atomic_fetch_add_16(ptr, val) _InterlockedExchangeAdd16((short volatile*)(ptr), (short)(val)) #define atomic_fetch_add_32(ptr, val) _InterlockedExchangeAdd((long volatile*)(ptr), (long)(val)) #define atomic_fetch_add_64(ptr, val) _InterlockedExchangeAdd64((__int64 volatile*)(ptr), (__int64)(val)) - #ifdef _WIN64 - #define atomic_fetch_add_ptr atomic_fetch_add_64 - #else - #define atomic_fetch_add_ptr atomic_fetch_add_32 - #endif - + #define atomic_sub_fetch_8(ptr, val) interlocked_add_fetch_8((char volatile*)(ptr), -(char)(val)) #define atomic_sub_fetch_16(ptr, val) interlocked_add_fetch_16((short volatile*)(ptr), -(short)(val)) #define atomic_sub_fetch_32(ptr, val) interlocked_add_fetch_32((long volatile*)(ptr), -(long)(val)) #define atomic_sub_fetch_64(ptr, val) interlocked_add_fetch_64((__int64 volatile*)(ptr), -(__int64)(val)) - #ifdef _WIN64 - #define atomic_sub_fetch_ptr atomic_sub_fetch_64 - #else - #define atomic_sub_fetch_ptr atomic_sub_fetch_32 - #endif #define atomic_fetch_sub_8(ptr, val) _InterlockedExchangeAdd8((char volatile*)(ptr), -(char)(val)) #define atomic_fetch_sub_16(ptr, val) _InterlockedExchangeAdd16((short volatile*)(ptr), -(short)(val)) #define atomic_fetch_sub_32(ptr, val) _InterlockedExchangeAdd((long volatile*)(ptr), -(long)(val)) #define atomic_fetch_sub_64(ptr, val) _InterlockedExchangeAdd64((__int64 volatile*)(ptr), -(__int64)(val)) - #ifdef _WIN64 - #define atomic_fetch_sub_ptr atomic_fetch_sub_64 - #else - #define atomic_fetch_sub_ptr atomic_fetch_sub_32 - #endif - #ifndef _TD_GO_DLL_ - char interlocked_and_fetch_8(char volatile* ptr, char val); - short interlocked_and_fetch_16(short volatile* ptr, short val); - #endif - long interlocked_and_fetch_32(long volatile* ptr, long val); - __int64 interlocked_and_fetch_64(__int64 volatile* ptr, __int64 val); - - #ifndef _TD_GO_DLL_ - #define atomic_and_fetch_8(ptr, val) interlocked_and_fetch_8((char volatile*)(ptr), (char)(val)) - #define atomic_and_fetch_16(ptr, val) interlocked_and_fetch_16((short volatile*)(ptr), (short)(val)) - #endif + #define atomic_and_fetch_8(ptr, val) interlocked_and_fetch_8((char volatile*)(ptr), (char)(val)) + #define atomic_and_fetch_16(ptr, val) interlocked_and_fetch_16((short volatile*)(ptr), (short)(val)) #define atomic_and_fetch_32(ptr, val) interlocked_and_fetch_32((long volatile*)(ptr), (long)(val)) #define atomic_and_fetch_64(ptr, val) interlocked_and_fetch_64((__int64 volatile*)(ptr), (__int64)(val)) - #ifdef _WIN64 - #define atomic_and_fetch_ptr atomic_and_fetch_64 - #else - #define atomic_and_fetch_ptr atomic_and_fetch_32 - #endif - #ifndef _TD_GO_DLL_ - #define atomic_fetch_and_8(ptr, val) _InterlockedAnd8((char volatile*)(ptr), (char)(val)) - #define atomic_fetch_and_16(ptr, val) _InterlockedAnd16((short volatile*)(ptr), (short)(val)) - #endif + + #define atomic_fetch_and_8(ptr, val) _InterlockedAnd8((char volatile*)(ptr), (char)(val)) + #define atomic_fetch_and_16(ptr, val) _InterlockedAnd16((short volatile*)(ptr), (short)(val)) #define atomic_fetch_and_32(ptr, val) _InterlockedAnd((long volatile*)(ptr), (long)(val)) + #define atomic_fetch_and_64(ptr, val) interlocked_fetch_and_64((__int64 volatile*)(ptr), (__int64)(val)) - #ifdef _M_IX86 - __int64 interlocked_fetch_and_64(__int64 volatile* ptr, __int64 val); - #define atomic_fetch_and_64(ptr, val) interlocked_fetch_and_64((__int64 volatile*)(ptr), (__int64)(val)) - #else - #define atomic_fetch_and_64(ptr, val) _InterlockedAnd64((__int64 volatile*)(ptr), (__int64)(val)) - #endif - - #ifdef _WIN64 - #define atomic_fetch_and_ptr atomic_fetch_and_64 - #else - #define atomic_fetch_and_ptr atomic_fetch_and_32 - #endif - #ifndef _TD_GO_DLL_ - char interlocked_or_fetch_8(char volatile* ptr, char val); - short interlocked_or_fetch_16(short volatile* ptr, short val); - #endif - long interlocked_or_fetch_32(long volatile* ptr, long val); - __int64 interlocked_or_fetch_64(__int64 volatile* ptr, __int64 val); - - #ifndef _TD_GO_DLL_ - #define atomic_or_fetch_8(ptr, val) interlocked_or_fetch_8((char volatile*)(ptr), (char)(val)) - #define atomic_or_fetch_16(ptr, val) interlocked_or_fetch_16((short volatile*)(ptr), (short)(val)) - #endif + #define atomic_or_fetch_8(ptr, val) interlocked_or_fetch_8((char volatile*)(ptr), (char)(val)) + #define atomic_or_fetch_16(ptr, val) interlocked_or_fetch_16((short volatile*)(ptr), (short)(val)) #define atomic_or_fetch_32(ptr, val) interlocked_or_fetch_32((long volatile*)(ptr), (long)(val)) #define atomic_or_fetch_64(ptr, val) interlocked_or_fetch_64((__int64 volatile*)(ptr), (__int64)(val)) - #ifdef _WIN64 - #define atomic_or_fetch_ptr atomic_or_fetch_64 - #else - #define atomic_or_fetch_ptr atomic_or_fetch_32 - #endif - #ifndef _TD_GO_DLL_ - #define atomic_fetch_or_8(ptr, val) _InterlockedOr8((char volatile*)(ptr), (char)(val)) - #define atomic_fetch_or_16(ptr, val) _InterlockedOr16((short volatile*)(ptr), (short)(val)) - #endif + + #define atomic_fetch_or_8(ptr, val) _InterlockedOr8((char volatile*)(ptr), (char)(val)) + #define atomic_fetch_or_16(ptr, val) _InterlockedOr16((short volatile*)(ptr), (short)(val)) #define atomic_fetch_or_32(ptr, val) _InterlockedOr((long volatile*)(ptr), (long)(val)) + #define atomic_fetch_or_64(ptr, val) interlocked_fetch_or_64((__int64 volatile*)(ptr), (__int64)(val)) - #ifdef _M_IX86 - __int64 interlocked_fetch_or_64(__int64 volatile* ptr, __int64 val); - #define atomic_fetch_or_64(ptr, val) interlocked_fetch_or_64((__int64 volatile*)(ptr), (__int64)(val)) - #else - #define atomic_fetch_or_64(ptr, val) _InterlockedOr64((__int64 volatile*)(ptr), (__int64)(val)) - #endif - - #ifdef _WIN64 - #define atomic_fetch_or_ptr atomic_fetch_or_64 - #else - #define atomic_fetch_or_ptr atomic_fetch_or_32 - #endif - - #ifndef _TD_GO_DLL_ - char interlocked_xor_fetch_8(char volatile* ptr, char val); - short interlocked_xor_fetch_16(short volatile* ptr, short val); - #endif - long interlocked_xor_fetch_32(long volatile* ptr, long val); - __int64 interlocked_xor_fetch_64(__int64 volatile* ptr, __int64 val); - - #ifndef _TD_GO_DLL_ - #define atomic_xor_fetch_8(ptr, val) interlocked_xor_fetch_8((char volatile*)(ptr), (char)(val)) - #define atomic_xor_fetch_16(ptr, val) interlocked_xor_fetch_16((short volatile*)(ptr), (short)(val)) - #endif + #define atomic_xor_fetch_8(ptr, val) interlocked_xor_fetch_8((char volatile*)(ptr), (char)(val)) + #define atomic_xor_fetch_16(ptr, val) interlocked_xor_fetch_16((short volatile*)(ptr), (short)(val)) #define atomic_xor_fetch_32(ptr, val) interlocked_xor_fetch_32((long volatile*)(ptr), (long)(val)) #define atomic_xor_fetch_64(ptr, val) interlocked_xor_fetch_64((__int64 volatile*)(ptr), (__int64)(val)) - #ifdef _WIN64 - #define atomic_xor_fetch_ptr atomic_xor_fetch_64 - #else - #define atomic_xor_fetch_ptr atomic_xor_fetch_32 - #endif - #ifndef _TD_GO_DLL_ - #define atomic_fetch_xor_8(ptr, val) _InterlockedXor8((char volatile*)(ptr), (char)(val)) - #define atomic_fetch_xor_16(ptr, val) _InterlockedXor16((short volatile*)(ptr), (short)(val)) - #endif + #define atomic_fetch_xor_8(ptr, val) _InterlockedXor8((char volatile*)(ptr), (char)(val)) + #define atomic_fetch_xor_16(ptr, val) _InterlockedXor16((short volatile*)(ptr), (short)(val)) #define atomic_fetch_xor_32(ptr, val) _InterlockedXor((long volatile*)(ptr), (long)(val)) - - #ifdef _M_IX86 - __int64 interlocked_fetch_xor_64(__int64 volatile* ptr, __int64 val); - #define atomic_fetch_xor_64(ptr, val) interlocked_fetch_xor_64((__int64 volatile*)(ptr), (__int64)(val)) - #else - #define atomic_fetch_xor_64(ptr, val) _InterlockedXor64((__int64 volatile*)(ptr), (__int64)(val)) - #endif + #define atomic_fetch_xor_64(ptr, val) interlocked_fetch_xor_64((__int64 volatile*)(ptr), (__int64)(val)) #ifdef _WIN64 + #define atomic_add_fetch_ptr atomic_add_fetch_64 + #define atomic_fetch_add_ptr atomic_fetch_add_64 + #define atomic_sub_fetch_ptr atomic_sub_fetch_64 + #define atomic_fetch_sub_ptr atomic_fetch_sub_64 + #define atomic_and_fetch_ptr atomic_and_fetch_64 + #define atomic_fetch_and_ptr atomic_fetch_and_64 + #define atomic_or_fetch_ptr atomic_or_fetch_64 + #define atomic_fetch_or_ptr atomic_fetch_or_64 + #define atomic_xor_fetch_ptr atomic_xor_fetch_64 #define atomic_fetch_xor_ptr atomic_fetch_xor_64 #else + #define atomic_add_fetch_ptr atomic_add_fetch_32 + #define atomic_fetch_add_ptr atomic_fetch_add_32 + #define atomic_sub_fetch_ptr atomic_sub_fetch_32 + #define atomic_fetch_sub_ptr atomic_fetch_sub_32 + #define atomic_and_fetch_ptr atomic_and_fetch_32 + #define atomic_fetch_and_ptr atomic_fetch_and_32 + #define atomic_or_fetch_ptr atomic_or_fetch_32 + #define atomic_fetch_or_ptr atomic_fetch_or_32 + #define atomic_xor_fetch_ptr atomic_xor_fetch_32 #define atomic_fetch_xor_ptr atomic_fetch_xor_32 #endif - #ifdef __cplusplus } #endif diff --git a/src/os/src/detail/osDir.c b/src/os/src/detail/osDir.c index 9496b74405..7a537cdfea 100644 --- a/src/os/src/detail/osDir.c +++ b/src/os/src/detail/osDir.c @@ -50,18 +50,19 @@ int taosMkDir(const char *path, mode_t mode) { return code; } -void taosMvDir(char* destDir, char *srcDir) { +void taosRename(char* oldName, char *newName) { if (0 == tsEnableVnodeBak) { uInfo("vnode backup not enabled"); return; } - char shellCmd[1024+1] = {0}; - - //(void)snprintf(shellCmd, 1024, "cp -rf %s %s", srcDir, destDir); - (void)snprintf(shellCmd, 1024, "mv %s %s", srcDir, destDir); - taosSystem(shellCmd); - uInfo("shell cmd:%s is executed", shellCmd); + // if newName in not empty, rename return fail. + // the newName must be empty or does not exist + if (rename(oldName, newName)) { + uError("%s is modify to %s fail, reason:%s", oldName, newName, strerror(errno)); + } else { + uInfo("%s is modify to %s success!", oldName, newName); + } } -#endif \ No newline at end of file +#endif diff --git a/src/os/src/detail/osSemphone.c b/src/os/src/detail/osSemphone.c index 74f8859029..b91888845e 100644 --- a/src/os/src/detail/osSemphone.c +++ b/src/os/src/detail/osSemphone.c @@ -16,6 +16,18 @@ #define _DEFAULT_SOURCE #include "os.h" +#ifndef TAOS_OS_FUNC_SEMPHONE + +int tsem_wait(tsem_t* sem) { + int ret = 0; + do { + ret = sem_wait(sem); + } while (ret != 0 && errno == EINTR); + return ret; +} + +#endif + #ifndef TAOS_OS_FUNC_SEMPHONE_PTHREAD bool taosCheckPthreadValid(pthread_t thread) { return thread != 0; } diff --git a/src/os/src/linux/ningsi.c b/src/os/src/linux/ningsi.c new file mode 100644 index 0000000000..793ccac84a --- /dev/null +++ b/src/os/src/linux/ningsi.c @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" + +#ifdef _TD_NINGSI_60_ +void* atomic_exchange_ptr_impl(void** ptr, void* val ) { + void *old; + do { + old = *ptr; + } while( !__sync_bool_compare_and_swap(ptr, old, val) ); + return old; +} +int8_t atomic_exchange_8_impl(int8_t* ptr, int8_t val ) { + int8_t old; + do { + old = *ptr; + } while( !__sync_bool_compare_and_swap(ptr, old, val) ); + return old; +} +int16_t atomic_exchange_16_impl(int16_t* ptr, int16_t val ) { + int16_t old; + do { + old = *ptr; + } while( !__sync_bool_compare_and_swap(ptr, old, val) ); + return old; +} +int32_t atomic_exchange_32_impl(int32_t* ptr, int32_t val ) { + int32_t old; + do { + old = *ptr; + } while( !__sync_bool_compare_and_swap(ptr, old, val) ); + return old; +} +int64_t atomic_exchange_64_impl(int64_t* ptr, int64_t val ) { + int64_t old; + do { + old = *ptr; + } while( !__sync_bool_compare_and_swap(ptr, old, val) ); + return old; +} +#endif + diff --git a/src/os/src/windows/CMakeLists.txt b/src/os/src/windows/CMakeLists.txt index 588d3b7f68..9dcc9e7e6d 100644 --- a/src/os/src/windows/CMakeLists.txt +++ b/src/os/src/windows/CMakeLists.txt @@ -4,4 +4,4 @@ PROJECT(TDengine) AUX_SOURCE_DIRECTORY(. SRC) ADD_LIBRARY(os ${SRC}) -TARGET_LINK_LIBRARIES(os winmm IPHLPAPI ws2_32 MsvcLibXw64) +TARGET_LINK_LIBRARIES(os winmm IPHLPAPI ws2_32 MsvcLibXw) diff --git a/src/os/src/windows/w64Atomic.c b/src/os/src/windows/wAtomic.c similarity index 79% rename from src/os/src/windows/w64Atomic.c rename to src/os/src/windows/wAtomic.c index 9fc3eae672..a025cb8f0e 100644 --- a/src/os/src/windows/w64Atomic.c +++ b/src/os/src/windows/wAtomic.c @@ -23,19 +23,19 @@ // add char interlocked_add_fetch_8(char volatile* ptr, char val) { - #ifdef _TD_GO_DLL_ - return __sync_fetch_and_add(ptr, val) + val; - #else - return _InterlockedExchangeAdd8(ptr, val) + val; - #endif +#ifdef _TD_GO_DLL_ + return __sync_fetch_and_add(ptr, val) + val; +#else + return _InterlockedExchangeAdd8(ptr, val) + val; +#endif } short interlocked_add_fetch_16(short volatile* ptr, short val) { - #ifdef _TD_GO_DLL_ - return __sync_fetch_and_add(ptr, val) + val; - #else - return _InterlockedExchangeAdd16(ptr, val) + val; - #endif +#ifdef _TD_GO_DLL_ + return __sync_fetch_and_add(ptr, val) + val; +#else + return _InterlockedExchangeAdd16(ptr, val) + val; +#endif } long interlocked_add_fetch_32(long volatile* ptr, long val) { @@ -43,15 +43,13 @@ long interlocked_add_fetch_32(long volatile* ptr, long val) { } __int64 interlocked_add_fetch_64(__int64 volatile* ptr, __int64 val) { -#ifdef _WIN64 +//#ifdef _WIN64 return _InterlockedExchangeAdd64(ptr, val) + val; -#else - return _InterlockedExchangeAdd(ptr, val) + val; -#endif +//#else +// return _InterlockedExchangeAdd(ptr, val) + val; +//#endif } -// and -#ifndef _TD_GO_DLL_ char interlocked_and_fetch_8(char volatile* ptr, char val) { return _InterlockedAnd8(ptr, val) & val; } @@ -59,41 +57,37 @@ char interlocked_and_fetch_8(char volatile* ptr, char val) { short interlocked_and_fetch_16(short volatile* ptr, short val) { return _InterlockedAnd16(ptr, val) & val; } -#endif long interlocked_and_fetch_32(long volatile* ptr, long val) { return _InterlockedAnd(ptr, val) & val; } + +__int64 interlocked_and_fetch_64(__int64 volatile* ptr, __int64 val) { #ifndef _M_IX86 - -__int64 interlocked_and_fetch_64(__int64 volatile* ptr, __int64 val) { return _InterlockedAnd64(ptr, val) & val; -} - #else - -__int64 interlocked_and_fetch_64(__int64 volatile* ptr, __int64 val) { __int64 old, res; do { old = *ptr; res = old & val; - } while(_InterlockedCompareExchange64(ptr, res, old) != old); + } while (_InterlockedCompareExchange64(ptr, res, old) != old); return res; +#endif } __int64 interlocked_fetch_and_64(__int64 volatile* ptr, __int64 val) { +#ifdef _M_IX86 __int64 old; do { old = *ptr; - } while(_InterlockedCompareExchange64(ptr, old & val, old) != old); + } while (_InterlockedCompareExchange64(ptr, old & val, old) != old); return old; +#else + return _InterlockedAnd64((__int64 volatile*)(ptr), (__int64)(val)); +#endif } -#endif - -// or -#ifndef _TD_GO_DLL_ char interlocked_or_fetch_8(char volatile* ptr, char val) { return _InterlockedOr8(ptr, val) | val; } @@ -101,40 +95,36 @@ char interlocked_or_fetch_8(char volatile* ptr, char val) { short interlocked_or_fetch_16(short volatile* ptr, short val) { return _InterlockedOr16(ptr, val) | val; } -#endif + long interlocked_or_fetch_32(long volatile* ptr, long val) { return _InterlockedOr(ptr, val) | val; } -#ifndef _M_IX86 - -__int64 interlocked_or_fetch_64(__int64 volatile* ptr, __int64 val) { - return _InterlockedOr64(ptr, val) & val; -} - -#else - __int64 interlocked_or_fetch_64(__int64 volatile* ptr, __int64 val) { +#ifdef _M_IX86 __int64 old, res; do { old = *ptr; res = old | val; } while(_InterlockedCompareExchange64(ptr, res, old) != old); return res; +#else + return _InterlockedOr64(ptr, val) & val; +#endif } __int64 interlocked_fetch_or_64(__int64 volatile* ptr, __int64 val) { +#ifdef _M_IX86 __int64 old; do { old = *ptr; } while(_InterlockedCompareExchange64(ptr, old | val, old) != old); return old; +#else + return _InterlockedOr64((__int64 volatile*)(ptr), (__int64)(val)); +#endif } -#endif - -// xor -#ifndef _TD_GO_DLL_ char interlocked_xor_fetch_8(char volatile* ptr, char val) { return _InterlockedXor8(ptr, val) ^ val; } @@ -142,35 +132,33 @@ char interlocked_xor_fetch_8(char volatile* ptr, char val) { short interlocked_xor_fetch_16(short volatile* ptr, short val) { return _InterlockedXor16(ptr, val) ^ val; } -#endif + long interlocked_xor_fetch_32(long volatile* ptr, long val) { return _InterlockedXor(ptr, val) ^ val; } -#ifndef _M_IX86 - -__int64 interlocked_xor_fetch_64(__int64 volatile* ptr, __int64 val) { - return _InterlockedXor64(ptr, val) ^ val; -} - -#else - __int64 interlocked_xor_fetch_64(__int64 volatile* ptr, __int64 val) { +#ifdef _M_IX86 __int64 old, res; do { old = *ptr; res = old ^ val; } while(_InterlockedCompareExchange64(ptr, res, old) != old); return res; +#else + return _InterlockedXor64(ptr, val) ^ val; +#endif } __int64 interlocked_fetch_xor_64(__int64 volatile* ptr, __int64 val) { +#ifdef _M_IX86 __int64 old; do { old = *ptr; - } while(_InterlockedCompareExchange64(ptr, old ^ val, old) != old); + } while (_InterlockedCompareExchange64(ptr, old ^ val, old) != old); return old; +#else + return _InterlockedXor64((__int64 volatile*)(ptr), (__int64)(val)); +#endif } -#endif - diff --git a/src/os/src/windows/w64Dir.c b/src/os/src/windows/wDir.c similarity index 100% rename from src/os/src/windows/w64Dir.c rename to src/os/src/windows/wDir.c diff --git a/src/os/src/windows/w64Env.c b/src/os/src/windows/wEnv.c similarity index 100% rename from src/os/src/windows/w64Env.c rename to src/os/src/windows/wEnv.c diff --git a/src/os/src/windows/w64File.c b/src/os/src/windows/wFile.c similarity index 100% rename from src/os/src/windows/w64File.c rename to src/os/src/windows/wFile.c diff --git a/src/os/src/windows/w64Getline.c b/src/os/src/windows/wGetline.c similarity index 100% rename from src/os/src/windows/w64Getline.c rename to src/os/src/windows/wGetline.c diff --git a/src/os/src/windows/w64Godll.c b/src/os/src/windows/wGodll.c similarity index 100% rename from src/os/src/windows/w64Godll.c rename to src/os/src/windows/wGodll.c diff --git a/src/os/src/windows/w64Lz4.c b/src/os/src/windows/wLz4.c similarity index 100% rename from src/os/src/windows/w64Lz4.c rename to src/os/src/windows/wLz4.c diff --git a/src/os/src/windows/w64Semphone.c b/src/os/src/windows/wSemphone.c similarity index 100% rename from src/os/src/windows/w64Semphone.c rename to src/os/src/windows/wSemphone.c diff --git a/src/os/src/windows/w64Socket.c b/src/os/src/windows/wSocket.c similarity index 100% rename from src/os/src/windows/w64Socket.c rename to src/os/src/windows/wSocket.c diff --git a/src/os/src/windows/w64String.c b/src/os/src/windows/wString.c similarity index 100% rename from src/os/src/windows/w64String.c rename to src/os/src/windows/wString.c diff --git a/src/os/src/windows/w64Strptime.c b/src/os/src/windows/wStrptime.c similarity index 100% rename from src/os/src/windows/w64Strptime.c rename to src/os/src/windows/wStrptime.c diff --git a/src/os/src/windows/w64Sysinfo.c b/src/os/src/windows/wSysinfo.c similarity index 96% rename from src/os/src/windows/w64Sysinfo.c rename to src/os/src/windows/wSysinfo.c index eb252a0b86..61adc3ee14 100644 --- a/src/os/src/windows/w64Sysinfo.c +++ b/src/os/src/windows/wSysinfo.c @@ -154,13 +154,15 @@ bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) { bool taosGetProcMemory(float *memoryUsedMB) { unsigned bytes_used = 0; +#if 0 #if defined(_WIN32) && defined(_MSC_VER) - PROCESS_MEMORY_COUNTERS pmc; - HANDLE cur_proc = GetCurrentProcess(); + PROCESS_MEMORY_COUNTERS pmc; + HANDLE cur_proc = GetCurrentProcess(); - if (GetProcessMemoryInfo(cur_proc, &pmc, sizeof(pmc))) { - bytes_used = (unsigned)(pmc.WorkingSetSize + pmc.PagefileUsage); - } + if (GetProcessMemoryInfo(cur_proc, &pmc, sizeof(pmc))) { + bytes_used = (unsigned)(pmc.WorkingSetSize + pmc.PagefileUsage); + } +#endif #endif *memoryUsedMB = (float)bytes_used / 1024 / 1024; diff --git a/src/os/src/windows/w64Time.c b/src/os/src/windows/wTime.c similarity index 100% rename from src/os/src/windows/w64Time.c rename to src/os/src/windows/wTime.c diff --git a/src/os/src/windows/w64Timer.c b/src/os/src/windows/wTimer.c similarity index 100% rename from src/os/src/windows/w64Timer.c rename to src/os/src/windows/wTimer.c diff --git a/src/os/src/windows/w64Wordexp.c b/src/os/src/windows/wWordexp.c similarity index 100% rename from src/os/src/windows/w64Wordexp.c rename to src/os/src/windows/wWordexp.c diff --git a/src/plugins/http/CMakeLists.txt b/src/plugins/http/CMakeLists.txt index 94f5305f28..2c3cbf636f 100644 --- a/src/plugins/http/CMakeLists.txt +++ b/src/plugins/http/CMakeLists.txt @@ -11,7 +11,12 @@ AUX_SOURCE_DIRECTORY(src SRC) IF (TD_LINUX) ADD_LIBRARY(http ${SRC}) - TARGET_LINK_LIBRARIES(http taos z) + + IF (TD_SOMODE_STATIC) + TARGET_LINK_LIBRARIES(http taos_static z) + ELSE () + TARGET_LINK_LIBRARIES(http taos z) + ENDIF () IF (TD_ADMIN) TARGET_LINK_LIBRARIES(http admin) diff --git a/src/plugins/monitor/CMakeLists.txt b/src/plugins/monitor/CMakeLists.txt index ba30edd0e4..26a7775e9c 100644 --- a/src/plugins/monitor/CMakeLists.txt +++ b/src/plugins/monitor/CMakeLists.txt @@ -8,5 +8,10 @@ IF (TD_LINUX) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc) ADD_LIBRARY(monitor ${SRC}) - TARGET_LINK_LIBRARIES(monitor taos) + + IF (TD_SOMODE_STATIC) + TARGET_LINK_LIBRARIES(monitor taos_static) + ELSE () + TARGET_LINK_LIBRARIES(monitor taos) + ENDIF () ENDIF () diff --git a/src/plugins/mqtt/CMakeLists.txt b/src/plugins/mqtt/CMakeLists.txt index 72312ffcec..2467af588c 100644 --- a/src/plugins/mqtt/CMakeLists.txt +++ b/src/plugins/mqtt/CMakeLists.txt @@ -10,8 +10,12 @@ IF (TD_LINUX) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/MQTT-C/include) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/MQTT-C/examples/templates) ADD_LIBRARY(mqtt ${SRC}) - TARGET_LINK_LIBRARIES(mqtt taos cJson mqttc) + IF (TD_SOMODE_STATIC) + TARGET_LINK_LIBRARIES(mqtt taos_static cJson mqttc) + ELSE () + TARGET_LINK_LIBRARIES(mqtt taos cJson mqttc) + ENDIF () IF (TD_ADMIN) TARGET_LINK_LIBRARIES(mqtt admin cJson) ENDIF () diff --git a/src/query/inc/qAst.h b/src/query/inc/qAst.h index ec568a6cdb..547616dee6 100644 --- a/src/query/inc/qAst.h +++ b/src/query/inc/qAst.h @@ -80,7 +80,7 @@ void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, S void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order, char *(*cb)(void *, const char*, int32_t)); -uint8_t getBinaryExprOptr(SSQLToken *pToken); +uint8_t getBinaryExprOptr(SStrToken *pToken); void tExprNodeDestroy(tExprNode *pNode, void (*fp)(void *)); void exprTreeToBinary(SBufferWriter* bw, tExprNode* pExprTree); diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index b5487561b2..7093495763 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -33,6 +33,17 @@ struct SColumnFilterElem; typedef bool (*__filter_func_t)(struct SColumnFilterElem* pFilter, char* val1, char* val2); typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order); +typedef struct SPosInfo { + int32_t pageId:20; + int32_t rowId:12; +} SPosInfo; + +typedef struct SGroupResInfo { + int32_t groupId; + int32_t numOfDataPages; + SPosInfo pos; +} SGroupResInfo; + typedef struct SSqlGroupbyExpr { int16_t tableIndex; SArray* columnInfo; // SArray, group by columns information @@ -41,21 +52,12 @@ typedef struct SSqlGroupbyExpr { int16_t orderType; // order by type: asc/desc } SSqlGroupbyExpr; -typedef struct SPosInfo { - int32_t pageId; - int32_t rowId; -} SPosInfo; - -typedef struct SWindowStatus { - bool closed; -} SWindowStatus; - typedef struct SWindowResult { - uint16_t numOfRows; // number of rows of current time window - SWindowStatus status; // this result status: closed or opened SPosInfo pos; // Position of current result in disk-based output buffer + uint16_t numOfRows; // number of rows of current time window + bool closed; // this result status: closed or opened SResultInfo* resultInfo; // For each result column, there is a resultInfo - STimeWindow window; // The time window that current result covers. + TSKEY skey; // start key of current time window } SWindowResult; /** @@ -79,6 +81,7 @@ typedef struct SWindowResInfo { int64_t startTime; // start time of the first time window for sliding query int64_t prevSKey; // previous (not completed) sliding window start key int64_t threshold; // threshold to halt query and return the generated results. + int64_t interval; // time window interval } SWindowResInfo; typedef struct SColumnFilterElem { @@ -98,7 +101,7 @@ typedef struct STableQueryInfo { // todo merge with the STableQueryInfo struct TSKEY lastKey; int32_t groupIndex; // group id in table list int16_t queryRangeSet; // denote if the query range is set, only available for interval query - int64_t tag; + tVariant tag; STimeWindow win; STSCursor cur; void* pTable; // for retrieve the page id list @@ -121,8 +124,9 @@ typedef struct SQueryCostInfo { uint32_t loadBlockStatis; uint32_t discardBlocks; uint64_t elapsedTime; - uint64_t computTime; + uint64_t firstStageMergeTime; uint64_t internalSupSize; + uint64_t numOfTimeWindows; } SQueryCostInfo; typedef struct SQuery { @@ -189,21 +193,18 @@ typedef struct SQInfo { int64_t owner; // if it is in execution void* tsdb; int32_t vgId; - STableGroupInfo tableGroupInfo; // table id list < only includes the STable list> + STableGroupInfo tableGroupInfo; // table list SArray STableGroupInfo tableqinfoGroupInfo; // this is a group array list, including SArray structure SQueryRuntimeEnv runtimeEnv; - int32_t groupIndex; - int32_t offset; // offset in group result set of subgroup, todo refactor SArray* arrTableIdInfo; + int32_t groupIndex; /* * the query is executed position on which meter of the whole list. * when the index reaches the last one of the list, it means the query is completed. - * We later may refactor to remove this attribution by using another flag to denote - * whether a multimeter query is completed or not. */ int32_t tableIndex; - int32_t numOfGroupResultPages; + SGroupResInfo groupResInfo; void* pBuf; // allocated buffer for STableQueryInfo, sizeof(STableQueryInfo)*numOfTables; pthread_mutex_t lock; // used to synchronize the rsp/query threads diff --git a/src/query/inc/qSqlparser.h b/src/query/inc/qSqlparser.h index cd9618b223..69fc0bc7ef 100644 --- a/src/query/inc/qSqlparser.h +++ b/src/query/inc/qSqlparser.h @@ -26,7 +26,7 @@ extern "C" { #include "tstoken.h" #include "tvariant.h" -#define ParseTOKENTYPE SSQLToken +#define ParseTOKENTYPE SStrToken extern char tTokenTypeSwitcher[13]; #define toTSDBType(x) \ @@ -71,16 +71,16 @@ typedef struct SQuerySQL { struct tSQLExpr * pWhere; // where clause [optional] tVariantList * pGroupby; // groupby clause, only for tags[optional] tVariantList * pSortOrder; // orderby [optional] - SSQLToken interval; // interval [optional] - SSQLToken sliding; // sliding window [optional] + SStrToken interval; // interval [optional] + SStrToken sliding; // sliding window [optional] SLimitVal limit; // limit offset [optional] SLimitVal slimit; // group limit offset [optional] tVariantList * fillType; // fill type[optional] - SSQLToken selectToken; // sql string + SStrToken selectToken; // sql string } SQuerySQL; typedef struct SCreateTableSQL { - struct SSQLToken name; // meter name, create table [meterName] xxx + struct SStrToken name; // meter name, create table [meterName] xxx bool existCheck; int8_t type; // create normal table/from super table/ stream @@ -90,7 +90,7 @@ typedef struct SCreateTableSQL { } colInfo; struct { - SSQLToken stableName; // super table name, for using clause + SStrToken stableName; // super table name, for using clause tVariantList *pTagVals; // create by using metric, tag value STagData tagdata; } usingInfo; @@ -99,7 +99,7 @@ typedef struct SCreateTableSQL { } SCreateTableSQL; typedef struct SAlterTableSQL { - SSQLToken name; + SStrToken name; int16_t type; STagData tagData; @@ -108,7 +108,7 @@ typedef struct SAlterTableSQL { } SAlterTableSQL; typedef struct SCreateDBInfo { - SSQLToken dbname; + SStrToken dbname; int32_t replica; int32_t cacheBlockSize; int32_t maxTablesPerVnode; @@ -121,7 +121,7 @@ typedef struct SCreateDBInfo { int32_t walLevel; int32_t quorum; int32_t compressionLevel; - SSQLToken precision; + SStrToken precision; bool ignoreExists; tVariantList *keep; @@ -136,33 +136,33 @@ typedef struct SCreateAcctSQL { int64_t maxStorage; int64_t maxQueryTime; int32_t maxConnections; - SSQLToken stat; + SStrToken stat; } SCreateAcctSQL; typedef struct SShowInfo { uint8_t showType; - SSQLToken prefix; - SSQLToken pattern; + SStrToken prefix; + SStrToken pattern; } SShowInfo; typedef struct SUserInfo { - SSQLToken user; - SSQLToken passwd; - SSQLToken privilege; + SStrToken user; + SStrToken passwd; + SStrToken privilege; int16_t type; } SUserInfo; typedef struct tDCLSQL { int32_t nTokens; /* Number of expressions on the list */ int32_t nAlloc; /* Number of entries allocated below */ - SSQLToken *a; /* one entry for element */ + SStrToken *a; /* one entry for element */ bool existsCheck; union { SCreateDBInfo dbOpt; SCreateAcctSQL acctOpt; SShowInfo showOpt; - SSQLToken ip; + SStrToken ip; }; SUserInfo user; @@ -194,14 +194,14 @@ typedef struct tSQLExpr { // the full sql string of function(col, param), which is actually the raw // field name, since the function name is kept in nSQLOptr already - SSQLToken operand; - SSQLToken colInfo; // field id + SStrToken operand; + SStrToken colInfo; // field id tVariant val; // value only for string, float, int struct tSQLExpr *pLeft; // left child struct tSQLExpr *pRight; // right child - struct tSQLExprList *pParam; // function parameters + SStrToken token; // original sql expr string } tSQLExpr; // used in select clause. select from xxx @@ -210,6 +210,7 @@ typedef struct tSQLExprItem { char * aliasName; // alias name, null-terminated string } tSQLExprItem; +// todo refactor by using SArray typedef struct tSQLExprList { int32_t nExpr; /* Number of expressions on the list */ int32_t nAlloc; /* Number of entries allocated below */ @@ -242,7 +243,7 @@ tVariantList *tVariantListAppend(tVariantList *pList, tVariant *pVar, uint8_t so tVariantList *tVariantListInsert(tVariantList *pList, tVariant *pVar, uint8_t sortOrder, int32_t index); -tVariantList *tVariantListAppendToken(tVariantList *pList, SSQLToken *pAliasToken, uint8_t sortOrder); +tVariantList *tVariantListAppendToken(tVariantList *pList, SStrToken *pAliasToken, uint8_t sortOrder); void tVariantListDestroy(tVariantList *pList); tFieldList *tFieldListAppend(tFieldList *pList, TAOS_FIELD *pField); @@ -253,61 +254,61 @@ tSQLExpr *tSQLExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optType); void tSQLExprDestroy(tSQLExpr *); -tSQLExprList *tSQLExprListAppend(tSQLExprList *pList, tSQLExpr *pNode, SSQLToken *pToken); +tSQLExprList *tSQLExprListAppend(tSQLExprList *pList, tSQLExpr *pNode, SStrToken *pToken); void tSQLExprListDestroy(tSQLExprList *pList); -SQuerySQL *tSetQuerySQLElems(SSQLToken *pSelectToken, tSQLExprList *pSelection, tVariantList *pFrom, tSQLExpr *pWhere, - tVariantList *pGroupby, tVariantList *pSortOrder, SSQLToken *pInterval, - SSQLToken *pSliding, tVariantList *pFill, SLimitVal *pLimit, SLimitVal *pGLimit); +SQuerySQL *tSetQuerySQLElems(SStrToken *pSelectToken, tSQLExprList *pSelection, tVariantList *pFrom, tSQLExpr *pWhere, + tVariantList *pGroupby, tVariantList *pSortOrder, SStrToken *pInterval, + SStrToken *pSliding, tVariantList *pFill, SLimitVal *pLimit, SLimitVal *pGLimit); -SCreateTableSQL *tSetCreateSQLElems(tFieldList *pCols, tFieldList *pTags, SSQLToken *pMetricName, +SCreateTableSQL *tSetCreateSQLElems(tFieldList *pCols, tFieldList *pTags, SStrToken *pMetricName, tVariantList *pTagVals, SQuerySQL *pSelect, int32_t type); void tSQLExprNodeDestroy(tSQLExpr *pExpr); tSQLExpr *tSQLExprNodeClone(tSQLExpr *pExpr); -SAlterTableSQL *tAlterTableSQLElems(SSQLToken *pMeterName, tFieldList *pCols, tVariantList *pVals, int32_t type); +SAlterTableSQL *tAlterTableSQLElems(SStrToken *pMeterName, tFieldList *pCols, tVariantList *pVals, int32_t type); tSQLExprListList *tSQLListListAppend(tSQLExprListList *pList, tSQLExprList *pExprList); void destroyAllSelectClause(SSubclauseInfo *pSql); void doDestroyQuerySql(SQuerySQL *pSql); -SSqlInfo * setSQLInfo(SSqlInfo *pInfo, void *pSqlExprInfo, SSQLToken *pMeterName, int32_t type); +SSqlInfo * setSQLInfo(SSqlInfo *pInfo, void *pSqlExprInfo, SStrToken *pMeterName, int32_t type); SSubclauseInfo *setSubclause(SSubclauseInfo *pClause, void *pSqlExprInfo); SSubclauseInfo *appendSelectClause(SSubclauseInfo *pInfo, void *pSubclause); -void setCreatedTableName(SSqlInfo *pInfo, SSQLToken *pMeterName, SSQLToken *pIfNotExists); +void setCreatedTableName(SSqlInfo *pInfo, SStrToken *pMeterName, SStrToken *pIfNotExists); void SQLInfoDestroy(SSqlInfo *pInfo); void setDCLSQLElems(SSqlInfo *pInfo, int32_t type, int32_t nParams, ...); -void setDropDBTableInfo(SSqlInfo *pInfo, int32_t type, SSQLToken* pToken, SSQLToken* existsCheck); -void setShowOptions(SSqlInfo *pInfo, int32_t type, SSQLToken* prefix, SSQLToken* pPatterns); +void setDropDBTableInfo(SSqlInfo *pInfo, int32_t type, SStrToken* pToken, SStrToken* existsCheck); +void setShowOptions(SSqlInfo *pInfo, int32_t type, SStrToken* prefix, SStrToken* pPatterns); -tDCLSQL *tTokenListAppend(tDCLSQL *pTokenList, SSQLToken *pToken); +tDCLSQL *tTokenListAppend(tDCLSQL *pTokenList, SStrToken *pToken); -void setCreateDBSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *pToken, SCreateDBInfo *pDB, SSQLToken *pIgExists); +void setCreateDBSQL(SSqlInfo *pInfo, int32_t type, SStrToken *pToken, SCreateDBInfo *pDB, SStrToken *pIgExists); -void setCreateAcctSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *pName, SSQLToken *pPwd, SCreateAcctSQL *pAcctInfo); -void setCreateUserSQL(SSqlInfo *pInfo, SSQLToken *pName, SSQLToken *pPasswd); -void setKillSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *ip); -void setAlterUserSQL(SSqlInfo *pInfo, int16_t type, SSQLToken *pName, SSQLToken* pPwd, SSQLToken *pPrivilege); +void setCreateAcctSQL(SSqlInfo *pInfo, int32_t type, SStrToken *pName, SStrToken *pPwd, SCreateAcctSQL *pAcctInfo); +void setCreateUserSQL(SSqlInfo *pInfo, SStrToken *pName, SStrToken *pPasswd); +void setKillSQL(SSqlInfo *pInfo, int32_t type, SStrToken *ip); +void setAlterUserSQL(SSqlInfo *pInfo, int16_t type, SStrToken *pName, SStrToken* pPwd, SStrToken *pPrivilege); void setDefaultCreateDbOption(SCreateDBInfo *pDBInfo); // prefix show db.tables; -void setDBName(SSQLToken *pCpxName, SSQLToken *pDB); +void setDBName(SStrToken *pCpxName, SStrToken *pDB); -tSQLExpr *tSQLExprIdValueCreate(SSQLToken *pToken, int32_t optType); +tSQLExpr *tSQLExprIdValueCreate(SStrToken *pToken, int32_t optType); -tSQLExpr *tSQLExprCreateFunction(tSQLExprList *pList, SSQLToken *pFuncToken, SSQLToken *endToken, int32_t optType); +tSQLExpr *tSQLExprCreateFunction(tSQLExprList *pList, SStrToken *pFuncToken, SStrToken *endToken, int32_t optType); -void tSQLSetColumnInfo(TAOS_FIELD *pField, SSQLToken *pName, TAOS_FIELD *pType); +void tSQLSetColumnInfo(TAOS_FIELD *pField, SStrToken *pName, TAOS_FIELD *pType); -void tSQLSetColumnType(TAOS_FIELD *pField, SSQLToken *pToken); +void tSQLSetColumnType(TAOS_FIELD *pField, SStrToken *pToken); void *ParseAlloc(void *(*mallocProc)(size_t)); diff --git a/src/query/inc/qTsbuf.h b/src/query/inc/qTsbuf.h index 59b224e096..46e6f79014 100644 --- a/src/query/inc/qTsbuf.h +++ b/src/query/inc/qTsbuf.h @@ -22,6 +22,7 @@ extern "C" { #include "os.h" #include "taosdef.h" +#include "tvariant.h" #define MEM_BUF_SIZE (1 << 20) #define TS_COMP_FILE_MAGIC 0x87F5EC4C @@ -42,9 +43,9 @@ typedef struct STSRawBlock { } STSRawBlock; typedef struct STSElem { - TSKEY ts; - int64_t tag; - int32_t vnode; + TSKEY ts; + tVariant tag; + int32_t vnode; } STSElem; typedef struct STSCursor { @@ -55,11 +56,11 @@ typedef struct STSCursor { } STSCursor; typedef struct STSBlock { - int64_t tag; // tag value - int32_t numOfElem; // number of elements - int32_t compLen; // size after compressed - int32_t padding; // 0xFFFFFFFF by default, after the payload - char* payload; // actual data that is compressed + tVariant tag; // tag value + int32_t numOfElem; // number of elements + int32_t compLen; // size after compressed + int32_t padding; // 0xFFFFFFFF by default, after the payload + char* payload; // actual data that is compressed } STSBlock; /* @@ -84,8 +85,8 @@ typedef struct STSBuf { uint32_t fileSize; STSVnodeBlockInfoEx* pData; - int32_t numOfAlloc; - int32_t numOfVnodes; + uint32_t numOfAlloc; + uint32_t numOfVnodes; char* assistBuf; int32_t bufSize; @@ -109,7 +110,7 @@ STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_ void* tsBufDestroy(STSBuf* pTSBuf); -void tsBufAppend(STSBuf* pTSBuf, int32_t vnodeId, int64_t tag, const char* pData, int32_t len); +void tsBufAppend(STSBuf* pTSBuf, int32_t vnodeId, tVariant* tag, const char* pData, int32_t len); int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf, int32_t vnodeIdx); STSBuf* tsBufClone(STSBuf* pTSBuf); @@ -122,7 +123,7 @@ void tsBufResetPos(STSBuf* pTSBuf); STSElem tsBufGetElem(STSBuf* pTSBuf); bool tsBufNextPos(STSBuf* pTSBuf); -STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t vnodeId, int64_t tag); +STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t vnodeId, tVariant* tag); STSCursor tsBufGetCursor(STSBuf* pTSBuf); void tsBufSetTraverseOrder(STSBuf* pTSBuf, int32_t order); diff --git a/src/query/inc/qUtil.h b/src/query/inc/qUtil.h index b7f57bf1b5..6de3c7c0e5 100644 --- a/src/query/inc/qUtil.h +++ b/src/query/inc/qUtil.h @@ -38,7 +38,8 @@ static FORCE_INLINE SWindowResult *getWindowResult(SWindowResInfo *pWindowResInf return &pWindowResInfo->pResult[slot]; } -#define curTimeWindow(_winres) ((_winres)->curIndex) +#define curTimeWindowIndex(_winres) ((_winres)->curIndex) +#define GET_TIMEWINDOW(_winresInfo, _win) (STimeWindow) {(_win)->skey, ((_win)->skey + (_winresInfo)->interval - 1)} #define GET_ROW_PARAM_FOR_MULTIOUTPUT(_q, tbq, sq) (((tbq) && (!sq))? (_q)->pSelectExpr[1].base.arg->argValue.i64:1) bool isWindowResClosed(SWindowResInfo *pWindowResInfo, int32_t slot); diff --git a/src/query/inc/sql.y b/src/query/inc/sql.y index 2b2a967edf..7500dcba56 100644 --- a/src/query/inc/sql.y +++ b/src/query/inc/sql.y @@ -2,8 +2,8 @@ //usage: lemon sql.y %token_prefix TK_ -%token_type {SSQLToken} -%default_type {SSQLToken} +%token_type {SStrToken} +%default_type {SStrToken} %extra_argument {SSqlInfo* pInfo} %fallback ID BOOL TINYINT SMALLINT INTEGER BIGINT FLOAT DOUBLE STRING TIMESTAMP BINARY NCHAR. @@ -73,18 +73,18 @@ cmd ::= SHOW MODULES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_MODULE, 0, 0); cmd ::= SHOW QUERIES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_QUERIES, 0, 0); } cmd ::= SHOW CONNECTIONS.{ setShowOptions(pInfo, TSDB_MGMT_TABLE_CONNS, 0, 0);} cmd ::= SHOW STREAMS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_STREAMS, 0, 0); } -cmd ::= SHOW CONFIGS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_CONFIGS, 0, 0); } +cmd ::= SHOW VARIABLES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_VARIABLES, 0, 0); } cmd ::= SHOW SCORES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_SCORES, 0, 0); } cmd ::= SHOW GRANTS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_GRANTS, 0, 0); } cmd ::= SHOW VNODES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_VNODES, 0, 0); } cmd ::= SHOW VNODES IPTOKEN(X). { setShowOptions(pInfo, TSDB_MGMT_TABLE_VNODES, &X, 0); } -%type dbPrefix {SSQLToken} +%type dbPrefix {SStrToken} dbPrefix(A) ::=. {A.n = 0; A.type = 0;} dbPrefix(A) ::= ids(X) DOT. {A = X; } -%type cpxName {SSQLToken} +%type cpxName {SStrToken} cpxName(A) ::= . {A.n = 0; } cpxName(A) ::= DOT ids(Y). {A = Y; A.n += 1; } @@ -101,19 +101,19 @@ cmd ::= SHOW dbPrefix(X) STABLES. { } cmd ::= SHOW dbPrefix(X) STABLES LIKE ids(Y). { - SSQLToken token; + SStrToken token; setDBName(&token, &X); setShowOptions(pInfo, TSDB_MGMT_TABLE_METRIC, &token, &Y); } cmd ::= SHOW dbPrefix(X) VGROUPS. { - SSQLToken token; + SStrToken token; setDBName(&token, &X); setShowOptions(pInfo, TSDB_MGMT_TABLE_VGROUP, &token, 0); } cmd ::= SHOW dbPrefix(X) VGROUPS ids(Y). { - SSQLToken token; + SStrToken token; setDBName(&token, &X); setShowOptions(pInfo, TSDB_MGMT_TABLE_VGROUP, &token, &Y); } @@ -145,7 +145,7 @@ cmd ::= ALTER DNODE ids(X) ids(Y). { setDCLSQLElems(pInfo, TSDB_SQL cmd ::= ALTER DNODE ids(X) ids(Y) ids(Z). { setDCLSQLElems(pInfo, TSDB_SQL_CFG_DNODE, 3, &X, &Y, &Z); } cmd ::= ALTER LOCAL ids(X). { setDCLSQLElems(pInfo, TSDB_SQL_CFG_LOCAL, 1, &X); } cmd ::= ALTER LOCAL ids(X) ids(Y). { setDCLSQLElems(pInfo, TSDB_SQL_CFG_LOCAL, 2, &X, &Y); } -cmd ::= ALTER DATABASE ids(X) alter_db_optr(Y). { SSQLToken t = {0}; setCreateDBSQL(pInfo, TSDB_SQL_ALTER_DB, &X, &Y, &t);} +cmd ::= ALTER DATABASE ids(X) alter_db_optr(Y). { SStrToken t = {0}; setCreateDBSQL(pInfo, TSDB_SQL_ALTER_DB, &X, &Y, &t);} cmd ::= ALTER ACCOUNT ids(X) acct_optr(Z). { setCreateAcctSQL(pInfo, TSDB_SQL_ALTER_ACCT, &X, NULL, &Z);} cmd ::= ALTER ACCOUNT ids(X) PASS ids(Y) acct_optr(Z). { setCreateAcctSQL(pInfo, TSDB_SQL_ALTER_ACCT, &X, &Y, &Z);} @@ -153,15 +153,15 @@ cmd ::= ALTER ACCOUNT ids(X) PASS ids(Y) acct_optr(Z). { setCreateAcctSQL(p // An IDENTIFIER can be a generic identifier, or one of several keywords. // Any non-standard keyword can also be an identifier. // And "ids" is an identifer-or-string. -%type ids {SSQLToken} +%type ids {SStrToken} ids(A) ::= ID(X). {A = X; } ids(A) ::= STRING(X). {A = X; } -%type ifexists {SSQLToken} +%type ifexists {SStrToken} ifexists(X) ::= IF EXISTS. {X.n = 1;} ifexists(X) ::= . {X.n = 0;} -%type ifnotexists {SSQLToken} +%type ifnotexists {SStrToken} ifnotexists(X) ::= IF NOT EXISTS. {X.n = 1;} ifnotexists(X) ::= . {X.n = 0;} @@ -416,7 +416,7 @@ selcollist(A) ::= sclp(P) STAR. { // An option "AS " phrase that can follow one of the expressions that // define the result set, or one of the tables in the FROM clause. // -%type as {SSQLToken} +%type as {SStrToken} as(X) ::= AS ids(Y). { X = Y; } as(X) ::= ids(Y). { X = Y; } as(X) ::= . { X.n = 0; } @@ -458,10 +458,10 @@ tablelist(A) ::= tablelist(Y) COMMA ids(X) cpxName(Z) ids(F). { } // The value of interval should be the form of "number+[a,s,m,h,d,n,y]" or "now" -%type tmvar {SSQLToken} +%type tmvar {SStrToken} tmvar(A) ::= VARIABLE(X). {A = X;} -%type interval_opt {SSQLToken} +%type interval_opt {SStrToken} interval_opt(N) ::= INTERVAL LP tmvar(E) RP. {N = E; } interval_opt(N) ::= . {N.n = 0; N.z = NULL; N.type = 0; } @@ -482,7 +482,7 @@ fill_opt(N) ::= FILL LP ID(Y) RP. { N = tVariantListAppendToken(NULL, &Y, -1); } -%type sliding_opt {SSQLToken} +%type sliding_opt {SStrToken} sliding_opt(K) ::= SLIDING LP tmvar(E) RP. {K = E; } sliding_opt(K) ::= . {K.n = 0; K.z = NULL; K.type = 0; } diff --git a/src/query/inc/tsqlfunction.h b/src/query/inc/tsqlfunction.h index 58d2ff260e..65ab82883b 100644 --- a/src/query/inc/tsqlfunction.h +++ b/src/query/inc/tsqlfunction.h @@ -91,16 +91,16 @@ extern "C" { #define QUERY_COND_REL_PREFIX_IN "IN|" #define QUERY_COND_REL_PREFIX_LIKE "LIKE|" -#define QUERY_COND_REL_PREFIX_IN_LEN 3 +#define QUERY_COND_REL_PREFIX_IN_LEN 3 #define QUERY_COND_REL_PREFIX_LIKE_LEN 5 -#define QUERY_ASC_FORWARD_STEP 1 +#define QUERY_ASC_FORWARD_STEP 1 #define QUERY_DESC_FORWARD_STEP -1 #define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP) -#define MAX_RETRIEVE_ROWS_IN_INTERVAL_QUERY 10000000 -#define TOP_BOTTOM_QUERY_LIMIT 100 +#define MAX_INTERVAL_TIME_WINDOW 10000000 +#define TOP_BOTTOM_QUERY_LIMIT 100 enum { MASTER_SCAN = 0x0u, @@ -137,13 +137,13 @@ typedef struct SInterpInfoDetail { } SInterpInfoDetail; typedef struct SResultInfo { - int8_t hasResult; // result generated, not NULL value - bool initialized; // output buffer has been initialized - bool complete; // query has completed - bool superTableQ; // is super table query - int32_t numOfRes; // num of output result in current buffer - int32_t bufLen; // buffer size - void* interResultBuf; // output result buffer + int8_t hasResult; // result generated, not NULL value + bool initialized; // output buffer has been initialized + bool complete; // query has completed + bool superTableQ; // is super table query + uint32_t bufLen; // buffer size + uint64_t numOfRes; // num of output result in current buffer + void* interResultBuf; // output result buffer } SResultInfo; struct SQLFunctionCtx; diff --git a/src/query/src/qAst.c b/src/query/src/qAst.c index afa3618a96..634f014d97 100644 --- a/src/query/src/qAst.c +++ b/src/query/src/qAst.c @@ -44,7 +44,7 @@ * ver 0.3, pipeline filter in the form of: (a+2)/9 > 14 * */ -static tExprNode *tExprNodeCreate(SSchema *pSchema, int32_t numOfCols, SSQLToken *pToken); +static tExprNode *tExprNodeCreate(SSchema *pSchema, int32_t numOfCols, SStrToken *pToken); static tExprNode *createSyntaxTree(SSchema *pSchema, int32_t numOfCols, char *str, int32_t *i); static void destroySyntaxTree(tExprNode *); @@ -103,7 +103,7 @@ static void reviseBinaryExprIfNecessary(tExprNode **pLeft, tExprNode **pRight, u } } -static tExprNode *tExprNodeCreate(SSchema *pSchema, int32_t numOfCols, SSQLToken *pToken) { +static tExprNode *tExprNodeCreate(SSchema *pSchema, int32_t numOfCols, SStrToken *pToken) { /* if the token is not a value, return false */ if (pToken->type == TK_RP || (pToken->type != TK_INTEGER && pToken->type != TK_FLOAT && pToken->type != TK_ID && pToken->type != TK_TBNAME && pToken->type != TK_STRING && pToken->type != TK_BOOL)) { @@ -117,7 +117,7 @@ static tExprNode *tExprNodeCreate(SSchema *pSchema, int32_t numOfCols, SSQLToken int32_t i = 0; if (pToken->type == TK_ID) { do { - SSQLToken tableToken = {0}; + SStrToken tableToken = {0}; extractTableNameFromToken(pToken, &tableToken); size_t len = strlen(pSchema[i].name); @@ -157,7 +157,7 @@ static tExprNode *tExprNodeCreate(SSchema *pSchema, int32_t numOfCols, SSQLToken return pNode; } -uint8_t getBinaryExprOptr(SSQLToken *pToken) { +uint8_t getBinaryExprOptr(SStrToken *pToken) { switch (pToken->type) { case TK_LT: return TSDB_RELATION_LESS; @@ -234,7 +234,7 @@ uint8_t isQueryOnPrimaryKey(const char *primaryColumnName, const tExprNode *pLef } static tExprNode *createSyntaxTree(SSchema *pSchema, int32_t numOfCols, char *str, int32_t *i) { - SSQLToken t0 = tStrGetToken(str, i, false, 0, NULL); + SStrToken t0 = tStrGetToken(str, i, false, 0, NULL); if (t0.n == 0) { return NULL; } diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index e0eaf5eedf..4e2e31d269 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -27,7 +27,8 @@ #include "query.h" #include "queryLog.h" #include "tlosertree.h" -#include "tscompression.h" + +#define MAX_ROWS_PER_RESBUF_PAGE ((1u<<12) - 1) /** * check if the primary column is load by default, otherwise, the program will @@ -35,6 +36,9 @@ */ #define Q_STATUS_EQUAL(p, s) (((p) & (s)) != 0) #define TSDB_COL_IS_TAG(f) (((f)&TSDB_COL_TAG) != 0) +#define TSDB_COL_IS_NORMAL_COL(f) ((f) == TSDB_COL_NORMAL) +#define TSDB_COL_IS_UD_COL(f) ((f) == TSDB_COL_UDC) + #define QUERY_IS_ASC_QUERY(q) (GET_FORWARD_DIRECTION_FACTOR((q)->order.order) == QUERY_ASC_FORWARD_STEP) #define IS_MASTER_SCAN(runtime) ((runtime)->scanFlag == MASTER_SCAN) @@ -93,7 +97,8 @@ typedef struct { #if 0 static UNUSED_FUNC void *u_malloc (size_t __size) { uint32_t v = rand(); - if (v % 5 <= 1) { + + if (v % 1000 <= 0) { return NULL; } else { return malloc(__size); @@ -102,15 +107,25 @@ static UNUSED_FUNC void *u_malloc (size_t __size) { static UNUSED_FUNC void* u_calloc(size_t num, size_t __size) { uint32_t v = rand(); - if (v % 5 <= 1) { + if (v % 1000 <= 0) { return NULL; } else { return calloc(num, __size); } } +static UNUSED_FUNC void* u_realloc(void* p, size_t __size) { + uint32_t v = rand(); + if (v % 5 <= 1) { + return NULL; + } else { + return realloc(p, __size); + } +} + #define calloc u_calloc #define malloc u_malloc +#define realloc u_realloc #endif #define CLEAR_QUERY_STATUS(q, st) ((q)->status &= (~(st))) @@ -151,7 +166,7 @@ static bool hasMainOutput(SQuery *pQuery); static void buildTagQueryResult(SQInfo *pQInfo); static int32_t setAdditionalInfo(SQInfo *pQInfo, void *pTable, STableQueryInfo *pTableQueryInfo); -static int32_t flushFromResultBuf(SQInfo *pQInfo); +static int32_t flushFromResultBuf(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo); bool doFilterData(SQuery *pQuery, int32_t elemPos) { for (int32_t k = 0; k < pQuery->numOfFilterCols; ++k) { @@ -212,22 +227,22 @@ int64_t getNumOfResult(SQueryRuntimeEnv *pRuntimeEnv) { */ void updateNumOfResult(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOfRes) { SQuery *pQuery = pRuntimeEnv->pQuery; - + for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { SResultInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]); - + int16_t functionId = pRuntimeEnv->pCtx[j].functionId; if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TAGPRJ || functionId == TSDB_FUNC_TS_DUMMY) { continue; } - + assert(pResInfo->numOfRes > numOfRes); pResInfo->numOfRes = numOfRes; } } -static int32_t getGroupResultId(int32_t groupIndex) { +static UNUSED_FUNC int32_t getGroupResultId(int32_t groupIndex) { int32_t base = 20000000; return base + (groupIndex * 10000); } @@ -315,10 +330,10 @@ bool isTSCompQuery(SQuery *pQuery) { return pQuery->pSelectExpr[0].base.function static bool limitResults(SQueryRuntimeEnv* pRuntimeEnv) { SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv); SQuery* pQuery = pRuntimeEnv->pQuery; - + if ((pQuery->limit.limit > 0) && (pQuery->rec.total + pQuery->rec.rows > pQuery->limit.limit)) { pQuery->rec.rows = pQuery->limit.limit - pQuery->rec.total; - + qDebug("QInfo:%p discard remain data due to result limitation, limit:%"PRId64", current return:%" PRId64 ", total:%"PRId64, pQInfo, pQuery->limit.limit, pQuery->rec.rows, pQuery->rec.total + pQuery->rec.rows); assert(pQuery->rec.rows >= 0); @@ -371,14 +386,14 @@ static bool hasTagValOutput(SQuery* pQuery) { * @return */ static bool hasNullValue(SColIndex* pColIndex, SDataStatis *pStatis, SDataStatis **pColStatis) { - if (pStatis != NULL && !TSDB_COL_IS_TAG(pColIndex->flag)) { + if (pStatis != NULL && TSDB_COL_IS_NORMAL_COL(pColIndex->flag)) { *pColStatis = &pStatis[pColIndex->colIndex]; assert((*pColStatis)->colId == pColIndex->colId); } else { *pColStatis = NULL; } - if (TSDB_COL_IS_TAG(pColIndex->flag) || pColIndex->colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + if (TSDB_COL_IS_TAG(pColIndex->flag) || TSDB_COL_IS_UD_COL(pColIndex->flag) || pColIndex->colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { return false; } @@ -410,8 +425,9 @@ static SWindowResult *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWin newCap = (int64_t)(pWindowResInfo->capacity * 1.5); } - char *t = realloc(pWindowResInfo->pResult, newCap * sizeof(SWindowResult)); + char *t = realloc(pWindowResInfo->pResult, (size_t)(newCap * sizeof(SWindowResult))); pRuntimeEnv->summary.internalSupSize += (newCap - pWindowResInfo->capacity) * sizeof(SWindowResult); + pRuntimeEnv->summary.numOfTimeWindows += (newCap - pWindowResInfo->capacity); if (t == NULL) { longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -425,7 +441,10 @@ static SWindowResult *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWin pRuntimeEnv->summary.internalSupSize += (pQuery->numOfOutput * sizeof(SResultInfo) + pRuntimeEnv->interBufSize) * inc; for (int32_t i = pWindowResInfo->capacity; i < newCap; ++i) { - createQueryResultInfo(pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, pRuntimeEnv->interBufSize); + int32_t ret = createQueryResultInfo(pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, pRuntimeEnv->interBufSize); + if (ret != TSDB_CODE_SUCCESS) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } } pWindowResInfo->capacity = (int32_t)newCap; @@ -436,6 +455,11 @@ static SWindowResult *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWin taosHashPut(pWindowResInfo->hashList, pData, bytes, (char *)&pWindowResInfo->curIndex, sizeof(int32_t)); } + // too many time window in query + if (pWindowResInfo->size > MAX_INTERVAL_TIME_WINDOW) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW); + } + return getWindowResult(pWindowResInfo, pWindowResInfo->curIndex); } @@ -447,8 +471,9 @@ static STimeWindow getActiveTimeWindow(SWindowResInfo *pWindowResInfo, int64_t t w.skey = pWindowResInfo->prevSKey; w.ekey = w.skey + pQuery->intervalTime - 1; } else { - int32_t slot = curTimeWindow(pWindowResInfo); - w = getWindowResult(pWindowResInfo, slot)->window; + int32_t slot = curTimeWindowIndex(pWindowResInfo); + SWindowResult* pWindowRes = getWindowResult(pWindowResInfo, slot); + w = GET_TIMEWINDOW(pWindowResInfo, pWindowRes); } if (w.skey > ts || w.ekey < ts) { @@ -549,15 +574,15 @@ static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowRes } // set time window for current result - pWindowRes->window = *win; + pWindowRes->skey = win->skey; setWindowResOutputBufInitCtx(pRuntimeEnv, pWindowRes); return TSDB_CODE_SUCCESS; } -static SWindowStatus *getTimeWindowResStatus(SWindowResInfo *pWindowResInfo, int32_t slot) { +static bool getTimeWindowResStatus(SWindowResInfo *pWindowResInfo, int32_t slot) { assert(slot >= 0 && slot < pWindowResInfo->size); - return &pWindowResInfo->pResult[slot].status; + return pWindowResInfo->pResult[slot].closed; } static FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey, int16_t pos, @@ -599,7 +624,7 @@ static int32_t doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKe // no qualified results exist, abort check int32_t numOfClosed = 0; - + if (pWindowResInfo->size == 0) { return pWindowResInfo->size; } @@ -617,16 +642,17 @@ static int32_t doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKe for (i = 0; i < pWindowResInfo->size; ++i) { SWindowResult *pResult = &pWindowResInfo->pResult[i]; - if (pResult->status.closed) { + if (pResult->closed) { numOfClosed += 1; continue; } - if ((pResult->window.ekey <= lastKey && QUERY_IS_ASC_QUERY(pQuery)) || - (pResult->window.skey >= lastKey && !QUERY_IS_ASC_QUERY(pQuery))) { + TSKEY ekey = pResult->skey + pWindowResInfo->interval; + if ((ekey <= lastKey && QUERY_IS_ASC_QUERY(pQuery)) || + (pResult->skey >= lastKey && !QUERY_IS_ASC_QUERY(pQuery))) { closeTimeWindow(pWindowResInfo, i); } else { - skey = pResult->window.skey; + skey = pResult->skey; break; } } @@ -639,26 +665,26 @@ static int32_t doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKe pWindowResInfo->curIndex = i; } - pWindowResInfo->prevSKey = pWindowResInfo->pResult[pWindowResInfo->curIndex].window.skey; + pWindowResInfo->prevSKey = pWindowResInfo->pResult[pWindowResInfo->curIndex].skey; // the number of completed slots are larger than the threshold, return current generated results to client. if (numOfClosed > pWindowResInfo->threshold) { qDebug("QInfo:%p total result window:%d closed:%d, reached the output threshold %d, return", GET_QINFO_ADDR(pRuntimeEnv), pWindowResInfo->size, numOfClosed, pQuery->rec.threshold); - + setQueryStatus(pQuery, QUERY_RESBUF_FULL); } else { qDebug("QInfo:%p total result window:%d already closed:%d", GET_QINFO_ADDR(pRuntimeEnv), pWindowResInfo->size, numOfClosed); } } - + // output has reached the limitation, set query completed if (pQuery->limit.limit > 0 && (pQuery->limit.limit + pQuery->limit.offset) <= numOfClosed && pRuntimeEnv->scanFlag == MASTER_SCAN) { setQueryStatus(pQuery, QUERY_COMPLETED); } - + assert(pWindowResInfo->prevSKey != TSKEY_INITIAL_VAL); return numOfClosed; } @@ -672,7 +698,7 @@ static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SDataBlockInfo *pDataBlo int32_t step = GET_FORWARD_DIRECTION_FACTOR(order); STableQueryInfo* item = pQuery->current; - + if (QUERY_IS_ASC_QUERY(pQuery)) { if (ekey < pDataBlockInfo->window.ekey) { num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn); @@ -703,12 +729,12 @@ static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SDataBlockInfo *pDataBlo return num; } -static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SWindowStatus *pStatus, STimeWindow *pWin, +static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, bool closed, STimeWindow *pWin, int32_t offset, int32_t forwardStep, TSKEY *tsBuf, int32_t numOfTotal) { SQuery * pQuery = pRuntimeEnv->pQuery; SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; - if (IS_MASTER_SCAN(pRuntimeEnv) || pStatus->closed) { + if (IS_MASTER_SCAN(pRuntimeEnv) || closed) { for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { int32_t functionId = pQuery->pSelectExpr[k].base.functionId; @@ -732,12 +758,11 @@ static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SWindowStat } } -static void doRowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SWindowStatus *pStatus, STimeWindow *pWin, - int32_t offset) { +static void doRowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, bool closed, STimeWindow *pWin, int32_t offset) { SQuery * pQuery = pRuntimeEnv->pQuery; SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; - if (IS_MASTER_SCAN(pRuntimeEnv) || pStatus->closed) { + if (IS_MASTER_SCAN(pRuntimeEnv) || closed) { for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { pCtx[k].nStartQueryTimestamp = pWin->skey; @@ -822,14 +847,14 @@ static FORCE_INLINE TSKEY reviseWindowEkey(SQuery *pQuery, STimeWindow *pWindow) //todo binary search static void* getDataBlockImpl(SArray* pDataBlock, int32_t colId) { int32_t numOfCols = (int32_t)taosArrayGetSize(pDataBlock); - + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData *p = taosArrayGet(pDataBlock, i); if (colId == p->info.colId) { return p->pData; } } - + return NULL; } @@ -879,19 +904,19 @@ static char *getDataBlock(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas } assert(dataBlock != NULL); - sas->data[i] = dataBlock/* + pQuery->colList[i].bytes*/; // start from the offset + sas->data[i] = dataBlock; // start from the offset } } else { // other type of query function SColIndex *pCol = &pQuery->pSelectExpr[col].base.colInfo; - if (TSDB_COL_IS_TAG(pCol->flag)) { - dataBlock = NULL; - } else { + if (TSDB_COL_IS_NORMAL_COL(pCol->flag)) { SColIndex* pColIndex = &pQuery->pSelectExpr[col].base.colInfo; SColumnInfoData *p = taosArrayGet(pDataBlock, pColIndex->colIndex); assert(p->info.colId == pColIndex->colId); dataBlock = p->pData; + } else { + dataBlock = NULL; } } @@ -958,7 +983,7 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * TSKEY ekey = reviseWindowEkey(pQuery, &win); forwardStep = getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, tsCols, pQuery->pos, ekey, searchFn, true); - SWindowStatus *pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo)); + bool pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); doBlockwiseApplyFunctions(pRuntimeEnv, pStatus, &win, startPos, forwardStep, tsCols, pDataBlockInfo->rows); } @@ -986,8 +1011,8 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * TSKEY ekey = reviseWindowEkey(pQuery, &nextWin); forwardStep = getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, tsCols, startPos, ekey, searchFn, true); - SWindowStatus *pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo)); - doBlockwiseApplyFunctions(pRuntimeEnv, pStatus, &nextWin, startPos, forwardStep, tsCols, pDataBlockInfo->rows); + bool closed = getTimeWindowResStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); + doBlockwiseApplyFunctions(pRuntimeEnv, closed, &nextWin, startPos, forwardStep, tsCols, pDataBlockInfo->rows); } pWindowResInfo->curIndex = index; @@ -1040,8 +1065,8 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat return -1; } - pWindowRes->window.skey = v; - pWindowRes->window.ekey = v; + pWindowRes->skey = v; + assert(pRuntimeEnv->windowResInfo.interval == 0); if (pWindowRes->pos.pageId == -1) { int32_t ret = addNewWindowResultBuf(pWindowRes, pResultBuf, GROUPRESULTID, pRuntimeEnv->numOfRowsPerPage); @@ -1103,7 +1128,7 @@ static int32_t doTSJoinFilter(SQueryRuntimeEnv *pRuntimeEnv, int32_t offset) { SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; // compare tag first - if (pCtx[0].tag.i64Key != elem.tag) { + if (tVariantCompare(&pCtx[0].tag, &elem.tag) != 0) { return TS_JOIN_TAG_NOT_EQUALS; } @@ -1140,7 +1165,7 @@ static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx if (functionId == TSDB_FUNC_TS) { return true; } - + if (pResInfo->complete || functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TS_DUMMY) { return false; } @@ -1247,8 +1272,8 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS continue; } - SWindowStatus *pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo)); - doRowwiseApplyFunctions(pRuntimeEnv, pStatus, &win, offset); + bool closed = getTimeWindowResStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); + doRowwiseApplyFunctions(pRuntimeEnv, closed, &win, offset); STimeWindow nextWin = win; int32_t index = pWindowResInfo->curIndex; @@ -1271,8 +1296,8 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS } if (hasTimeWindow) { - pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo)); - doRowwiseApplyFunctions(pRuntimeEnv, pStatus, &nextWin, offset); + closed = getTimeWindowResStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); + doRowwiseApplyFunctions(pRuntimeEnv, closed, &nextWin, offset); } } @@ -1327,10 +1352,10 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pDataBlockInfo, SDataStatis *pStatis, __block_search_fn_t searchFn, SArray *pDataBlock) { SQuery *pQuery = pRuntimeEnv->pQuery; - + STableQueryInfo* pTableQInfo = pQuery->current; SWindowResInfo* pWindowResInfo = &pRuntimeEnv->windowResInfo; - + if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL || pRuntimeEnv->groupbyNormalCol) { rowwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, pDataBlock); } else { @@ -1373,10 +1398,10 @@ static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBl void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY *tsCol, SDataBlockInfo* pBlockInfo, SDataStatis *pStatis, void *param, int32_t colIndex) { - + int32_t functionId = pQuery->pSelectExpr[colIndex].base.functionId; int32_t colId = pQuery->pSelectExpr[colIndex].base.colInfo.colId; - + SDataStatis *tpField = NULL; pCtx->hasNull = hasNullValue(&pQuery->pSelectExpr[colIndex].base.colInfo, pStatis, &tpField); pCtx->aInputElemBuf = inputData; @@ -1432,7 +1457,7 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY pInterpInfo->type = (int8_t)pQuery->fillType; pInterpInfo->ts = pQuery->window.skey; pInterpInfo->primaryCol = (colId == PRIMARYKEY_TIMESTAMP_COL_INDEX); - + if (pQuery->fillVal != NULL) { if (isNull((const char*) &pQuery->fillVal[colIndex], pCtx->inputType)) { pCtx->param[1].nType = TSDB_DATA_TYPE_NULL; @@ -1459,19 +1484,22 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY } // set the output buffer for the selectivity + tag query -static void setCtxTagColumnInfo(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx) { +static int32_t setCtxTagColumnInfo(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx) { SQuery* pQuery = pRuntimeEnv->pQuery; if (isSelectivityWithTagsQuery(pQuery)) { int32_t num = 0; int16_t tagLen = 0; - + SQLFunctionCtx *p = NULL; SQLFunctionCtx **pTagCtx = calloc(pQuery->numOfOutput, POINTER_BYTES); + if (pTagCtx == NULL) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { SSqlFuncMsg *pSqlFuncMsg = &pQuery->pSelectExpr[i].base; - + if (pSqlFuncMsg->functionId == TSDB_FUNC_TAG_DUMMY || pSqlFuncMsg->functionId == TSDB_FUNC_TS_DUMMY) { tagLen += pCtx[i].outputBytes; pTagCtx[num++] = &pCtx[i]; @@ -1490,9 +1518,11 @@ static void setCtxTagColumnInfo(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *p p->tagInfo.numOfTagCols = num; p->tagInfo.tagsLen = tagLen; } else { - taosTFree(pTagCtx); + taosTFree(pTagCtx); } } + + return TSDB_CODE_SUCCESS; } static FORCE_INLINE void setWindowResultInfo(SResultInfo *pResultInfo, SQuery *pQuery, bool isStableQuery, char* buf) { @@ -1536,12 +1566,14 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order pCtx->inputBytes = pQuery->tagColList[index].bytes; pCtx->inputType = pQuery->tagColList[index].type; } - + } else if (TSDB_COL_IS_UD_COL(pIndex->flag)) { + pCtx->inputBytes = pSqlFuncMsg->arg[0].argBytes; + pCtx->inputType = pSqlFuncMsg->arg[0].argType; } else { pCtx->inputBytes = pQuery->colList[index].bytes; pCtx->inputType = pQuery->colList[index].type; } - + assert(isValidDataType(pCtx->inputType)); pCtx->ptsOutputBuf = NULL; @@ -1556,7 +1588,7 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order int16_t type = pSqlFuncMsg->arg[j].argType; int16_t bytes = pSqlFuncMsg->arg[j].argBytes; if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - tVariantCreateFromBinary(&pCtx->param[j], pSqlFuncMsg->arg->argValue.pz, bytes, type); + tVariantCreateFromBinary(&pCtx->param[j], pSqlFuncMsg->arg[j].argValue.pz, bytes, type); } else { tVariantCreateFromBinary(&pCtx->param[j], (char *)&pSqlFuncMsg->arg[j].argValue.i64, bytes, type); } @@ -1592,7 +1624,9 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order resetCtxOutputBuf(pRuntimeEnv); } - setCtxTagColumnInfo(pRuntimeEnv, pRuntimeEnv->pCtx); + if (setCtxTagColumnInfo(pRuntimeEnv, pRuntimeEnv->pCtx) != TSDB_CODE_SUCCESS) { + goto _clean; + } qDebug("QInfo:%p init runtime completed", GET_QINFO_ADDR(pRuntimeEnv)); return TSDB_CODE_SUCCESS; @@ -1736,13 +1770,20 @@ static bool needReverseScan(SQuery *pQuery) { return false; } +/** + * The following 4 kinds of query are treated as the tags query + * tagprj, tid_tag query, count(tbname), 'abc' (user defined constant value column) query + */ static bool onlyQueryTags(SQuery* pQuery) { for(int32_t i = 0; i < pQuery->numOfOutput; ++i) { SExprInfo* pExprInfo = &pQuery->pSelectExpr[i]; int32_t functionId = pExprInfo->base.functionId; - if (functionId != TSDB_FUNC_TAGPRJ && functionId != TSDB_FUNC_TID_TAG && - (!(functionId == TSDB_FUNC_COUNT && pExprInfo->base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX))) { + + if (functionId != TSDB_FUNC_TAGPRJ && + functionId != TSDB_FUNC_TID_TAG && + (!(functionId == TSDB_FUNC_COUNT && pExprInfo->base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX)) && + (!(functionId == TSDB_FUNC_PRJ && pExprInfo->base.colInfo.flag == TSDB_COL_UDC))) { return false; } } @@ -1803,7 +1844,7 @@ bool colIdCheck(SQuery *pQuery) { return false; } } - + return true; } @@ -1859,16 +1900,12 @@ static void changeExecuteScanOrder(SQInfo *pQInfo, bool stableQuery) { // descending order query for last_row query if (isFirstLastRowQuery(pQuery)) { qDebug("QInfo:%p scan order changed for last_row query, old:%d, new:%d", GET_QINFO_ADDR(pQuery), - pQuery->order.order, TSDB_ORDER_DESC); - - pQuery->order.order = TSDB_ORDER_DESC; - - int64_t skey = MIN(pQuery->window.skey, pQuery->window.ekey); - int64_t ekey = MAX(pQuery->window.skey, pQuery->window.ekey); - - pQuery->window.skey = ekey; - pQuery->window.ekey = skey; + pQuery->order.order, TSDB_ORDER_ASC); + pQuery->order.order = TSDB_ORDER_ASC; + if (pQuery->window.skey > pQuery->window.ekey) { + SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); + } return; } @@ -1963,7 +2000,7 @@ static void getIntermediateBufInfo(SQueryRuntimeEnv* pRuntimeEnv, int32_t* ps, i } pRuntimeEnv->numOfRowsPerPage = ((*ps) - sizeof(tFilePage)) / (*rowsize); - + assert(pRuntimeEnv->numOfRowsPerPage <= MAX_ROWS_PER_RESBUF_PAGE); } #define IS_PREFILTER_TYPE(_t) ((_t) != TSDB_DATA_TYPE_BINARY && (_t) != TSDB_DATA_TYPE_NCHAR) @@ -2117,21 +2154,21 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, void* pQueryHandle, if (tsdbRetrieveDataBlockStatisInfo(pQueryHandle, pStatis) != TSDB_CODE_SUCCESS) { // return DISK_DATA_LOAD_FAILED; } - + pRuntimeEnv->summary.loadBlockStatis += 1; - + if (*pStatis == NULL) { // data block statistics does not exist, load data block *pDataBlock = tsdbRetrieveDataBlock(pQueryHandle, NULL); pRuntimeEnv->summary.totalCheckedRows += pBlockInfo->rows; } } else { assert(status == BLK_DATA_ALL_NEEDED); - + // load the data block statistics to perform further filter pRuntimeEnv->summary.loadBlockStatis += 1; if (tsdbRetrieveDataBlockStatisInfo(pQueryHandle, pStatis) != TSDB_CODE_SUCCESS) { } - + if (!needToLoadDataBlock(pRuntimeEnv, *pStatis, pRuntimeEnv->pCtx, pBlockInfo->rows)) { // current block has been discard due to filter applied pRuntimeEnv->summary.discardBlocks += 1; @@ -2139,7 +2176,7 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, void* pQueryHandle, pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); return BLK_DATA_DISCARD; } - + pRuntimeEnv->summary.totalCheckedRows += pBlockInfo->rows; pRuntimeEnv->summary.loadBlocks += 1; *pDataBlock = tsdbRetrieveDataBlock(pQueryHandle, NULL); @@ -2224,7 +2261,7 @@ static void ensureOutputBufferSimple(SQueryRuntimeEnv* pRuntimeEnv, int32_t capa char *tmp = realloc(pQuery->sdata[i], bytes * capacity + sizeof(tFilePage)); if (tmp == NULL) { // todo handle the oom - assert(0); + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } else { pQuery->sdata[i] = (tFilePage *)tmp; } @@ -2244,20 +2281,20 @@ static void ensureOutputBuffer(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pB SQuery* pQuery = pRuntimeEnv->pQuery; if (!QUERY_IS_INTERVAL_QUERY(pQuery) && !pRuntimeEnv->groupbyNormalCol && !isFixedOutputQuery(pRuntimeEnv) && !isTSCompQuery(pQuery)) { SResultRec *pRec = &pQuery->rec; - + if (pQuery->rec.capacity - pQuery->rec.rows < pBlockInfo->rows) { int32_t remain = (int32_t)(pRec->capacity - pRec->rows); int32_t newSize = (int32_t)(pRec->capacity + (pBlockInfo->rows - remain)); - + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t bytes = pQuery->pSelectExpr[i].bytes; assert(bytes > 0 && newSize > 0); char *tmp = realloc(pQuery->sdata[i], bytes * newSize + sizeof(tFilePage)); if (tmp == NULL) { // todo handle the oom - assert(0); + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } else { - memset(tmp + sizeof(tFilePage) + bytes * pRec->rows, 0, (newSize - pRec->rows) * bytes); + memset(tmp + sizeof(tFilePage) + bytes * pRec->rows, 0, (size_t)((newSize - pRec->rows) * bytes)); pQuery->sdata[i] = (tFilePage *)tmp; } @@ -2272,7 +2309,7 @@ static void ensureOutputBuffer(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pB qDebug("QInfo:%p realloc output buffer, new size: %d rows, old:%" PRId64 ", remain:%" PRId64, GET_QINFO_ADDR(pRuntimeEnv), newSize, pRec->capacity, newSize - pRec->rows); - + pRec->capacity = newSize; } } @@ -2382,7 +2419,7 @@ static void doSetTagValueInParam(void *tsdb, void* pTable, int32_t tagColId, tVa if (tagColId == TSDB_TBNAME_COLUMN_INDEX) { char* val = tsdbGetTableName(pTable); assert(val != NULL); - + tVariantCreateFromBinary(tag, varDataVal(val), varDataLen(val), TSDB_DATA_TYPE_BINARY); } else { char* val = tsdbGetTableTagVal(pTable, tagColId, type, bytes); @@ -2390,7 +2427,7 @@ static void doSetTagValueInParam(void *tsdb, void* pTable, int32_t tagColId, tVa tag->nType = TSDB_DATA_TYPE_NULL; return; } - + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { if (isNull(val, type)) { tag->nType = TSDB_DATA_TYPE_NULL; @@ -2437,7 +2474,7 @@ void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, void *pTable, void *tsdb) { // set tag value, by which the results are aggregated. for (int32_t idx = 0; idx < pQuery->numOfOutput; ++idx) { SExprInfo* pLocalExprInfo = &pQuery->pSelectExpr[idx]; - + // ts_comp column required the tag value for join filter if (!TSDB_COL_IS_TAG(pLocalExprInfo->base.colInfo.flag)) { continue; @@ -2487,14 +2524,14 @@ static void doMerge(SQueryRuntimeEnv *pRuntimeEnv, int64_t timestamp, SWindowRes // in case of tag column, the tag information should be extracted from input buffer if (functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TAG) { tVariantDestroy(&pCtx[i].tag); - + int32_t type = pCtx[i].outputType; if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { tVariantCreateFromBinary(&pCtx[i].tag, varDataVal(pCtx[i].aInputElemBuf), varDataLen(pCtx[i].aInputElemBuf), type); } else { tVariantCreateFromBinary(&pCtx[i].tag, pCtx[i].aInputElemBuf, pCtx[i].inputBytes, pCtx[i].inputType); } - + } } @@ -2584,7 +2621,7 @@ void UNUSED_FUNC displayInterResult(tFilePage **pdata, SQueryRuntimeEnv* pRuntim for (int32_t j = 0; j < numOfRows; ++j) { for (int32_t i = 0; i < numOfCols; ++i) { - + switch (pQuery->pSelectExpr[i].type) { case TSDB_DATA_TYPE_BINARY: { int32_t type = pQuery->pSelectExpr[i].type; @@ -2659,7 +2696,7 @@ int32_t tableResultComparFn(const void *pLeft, const void *pRight, void *param) } int32_t mergeIntoGroupResult(SQInfo *pQInfo) { - int64_t st = taosGetTimestampMs(); + int64_t st = taosGetTimestampUs(); int32_t ret = TSDB_CODE_SUCCESS; int32_t numOfGroups = (int32_t)(GET_NUM_OF_TABLEGROUP(pQInfo)); @@ -2678,23 +2715,30 @@ int32_t mergeIntoGroupResult(SQInfo *pQInfo) { break; } - assert(pQInfo->numOfGroupResultPages == 0); + assert(pQInfo->groupResInfo.numOfDataPages == 0); qDebug("QInfo:%p no result in group %d, continue", pQInfo, pQInfo->groupIndex - 1); } - if (pQInfo->groupIndex == numOfGroups && pQInfo->offset == pQInfo->numOfGroupResultPages) { + SGroupResInfo* info = &pQInfo->groupResInfo; + if (pQInfo->groupIndex == numOfGroups && info->pos.pageId == info->numOfDataPages) { SET_STABLE_QUERY_OVER(pQInfo); } - qDebug("QInfo:%p merge res data into group, index:%d, total group:%d, elapsed time:%" PRId64 "ms", pQInfo, - pQInfo->groupIndex - 1, numOfGroups, taosGetTimestampMs() - st); + int64_t elapsedTime = taosGetTimestampUs() - st; + qDebug("QInfo:%p merge res data into group, index:%d, total group:%d, elapsed time:%" PRId64 "us", pQInfo, + pQInfo->groupIndex - 1, numOfGroups, elapsedTime); + pQInfo->runtimeEnv.summary.firstStageMergeTime += elapsedTime; return TSDB_CODE_SUCCESS; } void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { - if (pQInfo->offset == pQInfo->numOfGroupResultPages) { - pQInfo->numOfGroupResultPages = 0; + SGroupResInfo* pGroupResInfo = &pQInfo->groupResInfo; + + // all results have been return to client, try next group + if (pGroupResInfo->pos.pageId == pGroupResInfo->numOfDataPages) { + pGroupResInfo->numOfDataPages = 0; + pGroupResInfo->pos.rowId = 0; // current results of group has been sent to client, try next group if (mergeIntoGroupResult(pQInfo) != TSDB_CODE_SUCCESS) { @@ -2703,7 +2747,7 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { // check if all results has been sent to client int32_t numOfGroup = (int32_t)(GET_NUM_OF_TABLEGROUP(pQInfo)); - if (pQInfo->numOfGroupResultPages == 0 && pQInfo->groupIndex == numOfGroup) { + if (pGroupResInfo->numOfDataPages == 0 && pQInfo->groupIndex == numOfGroup) { SET_STABLE_QUERY_OVER(pQInfo); return; } @@ -2712,29 +2756,50 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { SQueryRuntimeEnv * pRuntimeEnv = &pQInfo->runtimeEnv; SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; - int32_t id = getGroupResultId(pQInfo->groupIndex - 1); - SIDList list = getDataBufPagesIdList(pResultBuf, pQInfo->offset + id); - - int32_t size = (int32_t)(taosArrayGetSize(list)); + int32_t id = pQInfo->groupResInfo.groupId; + SIDList list = getDataBufPagesIdList(pResultBuf, id); int32_t offset = 0; - for (int32_t j = 0; j < size; ++j) { + int32_t numOfCopiedRows = 0; + + size_t size = taosArrayGetSize(list); + assert(size == pGroupResInfo->numOfDataPages); + + bool done = false; + for (int32_t j = pGroupResInfo->pos.pageId; j < size; ++j) { SPageInfo* pi = *(SPageInfo**) taosArrayGet(list, j); - tFilePage *pData = getResBufPage(pResultBuf, pi->pageId); + tFilePage* pData = getResBufPage(pResultBuf, pi->pageId); + + assert(pData->num > 0 && pData->num <= pRuntimeEnv->numOfRowsPerPage && pGroupResInfo->pos.rowId < pData->num); + int32_t numOfRes = (int32_t)(pData->num - pGroupResInfo->pos.rowId); + + if (numOfRes > pQuery->rec.capacity - offset) { + numOfCopiedRows = (int32_t)(pQuery->rec.capacity - offset); + pGroupResInfo->pos.rowId += numOfCopiedRows; + done = true; + } else { + numOfCopiedRows = (int32_t)pData->num; + + pGroupResInfo->pos.pageId += 1; + pGroupResInfo->pos.rowId = 0; + } for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; char * pDest = pQuery->sdata[i]->data; - memcpy(pDest + offset * bytes, pData->data + pRuntimeEnv->offset[i] * pData->num, bytes * pData->num); + + memcpy(pDest + offset * bytes, pData->data + pRuntimeEnv->offset[i] * pRuntimeEnv->numOfRowsPerPage, + (size_t)bytes * numOfCopiedRows); } - offset += (int32_t)pData->num; + offset += numOfCopiedRows; + if (done) { + break; + } } assert(pQuery->rec.rows == 0); - pQuery->rec.rows += offset; - pQInfo->offset += 1; } int64_t getNumOfResultWindowRes(SQuery *pQuery, SWindowResult *pWindowRes) { @@ -2767,7 +2832,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { size_t size = taosArrayGetSize(pGroup); tFilePage **buffer = pQuery->sdata; - int32_t* posList = calloc(size, sizeof(int32_t)); + int32_t *posList = calloc(size, sizeof(int32_t)); STableQueryInfo **pTableList = malloc(POINTER_BYTES * size); if (pTableList == NULL || posList == NULL) { @@ -2780,23 +2845,38 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { // todo opt for the case of one table per group int32_t numOfTables = 0; + SIDList pageList = NULL; + int32_t tid = -1; + for (int32_t i = 0; i < size; ++i) { STableQueryInfo *item = taosArrayGetP(pGroup, i); SIDList list = getDataBufPagesIdList(pRuntimeEnv->pResultBuf, TSDB_TABLEID(item->pTable)->tid); + pageList = list; + tid = TSDB_TABLEID(item->pTable)->tid; + if (taosArrayGetSize(list) > 0 && item->windowResInfo.size > 0) { - pTableList[numOfTables] = item; - numOfTables += 1; + pTableList[numOfTables++] = item; } } + // there is no data in current group if (numOfTables == 0) { taosTFree(posList); taosTFree(pTableList); - - assert(pQInfo->numOfGroupResultPages == 0); return 0; } else if (numOfTables == 1) { // no need to merge results since only one table in each group + taosTFree(posList); + taosTFree(pTableList); + + SGroupResInfo* pGroupResInfo = &pQInfo->groupResInfo; + + pGroupResInfo->numOfDataPages = (int32_t)taosArrayGetSize(pageList); + pGroupResInfo->groupId = tid; + pGroupResInfo->pos.pageId = 0; + pGroupResInfo->pos.rowId = 0; + + return pGroupResInfo->numOfDataPages; } SCompSupporter cs = {pTableList, posList, pQInfo}; @@ -2810,9 +2890,15 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { } char* buf = calloc(1, pRuntimeEnv->interBufSize); + if (buf == NULL) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + setWindowResultInfo(pResultInfo, pQuery, pRuntimeEnv->stableQuery, buf); resetMergeResultBuf(pQuery, pRuntimeEnv->pCtx, pResultInfo); + pQInfo->groupResInfo.groupId = getGroupResultId(pQInfo->groupIndex); + // todo add windowRes iterator int64_t lastTimestamp = -1; int64_t startt = taosGetTimestampMs(); @@ -2820,6 +2906,13 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { while (1) { if (IS_QUERY_KILLED(pQInfo)) { qDebug("QInfo:%p it is already killed, abort", pQInfo); + + taosTFree(pTableList); + taosTFree(posList); + taosTFree(pTree); + taosTFree(pResultInfo); + taosTFree(buf); + longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); } @@ -2832,7 +2925,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { char *b = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes, page); TSKEY ts = GET_INT64_VAL(b); - assert(ts == pWindowRes->window.skey); + assert(ts == pWindowRes->skey); int64_t num = getNumOfResultWindowRes(pQuery, pWindowRes); if (num <= 0) { cs.position[pos] += 1; @@ -2850,7 +2943,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { doMerge(pRuntimeEnv, ts, pWindowRes, true); } else { // copy data to disk buffer if (buffer[0]->num == pQuery->rec.capacity) { - if (flushFromResultBuf(pQInfo) != TSDB_CODE_SUCCESS) { + if (flushFromResultBuf(pRuntimeEnv, &pQInfo->groupResInfo) != TSDB_CODE_SUCCESS) { return -1; } @@ -2887,7 +2980,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { } if (buffer[0]->num != 0) { // there are data in buffer - if (flushFromResultBuf(pQInfo) != TSDB_CODE_SUCCESS) { + if (flushFromResultBuf(pRuntimeEnv, &pQInfo->groupResInfo) != TSDB_CODE_SUCCESS) { qError("QInfo:%p failed to flush data into temp file, abort query", pQInfo); taosTFree(pTree); @@ -2911,16 +3004,14 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { taosTFree(posList); taosTFree(pTree); - pQInfo->offset = 0; - taosTFree(pResultInfo); taosTFree(buf); - return pQInfo->numOfGroupResultPages; + + return pQInfo->groupResInfo.numOfDataPages; } -int32_t flushFromResultBuf(SQInfo *pQInfo) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery * pQuery = pRuntimeEnv->pQuery; +int32_t flushFromResultBuf(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo) { + SQuery *pQuery = pRuntimeEnv->pQuery; SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; @@ -2928,32 +3019,32 @@ int32_t flushFromResultBuf(SQInfo *pQInfo) { int32_t pageId = -1; int32_t capacity = pResultBuf->numOfRowsPerPage; - int32_t remain = (int32_t)pQuery->sdata[0]->num; + int32_t remain = (int32_t) pQuery->sdata[0]->num; int32_t offset = 0; while (remain > 0) { - int32_t r = remain; - if (r > capacity) { - r = capacity; - } + int32_t rows = (remain > capacity)? capacity:remain; + assert(rows > 0); - int32_t id = getGroupResultId(pQInfo->groupIndex) + pQInfo->numOfGroupResultPages; - tFilePage *buf = getNewDataBuf(pResultBuf, id, &pageId); + // get the output buffer page + tFilePage *buf = getNewDataBuf(pResultBuf, pGroupResInfo->groupId, &pageId); + buf->num = rows; - // pagewise copy to dest buffer + // pagewisely copy to dest buffer for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; - buf->num = r; - memcpy(buf->data + pRuntimeEnv->offset[i] * buf->num, ((char *)pQuery->sdata[i]->data) + offset * bytes, - buf->num * bytes); + char* output = buf->data + pRuntimeEnv->offset[i] * pRuntimeEnv->numOfRowsPerPage; + char* src = ((char *) pQuery->sdata[i]->data) + offset * bytes; + memcpy(output, src, (size_t)(buf->num * bytes)); } - offset += r; - remain -= r; + offset += rows; + remain -= rows; + + pGroupResInfo->numOfDataPages += 1; } - pQInfo->numOfGroupResultPages += 1; return TSDB_CODE_SUCCESS; } @@ -2972,10 +3063,10 @@ static void updateTableQueryInfoForReverseScan(SQuery *pQuery, STableQueryInfo * if (pTableQueryInfo == NULL) { return; } - + // order has changed already int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); - + // TODO validate the assertion // if (!QUERY_IS_ASC_QUERY(pQuery)) { // assert(pTableQueryInfo->win.ekey >= pTableQueryInfo->lastKey + step); @@ -2991,7 +3082,7 @@ static void updateTableQueryInfoForReverseScan(SQuery *pQuery, STableQueryInfo * SWAP(pTableQueryInfo->win.skey, pTableQueryInfo->win.ekey, TSKEY); pTableQueryInfo->lastKey = pTableQueryInfo->win.skey; - + SWITCH_ORDER(pTableQueryInfo->cur.order); pTableQueryInfo->cur.vgroupIndex = -1; @@ -3001,10 +3092,10 @@ static void updateTableQueryInfoForReverseScan(SQuery *pQuery, STableQueryInfo * static void disableFuncInReverseScanImpl(SQInfo* pQInfo, SWindowResInfo *pWindowResInfo, int32_t order) { SQuery* pQuery = pQInfo->runtimeEnv.pQuery; - + for (int32_t i = 0; i < pWindowResInfo->size; ++i) { - SWindowStatus *pStatus = getTimeWindowResStatus(pWindowResInfo, i); - if (!pStatus->closed) { + bool closed = getTimeWindowResStatus(pWindowResInfo, i); + if (!closed) { continue; } @@ -3065,7 +3156,8 @@ static void setupQueryRangeForReverseScan(SQInfo* pQInfo) { STableQueryInfo *pCheckInfo = taosArrayGetP(group, j); updateTableQueryInfoForReverseScan(pQuery, pCheckInfo); - // update the last key in tableKeyInfo list + // update the last key in tableKeyInfo list, the tableKeyInfo is used to build the tsdbQueryHandle and decide + // the start check timestamp of tsdbQueryHandle STableKeyInfo *pTableKeyInfo = taosArrayGet(tableKeyGroup, j); pTableKeyInfo->lastKey = pCheckInfo->lastKey; @@ -3119,7 +3211,7 @@ void resetCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { pCtx->ptsOutputBuf = pRuntimeEnv->pCtx[0].aOutputBuf; } - memset(pQuery->sdata[i]->data, 0, (size_t)pQuery->pSelectExpr[i].bytes * pQuery->rec.capacity); + memset(pQuery->sdata[i]->data, 0, (size_t)(pQuery->pSelectExpr[i].bytes * pQuery->rec.capacity)); } initCtxOutputBuf(pRuntimeEnv); @@ -3178,7 +3270,7 @@ void skipResults(SQueryRuntimeEnv *pRuntimeEnv) { if (pQuery->rec.rows <= pQuery->limit.offset) { qDebug("QInfo:%p skip rows:%" PRId64 ", new offset:%" PRIu64, GET_QINFO_ADDR(pRuntimeEnv), pQuery->rec.rows, pQuery->limit.offset - pQuery->rec.rows); - + pQuery->limit.offset -= pQuery->rec.rows; pQuery->rec.rows = 0; @@ -3190,15 +3282,15 @@ void skipResults(SQueryRuntimeEnv *pRuntimeEnv) { int64_t numOfSkip = pQuery->limit.offset; pQuery->rec.rows -= numOfSkip; pQuery->limit.offset = 0; - + qDebug("QInfo:%p skip row:%"PRId64", new offset:%d, numOfRows remain:%" PRIu64, GET_QINFO_ADDR(pRuntimeEnv), numOfSkip, 0, pQuery->rec.rows); - + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t functionId = pQuery->pSelectExpr[i].base.functionId; int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; - memmove(pQuery->sdata[i]->data, (char*) pQuery->sdata[i]->data + bytes * numOfSkip, pQuery->rec.rows * bytes); + memmove(pQuery->sdata[i]->data, (char*)pQuery->sdata[i]->data + bytes * numOfSkip, (size_t)(pQuery->rec.rows * bytes)); pRuntimeEnv->pCtx[i].aOutputBuf = ((char*) pQuery->sdata[i]->data) + pQuery->rec.rows * bytes; if (functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) { @@ -3230,7 +3322,7 @@ bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) { for (int32_t i = 0; i < pWindowResInfo->size; ++i) { SWindowResult *pResult = getWindowResult(pWindowResInfo, i); - if (!pResult->status.closed) { + if (!pResult->closed) { continue; } @@ -3268,10 +3360,10 @@ bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) { static SQueryStatusInfo getQueryStatusInfo(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { SQuery *pQuery = pRuntimeEnv->pQuery; STableQueryInfo* pTableQueryInfo = pQuery->current; - + assert((start <= pTableQueryInfo->lastKey && QUERY_IS_ASC_QUERY(pQuery)) || (start >= pTableQueryInfo->lastKey && !QUERY_IS_ASC_QUERY(pQuery))); - + SQueryStatusInfo info = { .status = pQuery->status, .windowIndex = pRuntimeEnv->windowResInfo.curIndex, @@ -3293,7 +3385,8 @@ static void setEnvBeforeReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatusI pStatus->cur = tsBufGetCursor(pRuntimeEnv->pTSBuf); // save the cursor if (pRuntimeEnv->pTSBuf) { SWITCH_ORDER(pRuntimeEnv->pTSBuf->cur.order); - tsBufNextPos(pRuntimeEnv->pTSBuf); + bool ret = tsBufNextPos(pRuntimeEnv->pTSBuf); + assert(ret); } // reverse order time range @@ -3351,7 +3444,7 @@ static void clearEnvAfterReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatus // update the pQuery->window.skey and pQuery->window.ekey to limit the scan scope of sliding query during reverse scan pTableQueryInfo->lastKey = pStatus->lastKey; pQuery->status = pStatus->status; - + pTableQueryInfo->win = pStatus->w; pQuery->window = pTableQueryInfo->win; } @@ -3367,7 +3460,7 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { SQInfo *pQInfo = (SQInfo *) GET_QINFO_ADDR(pRuntimeEnv); SQuery *pQuery = pRuntimeEnv->pQuery; STableQueryInfo *pTableQueryInfo = pQuery->current; - + setQueryStatus(pQuery, QUERY_NOT_COMPLETED); // store the start query position @@ -3420,7 +3513,7 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { pRuntimeEnv->windowResInfo.curIndex = qstatus.windowIndex; setQueryStatus(pQuery, QUERY_NOT_COMPLETED); pRuntimeEnv->scanFlag = REPEAT_SCAN; - + qDebug("QInfo:%p start to repeat scan data blocks due to query func required, qrange:%"PRId64"-%"PRId64, pQInfo, cond.twindow.skey, cond.twindow.ekey); @@ -3525,12 +3618,6 @@ void destroyTableQueryInfo(STableQueryInfo *pTableQueryInfo) { cleanupTimeWindowInfo(&pTableQueryInfo->windowResInfo); } -#define CHECK_QUERY_TIME_RANGE(_q, _tableInfo) \ - do { \ - assert((((_tableInfo)->lastKey >= (_tableInfo)->win.skey) && QUERY_IS_ASC_QUERY(_q)) || \ - (((_tableInfo)->lastKey <= (_tableInfo)->win.skey) && !QUERY_IS_ASC_QUERY(_q))); \ - } while (0) - /** * set output buffer for different group * @param pRuntimeEnv @@ -3644,9 +3731,8 @@ int32_t setAdditionalInfo(SQInfo *pQInfo, void* pTable, STableQueryInfo *pTableQ // both the master and supplement scan needs to set the correct ts comp start position if (pRuntimeEnv->pTSBuf != NULL) { if (pTableQueryInfo->cur.vgroupIndex == -1) { - pTableQueryInfo->tag = pRuntimeEnv->pCtx[0].tag.i64Key; - - tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, 0, pTableQueryInfo->tag); + tVariantAssign(&pTableQueryInfo->tag, &pRuntimeEnv->pCtx[0].tag); + tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, 0, &pTableQueryInfo->tag); // keep the cursor info of current meter pTableQueryInfo->cur = pRuntimeEnv->pTSBuf->cur; @@ -3671,7 +3757,7 @@ void setIntervalQueryRange(SQInfo *pQInfo, TSKEY key) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; STableQueryInfo *pTableQueryInfo = pQuery->current; - + if (pTableQueryInfo->queryRangeSet) { pTableQueryInfo->lastKey = key; } else { @@ -3702,7 +3788,7 @@ void setIntervalQueryRange(SQInfo *pQInfo, TSKEY key) { if (!QUERY_IS_ASC_QUERY(pQuery)) { assert(win.ekey == pQuery->window.ekey); } - + pWindowResInfo->prevSKey = w.skey; } @@ -3728,7 +3814,7 @@ bool needPrimaryTimestampCol(SQuery *pQuery, SDataBlockInfo *pDataBlockInfo) { */ STimeWindow *w = &pDataBlockInfo->window; STableQueryInfo* pTableQueryInfo = pQuery->current; - + bool loadPrimaryTS = (pTableQueryInfo->lastKey >= w->skey && pTableQueryInfo->lastKey <= w->ekey) || (pQuery->window.ekey >= w->skey && pQuery->window.ekey <= w->ekey) || requireTimestamp(pQuery); @@ -3755,27 +3841,27 @@ static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResInfo *pResultInfo, int32_ step = -1; } + SGroupResInfo* pGroupResInfo = &pQInfo->groupResInfo; + for (int32_t i = startIdx; (i < totalSet) && (i >= 0); i += step) { if (result[i].numOfRows == 0) { - pQInfo->offset = 0; pQInfo->groupIndex += 1; + pGroupResInfo->pos.rowId = 0; continue; } - assert(pQInfo->offset <= 1); - - int32_t numOfRowsToCopy = result[i].numOfRows - pQInfo->offset; - int32_t oldOffset = pQInfo->offset; + int32_t numOfRowsToCopy = result[i].numOfRows - pGroupResInfo->pos.rowId; + int32_t oldOffset = pGroupResInfo->pos.rowId; /* - * current output space is not enough to keep all the result data of this group, only copy partial results - * to SQuery object's result buffer + * current output space is not enough to accommodate all data of this page, only partial results + * will be copied to SQuery object's result buffer */ if (numOfRowsToCopy > pQuery->rec.capacity - numOfResult) { - numOfRowsToCopy = (int32_t)pQuery->rec.capacity - numOfResult; - pQInfo->offset += numOfRowsToCopy; + numOfRowsToCopy = (int32_t) pQuery->rec.capacity - numOfResult; + pGroupResInfo->pos.rowId += numOfRowsToCopy; } else { - pQInfo->offset = 0; + pGroupResInfo->pos.rowId = 0; pQInfo->groupIndex += 1; } @@ -3840,7 +3926,7 @@ static void updateWindowResNumOfRes(SQueryRuntimeEnv *pRuntimeEnv) { continue; } - pResult->numOfRows = MAX(pResult->numOfRows, pResult->resultInfo[j].numOfRes); + pResult->numOfRows = (uint16_t)(MAX(pResult->numOfRows, pResult->resultInfo[j].numOfRes)); } } } @@ -3849,7 +3935,7 @@ static void stableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBloc SArray *pDataBlock, __block_search_fn_t searchFn) { SQuery * pQuery = pRuntimeEnv->pQuery; STableQueryInfo* pTableQueryInfo = pQuery->current; - + SWindowResInfo * pWindowResInfo = &pTableQueryInfo->windowResInfo; pQuery->pos = QUERY_IS_ASC_QUERY(pQuery)? 0 : pDataBlockInfo->rows - 1; @@ -3941,10 +4027,10 @@ int32_t doFillGapsInResults(SQueryRuntimeEnv* pRuntimeEnv, tFilePage **pDst, int SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv); SQuery *pQuery = pRuntimeEnv->pQuery; SFillInfo* pFillInfo = pRuntimeEnv->pFillInfo; - + while (1) { int32_t ret = (int32_t)taosGenerateDataBlock(pFillInfo, (tFilePage**)pQuery->sdata, (int32_t)pQuery->rec.capacity); - + // todo apply limit output function /* reached the start position of according to offset value, return immediately */ if (pQuery->limit.offset == 0) { @@ -3955,7 +4041,7 @@ int32_t doFillGapsInResults(SQueryRuntimeEnv* pRuntimeEnv, tFilePage **pDst, int if (pQuery->limit.offset < ret) { qDebug("QInfo:%p initial numOfRows:%d, generate filled result:%d rows, offset:%" PRId64 ". Discard due to offset, remain:%" PRId64 ", new offset:%d", pQInfo, pFillInfo->numOfRows, ret, pQuery->limit.offset, ret - pQuery->limit.offset, 0); - + ret -= (int32_t)pQuery->limit.offset; // todo !!!!there exactly number of interpo is not valid. // todo refactor move to the beginning of buffer @@ -3963,14 +4049,14 @@ int32_t doFillGapsInResults(SQueryRuntimeEnv* pRuntimeEnv, tFilePage **pDst, int memmove(pDst[i]->data, pDst[i]->data + pQuery->pSelectExpr[i].bytes * pQuery->limit.offset, ret * pQuery->pSelectExpr[i].bytes); } - + pQuery->limit.offset = 0; return ret; } else { qDebug("QInfo:%p initial numOfRows:%d, generate filled result:%d rows, offset:%" PRId64 ". Discard due to offset, " "remain:%d, new offset:%" PRId64, pQInfo, pFillInfo->numOfRows, ret, pQuery->limit.offset, 0, pQuery->limit.offset - ret); - + pQuery->limit.offset -= ret; pQuery->rec.rows = 0; ret = 0; @@ -3986,18 +4072,22 @@ static void queryCostStatis(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQueryCostInfo *pSummary = &pRuntimeEnv->summary; - qDebug("QInfo:%p :cost summary: elapsed time:%"PRId64" us, total blocks:%d, load block statis:%d," - " load data block:%d, total rows:%"PRId64 ", check rows:%"PRId64, - pQInfo, pSummary->elapsedTime, pSummary->totalBlocks, pSummary->loadBlockStatis, + // add the merge time + pSummary->elapsedTime += pSummary->firstStageMergeTime; + + qDebug("QInfo:%p :cost summary: elapsed time:%"PRId64" us, first merge:%"PRId64" us, total blocks:%d, " + "load block statis:%d, load data block:%d, total rows:%"PRId64 ", check rows:%"PRId64, + pQInfo, pSummary->elapsedTime, pSummary->firstStageMergeTime, pSummary->totalBlocks, pSummary->loadBlockStatis, pSummary->loadBlocks, pSummary->totalRows, pSummary->totalCheckedRows); - qDebug("QInfo:%p :cost summary: internal size:%"PRId64, pQInfo, pSummary->internalSupSize); + qDebug("QInfo:%p :cost summary: internal size:%"PRId64"B, numOfWin:%"PRId64, pQInfo, pSummary->internalSupSize, + pSummary->numOfTimeWindows); } static void updateOffsetVal(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBlockInfo) { SQuery *pQuery = pRuntimeEnv->pQuery; STableQueryInfo* pTableQueryInfo = pQuery->current; - + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); if (pQuery->limit.offset == pBlockInfo->rows) { // current block will ignore completed @@ -4087,7 +4177,7 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) { assert(pRuntimeEnv->windowResInfo.prevSKey == TSKEY_INITIAL_VAL); STimeWindow w = TSWINDOW_INITIALIZER; - + SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; STableQueryInfo *pTableQueryInfo = pQuery->current; @@ -4136,21 +4226,21 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) { // set the abort info pQuery->pos = startPos; - + // reset the query start timestamp pTableQueryInfo->win.skey = ((TSKEY *)pColInfoData->pData)[startPos]; pQuery->window.skey = pTableQueryInfo->win.skey; *start = pTableQueryInfo->win.skey; - + pWindowResInfo->prevSKey = tw.skey; int32_t index = pRuntimeEnv->windowResInfo.curIndex; - + int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, NULL, binarySearchForKey, pDataBlock); pRuntimeEnv->windowResInfo.curIndex = index; // restore the window index - + qDebug("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, numOfRes:%d, lastKey:%"PRId64, GET_QINFO_ADDR(pRuntimeEnv), blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, numOfRes, pQuery->current->lastKey); - + return true; } else { // do nothing *start = tw.skey; @@ -4218,7 +4308,7 @@ static int32_t setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery) if (!isSTableQuery && (pQInfo->tableqinfoGroupInfo.numOfTables == 1) - && (cond.order == TSDB_ORDER_ASC) + && (cond.order == TSDB_ORDER_ASC) && (!QUERY_IS_INTERVAL_QUERY(pQuery)) && (!isGroupbyNormalCol(pQuery->pGroupbyExpr)) && (!isFixedOutputQuery(pRuntimeEnv)) @@ -4231,6 +4321,22 @@ static int32_t setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery) terrno = TSDB_CODE_SUCCESS; if (isFirstLastRowQuery(pQuery)) { pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo); + + // update the query time window + pQuery->window = cond.twindow; + + size_t numOfGroups = GET_NUM_OF_TABLEGROUP(pQInfo); + for(int32_t i = 0; i < numOfGroups; ++i) { + SArray *group = GET_TABLEGROUP(pQInfo, i); + + size_t t = taosArrayGetSize(group); + for (int32_t j = 0; j < t; ++j) { + STableQueryInfo *pCheckInfo = taosArrayGetP(group, j); + + pCheckInfo->win = pQuery->window; + pCheckInfo->lastKey = pCheckInfo->win.skey; + } + } } else if (isPointInterpoQuery(pQuery)) { pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo); } else { @@ -4243,21 +4349,25 @@ static int32_t setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery) static SFillColInfo* taosCreateFillColInfo(SQuery* pQuery) { int32_t numOfCols = pQuery->numOfOutput; int32_t offset = 0; - + SFillColInfo* pFillCol = calloc(numOfCols, sizeof(SFillColInfo)); + if (pFillCol == NULL) { + return NULL; + } + for(int32_t i = 0; i < numOfCols; ++i) { SExprInfo* pExprInfo = &pQuery->pSelectExpr[i]; - + pFillCol[i].col.bytes = pExprInfo->bytes; pFillCol[i].col.type = (int8_t)pExprInfo->type; pFillCol[i].col.offset = offset; pFillCol[i].flag = TSDB_COL_NORMAL; // always be ta normal column for table query pFillCol[i].functionId = pExprInfo->base.functionId; pFillCol[i].fillVal.i = pQuery->fillVal[i]; - + offset += pExprInfo->bytes; } - + return pFillCol; } @@ -4272,13 +4382,16 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo pRuntimeEnv->hasTagResults = hasTagValOutput(pQuery); setScanLimitationByResultBuffer(pQuery); + + // NOTE: pTableCheckInfo need to update the query time range and the lastKey info + // TODO fixme changeExecuteScanOrder(pQInfo, false); code = setupQueryHandle(tsdb, pQInfo, isSTableQuery); if (code != TSDB_CODE_SUCCESS) { return code; } - + pQInfo->tsdb = tsdb; pQInfo->vgId = vgId; @@ -4423,7 +4536,17 @@ static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) { } pQuery->current = *pTableQueryInfo; - CHECK_QUERY_TIME_RANGE(pQuery, *pTableQueryInfo); + if (QUERY_IS_ASC_QUERY(pQuery)) { + assert( + ((*pTableQueryInfo)->win.skey <= (*pTableQueryInfo)->win.ekey) && + ((*pTableQueryInfo)->lastKey >= (*pTableQueryInfo)->win.skey) && + ((*pTableQueryInfo)->win.skey >= pQuery->window.skey && (*pTableQueryInfo)->win.ekey <= pQuery->window.ekey)); + } else { + assert( + ((*pTableQueryInfo)->win.skey >= (*pTableQueryInfo)->win.ekey) && + ((*pTableQueryInfo)->lastKey <= (*pTableQueryInfo)->win.skey) && + ((*pTableQueryInfo)->win.skey <= pQuery->window.skey && (*pTableQueryInfo)->win.ekey >= pQuery->window.ekey)); + } if (!pRuntimeEnv->groupbyNormalCol) { setEnvForEachBlock(pQInfo, *pTableQueryInfo, &blockInfo); @@ -4439,8 +4562,10 @@ static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) { summary->totalRows += blockInfo.rows; stableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, pStatis, pDataBlock, binarySearchForKey); - qDebug("QInfo:%p check data block, uid:%"PRId64", tid:%d, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, lastKey:%" PRId64, - pQInfo, blockInfo.uid, blockInfo.tid, blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, pQuery->current->lastKey); + qDebug("QInfo:%p check data block completed, uid:%"PRId64", tid:%d, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, " + "lastKey:%" PRId64, + pQInfo, blockInfo.uid, blockInfo.tid, blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, + pQuery->current->lastKey); } if (terrno != TSDB_CODE_SUCCESS) { @@ -4501,8 +4626,7 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { if (pRuntimeEnv->pTSBuf != NULL) { if (pRuntimeEnv->cur.vgroupIndex == -1) { - int64_t tag = pRuntimeEnv->pCtx[0].tag.i64Key; - STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, 0, tag); + STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, 0, &pRuntimeEnv->pCtx[0].tag); // failed to find data with the specified tag value if (elem.vnode < 0) { @@ -4552,7 +4676,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { SArray *g1 = taosArrayInit(1, POINTER_BYTES); SArray *tx = taosArrayClone(group); taosArrayPush(g1, &tx); - + STableGroupInfo gp = {.numOfTables = taosArrayGetSize(tx), .pGroupList = g1}; // include only current table @@ -4560,9 +4684,9 @@ static void sequentialTableProcess(SQInfo *pQInfo) { tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle); pRuntimeEnv->pQueryHandle = NULL; } - + if (isFirstLastRowQuery(pQuery)) { - pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(pQInfo->tsdb, &cond, &gp, pQInfo); + assert(0); // last_row query switch to other routine to handle } else { pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(pQInfo->tsdb, &cond, &gp, pQInfo); } @@ -4574,10 +4698,10 @@ static void sequentialTableProcess(SQInfo *pQInfo) { } initCtxOutputBuf(pRuntimeEnv); - + SArray* s = tsdbGetQueriedTableList(pRuntimeEnv->pQueryHandle); assert(taosArrayGetSize(s) >= 1); - + setTagVal(pRuntimeEnv, taosArrayGetP(s, 0), pQInfo->tsdb); if (isFirstLastRowQuery(pQuery)) { assert(taosArrayGetSize(s) == 1); @@ -4590,13 +4714,13 @@ static void sequentialTableProcess(SQInfo *pQInfo) { pQuery->current = taosArrayGetP(first, 0); scanOneTableDataBlocks(pRuntimeEnv, pQuery->current->lastKey); - + int64_t numOfRes = getNumOfResult(pRuntimeEnv); if (numOfRes > 0) { pQuery->rec.rows += numOfRes; forwardCtxOutputBuf(pRuntimeEnv, numOfRes); } - + skipResults(pRuntimeEnv); pQInfo->groupIndex += 1; @@ -4655,18 +4779,17 @@ static void sequentialTableProcess(SQInfo *pQInfo) { SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; // no results generated for current group, continue to try the next group - taosArrayDestroy(s); + taosArrayDestroy(s); if (pWindowResInfo->size <= 0) { continue; } for (int32_t i = 0; i < pWindowResInfo->size; ++i) { - SWindowStatus *pStatus = &pWindowResInfo->pResult[i].status; - pStatus->closed = true; // enable return all results for group by normal columns + pWindowResInfo->pResult[i].closed = true; // enable return all results for group by normal columns SWindowResult *pResult = &pWindowResInfo->pResult[i]; for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { - pResult->numOfRows = MAX(pResult->numOfRows, pResult->resultInfo[j].numOfRes); + pResult->numOfRows = (uint16_t)(MAX(pResult->numOfRows, pResult->resultInfo[j].numOfRes)); } } @@ -4816,11 +4939,11 @@ static void doSaveContext(SQInfo *pQInfo) { SET_REVERSE_SCAN_FLAG(pRuntimeEnv); SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); SWITCH_ORDER(pQuery->order.order); - + if (pRuntimeEnv->pTSBuf != NULL) { pRuntimeEnv->pTSBuf->cur.order = pQuery->order.order; } - + STsdbQueryCond cond = { .order = pQuery->order.order, .colList = pQuery->colList, @@ -4924,7 +5047,6 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { el = scanMultiTableDataBlocks(pQInfo); qDebug("QInfo:%p reversed scan completed, elapsed time: %" PRId64 "ms", pQInfo, el); -// doCloseAllTimeWindowAfterScan(pQInfo); doRestoreContext(pQInfo); } else { qDebug("QInfo:%p no need to do reversed scan, query completed", pQInfo); @@ -4962,14 +5084,14 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { */ static void tableFixedOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - + SQuery *pQuery = pRuntimeEnv->pQuery; if (!pRuntimeEnv->topBotQuery && pQuery->limit.offset > 0) { // no need to execute, since the output will be ignore. return; } - + pQuery->current = pTableInfo; // set current query table info - + scanOneTableDataBlocks(pRuntimeEnv, pTableInfo->lastKey); finalizeQueryResult(pRuntimeEnv); @@ -4987,10 +5109,10 @@ static void tableFixedOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) static void tableMultiOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - + SQuery *pQuery = pRuntimeEnv->pQuery; pQuery->current = pTableInfo; - + // for ts_comp query, re-initialized is not allowed if (!isTSCompQuery(pQuery)) { resetCtxOutputBuf(pRuntimeEnv); @@ -5081,7 +5203,7 @@ static void tableIntervalProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { int32_t numOfFilled = 0; TSKEY newStartKey = TSKEY_INITIAL_VAL; - + // skip blocks without load the actual data block from file if no filter condition present skipTimeInterval(pRuntimeEnv, &newStartKey); if (pQuery->limit.offset > 0 && pQuery->numOfFilterCols == 0 && pRuntimeEnv->pFillInfo == NULL) { @@ -5108,7 +5230,7 @@ static void tableIntervalProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { taosFillSetStartInfo(pRuntimeEnv->pFillInfo, (int32_t)pQuery->rec.rows, pQuery->window.ekey); taosFillCopyInputDataFromFilePage(pRuntimeEnv->pFillInfo, (tFilePage**) pQuery->sdata); numOfFilled = 0; - + pQuery->rec.rows = doFillGapsInResults(pRuntimeEnv, (tFilePage **)pQuery->sdata, &numOfFilled); if (pQuery->rec.rows > 0 || Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { limitResults(pRuntimeEnv); @@ -5174,11 +5296,11 @@ static void tableQueryImpl(SQInfo *pQInfo) { // number of points returned during this query pQuery->rec.rows = 0; int64_t st = taosGetTimestampUs(); - + assert(pQInfo->tableqinfoGroupInfo.numOfTables == 1); SArray* g = GET_TABLEGROUP(pQInfo, 0); STableQueryInfo* item = taosArrayGetP(g, 0); - + // group by normal column, sliding window query, interval query are handled by interval query processor if (QUERY_IS_INTERVAL_QUERY(pQuery) || pRuntimeEnv->groupbyNormalCol) { // interval (down sampling operation) tableIntervalProcess(pQInfo, item); @@ -5202,15 +5324,13 @@ static void stableQueryImpl(SQInfo *pQInfo) { int64_t st = taosGetTimestampUs(); if (QUERY_IS_INTERVAL_QUERY(pQuery) || - (isFixedOutputQuery(pRuntimeEnv) && (!isPointInterpoQuery(pQuery)) && !pRuntimeEnv->groupbyNormalCol && - !isFirstLastRowQuery(pQuery))) { + (isFixedOutputQuery(pRuntimeEnv) && (!isPointInterpoQuery(pQuery)) && (!pRuntimeEnv->groupbyNormalCol))) { multiTableQueryProcess(pQInfo); } else { assert((pQuery->checkBuffer == 1 && pQuery->intervalTime == 0) || isPointInterpoQuery(pQuery) || isFirstLastRowQuery(pQuery) || pRuntimeEnv->groupbyNormalCol); sequentialTableProcess(pQInfo); - } // record the total elapsed time @@ -5222,7 +5342,7 @@ static int32_t getColumnIndexInSource(SQueryTableMsg *pQueryMsg, SSqlFuncMsg *pE if (TSDB_COL_IS_TAG(pExprMsg->colInfo.flag)) { if (pExprMsg->colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) { - return -1; + return TSDB_TBNAME_COLUMN_INDEX; } while(j < pQueryMsg->numOfTags) { @@ -5233,6 +5353,8 @@ static int32_t getColumnIndexInSource(SQueryTableMsg *pQueryMsg, SSqlFuncMsg *pE j += 1; } + } else if (pExprMsg->colInfo.flag == TSDB_COL_UDC) { // user specified column data + return TSDB_UD_COLUMN_INDEX; } else { while (j < pQueryMsg->numOfCols) { if (pExprMsg->colInfo.colId == pQueryMsg->colList[j].colId) { @@ -5371,11 +5493,15 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, int32_t numOfFilters = pColInfo->numOfFilters; if (numOfFilters > 0) { pColInfo->filters = calloc(numOfFilters, sizeof(SColumnFilterInfo)); + if (pColInfo->filters == NULL) { + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _cleanup; + } } for (int32_t f = 0; f < numOfFilters; ++f) { SColumnFilterInfo *pFilterMsg = (SColumnFilterInfo *)pMsg; - + SColumnFilterInfo *pColFilter = &pColInfo->filters[f]; pColFilter->filterstr = htons(pFilterMsg->filterstr); @@ -5384,8 +5510,13 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, if (pColFilter->filterstr) { pColFilter->len = htobe64(pFilterMsg->len); - pColFilter->pz = (int64_t) calloc(1, pColFilter->len + 1 * TSDB_NCHAR_SIZE); // note: null-terminator - memcpy((void *)pColFilter->pz, pMsg, pColFilter->len); + pColFilter->pz = (int64_t)calloc(1, (size_t)(pColFilter->len + 1 * TSDB_NCHAR_SIZE)); // note: null-terminator + if (pColFilter->pz == 0) { + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _cleanup; + } + + memcpy((void *)pColFilter->pz, pMsg, (size_t)pColFilter->len); pMsg += (pColFilter->len + 1); } else { pColFilter->lowerBndi = htobe64(pFilterMsg->lowerBndi); @@ -5398,6 +5529,11 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, } *pExpr = calloc(pQueryMsg->numOfOutput, POINTER_BYTES); + if (*pExpr == NULL) { + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _cleanup; + } + SSqlFuncMsg *pExprMsg = (SSqlFuncMsg *)pMsg; for (int32_t i = 0; i < pQueryMsg->numOfOutput; ++i) { @@ -5484,6 +5620,11 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, if (pQueryMsg->numOfTags > 0) { (*tagCols) = calloc(1, sizeof(SColumnInfo) * pQueryMsg->numOfTags); + if (*tagCols == NULL) { + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _cleanup; + } + for (int32_t i = 0; i < pQueryMsg->numOfTags; ++i) { SColumnInfo* pTagCol = (SColumnInfo*) pMsg; @@ -5500,6 +5641,12 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, // the tag query condition expression string is located at the end of query msg if (pQueryMsg->tagCondLen > 0) { *tagCond = calloc(1, pQueryMsg->tagCondLen); + + if (*tagCond == NULL) { + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _cleanup; + + } memcpy(*tagCond, pMsg, pQueryMsg->tagCondLen); pMsg += pQueryMsg->tagCondLen; } @@ -5591,9 +5738,19 @@ static int32_t createQFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SExprInfo * bytes = tDataTypeDesc[type].nSize; } else if (pExprs[i].base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX && pExprs[i].base.functionId == TSDB_FUNC_TAGPRJ) { // parse the normal column SSchema s = tGetTableNameColumnSchema(); - type = s.type; + type = s.type; bytes = s.bytes; - } else{ + } else if (pExprs[i].base.colInfo.colId <= TSDB_UD_COLUMN_INDEX) { + // it is a user-defined constant value column + assert(pExprs[i].base.functionId == TSDB_FUNC_PRJ); + + type = pExprs[i].base.arg[1].argType; + bytes = pExprs[i].base.arg[1].argBytes; + + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { + bytes += VARSTR_HEADER_SIZE; + } + } else { int32_t j = getColumnIndexInSource(pQueryMsg, &pExprs[i].base, pTagCols); assert(j < pQueryMsg->numOfCols || j < pQueryMsg->numOfTags); @@ -5629,18 +5786,19 @@ static int32_t createQFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SExprInfo * if (functId == TSDB_FUNC_TOP || functId == TSDB_FUNC_BOTTOM) { int32_t j = getColumnIndexInSource(pQueryMsg, &pExprs[i].base, pTagCols); - assert(j < pQueryMsg->numOfCols); - - SColumnInfo *pCol = &pQueryMsg->colList[j]; - - int32_t ret = - getResultDataInfo(pCol->type, pCol->bytes, functId, (int32_t)pExprs[i].base.arg[0].argValue.i64, - &pExprs[i].type, &pExprs[i].bytes, &pExprs[i].interBytes, tagLen, isSuperTable); - assert(ret == TSDB_CODE_SUCCESS); + if (j < 0 || j >= pQueryMsg->numOfCols) { + assert(0); + } else { + SColumnInfo *pCol = &pQueryMsg->colList[j]; + int32_t ret = + getResultDataInfo(pCol->type, pCol->bytes, functId, (int32_t)pExprs[i].base.arg[0].argValue.i64, + &pExprs[i].type, &pExprs[i].bytes, &pExprs[i].interBytes, tagLen, isSuperTable); + assert(ret == TSDB_CODE_SUCCESS); + } } } - *pExprInfo = pExprs; + *pExprInfo = pExprs; return TSDB_CODE_SUCCESS; } @@ -5680,6 +5838,9 @@ static int32_t createFilterInfo(void *pQInfo, SQuery *pQuery) { } pQuery->pFilterInfo = calloc(1, sizeof(SSingleColumnFilterInfo) * pQuery->numOfFilterCols); + if (pQuery->pFilterInfo == NULL) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } for (int32_t i = 0, j = 0; i < pQuery->numOfCols; ++i) { if (pQuery->colList[i].numOfFilters > 0) { @@ -5690,6 +5851,9 @@ static int32_t createFilterInfo(void *pQInfo, SQuery *pQuery) { pFilterInfo->numOfFilters = pQuery->colList[i].numOfFilters; pFilterInfo->pFilters = calloc(pFilterInfo->numOfFilters, sizeof(SColumnFilterElem)); + if (pFilterInfo->pFilters == NULL) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } for (int32_t f = 0; f < pFilterInfo->numOfFilters; ++f) { SColumnFilterElem *pSingleColFilter = &pFilterInfo->pFilters[f]; @@ -5766,7 +5930,7 @@ static void doUpdateExprColumnIndex(SQuery *pQuery) { // todo opt performance SColIndex *pColIndex = &pSqlExprMsg->colInfo; - if (!TSDB_COL_IS_TAG(pColIndex->flag)) { + if (TSDB_COL_IS_NORMAL_COL(pColIndex->flag)) { int32_t f = 0; for (f = 0; f < pQuery->numOfCols; ++f) { if (pColIndex->colId == pQuery->colList[f].colId) { @@ -5774,8 +5938,10 @@ static void doUpdateExprColumnIndex(SQuery *pQuery) { break; } } - - assert (f < pQuery->numOfCols); + + assert(f < pQuery->numOfCols); + } else if (pColIndex->colId <= TSDB_UD_COLUMN_INDEX) { + // do nothing for user-defined constant value result columns } else { int32_t f = 0; for (f = 0; f < pQuery->numOfTags; ++f) { @@ -5784,7 +5950,7 @@ static void doUpdateExprColumnIndex(SQuery *pQuery) { break; } } - + assert(f < pQuery->numOfTags || pColIndex->colId == TSDB_TBNAME_COLUMN_INDEX); } } @@ -5837,6 +6003,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList, if (pQuery == NULL) { goto _cleanup_query; } + pQInfo->runtimeEnv.pQuery = pQuery; pQuery->numOfCols = numOfCols; @@ -5889,7 +6056,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList, assert(pExprs[col].interBytes >= pExprs[col].bytes); // allocate additional memory for interResults that are usually larger then final results - size_t size = (pQuery->rec.capacity + 1) * pExprs[col].bytes + pExprs[col].interBytes + sizeof(tFilePage); + size_t size = (size_t)((pQuery->rec.capacity + 1) * pExprs[col].bytes + pExprs[col].interBytes + sizeof(tFilePage)); pQuery->sdata[col] = (tFilePage *)calloc(1, size); if (pQuery->sdata[col] == NULL) { goto _cleanup; @@ -5922,6 +6089,10 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList, pQInfo->runtimeEnv.interBufSize = getOutputInterResultBufSize(pQuery); pQInfo->pBuf = calloc(pTableGroupInfo->numOfTables, sizeof(STableQueryInfo)); + if (pQInfo->pBuf == NULL) { + goto _cleanup; + } + int32_t index = 0; for(int32_t i = 0; i < numOfGroups; ++i) { @@ -5932,20 +6103,18 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList, if (p1 == NULL) { goto _cleanup; } + taosArrayPush(pQInfo->tableqinfoGroupInfo.pGroupList, &p1); for(int32_t j = 0; j < s; ++j) { STableKeyInfo* info = taosArrayGet(pa, j); + STableId* id = TSDB_TABLEID(info->pTable); - STableIdInfo* pTableId = taosArraySearch(pTableIdList, id, compareTableIdInfo); - if (pTableId != NULL ) { - window.skey = pTableId->key; - } else { - window.skey = pQueryMsg->window.skey; - } + window.skey = (pTableId != NULL)? pTableId->key:pQueryMsg->window.skey; void* buf = (char*)pQInfo->pBuf + index * sizeof(STableQueryInfo); + STableQueryInfo* item = createTableQueryInfo(&pQInfo->runtimeEnv, info->pTable, window, buf); if (item == NULL) { goto _cleanup; @@ -6040,7 +6209,7 @@ static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQ if ((code = doInitQInfo(pQInfo, pTSBuf, tsdb, vgId, isSTable)) != TSDB_CODE_SUCCESS) { goto _error; } - + return code; _error: @@ -6066,35 +6235,59 @@ static void freeQInfo(SQInfo *pQInfo) { return; } - SQuery *pQuery = pQInfo->runtimeEnv.pQuery; qDebug("QInfo:%p start to free QInfo", pQInfo); - for (int32_t col = 0; col < pQuery->numOfOutput; ++col) { - taosTFree(pQuery->sdata[col]); - } teardownQueryRuntimeEnv(&pQInfo->runtimeEnv); - for (int32_t i = 0; i < pQuery->numOfFilterCols; ++i) { - SSingleColumnFilterInfo *pColFilter = &pQuery->pFilterInfo[i]; - if (pColFilter->numOfFilters > 0) { - taosTFree(pColFilter->pFilters); + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + if (pQuery != NULL) { + if (pQuery->sdata != NULL) { + for (int32_t col = 0; col < pQuery->numOfOutput; ++col) { + taosTFree(pQuery->sdata[col]); + } + taosTFree(pQuery->sdata); } - } - if (pQuery->pSelectExpr != NULL) { - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - SExprInfo* pExprInfo = &pQuery->pSelectExpr[i]; + if (pQuery->fillVal != NULL) { + taosTFree(pQuery->fillVal); + } - if (pExprInfo->pExpr != NULL) { - tExprTreeDestroy(&pExprInfo->pExpr, NULL); + for (int32_t i = 0; i < pQuery->numOfFilterCols; ++i) { + SSingleColumnFilterInfo *pColFilter = &pQuery->pFilterInfo[i]; + if (pColFilter->numOfFilters > 0) { + taosTFree(pColFilter->pFilters); } } - taosTFree(pQuery->pSelectExpr); - } + if (pQuery->pSelectExpr != NULL) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + SExprInfo *pExprInfo = &pQuery->pSelectExpr[i]; - if (pQuery->fillVal != NULL) { - taosTFree(pQuery->fillVal); + if (pExprInfo->pExpr != NULL) { + tExprTreeDestroy(&pExprInfo->pExpr, NULL); + } + } + + taosTFree(pQuery->pSelectExpr); + } + + if (pQuery->pGroupbyExpr != NULL) { + taosArrayDestroy(pQuery->pGroupbyExpr->columnInfo); + taosTFree(pQuery->pGroupbyExpr); + } + + taosTFree(pQuery->tagColList); + taosTFree(pQuery->pFilterInfo); + + if (pQuery->colList != NULL) { + for (int32_t i = 0; i < pQuery->numOfCols; i++) { + SColumnInfo *column = pQuery->colList + i; + freeColumnFilterInfo(column->filters, column->numOfFilters); + } + taosTFree(pQuery->colList); + } + + taosTFree(pQuery); } // todo refactor, extract method to destroytableDataInfo @@ -6118,25 +6311,8 @@ static void freeQInfo(SQInfo *pQInfo) { taosHashCleanup(pQInfo->tableqinfoGroupInfo.map); tsdbDestroyTableGroup(&pQInfo->tableGroupInfo); taosArrayDestroy(pQInfo->arrTableIdInfo); - - if (pQuery->pGroupbyExpr != NULL) { - taosArrayDestroy(pQuery->pGroupbyExpr->columnInfo); - taosTFree(pQuery->pGroupbyExpr); - } - taosTFree(pQuery->tagColList); - taosTFree(pQuery->pFilterInfo); - if (pQuery->colList != NULL) { - for (int32_t i = 0; i < pQuery->numOfCols; i++) { - SColumnInfo* column = pQuery->colList + i; - freeColumnFilterInfo(column->filters, column->numOfFilters); - } - taosTFree(pQuery->colList); - } - - taosTFree(pQuery->sdata); - taosTFree(pQuery); pQInfo->signature = 0; qDebug("QInfo:%p QInfo is freed", pQInfo); @@ -6162,7 +6338,7 @@ static size_t getResultSize(SQInfo *pQInfo, int64_t *numOfRows) { return 0; } } else { - return pQuery->rowSize * (*numOfRows); + return (size_t)(pQuery->rowSize * (*numOfRows)); } } @@ -6176,14 +6352,16 @@ static int32_t doDumpQueryResult(SQInfo *pQInfo, char *data) { // make sure file exist if (FD_VALID(fd)) { - int32_t s = lseek(fd, 0, SEEK_END); - UNUSED(s); - qDebug("QInfo:%p ts comp data return, file:%s, size:%d", pQInfo, pQuery->sdata[0]->data, s); + uint64_t s = lseek(fd, 0, SEEK_END); + + qDebug("QInfo:%p ts comp data return, file:%s, size:%"PRId64, pQInfo, pQuery->sdata[0]->data, s); if (lseek(fd, 0, SEEK_SET) >= 0) { - size_t sz = read(fd, data, s); - UNUSED(sz); + size_t sz = read(fd, data, (uint32_t)s); + if(sz < s) { // todo handle error + assert(0); + } } else { - // todo handle error + UNUSED(s); } close(fd); @@ -6193,7 +6371,7 @@ static int32_t doDumpQueryResult(SQInfo *pQInfo, char *data) { qError("QInfo:%p failed to open tmp file to send ts-comp data to client, path:%s, reason:%s", pQInfo, pQuery->sdata[0]->data, strerror(errno)); if (fd != -1) { - close(fd); + close(fd); } } @@ -6212,7 +6390,7 @@ static int32_t doDumpQueryResult(SQInfo *pQInfo, char *data) { qDebug("QInfo:%p results limitation reached, limitation:%"PRId64, pQInfo, pQuery->limit.limit); setQueryStatus(pQuery, QUERY_OVER); } - + return TSDB_CODE_SUCCESS; } @@ -6314,7 +6492,7 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qi pExprs = NULL; pGroupbyExpr = NULL; pTagColumnInfo = NULL; - + if ((*pQInfo) == NULL) { code = TSDB_CODE_QRY_OUT_OF_MEMORY; goto _over; @@ -6329,7 +6507,7 @@ _over: if (pGroupbyExpr != NULL) { taosArrayDestroy(pGroupbyExpr->columnInfo); free(pGroupbyExpr); - } + } free(pTagColumnInfo); free(pExprs); free(pExprMsg); @@ -6496,7 +6674,7 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co (*pRsp)->offset = 0; (*pRsp)->useconds = htobe64(pRuntimeEnv->summary.elapsedTime); } - + (*pRsp)->precision = htons(pQuery->precision); if (pQuery->rec.rows > 0 && pQInfo->code == TSDB_CODE_SUCCESS) { doDumpQueryResult(pQInfo, (*pRsp)->data); @@ -6573,7 +6751,7 @@ static void buildTagQueryResult(SQInfo* pQInfo) { if (numOfGroup == 0) { return; } - + SArray* pa = GET_TABLEGROUP(pQInfo, 0); size_t num = taosArrayGetSize(pa); @@ -6664,6 +6842,10 @@ static void buildTagQueryResult(SQInfo* pQInfo) { char *data = NULL, *dst = NULL; int16_t type = 0, bytes = 0; for(int32_t j = 0; j < pQuery->numOfOutput; ++j) { + // not assign value in case of user defined constant output column + if (pExprInfo[j].base.colInfo.flag == TSDB_COL_UDC) { + continue; + } if (pExprInfo[j].base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) { bytes = tbnameSchema.bytes; @@ -6674,7 +6856,7 @@ static void buildTagQueryResult(SQInfo* pQInfo) { } else { type = pExprInfo[j].type; bytes = pExprInfo[j].bytes; - + data = tsdbGetTableTagVal(item->pTable, pExprInfo[j].base.colInfo.colId, type, bytes); dst = pQuery->sdata[j]->data + count * pExprInfo[j].bytes; @@ -6710,12 +6892,16 @@ void freeqinfoFn(void *qhandle) { } void* qOpenQueryMgmt(int32_t vgId) { - const int32_t REFRESH_HANDLE_INTERVAL = 60; // every 30 seconds, refresh handle pool + const int32_t REFRESH_HANDLE_INTERVAL = 30; // every 30 seconds, refresh handle pool char cacheName[128] = {0}; sprintf(cacheName, "qhandle_%d", vgId); SQueryMgmt* pQueryMgmt = calloc(1, sizeof(SQueryMgmt)); + if (pQueryMgmt == NULL) { + terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + return NULL; + } pQueryMgmt->qinfoPool = taosCacheInit(TSDB_DATA_TYPE_BIGINT, REFRESH_HANDLE_INTERVAL, true, freeqinfoFn, cacheName); pQueryMgmt->closed = false; diff --git a/src/query/src/qExtbuffer.c b/src/query/src/qExtbuffer.c index a147780144..192a31ebf5 100644 --- a/src/query/src/qExtbuffer.c +++ b/src/query/src/qExtbuffer.c @@ -990,7 +990,7 @@ void tColModelCompact(SColumnModel *pModel, tFilePage *inputBuffer, int32_t maxE SSchemaEx* pSchemaEx = &pModel->pFields[i]; memmove(inputBuffer->data + pSchemaEx->offset * inputBuffer->num, inputBuffer->data + pSchemaEx->offset * maxElemsCapacity, - pSchemaEx->field.bytes * inputBuffer->num); + (size_t)(pSchemaEx->field.bytes * inputBuffer->num)); } } diff --git a/src/query/src/qFill.c b/src/query/src/qFill.c index d9fe67e1b7..c1cfab3ea2 100644 --- a/src/query/src/qFill.c +++ b/src/query/src/qFill.c @@ -153,7 +153,7 @@ void taosFillCopyInputDataFromOneFilePage(SFillInfo* pFillInfo, tFilePage* pInpu SFillColInfo* pCol = &pFillInfo->pFillCol[i]; char* data = pInput->data + pCol->col.offset * pInput->num; - memcpy(pFillInfo->pData[i], data, pInput->num * pCol->col.bytes); + memcpy(pFillInfo->pData[i], data, (size_t)(pInput->num * pCol->col.bytes)); if (pCol->flag == TSDB_COL_TAG) { // copy the tag value to tag value buffer for (int32_t j = 0; j < pFillInfo->numOfTags; ++j) { diff --git a/src/query/src/qParserImpl.c b/src/query/src/qParserImpl.c index d8f12da642..9629f24cc2 100644 --- a/src/query/src/qParserImpl.c +++ b/src/query/src/qParserImpl.c @@ -33,7 +33,7 @@ SSqlInfo qSQLParse(const char *pStr) { int32_t i = 0; while (1) { - SSQLToken t0 = {0}; + SStrToken t0 = {0}; if (pStr[i] == 0) { Parse(pParser, 0, t0, &sqlInfo); @@ -73,12 +73,12 @@ abort_parse: return sqlInfo; } -tSQLExprList *tSQLExprListAppend(tSQLExprList *pList, tSQLExpr *pNode, SSQLToken *pToken) { +tSQLExprList *tSQLExprListAppend(tSQLExprList *pList, tSQLExpr *pNode, SStrToken *pToken) { if (pList == NULL) { pList = calloc(1, sizeof(tSQLExprList)); } - if (pList->nAlloc <= pList->nExpr) { // + if (pList->nAlloc <= pList->nExpr) { pList->nAlloc = (pList->nAlloc << 1) + 4; pList->a = realloc(pList->a, pList->nAlloc * sizeof(pList->a[0])); if (pList->a == 0) { @@ -117,41 +117,45 @@ void tSQLExprListDestroy(tSQLExprList *pList) { free(pList); } -tSQLExpr *tSQLExprIdValueCreate(SSQLToken *pAliasToken, int32_t optrType) { - tSQLExpr *nodePtr = calloc(1, sizeof(tSQLExpr)); +tSQLExpr *tSQLExprIdValueCreate(SStrToken *pToken, int32_t optrType) { + tSQLExpr *pSQLExpr = calloc(1, sizeof(tSQLExpr)); + + if (pToken != NULL) { + pSQLExpr->token = *pToken; + } if (optrType == TK_INTEGER || optrType == TK_STRING || optrType == TK_FLOAT || optrType == TK_BOOL) { - toTSDBType(pAliasToken->type); + toTSDBType(pToken->type); - tVariantCreate(&nodePtr->val, pAliasToken); - nodePtr->nSQLOptr = optrType; + tVariantCreate(&pSQLExpr->val, pToken); + pSQLExpr->nSQLOptr = optrType; } else if (optrType == TK_NOW) { // default use microsecond - nodePtr->val.i64Key = taosGetTimestamp(TSDB_TIME_PRECISION_MICRO); - nodePtr->val.nType = TSDB_DATA_TYPE_BIGINT; - nodePtr->nSQLOptr = TK_TIMESTAMP; // TK_TIMESTAMP used to denote the time value is in microsecond + pSQLExpr->val.i64Key = taosGetTimestamp(TSDB_TIME_PRECISION_MICRO); + pSQLExpr->val.nType = TSDB_DATA_TYPE_BIGINT; + pSQLExpr->nSQLOptr = TK_TIMESTAMP; // TK_TIMESTAMP used to denote the time value is in microsecond } else if (optrType == TK_VARIABLE) { - int32_t ret = getTimestampInUsFromStr(pAliasToken->z, pAliasToken->n, &nodePtr->val.i64Key); + int32_t ret = getTimestampInUsFromStr(pToken->z, pToken->n, &pSQLExpr->val.i64Key); UNUSED(ret); - nodePtr->val.nType = TSDB_DATA_TYPE_BIGINT; - nodePtr->nSQLOptr = TK_TIMESTAMP; + pSQLExpr->val.nType = TSDB_DATA_TYPE_BIGINT; + pSQLExpr->nSQLOptr = TK_TIMESTAMP; } else { // it must be the column name (tk_id) if it is not the number assert(optrType == TK_ID || optrType == TK_ALL); - if (pAliasToken != NULL) { - nodePtr->colInfo = *pAliasToken; + if (pToken != NULL) { + pSQLExpr->colInfo = *pToken; } - nodePtr->nSQLOptr = optrType; + pSQLExpr->nSQLOptr = optrType; } - return nodePtr; + return pSQLExpr; } /* * pList is the parameters for function with id(optType) * function name is denoted by pFunctionToken */ -tSQLExpr *tSQLExprCreateFunction(tSQLExprList *pList, SSQLToken *pFuncToken, SSQLToken *endToken, int32_t optType) { +tSQLExpr *tSQLExprCreateFunction(tSQLExprList *pList, SStrToken *pFuncToken, SStrToken *endToken, int32_t optType) { if (pFuncToken == NULL) return NULL; tSQLExpr *pExpr = calloc(1, sizeof(tSQLExpr)); @@ -163,6 +167,8 @@ tSQLExpr *tSQLExprCreateFunction(tSQLExprList *pList, SSQLToken *pFuncToken, SSQ pExpr->operand.n = len; // raw field name pExpr->operand.type = pFuncToken->type; + + pExpr->token = pExpr->operand; return pExpr; } @@ -173,8 +179,14 @@ tSQLExpr *tSQLExprCreateFunction(tSQLExprList *pList, SSQLToken *pFuncToken, SSQ tSQLExpr *tSQLExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optrType) { tSQLExpr *pExpr = calloc(1, sizeof(tSQLExpr)); - if (optrType == TK_PLUS || optrType == TK_MINUS || optrType == TK_STAR || optrType == TK_DIVIDE || - optrType == TK_REM) { + if (pRight != NULL && pLeft != NULL) { + char* endPos = pRight->token.z + pRight->token.n; + pExpr->token.z = pLeft->token.z; + pExpr->token.n = (uint32_t)(endPos - pExpr->token.z); + pExpr->token.type = pLeft->token.type; + } + + if (optrType == TK_PLUS || optrType == TK_MINUS || optrType == TK_STAR || optrType == TK_DIVIDE || optrType == TK_REM) { /* * if a token is noted as the TK_TIMESTAMP, the time precision is microsecond * Otherwise, the time precision is adaptive, determined by the time precision from databases. @@ -373,7 +385,7 @@ void tVariantListDestroy(tVariantList *pList) { free(pList); } -tVariantList *tVariantListAppendToken(tVariantList *pList, SSQLToken *pAliasToken, uint8_t sortOrder) { +tVariantList *tVariantListAppendToken(tVariantList *pList, SStrToken *pToken, uint8_t sortOrder) { if (pList == NULL) { pList = calloc(1, sizeof(tVariantList)); } @@ -382,9 +394,9 @@ tVariantList *tVariantListAppendToken(tVariantList *pList, SSQLToken *pAliasToke return pList; } - if (pAliasToken) { + if (pToken) { tVariant t = {0}; - tVariantCreate(&t, pAliasToken); + tVariantCreate(&t, pToken); tVariantListItem *pItem = &pList->a[pList->nExpr++]; memcpy(pItem, &t, sizeof(tVariant)); @@ -420,7 +432,7 @@ void tFieldListDestroy(tFieldList *pList) { free(pList); } -void setDBName(SSQLToken *pCpxName, SSQLToken *pDB) { +void setDBName(SStrToken *pCpxName, SStrToken *pDB) { pCpxName->type = pDB->type; pCpxName->z = pDB->z; pCpxName->n = pDB->n; @@ -464,7 +476,7 @@ int32_t getTimestampInUsFromStrImpl(int64_t val, char unit, int64_t *result) { return 0; } -void tSQLSetColumnInfo(TAOS_FIELD *pField, SSQLToken *pName, TAOS_FIELD *pType) { +void tSQLSetColumnInfo(TAOS_FIELD *pField, SStrToken *pName, TAOS_FIELD *pType) { int32_t maxLen = sizeof(pField->name) / sizeof(pField->name[0]); // truncate the column name @@ -479,7 +491,7 @@ void tSQLSetColumnInfo(TAOS_FIELD *pField, SSQLToken *pName, TAOS_FIELD *pType) pField->bytes = pType->bytes; } -void tSQLSetColumnType(TAOS_FIELD *pField, SSQLToken *type) { +void tSQLSetColumnType(TAOS_FIELD *pField, SStrToken *type) { pField->type = -1; int32_t LENGTH_SIZE_OF_STR = 2; // in case of nchar and binary, there two bytes to keep the length of binary|nchar. @@ -517,9 +529,9 @@ void tSQLSetColumnType(TAOS_FIELD *pField, SSQLToken *type) { /* * extract the select info out of sql string */ -SQuerySQL *tSetQuerySQLElems(SSQLToken *pSelectToken, tSQLExprList *pSelection, tVariantList *pFrom, tSQLExpr *pWhere, - tVariantList *pGroupby, tVariantList *pSortOrder, SSQLToken *pInterval, - SSQLToken *pSliding, tVariantList *pFill, SLimitVal *pLimit, SLimitVal *pGLimit) { +SQuerySQL *tSetQuerySQLElems(SStrToken *pSelectToken, tSQLExprList *pSelection, tVariantList *pFrom, tSQLExpr *pWhere, + tVariantList *pGroupby, tVariantList *pSortOrder, SStrToken *pInterval, + SStrToken *pSliding, tVariantList *pFill, SLimitVal *pLimit, SLimitVal *pGLimit) { assert(pSelection != NULL); SQuerySQL *pQuery = calloc(1, sizeof(SQuerySQL)); @@ -611,7 +623,7 @@ void destroyAllSelectClause(SSubclauseInfo *pClause) { taosTFree(pClause->pClause); } -SCreateTableSQL *tSetCreateSQLElems(tFieldList *pCols, tFieldList *pTags, SSQLToken *pStableName, +SCreateTableSQL *tSetCreateSQLElems(tFieldList *pCols, tFieldList *pTags, SStrToken *pStableName, tVariantList *pTagVals, SQuerySQL *pSelect, int32_t type) { SCreateTableSQL *pCreate = calloc(1, sizeof(SCreateTableSQL)); @@ -644,7 +656,7 @@ SCreateTableSQL *tSetCreateSQLElems(tFieldList *pCols, tFieldList *pTags, SSQLTo return pCreate; } -SAlterTableSQL *tAlterTableSQLElems(SSQLToken *pMeterName, tFieldList *pCols, tVariantList *pVals, int32_t type) { +SAlterTableSQL *tAlterTableSQLElems(SStrToken *pMeterName, tFieldList *pCols, tVariantList *pVals, int32_t type) { SAlterTableSQL *pAlterTable = calloc(1, sizeof(SAlterTableSQL)); pAlterTable->name = *pMeterName; @@ -716,7 +728,7 @@ SSubclauseInfo* setSubclause(SSubclauseInfo* pSubclause, void *pSqlExprInfo) { return pSubclause; } -SSqlInfo* setSQLInfo(SSqlInfo *pInfo, void *pSqlExprInfo, SSQLToken *pMeterName, int32_t type) { +SSqlInfo* setSQLInfo(SSqlInfo *pInfo, void *pSqlExprInfo, SStrToken *pMeterName, int32_t type) { pInfo->type = type; if (type == TSDB_SQL_SELECT) { @@ -745,7 +757,7 @@ SSubclauseInfo* appendSelectClause(SSubclauseInfo *pQueryInfo, void *pSubclause) return pQueryInfo; } -void setCreatedTableName(SSqlInfo *pInfo, SSQLToken *pMeterName, SSQLToken *pIfNotExists) { +void setCreatedTableName(SSqlInfo *pInfo, SStrToken *pMeterName, SStrToken *pIfNotExists) { pInfo->pCreateTableInfo->name = *pMeterName; pInfo->pCreateTableInfo->existCheck = (pIfNotExists->n != 0); } @@ -760,7 +772,7 @@ void tTokenListBuyMoreSpace(tDCLSQL *pTokenList) { } } -tDCLSQL *tTokenListAppend(tDCLSQL *pTokenList, SSQLToken *pToken) { +tDCLSQL *tTokenListAppend(tDCLSQL *pTokenList, SStrToken *pToken) { if (pToken == NULL) return NULL; if (pTokenList == NULL) pTokenList = calloc(1, sizeof(tDCLSQL)); @@ -781,19 +793,19 @@ void setDCLSQLElems(SSqlInfo *pInfo, int32_t type, int32_t nParam, ...) { va_start(va, nParam); while (nParam-- > 0) { - SSQLToken *pToken = va_arg(va, SSQLToken *); + SStrToken *pToken = va_arg(va, SStrToken *); pInfo->pDCLInfo = tTokenListAppend(pInfo->pDCLInfo, pToken); } va_end(va); } -void setDropDBTableInfo(SSqlInfo *pInfo, int32_t type, SSQLToken* pToken, SSQLToken* existsCheck) { +void setDropDBTableInfo(SSqlInfo *pInfo, int32_t type, SStrToken* pToken, SStrToken* existsCheck) { pInfo->type = type; pInfo->pDCLInfo = tTokenListAppend(pInfo->pDCLInfo, pToken); pInfo->pDCLInfo->existsCheck = (existsCheck->n == 1); } -void setShowOptions(SSqlInfo *pInfo, int32_t type, SSQLToken* prefix, SSQLToken* pPatterns) { +void setShowOptions(SSqlInfo *pInfo, int32_t type, SStrToken* prefix, SStrToken* pPatterns) { if (pInfo->pDCLInfo == NULL) { pInfo->pDCLInfo = calloc(1, sizeof(tDCLSQL)); } @@ -816,7 +828,7 @@ void setShowOptions(SSqlInfo *pInfo, int32_t type, SSQLToken* prefix, SSQLToken* } } -void setCreateDBSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *pToken, SCreateDBInfo *pDB, SSQLToken *pIgExists) { +void setCreateDBSQL(SSqlInfo *pInfo, int32_t type, SStrToken *pToken, SCreateDBInfo *pDB, SStrToken *pIgExists) { pInfo->type = type; if (pInfo->pDCLInfo == NULL) { pInfo->pDCLInfo = calloc(1, sizeof(tDCLSQL)); @@ -827,7 +839,7 @@ void setCreateDBSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *pToken, SCreateDBI pInfo->pDCLInfo->dbOpt.ignoreExists = pIgExists->n; // sql.y has: ifnotexists(X) ::= IF NOT EXISTS. {X.n = 1;} } -void setCreateAcctSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *pName, SSQLToken *pPwd, SCreateAcctSQL *pAcctInfo) { +void setCreateAcctSQL(SSqlInfo *pInfo, int32_t type, SStrToken *pName, SStrToken *pPwd, SCreateAcctSQL *pAcctInfo) { pInfo->type = type; if (pInfo->pDCLInfo == NULL) { pInfo->pDCLInfo = calloc(1, sizeof(tDCLSQL)); @@ -843,7 +855,7 @@ void setCreateAcctSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *pName, SSQLToken } } -void setCreateUserSQL(SSqlInfo *pInfo, SSQLToken *pName, SSQLToken *pPasswd) { +void setCreateUserSQL(SSqlInfo *pInfo, SStrToken *pName, SStrToken *pPasswd) { pInfo->type = TSDB_SQL_CREATE_USER; if (pInfo->pDCLInfo == NULL) { pInfo->pDCLInfo = calloc(1, sizeof(tDCLSQL)); @@ -855,7 +867,7 @@ void setCreateUserSQL(SSqlInfo *pInfo, SSQLToken *pName, SSQLToken *pPasswd) { pInfo->pDCLInfo->user.passwd = *pPasswd; } -void setAlterUserSQL(SSqlInfo *pInfo, int16_t type, SSQLToken *pName, SSQLToken* pPwd, SSQLToken *pPrivilege) { +void setAlterUserSQL(SSqlInfo *pInfo, int16_t type, SStrToken *pName, SStrToken* pPwd, SStrToken *pPrivilege) { pInfo->type = TSDB_SQL_ALTER_USER; if (pInfo->pDCLInfo == NULL) { pInfo->pDCLInfo = calloc(1, sizeof(tDCLSQL)); @@ -880,7 +892,7 @@ void setAlterUserSQL(SSqlInfo *pInfo, int16_t type, SSQLToken *pName, SSQLToken* } } -void setKillSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *ip) { +void setKillSQL(SSqlInfo *pInfo, int32_t type, SStrToken *ip) { pInfo->type = type; if (pInfo->pDCLInfo == NULL) { pInfo->pDCLInfo = calloc(1, sizeof(tDCLSQL)); @@ -909,5 +921,5 @@ void setDefaultCreateDbOption(SCreateDBInfo *pDBInfo) { pDBInfo->quorum = -1; pDBInfo->keep = NULL; - memset(&pDBInfo->precision, 0, sizeof(SSQLToken)); + memset(&pDBInfo->precision, 0, sizeof(SStrToken)); } diff --git a/src/query/src/qResultbuf.c b/src/query/src/qResultbuf.c index 51ff892b33..b3e97459d3 100644 --- a/src/query/src/qResultbuf.c +++ b/src/query/src/qResultbuf.c @@ -137,8 +137,15 @@ static char* doFlushPageToDisk(SDiskbasedResultBuf* pResultBuf, SPageInfo* pg) { } //3. write to disk. - fseek(pResultBuf->file, pg->info.offset, SEEK_SET); - fwrite(t, size, 1, pResultBuf->file); + int32_t ret = fseek(pResultBuf->file, pg->info.offset, SEEK_SET); + if (ret != 0) { // todo handle the error case + + } + + ret = (int32_t)fwrite(t, size, 1, pResultBuf->file); + if (ret != size) { // todo handle the error case + + } if (pResultBuf->fileSize < pg->info.offset + pg->info.length) { pResultBuf->fileSize = pg->info.offset + pg->info.length; @@ -381,7 +388,7 @@ size_t getNumOfRowsPerPage(const SDiskbasedResultBuf* pResultBuf) { return pResu size_t getNumOfResultBufGroupId(const SDiskbasedResultBuf* pResultBuf) { return taosHashGetSize(pResultBuf->groupSet); } -size_t getResBufSize(const SDiskbasedResultBuf* pResultBuf) { return pResultBuf->totalBufSize; } +size_t getResBufSize(const SDiskbasedResultBuf* pResultBuf) { return (size_t)pResultBuf->totalBufSize; } SIDList getDataBufPagesIdList(SDiskbasedResultBuf* pResultBuf, int32_t groupId) { assert(pResultBuf != NULL); diff --git a/src/query/src/qTokenizer.c b/src/query/src/qTokenizer.c index 227aded5f1..5fdabca166 100644 --- a/src/query/src/qTokenizer.c +++ b/src/query/src/qTokenizer.c @@ -85,7 +85,7 @@ static SKeyword keywordTable[] = { {"QUERIES", TK_QUERIES}, {"CONNECTIONS", TK_CONNECTIONS}, {"STREAMS", TK_STREAMS}, - {"CONFIGS", TK_CONFIGS}, + {"VARIABLES", TK_VARIABLES}, {"SCORES", TK_SCORES}, {"GRANTS", TK_GRANTS}, {"DOT", TK_DOT}, @@ -580,8 +580,8 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenType) { return 0; } -SSQLToken tStrGetToken(char* str, int32_t* i, bool isPrevOptr, uint32_t numOfIgnoreToken, uint32_t* ignoreTokenTypes) { - SSQLToken t0 = {0}; +SStrToken tStrGetToken(char* str, int32_t* i, bool isPrevOptr, uint32_t numOfIgnoreToken, uint32_t* ignoreTokenTypes) { + SStrToken t0 = {0}; // here we reach the end of sql string, null-terminated string if (str[*i] == 0) { diff --git a/src/query/src/qTsbuf.c b/src/query/src/qTsbuf.c index 25eb33ff7d..b264f6cdc9 100644 --- a/src/query/src/qTsbuf.c +++ b/src/query/src/qTsbuf.c @@ -8,7 +8,6 @@ static void TSBufUpdateVnodeInfo(STSBuf* pTSBuf, int32_t index, STSVnodeBlockInf static STSBuf* allocResForTSBuf(STSBuf* pTSBuf); static int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader); - /** * todo error handling * support auto closeable tmp file @@ -76,7 +75,7 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { return NULL; } - if ((int32_t)header.numOfVnode > pTSBuf->numOfAlloc) { + if (header.numOfVnode > pTSBuf->numOfAlloc) { pTSBuf->numOfAlloc = header.numOfVnode; STSVnodeBlockInfoEx* tmp = realloc(pTSBuf->pData, sizeof(STSVnodeBlockInfoEx) * pTSBuf->numOfAlloc); if (tmp == NULL) { @@ -225,14 +224,15 @@ static void writeDataToDisk(STSBuf* pTSBuf) { } STSBlock* pBlock = &pTSBuf->block; - - pBlock->numOfElem = pTSBuf->tsData.len / TSDB_KEYSIZE; + STSList* pTsData = &pTSBuf->tsData; + + pBlock->numOfElem = pTsData->len / TSDB_KEYSIZE; pBlock->compLen = - tsCompressTimestamp(pTSBuf->tsData.rawBuf, pTSBuf->tsData.len, pTSBuf->tsData.len / TSDB_KEYSIZE, pBlock->payload, - pTSBuf->tsData.allocSize, TWO_STAGE_COMP, pTSBuf->assistBuf, pTSBuf->bufSize); + tsCompressTimestamp(pTsData->rawBuf, pTsData->len, pTsData->len/TSDB_KEYSIZE, pBlock->payload, pTsData->allocSize, + TWO_STAGE_COMP, pTSBuf->assistBuf, pTSBuf->bufSize); int64_t r = fseek(pTSBuf->f, pTSBuf->fileSize, SEEK_SET); - UNUSED(r); + assert(r == 0); /* * format for output data: @@ -241,16 +241,22 @@ static void writeDataToDisk(STSBuf* pTSBuf) { * * both side has the compressed length is used to support load data forwards/backwords. */ - fwrite(&pBlock->tag, sizeof(pBlock->tag), 1, pTSBuf->f); + int32_t metaLen = 0; + metaLen += (int32_t)fwrite(&pBlock->tag.nType, 1, sizeof(pBlock->tag.nType), pTSBuf->f); + metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); + + if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR) { + metaLen += (int32_t)fwrite(pBlock->tag.pz, 1, (size_t)pBlock->tag.nLen, pTSBuf->f); + } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { + metaLen += (int32_t)fwrite(&pBlock->tag.i64Key, 1, sizeof(int64_t), pTSBuf->f); + } + fwrite(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f); - fwrite(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f); - fwrite(pBlock->payload, (size_t)pBlock->compLen, 1, pTSBuf->f); - fwrite(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f); - int32_t blockSize = sizeof(pBlock->tag) + sizeof(pBlock->numOfElem) + sizeof(pBlock->compLen) * 2 + pBlock->compLen; + int32_t blockSize = metaLen + sizeof(pBlock->numOfElem) + sizeof(pBlock->compLen) * 2 + pBlock->compLen; pTSBuf->fileSize += blockSize; pTSBuf->tsData.len = 0; @@ -298,9 +304,24 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { ret = fseek(pTSBuf->f, -offset, SEEK_CUR); UNUSED(ret); } - - size_t sz = fread(&pBlock->tag, sizeof(pBlock->tag), 1, pTSBuf->f); - UNUSED(sz); + + fread(&pBlock->tag.nType, sizeof(pBlock->tag.nType), 1, pTSBuf->f); + fread(&pBlock->tag.nLen, sizeof(pBlock->tag.nLen), 1, pTSBuf->f); + + // NOTE: mix types tags are not supported + size_t sz = 0; + if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR) { + char* tp = realloc(pBlock->tag.pz, pBlock->tag.nLen + 1); + assert(tp != NULL); + + memset(tp, 0, pBlock->tag.nLen + 1); + pBlock->tag.pz = tp; + + sz = fread(pBlock->tag.pz, (size_t)pBlock->tag.nLen, 1, pTSBuf->f); + } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { + sz = fread(&pBlock->tag.i64Key, sizeof(int64_t), 1, pTSBuf->f); + } + sz = fread(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f); UNUSED(sz); sz = fread(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f); @@ -361,7 +382,7 @@ static int32_t setCheckTSOrder(STSBuf* pTSBuf, const char* pData, int32_t len) { return TSDB_CODE_SUCCESS; } -void tsBufAppend(STSBuf* pTSBuf, int32_t vnodeId, int64_t tag, const char* pData, int32_t len) { +void tsBufAppend(STSBuf* pTSBuf, int32_t vnodeId, tVariant* tag, const char* pData, int32_t len) { STSVnodeBlockInfoEx* pBlockInfo = NULL; STSList* ptsData = &pTSBuf->tsData; @@ -375,15 +396,15 @@ void tsBufAppend(STSBuf* pTSBuf, int32_t vnodeId, int64_t tag, const char* pData } assert(pBlockInfo->info.vnode == vnodeId); - - if (pTSBuf->block.tag != tag && ptsData->len > 0) { + + if ((tVariantCompare(&pTSBuf->block.tag, tag) != 0) && ptsData->len > 0) { // new arrived data with different tags value, save current value into disk first writeDataToDisk(pTSBuf); } else { expandBuffer(ptsData, len); } - pTSBuf->block.tag = tag; + tVariantAssign(&pTSBuf->block.tag, tag); memcpy(ptsData->rawBuf + ptsData->len, pData, (size_t)len); // todo check return value @@ -465,7 +486,7 @@ static int32_t tsBufFindBlock(STSBuf* pTSBuf, STSVnodeBlockInfo* pBlockInfo, int return 0; } -static int32_t tsBufFindBlockByTag(STSBuf* pTSBuf, STSVnodeBlockInfo* pBlockInfo, int64_t tag) { +static int32_t tsBufFindBlockByTag(STSBuf* pTSBuf, STSVnodeBlockInfo* pBlockInfo, tVariant* tag) { bool decomp = false; int64_t offset = 0; @@ -484,7 +505,7 @@ static int32_t tsBufFindBlockByTag(STSBuf* pTSBuf, STSVnodeBlockInfo* pBlockInfo return -1; } - if (pTSBuf->block.tag == tag) { + if (tVariantCompare(&pTSBuf->block.tag, tag) == 0) { return i; } } @@ -669,8 +690,8 @@ STSElem tsBufGetElem(STSBuf* pTSBuf) { elem1.vnode = pTSBuf->pData[pCur->vgroupIndex].info.vnode; elem1.ts = *(TSKEY*)(pTSBuf->tsData.rawBuf + pCur->tsIndex * TSDB_KEYSIZE); - elem1.tag = pBlock->tag; - + tVariantAssign(&elem1.tag, &pBlock->tag); + return elem1; } @@ -800,7 +821,7 @@ STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_ return pTSBuf; } -STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t vnodeId, int64_t tag) { +STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t vnodeId, tVariant* tag) { STSElem elem = {.vnode = -1}; if (pTSBuf == NULL) { @@ -881,7 +902,9 @@ void tsBufDisplay(STSBuf* pTSBuf) { while (tsBufNextPos(pTSBuf)) { STSElem elem = tsBufGetElem(pTSBuf); - printf("%d-%" PRId64 "-%" PRId64 "\n", elem.vnode, elem.tag, elem.ts); + if (elem.tag.nType == TSDB_DATA_TYPE_BIGINT) { + printf("%d-%" PRId64 "-%" PRId64 "\n", elem.vnode, elem.tag.i64Key, elem.ts); + } } pTSBuf->cur.order = old; diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index a01eb33ec7..509362863c 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -45,8 +45,8 @@ int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, SQueryRuntimeEnv *pRun pWindowResInfo->curIndex = -1; pWindowResInfo->size = 0; pWindowResInfo->prevSKey = TSKEY_INITIAL_VAL; - - pRuntimeEnv->summary.internalSupSize += sizeof(SWindowResult) * threshold; + + SQueryCostInfo* pSummary = &pRuntimeEnv->summary; // use the pointer arraylist pWindowResInfo->pResult = calloc(threshold, sizeof(SWindowResult)); @@ -54,8 +54,11 @@ int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, SQueryRuntimeEnv *pRun return TSDB_CODE_QRY_OUT_OF_MEMORY; } - pRuntimeEnv->summary.internalSupSize += sizeof(SWindowResult) * threshold; - pRuntimeEnv->summary.internalSupSize += (pRuntimeEnv->pQuery->numOfOutput * sizeof(SResultInfo) + pRuntimeEnv->interBufSize) * pWindowResInfo->capacity; + pWindowResInfo->interval = pRuntimeEnv->pQuery->intervalTime; + + pSummary->internalSupSize += sizeof(SWindowResult) * threshold; + pSummary->internalSupSize += (pRuntimeEnv->pQuery->numOfOutput * sizeof(SResultInfo) + pRuntimeEnv->interBufSize) * pWindowResInfo->capacity; + pSummary->numOfTimeWindows = threshold; for (int32_t i = 0; i < pWindowResInfo->capacity; ++i) { int32_t code = createQueryResultInfo(pRuntimeEnv->pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, pRuntimeEnv->interBufSize); @@ -126,8 +129,8 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { for (int32_t i = 0; i < num; ++i) { SWindowResult *pResult = &pWindowResInfo->pResult[i]; - if (pResult->status.closed) { // remove the window slot from hash table - taosHashRemove(pWindowResInfo->hashList, (const char *)&pResult->window.skey, pWindowResInfo->type); + if (pResult->closed) { // remove the window slot from hash table + taosHashRemove(pWindowResInfo->hashList, (const char *)&pResult->skey, pWindowResInfo->type); } else { break; } @@ -149,12 +152,12 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { pWindowResInfo->size = remain; for (int32_t k = 0; k < pWindowResInfo->size; ++k) { SWindowResult *pResult = &pWindowResInfo->pResult[k]; - int32_t *p = (int32_t *)taosHashGet(pWindowResInfo->hashList, (const char *)&pResult->window.skey, + int32_t *p = (int32_t *)taosHashGet(pWindowResInfo->hashList, (const char *)&pResult->skey, tDataTypeDesc[pWindowResInfo->type].nSize); assert(p != NULL); int32_t v = (*p - num); assert(v >= 0 && v <= pWindowResInfo->size); - taosHashPut(pWindowResInfo->hashList, (char *)&pResult->window.skey, tDataTypeDesc[pWindowResInfo->type].nSize, + taosHashPut(pWindowResInfo->hashList, (char *)&pResult->skey, tDataTypeDesc[pWindowResInfo->type].nSize, (char *)&v, sizeof(int32_t)); } @@ -173,7 +176,7 @@ void clearClosedTimeWindow(SQueryRuntimeEnv *pRuntimeEnv) { int32_t numOfClosedTimeWindow(SWindowResInfo *pWindowResInfo) { int32_t i = 0; - while (i < pWindowResInfo->size && pWindowResInfo->pResult[i].status.closed) { + while (i < pWindowResInfo->size && pWindowResInfo->pResult[i].closed) { ++i; } @@ -184,11 +187,11 @@ void closeAllTimeWindow(SWindowResInfo *pWindowResInfo) { assert(pWindowResInfo->size >= 0 && pWindowResInfo->capacity >= pWindowResInfo->size); for (int32_t i = 0; i < pWindowResInfo->size; ++i) { - if (pWindowResInfo->pResult[i].status.closed) { + if (pWindowResInfo->pResult[i].closed) { continue; } - pWindowResInfo->pResult[i].status.closed = true; + pWindowResInfo->pResult[i].closed = true; } } @@ -204,7 +207,7 @@ void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_ } // get the result order - int32_t resultOrder = (pWindowResInfo->pResult[0].window.skey < pWindowResInfo->pResult[1].window.skey)? 1:-1; + int32_t resultOrder = (pWindowResInfo->pResult[0].skey < pWindowResInfo->pResult[1].skey)? 1:-1; if (order != resultOrder) { return; @@ -212,11 +215,12 @@ void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_ int32_t i = 0; if (order == QUERY_ASC_FORWARD_STEP) { - while (i < pWindowResInfo->size && (pWindowResInfo->pResult[i].window.ekey < lastKey)) { + TSKEY ekey = pWindowResInfo->pResult[i].skey + pWindowResInfo->interval; + while (i < pWindowResInfo->size && (ekey < lastKey)) { ++i; } } else if (order == QUERY_DESC_FORWARD_STEP) { - while (i < pWindowResInfo->size && (pWindowResInfo->pResult[i].window.skey > lastKey)) { + while (i < pWindowResInfo->size && (pWindowResInfo->pResult[i].skey > lastKey)) { ++i; } } @@ -227,11 +231,11 @@ void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_ } bool isWindowResClosed(SWindowResInfo *pWindowResInfo, int32_t slot) { - return (getWindowResult(pWindowResInfo, slot)->status.closed == true); + return (getWindowResult(pWindowResInfo, slot)->closed == true); } void closeTimeWindow(SWindowResInfo *pWindowResInfo, int32_t slot) { - getWindowResult(pWindowResInfo, slot)->status.closed = true; + getWindowResult(pWindowResInfo, slot)->closed = true; } void clearTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pWindowRes) { @@ -253,8 +257,8 @@ void clearTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pWindow pWindowRes->numOfRows = 0; pWindowRes->pos = (SPosInfo){-1, -1}; - pWindowRes->status.closed = false; - pWindowRes->window = TSWINDOW_INITIALIZER; + pWindowRes->closed = false; + pWindowRes->skey = TSKEY_INITIAL_VAL; } /** @@ -264,8 +268,8 @@ void clearTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pWindow */ void copyTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *dst, const SWindowResult *src) { dst->numOfRows = src->numOfRows; - dst->window = src->window; - dst->status = src->status; + dst->skey = src->skey; + dst->closed = src->closed; int32_t nOutputCols = pRuntimeEnv->pQuery->numOfOutput; diff --git a/src/query/src/sql.c b/src/query/src/sql.c index 75ef2f3218..f3f55791b0 100644 --- a/src/query/src/sql.c +++ b/src/query/src/sql.c @@ -99,7 +99,7 @@ #define YYCODETYPE unsigned short int #define YYNOCODE 274 #define YYACTIONTYPE unsigned short int -#define ParseTOKENTYPE SSQLToken +#define ParseTOKENTYPE SStrToken typedef union { int yyinit; ParseTOKENTYPE yy0; @@ -488,7 +488,7 @@ static const YYCODETYPE yyFallback[] = { 0, /* QUERIES => nothing */ 0, /* CONNECTIONS => nothing */ 0, /* STREAMS => nothing */ - 0, /* CONFIGS => nothing */ + 0, /* VARIABLES => nothing */ 0, /* SCORES => nothing */ 0, /* GRANTS => nothing */ 0, /* VNODES => nothing */ @@ -781,7 +781,7 @@ static const char *const yyTokenName[] = { /* 51 */ "QUERIES", /* 52 */ "CONNECTIONS", /* 53 */ "STREAMS", - /* 54 */ "CONFIGS", + /* 54 */ "VARIABLES", /* 55 */ "SCORES", /* 56 */ "GRANTS", /* 57 */ "VNODES", @@ -1017,7 +1017,7 @@ static const char *const yyRuleName[] = { /* 7 */ "cmd ::= SHOW QUERIES", /* 8 */ "cmd ::= SHOW CONNECTIONS", /* 9 */ "cmd ::= SHOW STREAMS", - /* 10 */ "cmd ::= SHOW CONFIGS", + /* 10 */ "cmd ::= SHOW VARIABLES", /* 11 */ "cmd ::= SHOW SCORES", /* 12 */ "cmd ::= SHOW GRANTS", /* 13 */ "cmd ::= SHOW VNODES", @@ -1702,7 +1702,7 @@ static const struct { { 209, -2 }, /* (7) cmd ::= SHOW QUERIES */ { 209, -2 }, /* (8) cmd ::= SHOW CONNECTIONS */ { 209, -2 }, /* (9) cmd ::= SHOW STREAMS */ - { 209, -2 }, /* (10) cmd ::= SHOW CONFIGS */ + { 209, -2 }, /* (10) cmd ::= SHOW VARIABLES */ { 209, -2 }, /* (11) cmd ::= SHOW SCORES */ { 209, -2 }, /* (12) cmd ::= SHOW GRANTS */ { 209, -2 }, /* (13) cmd ::= SHOW VNODES */ @@ -2029,8 +2029,8 @@ static void yy_reduce( case 9: /* cmd ::= SHOW STREAMS */ { setShowOptions(pInfo, TSDB_MGMT_TABLE_STREAMS, 0, 0); } break; - case 10: /* cmd ::= SHOW CONFIGS */ -{ setShowOptions(pInfo, TSDB_MGMT_TABLE_CONFIGS, 0, 0); } + case 10: /* cmd ::= SHOW VARIABLES */ +{ setShowOptions(pInfo, TSDB_MGMT_TABLE_VARIABLES, 0, 0); } break; case 11: /* cmd ::= SHOW SCORES */ { setShowOptions(pInfo, TSDB_MGMT_TABLE_SCORES, 0, 0); } @@ -2074,21 +2074,21 @@ static void yy_reduce( break; case 22: /* cmd ::= SHOW dbPrefix STABLES LIKE ids */ { - SSQLToken token; + SStrToken token; setDBName(&token, &yymsp[-3].minor.yy0); setShowOptions(pInfo, TSDB_MGMT_TABLE_METRIC, &token, &yymsp[0].minor.yy0); } break; case 23: /* cmd ::= SHOW dbPrefix VGROUPS */ { - SSQLToken token; + SStrToken token; setDBName(&token, &yymsp[-1].minor.yy0); setShowOptions(pInfo, TSDB_MGMT_TABLE_VGROUP, &token, 0); } break; case 24: /* cmd ::= SHOW dbPrefix VGROUPS ids */ { - SSQLToken token; + SStrToken token; setDBName(&token, &yymsp[-2].minor.yy0); setShowOptions(pInfo, TSDB_MGMT_TABLE_VGROUP, &token, &yymsp[0].minor.yy0); } @@ -2139,7 +2139,7 @@ static void yy_reduce( { setDCLSQLElems(pInfo, TSDB_SQL_CFG_LOCAL, 2, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } break; case 38: /* cmd ::= ALTER DATABASE ids alter_db_optr */ -{ SSQLToken t = {0}; setCreateDBSQL(pInfo, TSDB_SQL_ALTER_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy268, &t);} +{ SStrToken t = {0}; setCreateDBSQL(pInfo, TSDB_SQL_ALTER_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy268, &t);} break; case 39: /* cmd ::= ALTER ACCOUNT ids acct_optr */ { setCreateAcctSQL(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-1].minor.yy0, NULL, &yymsp[0].minor.yy149);} @@ -2277,6 +2277,7 @@ static void yy_reduce( yymsp[-1].minor.yy268 = yylhsminor.yy268; break; case 95: /* db_optr ::= db_optr fsync */ + case 107: /* alter_db_optr ::= alter_db_optr fsync */ yytestcase(yyruleno==107); { yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.fsyncPeriod = strtol(yymsp[0].minor.yy0.z, NULL, 10); } yymsp[-1].minor.yy268 = yylhsminor.yy268; break; @@ -2297,10 +2298,6 @@ static void yy_reduce( case 99: /* alter_db_optr ::= */ { setDefaultCreateDbOption(&yymsp[1].minor.yy268);} break; - case 107: /* alter_db_optr ::= alter_db_optr fsync */ -{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.fsyncPeriod = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy268 = yylhsminor.yy268; - break; case 108: /* typename ::= ids */ { yymsp[0].minor.yy0.type = 0; diff --git a/src/query/tests/tsBufTest.cpp b/src/query/tests/tsBufTest.cpp index f8738eec9c..b78c5314f2 100644 --- a/src/query/tests/tsBufTest.cpp +++ b/src/query/tests/tsBufTest.cpp @@ -32,14 +32,16 @@ void simpleTest() { // write 10 ts points int32_t num = 10; - int64_t tag = 1; + tVariant t = {0}; + t.nType = TSDB_DATA_TYPE_BIGINT; + t.i64Key = 1; int64_t* list = createTsList(10, 10000000, 30); - tsBufAppend(pTSBuf, 0, tag, (const char*)list, num * sizeof(int64_t)); + tsBufAppend(pTSBuf, 0, &t, (const char*)list, num * sizeof(int64_t)); EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC); EXPECT_EQ(pTSBuf->tsData.len, sizeof(int64_t) * num); - EXPECT_EQ(pTSBuf->block.tag, tag); + EXPECT_EQ(tVariantCompare(&pTSBuf->block.tag, &t), 0); EXPECT_EQ(pTSBuf->numOfVnodes, 1); tsBufFlush(pTSBuf); @@ -57,14 +59,16 @@ void largeTSTest() { // write 10 ts points int32_t num = 1000000; - int64_t tag = 1; + tVariant t = {0}; + t.nType = TSDB_DATA_TYPE_BIGINT; + t.i64Key = 1; int64_t* list = createTsList(num, 10000000, 30); - tsBufAppend(pTSBuf, 0, tag, (const char*)list, num * sizeof(int64_t)); + tsBufAppend(pTSBuf, 0, &t, (const char*)list, num * sizeof(int64_t)); // the data has been flush to disk, no data in cache EXPECT_EQ(pTSBuf->tsData.len, 0); - EXPECT_EQ(pTSBuf->block.tag, tag); + EXPECT_EQ(tVariantCompare(&pTSBuf->block.tag, &t), 0); EXPECT_EQ(pTSBuf->numOfVnodes, 1); EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC); @@ -80,14 +84,18 @@ void multiTagsTest() { STSBuf* pTSBuf = tsBufCreate(true, TSDB_ORDER_ASC); int32_t num = 10000; - int64_t tag = 1; + tVariant t = {0}; + t.nType = TSDB_DATA_TYPE_BIGINT; + int64_t start = 10000000; int32_t numOfTags = 50; int32_t step = 30; for (int32_t i = 0; i < numOfTags; ++i) { int64_t* list = createTsList(num, start, step); - tsBufAppend(pTSBuf, 0, i, (const char*)list, num * sizeof(int64_t)); + t.i64Key = i; + + tsBufAppend(pTSBuf, 0, &t, (const char*)list, num * sizeof(int64_t)); free(list); start += step * num; @@ -96,7 +104,7 @@ void multiTagsTest() { EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC); EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); - EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1); + EXPECT_EQ(pTSBuf->block.tag.i64Key, numOfTags - 1); EXPECT_EQ(pTSBuf->numOfVnodes, 1); tsBufFlush(pTSBuf); @@ -118,9 +126,14 @@ void multiVnodeTagsTest() { for (int32_t j = 0; j < 20; ++j) { // vnodeId:0 start = 10000000; + tVariant t = {0}; + t.nType = TSDB_DATA_TYPE_BIGINT; + for (int32_t i = 0; i < numOfTags; ++i) { int64_t* list = createTsList(num, start, step); - tsBufAppend(pTSBuf, j, i, (const char*)list, num * sizeof(int64_t)); + t.i64Key = i; + + tsBufAppend(pTSBuf, j, &t, (const char*)list, num * sizeof(int64_t)); free(list); start += step * num; @@ -131,11 +144,11 @@ void multiVnodeTagsTest() { EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC); EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); - EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1); + EXPECT_EQ(pTSBuf->block.tag.i64Key, numOfTags - 1); EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); - EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1); + EXPECT_EQ(pTSBuf->block.tag.i64Key, numOfTags - 1); tsBufFlush(pTSBuf); EXPECT_EQ(pTSBuf->tsData.len, 0); @@ -157,9 +170,14 @@ void loadDataTest() { for (int32_t j = 0; j < numOfVnode; ++j) { // vnodeId:0 int64_t start = 10000000; + tVariant t = {0}; + t.nType = TSDB_DATA_TYPE_BIGINT; + for (int32_t i = 0; i < numOfTags; ++i) { int64_t* list = createTsList(num, start, step); - tsBufAppend(pTSBuf, j, i, (const char*)list, num * sizeof(int64_t)); + t.i64Key = i; + + tsBufAppend(pTSBuf, j, &t, (const char*)list, num * sizeof(int64_t)); printf("%d - %" PRIu64 "\n", i, list[0]); free(list); @@ -172,11 +190,11 @@ void loadDataTest() { EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC); EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); - EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1); + EXPECT_EQ(pTSBuf->block.tag.i64Key, numOfTags - 1); EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); - EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1); + EXPECT_EQ(pTSBuf->block.tag.i64Key, numOfTags - 1); tsBufFlush(pTSBuf); EXPECT_EQ(pTSBuf->tsData.len, 0); @@ -230,16 +248,21 @@ void TSTraverse() { for (int32_t j = 0; j < numOfVnode; ++j) { // vnodeId:0 int64_t start = 10000000; + tVariant t = {0}; + t.nType = TSDB_DATA_TYPE_BIGINT; + for (int32_t i = 0; i < numOfTags; ++i) { int64_t* list = createTsList(num, start, step); - tsBufAppend(pTSBuf, j, i, (const char*)list, num * sizeof(int64_t)); + t.i64Key = i; + + tsBufAppend(pTSBuf, j, &t, (const char*)list, num * sizeof(int64_t)); printf("%d - %d - %" PRIu64 ", %" PRIu64 "\n", j, i, list[0], list[num - 1]); free(list); start += step * num; list = createTsList(num, start, step); - tsBufAppend(pTSBuf, j, i, (const char*)list, num * sizeof(int64_t)); + tsBufAppend(pTSBuf, j, &t, (const char*)list, num * sizeof(int64_t)); printf("%d - %d - %" PRIu64 ", %" PRIu64 "\n", j, i, list[0], list[num - 1]); free(list); @@ -272,12 +295,16 @@ void TSTraverse() { int32_t startVnode = 1; int32_t startTag = 2; - tsBufGetElemStartPos(pTSBuf, startVnode, startTag); + tVariant t = {0}; + t.nType = TSDB_DATA_TYPE_BIGINT; + t.i64Key = startTag; + + tsBufGetElemStartPos(pTSBuf, startVnode, &t); int32_t totalOutput = 10; while (1) { STSElem elem = tsBufGetElem(pTSBuf); - printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag, elem.ts); + printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag.i64Key, elem.ts); if (!tsBufNextPos(pTSBuf)) { break; @@ -286,7 +313,9 @@ void TSTraverse() { if (--totalOutput <= 0) { totalOutput = 10; - tsBufGetElemStartPos(pTSBuf, startVnode, --startTag); + startTag -= 1; + t.i64Key = startTag; + tsBufGetElemStartPos(pTSBuf, startVnode, &t); if (startTag == 0) { startVnode -= 1; @@ -316,13 +345,14 @@ void TSTraverse() { startVnode = 1; startTag = 2; + t.i64Key = startTag; - tsBufGetElemStartPos(pTSBuf, startVnode, startTag); + tsBufGetElemStartPos(pTSBuf, startVnode, &t); totalOutput = 10; while (1) { STSElem elem = tsBufGetElem(pTSBuf); - printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag, elem.ts); + printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag.i64Key, elem.ts); if (!tsBufNextPos(pTSBuf)) { break; @@ -331,7 +361,9 @@ void TSTraverse() { if (--totalOutput <= 0) { totalOutput = 10; - tsBufGetElemStartPos(pTSBuf, startVnode, --startTag); + startTag -= 1; + t.i64Key = startTag; + tsBufGetElemStartPos(pTSBuf, startVnode, &t); if (startTag < 0) { startVnode -= 1; @@ -375,12 +407,17 @@ void mergeDiffVnodeBufferTest() { int32_t num = 1000; int32_t numOfTags = 10; + tVariant t = {0}; + t.nType = TSDB_DATA_TYPE_BIGINT; + // vnodeId:0 int64_t start = 10000000; for (int32_t i = 0; i < numOfTags; ++i) { int64_t* list = createTsList(num, start, step); - tsBufAppend(pTSBuf1, 0, i, (const char*)list, num * sizeof(int64_t)); - tsBufAppend(pTSBuf2, 0, i, (const char*)list, num * sizeof(int64_t)); + t.i64Key = i; + + tsBufAppend(pTSBuf1, 0, &t, (const char*)list, num * sizeof(int64_t)); + tsBufAppend(pTSBuf2, 0, &t, (const char*)list, num * sizeof(int64_t)); free(list); @@ -403,6 +440,9 @@ void mergeIdenticalVnodeBufferTest() { STSBuf* pTSBuf1 = tsBufCreate(true, TSDB_ORDER_ASC); STSBuf* pTSBuf2 = tsBufCreate(true, TSDB_ORDER_ASC); + tVariant t = {0}; + t.nType = TSDB_DATA_TYPE_BIGINT; + int32_t step = 30; int32_t num = 1000; int32_t numOfTags = 10; @@ -411,17 +451,21 @@ void mergeIdenticalVnodeBufferTest() { int64_t start = 10000000; for (int32_t i = 0; i < numOfTags; ++i) { int64_t* list = createTsList(num, start, step); + t.i64Key = i; - tsBufAppend(pTSBuf1, 12, i, (const char*)list, num * sizeof(int64_t)); + tsBufAppend(pTSBuf1, 12, &t, (const char*)list, num * sizeof(int64_t)); free(list); start += step * num; } + + for (int32_t i = numOfTags; i < numOfTags * 2; ++i) { int64_t* list = createTsList(num, start, step); - tsBufAppend(pTSBuf2, 77, i, (const char*)list, num * sizeof(int64_t)); + t.i64Key = i; + tsBufAppend(pTSBuf2, 77, &t, (const char*)list, num * sizeof(int64_t)); free(list); start += step * num; @@ -438,7 +482,7 @@ void mergeIdenticalVnodeBufferTest() { STSElem elem = tsBufGetElem(pTSBuf1); EXPECT_EQ(elem.vnode, 12); - printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag, elem.ts); + printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag.i64Key, elem.ts); } tsBufDestroy(pTSBuf1); @@ -446,6 +490,8 @@ void mergeIdenticalVnodeBufferTest() { } } // namespace + +//TODO add binary tag value test case TEST(testCase, tsBufTest) { simpleTest(); largeTSTest(); @@ -453,7 +499,7 @@ TEST(testCase, tsBufTest) { multiVnodeTagsTest(); loadDataTest(); invalidFileTest(); - // randomIncTsTest(); +// randomIncTsTest(); TSTraverse(); mergeDiffVnodeBufferTest(); mergeIdenticalVnodeBufferTest(); diff --git a/src/query/tests/unitTest.cpp b/src/query/tests/unitTest.cpp index b59e0783a2..b6e1170d0e 100644 --- a/src/query/tests/unitTest.cpp +++ b/src/query/tests/unitTest.cpp @@ -16,7 +16,7 @@ namespace { int32_t testValidateName(char* name) { - SSQLToken token = {0}; + SStrToken token = {0}; token.z = name; token.n = strlen(name); token.type = 0; @@ -720,8 +720,8 @@ TEST(testCase, tGetToken_Test) { EXPECT_FALSE(type == TK_HEX); } -static SSQLToken createStrToken(char* s) { - SSQLToken t = {0};//.type = TK_STRING, .z = s, .n = strlen(s)}; +static SStrToken createStrToken(char* s) { + SStrToken t = {0};//.type = TK_STRING, .z = s, .n = strlen(s)}; t.type = TK_STRING; t.z = s; t.n = strlen(s); @@ -730,7 +730,7 @@ static SSQLToken createStrToken(char* s) { } TEST(testCase, isValidNumber_test) { - SSQLToken t1 = createStrToken("123abc"); + SStrToken t1 = createStrToken("123abc"); EXPECT_EQ(isValidNumber(&t1), TK_ILLEGAL); diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index d0c57a34d0..3aecd127ef 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -538,7 +538,7 @@ void rpcCancelRequest(void *handle) { // signature is used to check if pContext is freed. // pContext may have been released just before app calls the rpcCancelRequest - if (pContext->signature != pContext) return; + if (pContext == NULL || pContext->signature != pContext) return; if (pContext->pConn) { tDebug("%s, app tries to cancel request", pContext->pConn->info); @@ -563,7 +563,7 @@ static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerFqdn, uint16_t peerPort, uint32_t peerIp = taosGetIpFromFqdn(peerFqdn); if (peerIp == 0xFFFFFFFF) { tError("%s, failed to resolve FQDN:%s", pRpc->label, peerFqdn); - terrno = TSDB_CODE_RPC_APP_ERROR; + terrno = TSDB_CODE_RPC_FQDN_ERROR; return NULL; } @@ -1114,13 +1114,16 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead) { if (pHead->code == TSDB_CODE_RPC_REDIRECT) { pContext->numOfTry = 0; - memcpy(&pContext->epSet, pHead->content, sizeof(pContext->epSet)); - tDebug("%s, redirect is received, numOfEps:%d", pConn->info, pContext->epSet.numOfEps); - for (int i=0; iepSet.numOfEps; ++i) - pContext->epSet.port[i] = htons(pContext->epSet.port[i]); + SRpcEpSet *pEpSet = (SRpcEpSet*)pHead->content; + if (pEpSet->numOfEps > 0) { + memcpy(&pContext->epSet, pHead->content, sizeof(pContext->epSet)); + tDebug("%s, redirect is received, numOfEps:%d", pConn->info, pContext->epSet.numOfEps); + for (int i=0; iepSet.numOfEps; ++i) + pContext->epSet.port[i] = htons(pContext->epSet.port[i]); + } rpcSendReqToServer(pRpc, pContext); rpcFreeCont(rpcMsg.pCont); - } else if (pHead->code == TSDB_CODE_RPC_NOT_READY) { + } else if (pHead->code == TSDB_CODE_RPC_NOT_READY || pHead->code == TSDB_CODE_APP_NOT_READY) { pContext->code = pHead->code; rpcProcessConnError(pContext, NULL); rpcFreeCont(rpcMsg.pCont); diff --git a/src/sync/inc/syncInt.h b/src/sync/inc/syncInt.h index be1e01cb23..cd1252f4b4 100644 --- a/src/sync/inc/syncInt.h +++ b/src/sync/inc/syncInt.h @@ -74,7 +74,7 @@ typedef struct { uint32_t magic; uint32_t index; uint64_t fversion; - int32_t size; + int64_t size; } SFileInfo; typedef struct { diff --git a/src/sync/inc/taosTcpPool.h b/src/sync/inc/taosTcpPool.h index 1e410acc26..5f7ca9ede5 100644 --- a/src/sync/inc/taosTcpPool.h +++ b/src/sync/inc/taosTcpPool.h @@ -26,7 +26,7 @@ typedef void* tthread_h; typedef struct { int numOfThreads; uint32_t serverIp; - short port; + int16_t port; int bufferSize; void (*processBrokenLink)(void *ahandle); int (*processIncomingMsg)(void *ahandle, void *buffer); diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index eaa073348a..b92ef4cf26 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -108,8 +108,7 @@ static void syncModuleInitFunc() { tstrncpy(tsNodeFqdn, tsLocalFqdn, sizeof(tsNodeFqdn)); } -void *syncStart(const SSyncInfo *pInfo) -{ +void *syncStart(const SSyncInfo *pInfo) { const SSyncCfg *pCfg = &pInfo->syncCfg; SSyncNode *pNode = (SSyncNode *) calloc(sizeof(SSyncNode), 1); @@ -189,9 +188,8 @@ void *syncStart(const SSyncInfo *pInfo) return pNode; } -void syncStop(void *param) -{ - SSyncNode *pNode = param; +void syncStop(void *param) { + SSyncNode * pNode = param; SSyncPeer *pPeer; if (pNode == NULL) return; @@ -215,9 +213,8 @@ void syncStop(void *param) syncDecNodeRef(pNode); } -int32_t syncReconfig(void *param, const SSyncCfg *pNewCfg) -{ - SSyncNode *pNode = param; +int32_t syncReconfig(void *param, const SSyncCfg *pNewCfg) { + SSyncNode * pNode = param; int i, j; if (pNode == NULL) return TSDB_CODE_SYN_INVALID_CONFIG; @@ -283,10 +280,9 @@ int32_t syncReconfig(void *param, const SSyncCfg *pNewCfg) return 0; } -int32_t syncForwardToPeer(void *param, void *data, void *mhandle, int qtype) -{ - SSyncNode *pNode = param; - SSyncPeer *pPeer; +int32_t syncForwardToPeer(void *param, void *data, void *mhandle, int qtype) { + SSyncNode * pNode = param; + SSyncPeer * pPeer; SSyncHead *pSyncHead; SWalHead *pWalHead = data; int fwdLen; @@ -334,9 +330,8 @@ int32_t syncForwardToPeer(void *param, void *data, void *mhandle, int qtype) return code; } -void syncConfirmForward(void *param, uint64_t version, int32_t code) -{ - SSyncNode *pNode = param; +void syncConfirmForward(void *param, uint64_t version, int32_t code) { + SSyncNode *pNode = param; if (pNode == NULL) return; if (pNode->quorum <= 1) return; @@ -387,10 +382,9 @@ void syncRecover(void *param) { pthread_mutex_unlock(&(pNode->mutex)); } -int syncGetNodesRole(void *param, SNodesRole *pNodesRole) -{ +int syncGetNodesRole(void *param, SNodesRole *pNodesRole) { SSyncNode *pNode = param; - + pNodesRole->selfIndex = pNode->selfIndex; for (int i=0; ireplica; ++i) { pNodesRole->nodeId[i] = pNode->peerInfo[i]->nodeId; @@ -400,8 +394,7 @@ int syncGetNodesRole(void *param, SNodesRole *pNodesRole) return 0; } -static void syncAddArbitrator(SSyncNode *pNode) -{ +static void syncAddArbitrator(SSyncNode *pNode) { SSyncPeer *pPeer = pNode->peerInfo[TAOS_SYNC_MAX_REPLICA]; // if not configured, return right away @@ -413,9 +406,11 @@ static void syncAddArbitrator(SSyncNode *pNode) SNodeInfo nodeInfo; nodeInfo.nodeId = 0; - taosGetFqdnPortFromEp(tsArbitrator, nodeInfo.nodeFqdn, &nodeInfo.nodePort); - nodeInfo.nodePort += TSDB_PORT_SYNC; - + int ret = taosGetFqdnPortFromEp(tsArbitrator, nodeInfo.nodeFqdn, &nodeInfo.nodePort); + if (-1 == ret) { + nodeInfo.nodePort = tsArbitratorPort; + } + if (pPeer) { if ((strcmp(nodeInfo.nodeFqdn, pPeer->fqdn) == 0) && (nodeInfo.nodePort == pPeer->port)) { return; @@ -454,13 +449,11 @@ static void syncDecNodeRef(SSyncNode *pNode) } } -void syncAddPeerRef(SSyncPeer *pPeer) -{ +void syncAddPeerRef(SSyncPeer *pPeer) { atomic_add_fetch_8(&pPeer->refCount, 1); } -int syncDecPeerRef(SSyncPeer *pPeer) -{ +int syncDecPeerRef(SSyncPeer *pPeer) { if (atomic_sub_fetch_8(&pPeer->refCount, 1) == 0) { syncDecNodeRef(pPeer->pSyncNode); @@ -473,18 +466,16 @@ int syncDecPeerRef(SSyncPeer *pPeer) return 1; } -static void syncClosePeerConn(SSyncPeer *pPeer) -{ +static void syncClosePeerConn(SSyncPeer *pPeer) { taosTmrStopA(&pPeer->timer); taosClose(pPeer->syncFd); - if (pPeer->peerFd >=0) { + if (pPeer->peerFd >= 0) { pPeer->peerFd = -1; taosFreeTcpConn(pPeer->pConn); } } -static void syncRemovePeer(SSyncPeer *pPeer) -{ +static void syncRemovePeer(SSyncPeer *pPeer) { sInfo("%s, it is removed", pPeer->id); pPeer->ip = 0; @@ -492,8 +483,7 @@ static void syncRemovePeer(SSyncPeer *pPeer) syncDecPeerRef(pPeer); } -static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) -{ +static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) { uint32_t ip = taosGetIpFromFqdn(pInfo->nodeFqdn); if (ip == -1) return NULL; @@ -523,25 +513,24 @@ static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) return pPeer; } -void syncBroadcastStatus(SSyncNode *pNode) -{ +void syncBroadcastStatus(SSyncNode *pNode) { SSyncPeer *pPeer; for (int i = 0; i < pNode->replica; ++i) { - if ( i == pNode->selfIndex ) continue; + if (i == pNode->selfIndex) continue; pPeer = pNode->peerInfo[i]; syncSendPeersStatusMsgToPeer(pPeer, 1); } -} +} static void syncResetFlowCtrl(SSyncNode *pNode) { - for (int i = 0; i < pNode->replica; ++i) { pNode->peerInfo[i]->numOfRetrieves = 0; } - if (pNode->notifyFlowCtrl) - (*pNode->notifyFlowCtrl)(pNode->ahandle, 0); + if (pNode->notifyFlowCtrl) { + (*pNode->notifyFlowCtrl)(pNode->ahandle, 0); + } } static void syncChooseMaster(SSyncNode *pNode) { @@ -598,9 +587,9 @@ static void syncChooseMaster(SSyncNode *pNode) { } else { sDebug("vgId:%d, failed to choose master", pNode->vgId); } -} - -static SSyncPeer *syncCheckMaster(SSyncNode *pNode ) { +} + +static SSyncPeer *syncCheckMaster(SSyncNode *pNode) { int onlineNum = 0; int index = -1; int replica = pNode->replica; @@ -617,7 +606,7 @@ static SSyncPeer *syncCheckMaster(SSyncNode *pNode ) { replica = pNode->replica + 1; } - if (onlineNum <= replica*0.5) { + if (onlineNum <= replica * 0.5) { if (nodeRole != TAOS_SYNC_ROLE_UNSYNCED) { nodeRole = TAOS_SYNC_ROLE_UNSYNCED; pNode->peerInfo[pNode->selfIndex]->role = nodeRole; @@ -625,13 +614,13 @@ static SSyncPeer *syncCheckMaster(SSyncNode *pNode ) { sInfo("vgId:%d, change to unsynced state, online:%d replica:%d", pNode->vgId, onlineNum, replica); } } else { - for (int i=0; ireplica; ++i) { + for (int i = 0; i < pNode->replica; ++i) { SSyncPeer *pTemp = pNode->peerInfo[i]; - if ( pTemp->role != TAOS_SYNC_ROLE_MASTER ) continue; - if ( index < 0 ) { + if (pTemp->role != TAOS_SYNC_ROLE_MASTER) continue; + if (index < 0) { index = i; - } else { // multiple masters, it shall not happen - if ( i == pNode->selfIndex ) { + } else { // multiple masters, it shall not happen + if (i == pNode->selfIndex) { sError("%s, peer is master, work as slave instead", pTemp->id); nodeRole = TAOS_SYNC_ROLE_SLAVE; (*pNode->notifyRole)(pNode->ahandle, nodeRole); @@ -640,7 +629,7 @@ static SSyncPeer *syncCheckMaster(SSyncNode *pNode ) { } } - SSyncPeer *pMaster = (index>=0) ? pNode->peerInfo[index]:NULL; + SSyncPeer *pMaster = (index >= 0) ? pNode->peerInfo[index] : NULL; return pMaster; } @@ -649,7 +638,7 @@ static int syncValidateMaster(SSyncPeer *pPeer) { int code = 0; if (nodeRole == TAOS_SYNC_ROLE_MASTER && nodeVersion < pPeer->version) { - sDebug("%s, slave has higher version, restart all connections!!!", pPeer->id); + sDebug("%s, slave has higher version, restart all connections!!!", pPeer->id); nodeRole = TAOS_SYNC_ROLE_UNSYNCED; (*pNode->notifyRole)(pNode->ahandle, nodeRole); code = -1; @@ -658,13 +647,12 @@ static int syncValidateMaster(SSyncPeer *pPeer) { if ( i == pNode->selfIndex ) continue; syncRestartPeer(pNode->peerInfo[i]); } - } + } return code; } -static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus peersStatus[], int8_t newRole) -{ +static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus peersStatus[], int8_t newRole) { SSyncNode *pNode = pPeer->pSyncNode; int8_t peerOldRole = pPeer->role; int8_t selfOldRole = nodeRole; @@ -686,14 +674,14 @@ static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus peersStatus[], int8_t ne if (syncValidateMaster(pPeer) < 0) return; if (nodeRole == TAOS_SYNC_ROLE_UNSYNCED) { - if ( nodeVersion < pMaster->version) { + if (nodeVersion < pMaster->version) { syncRequired = 1; } else { sInfo("%s is master, work as slave, ver:%" PRIu64, pMaster->id, pMaster->version); nodeRole = TAOS_SYNC_ROLE_SLAVE; (*pNode->notifyRole)(pNode->ahandle, nodeRole); } - } else if ( nodeRole == TAOS_SYNC_ROLE_SLAVE && pMaster == pPeer) { + } else if (nodeRole == TAOS_SYNC_ROLE_SLAVE && pMaster == pPeer) { // nodeVersion = pMaster->version; } } else { @@ -734,20 +722,18 @@ static void syncRestartPeer(SSyncPeer *pPeer) { pPeer->sstatus = TAOS_SYNC_STATUS_INIT; int ret = strcmp(pPeer->fqdn, tsNodeFqdn); - if (ret > 0 || (ret == 0 && pPeer->port > tsSyncPort) ) - taosTmrReset(syncCheckPeerConnection, tsSyncTimer*1000, pPeer, syncTmrCtrl, &pPeer->timer); + if (ret > 0 || (ret == 0 && pPeer->port > tsSyncPort)) + taosTmrReset(syncCheckPeerConnection, tsSyncTimer * 1000, pPeer, syncTmrCtrl, &pPeer->timer); } -void syncRestartConnection(SSyncPeer *pPeer) -{ +void syncRestartConnection(SSyncPeer *pPeer) { if (pPeer->ip == 0) return; syncRestartPeer(pPeer); syncCheckRole(pPeer, NULL, TAOS_SYNC_ROLE_OFFLINE); } -static void syncProcessSyncRequest(char *msg, SSyncPeer *pPeer) -{ +static void syncProcessSyncRequest(char *msg, SSyncPeer *pPeer) { SSyncNode *pNode = pPeer->pSyncNode; sDebug("%s, sync-req is received", pPeer->id); @@ -782,8 +768,7 @@ static void syncProcessSyncRequest(char *msg, SSyncPeer *pPeer) } } -static void syncNotStarted(void *param, void *tmrId) -{ +static void syncNotStarted(void *param, void *tmrId) { SSyncPeer *pPeer = param; SSyncNode *pNode = pPeer->pSyncNode; @@ -803,14 +788,13 @@ static void syncTryRecoverFromMaster(void *param, void *tmrId) { pthread_mutex_unlock(&(pNode->mutex)); } -static void syncRecoverFromMaster(SSyncPeer *pPeer) -{ - SSyncNode *pNode = pPeer->pSyncNode; +static void syncRecoverFromMaster(SSyncPeer *pPeer) { + SSyncNode *pNode = pPeer->pSyncNode; - if ( nodeSStatus != TAOS_SYNC_STATUS_INIT) { + if (nodeSStatus != TAOS_SYNC_STATUS_INIT) { sDebug("%s, sync is already started, status:%d", pPeer->id, nodeSStatus); return; - } + } taosTmrStopA(&pPeer->timer); if (tsSyncNum >= tsMaxSyncNum) { @@ -840,9 +824,8 @@ static void syncRecoverFromMaster(SSyncPeer *pPeer) return; } -static void syncProcessFwdResponse(char *cont, SSyncPeer *pPeer) -{ - SSyncNode *pNode = pPeer->pSyncNode; +static void syncProcessFwdResponse(char *cont, SSyncPeer *pPeer) { + SSyncNode * pNode = pPeer->pSyncNode; SFwdRsp *pFwdRsp = (SFwdRsp *) cont; SSyncFwds *pSyncFwds = pNode->pSyncFwds; SFwdInfo *pFwdInfo; @@ -862,10 +845,8 @@ static void syncProcessFwdResponse(char *cont, SSyncPeer *pPeer) } } - -static void syncProcessForwardFromPeer(char *cont, SSyncPeer *pPeer) -{ - SSyncNode *pNode = pPeer->pSyncNode; +static void syncProcessForwardFromPeer(char *cont, SSyncPeer *pPeer) { + SSyncNode * pNode = pPeer->pSyncNode; SWalHead *pHead = (SWalHead *)cont; sDebug("%s, forward is received, ver:%" PRIu64, pPeer->id, pHead->version); @@ -884,9 +865,8 @@ static void syncProcessForwardFromPeer(char *cont, SSyncPeer *pPeer) return; } -static void syncProcessPeersStatusMsg(char *cont, SSyncPeer *pPeer) -{ - SSyncNode *pNode = pPeer->pSyncNode; +static void syncProcessPeersStatusMsg(char *cont, SSyncPeer *pPeer) { + SSyncNode * pNode = pPeer->pSyncNode; SPeersStatus *pPeersStatus = (SPeersStatus *)cont; sDebug("%s, status msg received, self:%s ver:%" PRIu64 " peer:%s ver:%" PRIu64 ", ack:%d", pPeer->id, @@ -909,10 +889,10 @@ static int syncReadPeerMsg(SSyncPeer *pPeer, SSyncHead *pHead, char *cont) { } // head.len = htonl(head.len); - if (pHead->len <0) { + if (pHead->len < 0) { sError("%s, invalid pkt length, len:%d", pPeer->id, pHead->len); return -1; - } + } int bytes = taosReadMsg(pPeer->peerFd, cont, pHead->len); if (bytes != pHead->len) { @@ -923,9 +903,8 @@ static int syncReadPeerMsg(SSyncPeer *pPeer, SSyncHead *pHead, char *cont) { return 0; } -static int syncProcessPeerMsg(void *param, void *buffer) -{ - SSyncPeer *pPeer = param; +static int syncProcessPeerMsg(void *param, void *buffer) { + SSyncPeer * pPeer = param; SSyncHead head; char *cont = (char *)buffer; @@ -953,8 +932,7 @@ static int syncProcessPeerMsg(void *param, void *buffer) #define statusMsgLen sizeof(SSyncHead)+sizeof(SPeersStatus)+sizeof(SPeerStatus)*TAOS_SYNC_MAX_REPLICA -static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack) -{ +static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack) { SSyncNode *pNode = pPeer->pSyncNode; char msg[statusMsgLen] = {0}; @@ -1011,7 +989,7 @@ static void syncSetupPeerConnection(SSyncPeer *pPeer) { firstPkt.port = tsSyncPort; firstPkt.sourceId = pNode->vgId; // tell arbitrator its vgId - if ( write(connFd, &firstPkt, sizeof(firstPkt)) == sizeof(firstPkt)) { + if (write(connFd, &firstPkt, sizeof(firstPkt)) == sizeof(firstPkt)) { sDebug("%s, connection to peer server is setup", pPeer->id); pPeer->peerFd = connFd; pPeer->role = TAOS_SYNC_ROLE_UNSYNCED; @@ -1024,8 +1002,7 @@ static void syncSetupPeerConnection(SSyncPeer *pPeer) { } } -static void syncCheckPeerConnection(void *param, void *tmrId) -{ +static void syncCheckPeerConnection(void *param, void *tmrId) { SSyncPeer *pPeer = param; SSyncNode *pNode = pPeer->pSyncNode; @@ -1037,8 +1014,7 @@ static void syncCheckPeerConnection(void *param, void *tmrId) pthread_mutex_unlock(&(pNode->mutex)); } -static void syncCreateRestoreDataThread(SSyncPeer *pPeer) -{ +static void syncCreateRestoreDataThread(SSyncPeer *pPeer) { taosTmrStopA(&pPeer->timer); pthread_attr_t thattr; @@ -1059,8 +1035,7 @@ static void syncCreateRestoreDataThread(SSyncPeer *pPeer) } } -static void syncProcessIncommingConnection(int connFd, uint32_t sourceIp) -{ +static void syncProcessIncommingConnection(int connFd, uint32_t sourceIp) { char ipstr[24]; int i; @@ -1137,8 +1112,7 @@ static void syncProcessBrokenLink(void *param) { syncDecNodeRef(pNode); } -static void syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle) -{ +static void syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle) { SSyncFwds *pSyncFwds = pNode->pSyncFwds; uint64_t time = taosGetTimestampMs(); @@ -1160,8 +1134,7 @@ static void syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle) sDebug("vgId:%d, fwd info is saved, ver:%" PRIu64 " fwds:%d ", pNode->vgId, version, pSyncFwds->fwds); } -static void syncRemoveConfirmedFwdInfo(SSyncNode *pNode) -{ +static void syncRemoveConfirmedFwdInfo(SSyncNode *pNode) { SSyncFwds *pSyncFwds = pNode->pSyncFwds; int fwds = pSyncFwds->fwds; @@ -1178,8 +1151,7 @@ static void syncRemoveConfirmedFwdInfo(SSyncNode *pNode) } } -static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code) -{ +static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code) { int confirm = 0; if (pFwdInfo->code == 0) pFwdInfo->code = code; @@ -1200,8 +1172,7 @@ static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code } } -static void syncMonitorFwdInfos(void *param, void *tmrId) -{ +static void syncMonitorFwdInfos(void *param, void *tmrId) { SSyncNode *pNode = param; SSyncFwds *pSyncFwds = pNode->pSyncFwds; uint64_t time = taosGetTimestampMs(); @@ -1220,6 +1191,3 @@ static void syncMonitorFwdInfos(void *param, void *tmrId) pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, 300, pNode, syncTmrCtrl); } - - - diff --git a/src/sync/src/syncRestore.c b/src/sync/src/syncRestore.c index 505882b209..2a0bee3726 100644 --- a/src/sync/src/syncRestore.c +++ b/src/sync/src/syncRestore.c @@ -28,7 +28,7 @@ static void syncRemoveExtraFile(SSyncPeer *pPeer, uint32_t sindex, uint32_t eind char fname[TSDB_FILENAME_LEN*3] = {0}; uint32_t magic; uint64_t fversion; - int32_t size; + int64_t size; uint32_t index = sindex; SSyncNode *pNode = pPeer->pSyncNode; @@ -48,8 +48,7 @@ static void syncRemoveExtraFile(SSyncPeer *pPeer, uint32_t sindex, uint32_t eind } } -static int syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) -{ +static int syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { SSyncNode *pNode = pPeer->pSyncNode; SFileInfo minfo; memset(&minfo, 0, sizeof(minfo)); /* = {0}; */ // master file info SFileInfo sinfo; memset(&sinfo, 0, sizeof(sinfo)); /* = {0}; */ // slave file info @@ -113,7 +112,7 @@ static int syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) close(dfd); if (ret<0) break; - sDebug("%s, %s is received, size:%d", pPeer->id, minfo.name, minfo.size); + sDebug("%s, %s is received, size:%" PRId64, pPeer->id, minfo.name, minfo.size); } @@ -130,8 +129,7 @@ static int syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) return code; } -static int syncRestoreWal(SSyncPeer *pPeer) -{ +static int syncRestoreWal(SSyncPeer *pPeer) { SSyncNode *pNode = pPeer->pSyncNode; int ret, code = -1; @@ -172,8 +170,7 @@ static char *syncProcessOneBufferedFwd(SSyncPeer *pPeer, char *offset) return offset; } -static int syncProcessBufferedFwd(SSyncPeer *pPeer) -{ +static int syncProcessBufferedFwd(SSyncPeer *pPeer) { SSyncNode *pNode = pPeer->pSyncNode; SRecvBuffer *pRecv = pNode->pRecv; int forwards = 0; @@ -201,8 +198,7 @@ static int syncProcessBufferedFwd(SSyncPeer *pPeer) return pRecv->code; } -int syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead) -{ +int syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead) { SSyncNode *pNode = pPeer->pSyncNode; SRecvBuffer *pRecv = pNode->pRecv; @@ -222,8 +218,7 @@ int syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead) return pRecv->code; } -static void syncCloseRecvBuffer(SSyncNode *pNode) -{ +static void syncCloseRecvBuffer(SSyncNode *pNode) { if (pNode->pRecv) { taosTFree(pNode->pRecv->buffer); } @@ -231,8 +226,7 @@ static void syncCloseRecvBuffer(SSyncNode *pNode) taosTFree(pNode->pRecv); } -static int syncOpenRecvBuffer(SSyncNode *pNode) -{ +static int syncOpenRecvBuffer(SSyncNode *pNode) { syncCloseRecvBuffer(pNode); SRecvBuffer *pRecv = calloc(sizeof(SRecvBuffer), 1); @@ -253,8 +247,7 @@ static int syncOpenRecvBuffer(SSyncNode *pNode) return 0; } -static int syncRestoreDataStepByStep(SSyncPeer *pPeer) -{ +static int syncRestoreDataStepByStep(SSyncPeer *pPeer) { SSyncNode *pNode = pPeer->pSyncNode; nodeSStatus = TAOS_SYNC_STATUS_FILE; uint64_t fversion = 0; @@ -292,10 +285,9 @@ static int syncRestoreDataStepByStep(SSyncPeer *pPeer) return 0; } -void *syncRestoreData(void *param) -{ - SSyncPeer *pPeer = (SSyncPeer *)param; - SSyncNode *pNode = pPeer->pSyncNode; +void *syncRestoreData(void *param) { + SSyncPeer *pPeer = (SSyncPeer *)param; + SSyncNode *pNode = pPeer->pSyncNode; taosBlockSIGPIPE(); __sync_fetch_and_add(&tsSyncNum, 1); @@ -326,4 +318,3 @@ void *syncRestoreData(void *param) return NULL; } - diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c index 1dd1cda343..8aa317b1ac 100644 --- a/src/sync/src/syncRetrieve.c +++ b/src/sync/src/syncRetrieve.c @@ -27,11 +27,10 @@ #include "tsync.h" #include "syncInt.h" -static int syncAddIntoWatchList(SSyncPeer *pPeer, char *name) -{ +static int syncAddIntoWatchList(SSyncPeer *pPeer, char *name) { sDebug("%s, start to monitor:%s", pPeer->id, name); - if (pPeer->notifyFd <=0) { + if (pPeer->notifyFd <= 0) { pPeer->watchNum = 0; pPeer->notifyFd = inotify_init1(IN_NONBLOCK); if (pPeer->notifyFd < 0) { @@ -70,9 +69,8 @@ static int syncAddIntoWatchList(SSyncPeer *pPeer, char *name) return 0; } -static int syncAreFilesModified(SSyncPeer *pPeer) -{ - if (pPeer->notifyFd <=0) return 0; +static int syncAreFilesModified(SSyncPeer *pPeer) { + if (pPeer->notifyFd <= 0) return 0; char buf[2048]; int len = read(pPeer->notifyFd, buf, sizeof(buf)); @@ -96,12 +94,11 @@ static int syncAreFilesModified(SSyncPeer *pPeer) } } - return code; + return code; } -static int syncRetrieveFile(SSyncPeer *pPeer) -{ - SSyncNode *pNode = pPeer->pSyncNode; +static int syncRetrieveFile(SSyncPeer *pPeer) { + SSyncNode * pNode = pPeer->pSyncNode; SFileInfo fileInfo; SFileAck fileAck; int code = -1; @@ -128,7 +125,7 @@ static int syncRetrieveFile(SSyncPeer *pPeer) // wait for the ack from peer ret = taosReadMsg(pPeer->syncFd, &(fileAck), sizeof(fileAck)); - if (ret <0) break; + if (ret < 0) break; // set the peer sync version pPeer->sversion = fileInfo.fversion; @@ -148,13 +145,13 @@ static int syncRetrieveFile(SSyncPeer *pPeer) // send the file to peer int sfd = open(name, O_RDONLY); - if ( sfd < 0 ) break; + if (sfd < 0) break; ret = taosTSendFile(pPeer->syncFd, sfd, NULL, fileInfo.size); - close(sfd); - if (ret <0) break; + close(sfd); + if (ret < 0) break; - sDebug("%s, %s is sent, size:%d", pPeer->id, name, fileInfo.size); + sDebug("%s, %s is sent, size:%" PRId64, pPeer->id, name, fileInfo.size); fileInfo.index++; // check if processed files are modified @@ -170,8 +167,7 @@ static int syncRetrieveFile(SSyncPeer *pPeer) /* if only a partial record is read out, set the IN_MODIFY flag in event, so upper layer will reload the file to get a complete record */ -static int syncReadOneWalRecord(int sfd, SWalHead *pHead, uint32_t *pEvent) -{ +static int syncReadOneWalRecord(int sfd, SWalHead *pHead, uint32_t *pEvent) { int ret; ret = read(sfd, pHead, sizeof(SWalHead)); @@ -185,7 +181,7 @@ static int syncReadOneWalRecord(int sfd, SWalHead *pHead, uint32_t *pEvent) } ret = read(sfd, pHead->cont, pHead->len); - if (ret <0) return -1; + if (ret < 0) return -1; if (ret != pHead->len) { // file is not at end yet, it shall be reloaded @@ -194,10 +190,9 @@ static int syncReadOneWalRecord(int sfd, SWalHead *pHead, uint32_t *pEvent) } return sizeof(SWalHead) + pHead->len; -} +} -static int syncMonitorLastWal(SSyncPeer *pPeer, char *name) -{ +static int syncMonitorLastWal(SSyncPeer *pPeer, char *name) { pPeer->watchNum = 0; taosClose(pPeer->notifyFd); pPeer->notifyFd = inotify_init1(IN_NONBLOCK); @@ -221,18 +216,17 @@ static int syncMonitorLastWal(SSyncPeer *pPeer, char *name) return -1; } - return 0; + return 0; } -static uint32_t syncCheckLastWalChanges(SSyncPeer *pPeer, uint32_t *pEvent) -{ +static uint32_t syncCheckLastWalChanges(SSyncPeer *pPeer, uint32_t *pEvent) { char buf[2048]; int len = read(pPeer->notifyFd, buf, sizeof(buf)); - if (len <0 && errno != EAGAIN) { - sError("%s, failed to read notify FD(%s)", pPeer->id, strerror(errno)); + if (len < 0 && errno != EAGAIN) { + sError("%s, failed to read notify FD(%s)", pPeer->id, strerror(errno)); return -1; } - + if (len == 0) return 0; struct inotify_event *event; @@ -248,8 +242,7 @@ static uint32_t syncCheckLastWalChanges(SSyncPeer *pPeer, uint32_t *pEvent) return 0; } -static int syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion, int64_t offset, uint32_t *pEvent) -{ +static int syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion, int64_t offset, uint32_t *pEvent) { SWalHead *pHead = (SWalHead *) malloc(640000); int code = -1; int32_t bytes = 0; @@ -261,9 +254,12 @@ static int syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion, sDebug("%s, retrieve last wal, offset:%" PRId64 " fversion:%" PRIu64, pPeer->id, offset, fversion); while (1) { - int wsize = syncReadOneWalRecord(sfd, pHead, pEvent); - if (wsize <0) break; - if (wsize == 0) { code = 0; break; } + int wsize = syncReadOneWalRecord(sfd, pHead, pEvent); + if (wsize < 0) break; + if (wsize == 0) { + code = 0; + break; + } sDebug("%s, last wal is forwarded, ver:%" PRIu64, pPeer->id, pHead->version); int ret = taosWriteMsg(pPeer->syncFd, pHead, wsize); @@ -286,8 +282,7 @@ static int syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion, return -1; } -static int syncProcessLastWal(SSyncPeer *pPeer, char *wname, uint32_t index) -{ +static int syncProcessLastWal(SSyncPeer *pPeer, char *wname, uint32_t index) { SSyncNode *pNode = pPeer->pSyncNode; int code = -1; char fname[TSDB_FILENAME_LEN * 2]; // full path to wal file @@ -350,12 +345,16 @@ static int syncProcessLastWal(SSyncPeer *pPeer, char *wname, uint32_t index) } if (code < 0) break; - if (pPeer->sversion >= fversion && fversion > 0) break; + if (pPeer->sversion >= fversion && fversion > 0) break; - index++; wname[0] = 0; + index++; + wname[0] = 0; code = (*pNode->getWalInfo)(pNode->ahandle, wname, &index); - if ( code < 0) break; - if ( wname[0] == 0 ) {code = 0; break;} + if (code < 0) break; + if (wname[0] == 0) { + code = 0; + break; + } // current last wal is closed, there is a new one sDebug("%s, last wal is closed, try new one", pPeer->id); @@ -366,9 +365,8 @@ static int syncProcessLastWal(SSyncPeer *pPeer, char *wname, uint32_t index) return code; } -static int syncRetrieveWal(SSyncPeer *pPeer) -{ - SSyncNode *pNode = pPeer->pSyncNode; +static int syncRetrieveWal(SSyncPeer *pPeer) { + SSyncNode * pNode = pPeer->pSyncNode; char fname[TSDB_FILENAME_LEN * 3]; char wname[TSDB_FILENAME_LEN * 2]; int32_t size; @@ -396,7 +394,7 @@ static int syncRetrieveWal(SSyncPeer *pPeer) // send wal file, // inotify is not required, old wal file won't be modified, even remove is ok - if ( stat(fname, &fstat) < 0 ) break; + if (stat(fname, &fstat) < 0) break; size = fstat.st_size; sDebug("%s, retrieve wal:%s size:%d", pPeer->id, fname, size); @@ -425,9 +423,8 @@ static int syncRetrieveWal(SSyncPeer *pPeer) return code; } -static int syncRetrieveDataStepByStep(SSyncPeer *pPeer) -{ - SSyncNode *pNode = pPeer->pSyncNode; +static int syncRetrieveDataStepByStep(SSyncPeer *pPeer) { + SSyncNode *pNode = pPeer->pSyncNode; SFirstPkt firstPkt; memset(&firstPkt, 0, sizeof(firstPkt)); @@ -462,9 +459,8 @@ static int syncRetrieveDataStepByStep(SSyncPeer *pPeer) return 0; } -void *syncRetrieveData(void *param) -{ - SSyncPeer *pPeer = (SSyncPeer *)param; +void *syncRetrieveData(void *param) { + SSyncPeer * pPeer = (SSyncPeer *)param; SSyncNode *pNode = pPeer->pSyncNode; taosBlockSIGPIPE(); diff --git a/src/sync/src/taosTcpPool.c b/src/sync/src/taosTcpPool.c index b523728bf9..2f064ceb36 100644 --- a/src/sync/src/taosTcpPool.c +++ b/src/sync/src/taosTcpPool.c @@ -48,8 +48,7 @@ static void *taosProcessTcpData(void *param); static SThreadObj *taosGetTcpThread(SPoolObj *pPool); static void taosStopPoolThread(SThreadObj* pThread); -void *taosOpenTcpThreadPool(SPoolInfo *pInfo) -{ +void *taosOpenTcpThreadPool(SPoolInfo *pInfo) { pthread_attr_t thattr; SPoolObj *pPool = calloc(sizeof(SPoolObj), 1); @@ -89,8 +88,7 @@ void *taosOpenTcpThreadPool(SPoolInfo *pInfo) return pPool; } -void taosCloseTcpThreadPool(void *param) -{ +void taosCloseTcpThreadPool(void *param) { SPoolObj *pPool = (SPoolObj *)param; SThreadObj *pThread; @@ -107,8 +105,7 @@ void taosCloseTcpThreadPool(void *param) uDebug("%p TCP pool is closed", pPool); } -void *taosAllocateTcpConn(void *param, void *pPeer, int connFd) -{ +void *taosAllocateTcpConn(void *param, void *pPeer, int connFd) { struct epoll_event event; SPoolObj *pPool = (SPoolObj *)param; @@ -145,9 +142,8 @@ void *taosAllocateTcpConn(void *param, void *pPeer, int connFd) return pConn; } -void taosFreeTcpConn(void *param) -{ - SConnObj *pConn = (SConnObj *)param; +void taosFreeTcpConn(void *param) { + SConnObj * pConn = (SConnObj *)param; SThreadObj *pThread = pConn->pThread; uDebug("%p TCP connection will be closed, fd:%d", pThread, pConn->fd); diff --git a/src/sync/src/tarbitrator.c b/src/sync/src/tarbitrator.c index 3538391a94..b704b1ecae 100644 --- a/src/sync/src/tarbitrator.c +++ b/src/sync/src/tarbitrator.c @@ -45,7 +45,7 @@ int main(int argc, char *argv[]) { for (int i=1; itsdbMeta; STsdbFileH *pFileH = pRepo->tsdbFileH; diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 12199f491f..4c6d75ec14 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -908,7 +908,7 @@ static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkId // Add the sub-block if (pSCompBlock->numOfSubBlocks > 1) { - size_t tsize = pIdx->len - (pSCompBlock->offset + pSCompBlock->len); + size_t tsize = (size_t)(pIdx->len - (pSCompBlock->offset + pSCompBlock->len)); if (tsize > 0) { memmove((void *)((char *)(pHelper->pCompInfo) + pSCompBlock->offset + pSCompBlock->len + sizeof(SCompBlock)), (void *)((char *)(pHelper->pCompInfo) + pSCompBlock->offset + pSCompBlock->len), tsize); @@ -988,7 +988,7 @@ static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int // Delete the sub blocks it has if (pSCompBlock->numOfSubBlocks > 1) { - size_t tsize = pIdx->len - (pSCompBlock->offset + pSCompBlock->len); + size_t tsize = (size_t)(pIdx->len - (pSCompBlock->offset + pSCompBlock->len)); if (tsize > 0) { memmove(POINTER_SHIFT(pHelper->pCompInfo, pSCompBlock->offset), POINTER_SHIFT(pHelper->pCompInfo, pSCompBlock->offset + pSCompBlock->len), tsize); diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index ac3a6dac07..f8ff25ddab 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -28,6 +28,14 @@ #define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) #define QH_GET_NUM_OF_COLS(handle) ((size_t)(taosArrayGetSize((handle)->pColumns))) +#define GET_FILE_DATA_BLOCK_INFO(_checkInfo, _block) \ + ((SDataBlockInfo){.window = {.skey = (_block)->keyFirst, .ekey = (_block)->keyLast}, \ + .numOfCols = (_block)->numOfCols, \ + .rows = (_block)->numOfRows, \ + .tid = (_checkInfo)->tableId.tid, \ + .uid = (_checkInfo)->tableId.uid}) + + enum { TSDB_QUERY_TYPE_ALL = 1, TSDB_QUERY_TYPE_LAST = 2, @@ -119,7 +127,14 @@ typedef struct STsdbQueryHandle { SIOCostSummary cost; } STsdbQueryHandle; -static void changeQueryHandleForLastrowQuery(TsdbQueryHandleT pqHandle); +typedef struct STableGroupSupporter { + int32_t numOfCols; + SColIndex* pCols; + STSchema* pTagSchema; +} STableGroupSupporter; + +static STimeWindow changeTableGroupByLastrow(STableGroupInfo *groupList); + static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle); static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock); static int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order); @@ -242,10 +257,7 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab for (int32_t j = 0; j < gsize; ++j) { STableKeyInfo* pKeyInfo = (STableKeyInfo*) taosArrayGet(group, j); - STableCheckInfo info = { - .lastKey = pKeyInfo->lastKey, - .pTableObj = pKeyInfo->pTable, - }; + STableCheckInfo info = { .lastKey = pKeyInfo->lastKey, .pTableObj = pKeyInfo->pTable }; info.tableId = ((STable*)(pKeyInfo->pTable))->tableId; assert(info.pTableObj != NULL && (info.pTableObj->type == TSDB_NORMAL_TABLE || @@ -283,12 +295,9 @@ out_of_memory: } TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, void* qinfo) { + pCond->order = TSDB_ORDER_ASC; + pCond->twindow = changeTableGroupByLastrow(groupList); STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qinfo); - if (pQueryHandle != NULL) { - pQueryHandle->type = TSDB_QUERY_TYPE_LAST; - pQueryHandle->order = TSDB_ORDER_DESC; - changeQueryHandleForLastrowQuery(pQueryHandle); - } return pQueryHandle; } @@ -393,7 +402,7 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh SDataRow row = *(SDataRow *)SL_GET_NODE_DATA(node); TSKEY key = dataRowKey(row); // first timestamp in buffer - tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in mem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 + tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in imem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 "-%" PRId64 ", lastKey:%" PRId64 ", %p", pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, key, order, pIMem->keyFirst, pIMem->keyLast, pCheckInfo->lastKey, pHandle->qinfo); @@ -416,7 +425,7 @@ static void destroyTableMemIterator(STableCheckInfo* pCheckInfo) { tSkipListDestroyIter(pCheckInfo->iiter); } -SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order) { +static SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order) { SDataRow rmem = NULL, rimem = NULL; if (pCheckInfo->iter) { SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter); @@ -432,47 +441,46 @@ SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order) { } } - if (rmem != NULL && rimem != NULL) { - TSKEY r1 = dataRowKey(rmem); - TSKEY r2 = dataRowKey(rimem); - - if (r1 == r2) { // data ts are duplicated, ignore the data in mem - tSkipListIterNext(pCheckInfo->iter); - pCheckInfo->chosen = 1; - return rimem; - } else { - if (ASCENDING_TRAVERSE(order)) { - if (r1 < r2) { - pCheckInfo->chosen = 0; - return rmem; - } else { - pCheckInfo->chosen = 1; - return rimem; - } - } else { - if (r1 < r2) { - pCheckInfo->chosen = 1; - return rimem; - } else { - pCheckInfo->chosen = 0; - return rmem; - } - } - } + if (rmem == NULL && rimem == NULL) { + return NULL; } - // at least one (rmem or rimem) is absent here - if (rmem != NULL) { + if (rmem != NULL && rimem == NULL) { pCheckInfo->chosen = 0; return rmem; } - if (rimem != NULL) { + if (rmem == NULL && rimem != NULL) { pCheckInfo->chosen = 1; return rimem; } - return NULL; + TSKEY r1 = dataRowKey(rmem); + TSKEY r2 = dataRowKey(rimem); + + if (r1 == r2) { // data ts are duplicated, ignore the data in mem + tSkipListIterNext(pCheckInfo->iter); + pCheckInfo->chosen = 1; + return rimem; + } else { + if (ASCENDING_TRAVERSE(order)) { + if (r1 < r2) { + pCheckInfo->chosen = 0; + return rmem; + } else { + pCheckInfo->chosen = 1; + return rimem; + } + } else { + if (r1 < r2) { + pCheckInfo->chosen = 1; + return rimem; + } else { + pCheckInfo->chosen = 0; + return rmem; + } + } + } } static bool moveToNextRowInMem(STableCheckInfo* pCheckInfo) { @@ -672,14 +680,6 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo return code; } -#define GET_FILE_DATA_BLOCK_INFO(_checkInfo, _block) \ - ((SDataBlockInfo){.window = {.skey = (_block)->keyFirst, .ekey = (_block)->keyLast}, \ - .numOfCols = (_block)->numOfCols, \ - .rows = (_block)->numOfRows, \ - .tid = (_checkInfo)->tableId.tid, \ - .uid = (_checkInfo)->tableId.uid}) - - static int32_t doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo, int32_t slotIndex) { STsdbRepo *pRepo = pQueryHandle->pTsdb; int64_t st = taosGetTimestampUs(); @@ -736,6 +736,8 @@ static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBloc SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order); TSKEY key = (row != NULL)? dataRowKey(row):TSKEY_INITIAL_VAL; + tsdbDebug("%p key in mem:%"PRId64", %p", pQueryHandle, key, pQueryHandle->qinfo); + cur->pos = ASCENDING_TRAVERSE(pQueryHandle->order)? 0:(binfo.rows-1); if ((ASCENDING_TRAVERSE(pQueryHandle->order) && (key != TSKEY_INITIAL_VAL && key <= binfo.window.ekey)) || @@ -747,7 +749,8 @@ static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBloc // do not load file block into buffer int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order) ? 1 : -1; - cur->rows = tsdbReadRowsFromCache(pCheckInfo, binfo.window.skey - step, pQueryHandle->outputCapacity, &cur->win, pQueryHandle); + TSKEY maxKey = ASCENDING_TRAVERSE(pQueryHandle->order)? (binfo.window.skey - step):(binfo.window.ekey - step); + cur->rows = tsdbReadRowsFromCache(pCheckInfo, maxKey, pQueryHandle->outputCapacity, &cur->win, pQueryHandle); pQueryHandle->realNumOfRows = cur->rows; // update the last key value @@ -1072,8 +1075,8 @@ static void moveDataToFront(STsdbQueryHandle* pQueryHandle, int32_t numOfRows, i } } -static void getQualifiedRowsPos(STsdbQueryHandle* pQueryHandle, int32_t startPos, int32_t endPos, - int32_t numOfExisted, int32_t *start, int32_t *end) { +static void getQualifiedRowsPos(STsdbQueryHandle* pQueryHandle, int32_t startPos, int32_t endPos, int32_t numOfExisted, + int32_t* start, int32_t* end) { *start = -1; if (ASCENDING_TRAVERSE(pQueryHandle->order)) { @@ -1162,6 +1165,11 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* cur->mixBlock = true; } + tsdbDebug("%p uid:%" PRIu64",tid:%d start merge data block, file block range:%"PRIu64"-%"PRIu64" rows:%d, start:%d," + "end:%d, %p", + pQueryHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, blockInfo.window.skey, blockInfo.window.ekey, + blockInfo.rows, cur->pos, endPos, pQueryHandle->qinfo); + // compared with the data from in-memory buffer, to generate the correct timestamp array list int32_t numOfRows = 0; int32_t pos = cur->pos; @@ -1191,6 +1199,10 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* moveDataToFront(pQueryHandle, numOfRows, numOfCols); updateInfoAfterMerge(pQueryHandle, pCheckInfo, numOfRows, pos); doCheckGeneratedBlockRange(pQueryHandle); + + tsdbDebug("%p uid:%" PRIu64",tid:%d data block created, mixblock:%d, brange:%"PRIu64"-%"PRIu64" rows:%d, %p", + pQueryHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, cur->mixBlock, cur->win.skey, + cur->win.ekey, cur->rows, pQueryHandle->qinfo); return; } else if (pCheckInfo->iter != NULL || pCheckInfo->iiter != NULL) { SSkipListNode* node = NULL; @@ -1233,6 +1245,8 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* } int32_t end = doBinarySearchKey(pCols->cols[0].pData, pCols->numOfRows, key, order); + assert(end != -1); + if (tsArray[end] == key) { // the value of key in cache equals to the end timestamp value, ignore it moveToNextRowInMem(pCheckInfo); } @@ -1269,6 +1283,7 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* cur->win.ekey = ASCENDING_TRAVERSE(pQueryHandle->order)? tsArray[end]:tsArray[start]; cur->lastKey = cur->win.ekey + step; + cur->mixBlock = true; } } } @@ -1285,7 +1300,8 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* updateInfoAfterMerge(pQueryHandle, pCheckInfo, numOfRows, pos); doCheckGeneratedBlockRange(pQueryHandle); - tsdbDebug("%p uid:%" PRIu64",tid:%d data block created, brange:%"PRIu64"-%"PRIu64" rows:%d, %p", pQueryHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, cur->win.skey, + tsdbDebug("%p uid:%" PRIu64",tid:%d data block created, mixblock:%d, brange:%"PRIu64"-%"PRIu64" rows:%d, %p", + pQueryHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, cur->mixBlock, cur->win.skey, cur->win.ekey, cur->rows, pQueryHandle->qinfo); } @@ -1658,6 +1674,128 @@ static bool doHasDataInBuffer(STsdbQueryHandle* pQueryHandle) { return false; } +static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle) { + // filter the queried time stamp in the first place + STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle; + pQueryHandle->order = TSDB_ORDER_DESC; + + assert(pQueryHandle->window.skey == pQueryHandle->window.ekey); + + // starts from the buffer in case of descending timestamp order check data blocks + // todo consider the query time window, current last_row does not apply the query time window + size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + + int32_t i = 0; + while(i < numOfTables) { + STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); + if (pQueryHandle->window.skey <= pCheckInfo->pTableObj->lastKey && + pCheckInfo->pTableObj->lastKey != TSKEY_INITIAL_VAL) { + break; + } + + i++; + } + + // there are no data in all the tables + if (i == numOfTables) { + return; + } + + STableCheckInfo info = *(STableCheckInfo*) taosArrayGet(pQueryHandle->pTableCheckInfo, i); + taosArrayClear(pQueryHandle->pTableCheckInfo); + + info.lastKey = pQueryHandle->window.skey; + taosArrayPush(pQueryHandle->pTableCheckInfo, &info); + + // update the query time window according to the chosen last timestamp + pQueryHandle->window = (STimeWindow) {info.lastKey, TSKEY_INITIAL_VAL}; +} + +static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, STimeWindow* win, + STsdbQueryHandle* pQueryHandle) { + int numOfRows = 0; + int32_t numOfCols = (int32_t)taosArrayGetSize(pQueryHandle->pColumns); + win->skey = TSKEY_INITIAL_VAL; + + int64_t st = taosGetTimestampUs(); + STable* pTable = pCheckInfo->pTableObj; + + do { + SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order); + if (row == NULL) { + break; + } + + TSKEY key = dataRowKey(row); + if ((key > maxKey && ASCENDING_TRAVERSE(pQueryHandle->order)) || (key < maxKey && !ASCENDING_TRAVERSE(pQueryHandle->order))) { + tsdbDebug("%p key:%"PRIu64" beyond qrange:%"PRId64" - %"PRId64", no more data in buffer", pQueryHandle, key, pQueryHandle->window.skey, + pQueryHandle->window.ekey); + + break; + } + + if (win->skey == INT64_MIN) { + win->skey = key; + } + + win->ekey = key; + copyOneRowFromMem(pQueryHandle, maxRowsToRead, numOfRows, row, numOfCols, pTable); + + if (++numOfRows >= maxRowsToRead) { + moveToNextRowInMem(pCheckInfo); + break; + } + + } while(moveToNextRowInMem(pCheckInfo)); + + assert(numOfRows <= maxRowsToRead); + + // if the buffer is not full in case of descending order query, move the data in the front of the buffer + if (!ASCENDING_TRAVERSE(pQueryHandle->order) && numOfRows < maxRowsToRead) { + int32_t emptySize = maxRowsToRead - numOfRows; + + for(int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); + memmove((char*)pColInfo->pData, (char*)pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes); + } + } + + int64_t elapsedTime = taosGetTimestampUs() - st; + tsdbDebug("%p build data block from cache completed, elapsed time:%"PRId64" us, numOfRows:%d, numOfCols:%d, %p", pQueryHandle, + elapsedTime, numOfRows, numOfCols, pQueryHandle->qinfo); + + return numOfRows; +} + +static int32_t getAllTableList(STable* pSuperTable, SArray* list) { + SSkipListIterator* iter = tSkipListCreateIter(pSuperTable->pIndex); + while (tSkipListIterNext(iter)) { + SSkipListNode* pNode = tSkipListIterGet(iter); + + STable** pTable = (STable**) SL_GET_NODE_DATA((SSkipListNode*) pNode); + + STableKeyInfo info = {.pTable = *pTable, .lastKey = TSKEY_INITIAL_VAL}; + taosArrayPush(list, &info); + } + + tSkipListDestroyIter(iter); + return TSDB_CODE_SUCCESS; +} + +static void destroyHelper(void* param) { + if (param == NULL) { + return; + } + + + tQueryInfo* pInfo = (tQueryInfo*)param; + if (pInfo->optr != TSDB_RELATION_IN) { + taosTFree(pInfo->q); + } + + free(param); +} + // handle data in cache situation bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle; @@ -1698,6 +1836,11 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { return true; } else { STsdbQueryHandle* pSecQueryHandle = calloc(1, sizeof(STsdbQueryHandle)); + if (pSecQueryHandle == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return false; + } + pSecQueryHandle->order = TSDB_ORDER_ASC; pSecQueryHandle->window = (STimeWindow) {pQueryHandle->window.skey, INT64_MAX}; pSecQueryHandle->pTsdb = pQueryHandle->pTsdb; @@ -1709,6 +1852,7 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { pSecQueryHandle->outputCapacity = ((STsdbRepo*)pSecQueryHandle->pTsdb)->config.maxRowsPerFileBlock; if (tsdbInitReadHelper(&pSecQueryHandle->rhelper, (STsdbRepo*) pSecQueryHandle->pTsdb) != 0) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; free(pSecQueryHandle); return false; } @@ -1720,6 +1864,11 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { pSecQueryHandle->statis = calloc(numOfCols, sizeof(SDataStatis)); pSecQueryHandle->pColumns = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); + if (pSecQueryHandle->statis == NULL || pSecQueryHandle->pColumns == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + tsdbCleanupQueryHandle(pSecQueryHandle); + return false; + } for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData colInfo = {{0}, 0}; @@ -1727,6 +1876,12 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { colInfo.info = pCol->info; colInfo.pData = calloc(1, EXTRA_BYTES + pQueryHandle->outputCapacity * pCol->info.bytes); + if (colInfo.pData == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + tsdbCleanupQueryHandle(pSecQueryHandle); + return false; + } + taosArrayPush(pSecQueryHandle->pColumns, &colInfo); } @@ -1794,6 +1949,7 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { } if (pQueryHandle->checkFiles) { + // check if the query range overlaps with the file data block bool exists = true; int32_t code = getDataBlocksInFiles(pQueryHandle, &exists); @@ -1822,150 +1978,48 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { return ret; } -void changeQueryHandleForLastrowQuery(TsdbQueryHandleT pqHandle) { - STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pqHandle; - assert(!ASCENDING_TRAVERSE(pQueryHandle->order)); - - // starts from the buffer in case of descending timestamp order check data blocks +STimeWindow changeTableGroupByLastrow(STableGroupInfo *groupList) { + STimeWindow window = {INT64_MAX, INT64_MIN}; + // NOTE: starts from the buffer in case of descending timestamp order check data blocks // todo consider the query time window, current last_row does not apply the query time window - size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + size_t numOfGroups = taosArrayGetSize(groupList->pGroupList); + for(int32_t j = 0; j < numOfGroups; ++j) { + SArray* pGroup = taosArrayGetP(groupList->pGroupList, j); + TSKEY key = TSKEY_INITIAL_VAL; - TSKEY key = TSKEY_INITIAL_VAL; - int32_t index = -1; + STableKeyInfo keyInfo = {0}; - for(int32_t i = 0; i < numOfTables; ++i) { - STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); - if (pCheckInfo->pTableObj->lastKey > key) { - key = pCheckInfo->pTableObj->lastKey; - index = i; + size_t numOfTables = taosArrayGetSize(pGroup); + for(int32_t i = 0; i < numOfTables; ++i) { + STableKeyInfo* pKeyInfo = (STableKeyInfo*) taosArrayGet(pGroup, i); + TSKEY lastKey = ((STable*)(pKeyInfo->pTable))->lastKey; + + if (key < lastKey) { + key = lastKey; + + keyInfo.pTable = pKeyInfo->pTable; + keyInfo.lastKey = key; + pKeyInfo->lastKey = key; + + if (key < window.skey) { + window.skey = key; + } + + if (key > window.ekey) { + window.ekey = key; + } + } + } + + // more than one table in each group, only one table left for each group + if (numOfTables > 1) { + taosArrayClear(pGroup); + taosArrayPush(pGroup, &keyInfo); } } - if (index == -1) { - // todo add failure test cases - return; - } - - // erase all other elements in array list - size_t size = taosArrayGetSize(pQueryHandle->pTableCheckInfo); - for (int32_t i = 0; i < size; ++i) { - if (i == index) { - continue; - } - - STableCheckInfo* pTableCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); - tSkipListDestroyIter(pTableCheckInfo->iter); - - if (pTableCheckInfo->pDataCols != NULL) { - taosTFree(pTableCheckInfo->pDataCols->buf); - } - - taosTFree(pTableCheckInfo->pDataCols); - taosTFree(pTableCheckInfo->pCompInfo); - } - - STableCheckInfo info = *(STableCheckInfo*) taosArrayGet(pQueryHandle->pTableCheckInfo, index); - taosArrayClear(pQueryHandle->pTableCheckInfo); - - info.lastKey = key; - taosArrayPush(pQueryHandle->pTableCheckInfo, &info); - - // update the query time window according to the chosen last timestamp - pQueryHandle->window = (STimeWindow) {key, key}; -} - -static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle) { - // filter the queried time stamp in the first place - STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle; - pQueryHandle->order = TSDB_ORDER_DESC; - - assert(pQueryHandle->window.skey == pQueryHandle->window.ekey); - - // starts from the buffer in case of descending timestamp order check data blocks - // todo consider the query time window, current last_row does not apply the query time window - size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); - - int32_t i = 0; - while(i < numOfTables) { - STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); - if (pQueryHandle->window.skey <= pCheckInfo->pTableObj->lastKey && - pCheckInfo->pTableObj->lastKey != TSKEY_INITIAL_VAL) { - break; - } - - i++; - } - - // there are no data in all the tables - if (i == numOfTables) { - return; - } - - STableCheckInfo info = *(STableCheckInfo*) taosArrayGet(pQueryHandle->pTableCheckInfo, i); - taosArrayClear(pQueryHandle->pTableCheckInfo); - - info.lastKey = pQueryHandle->window.skey; - taosArrayPush(pQueryHandle->pTableCheckInfo, &info); - - // update the query time window according to the chosen last timestamp - pQueryHandle->window = (STimeWindow) {info.lastKey, TSKEY_INITIAL_VAL}; -} - -static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, STimeWindow* win, - STsdbQueryHandle* pQueryHandle) { - int numOfRows = 0; - int32_t numOfCols = (int32_t)taosArrayGetSize(pQueryHandle->pColumns); - win->skey = TSKEY_INITIAL_VAL; - - int64_t st = taosGetTimestampUs(); - STable* pTable = pCheckInfo->pTableObj; - - do { - SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order); - if (row == NULL) { - break; - } - - TSKEY key = dataRowKey(row); - if ((key > maxKey && ASCENDING_TRAVERSE(pQueryHandle->order)) || (key < maxKey && !ASCENDING_TRAVERSE(pQueryHandle->order))) { - tsdbDebug("%p key:%"PRIu64" beyond qrange:%"PRId64" - %"PRId64", no more data in buffer", pQueryHandle, key, pQueryHandle->window.skey, - pQueryHandle->window.ekey); - - break; - } - - if (win->skey == INT64_MIN) { - win->skey = key; - } - - win->ekey = key; - copyOneRowFromMem(pQueryHandle, maxRowsToRead, numOfRows, row, numOfCols, pTable); - - if (++numOfRows >= maxRowsToRead) { - moveToNextRowInMem(pCheckInfo); - break; - } - - } while(moveToNextRowInMem(pCheckInfo)); - - assert(numOfRows <= maxRowsToRead); - - // if the buffer is not full in case of descending order query, move the data in the front of the buffer - if (!ASCENDING_TRAVERSE(pQueryHandle->order) && numOfRows < maxRowsToRead) { - int32_t emptySize = maxRowsToRead - numOfRows; - - for(int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); - memmove((char*)pColInfo->pData, (char*)pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes); - } - } - - int64_t elapsedTime = taosGetTimestampUs() - st; - tsdbDebug("%p build data block from cache completed, elapsed time:%"PRId64" us, numOfRows:%d, numOfCols:%d, %p", pQueryHandle, - elapsedTime, numOfRows, numOfCols, pQueryHandle->qinfo); - - return numOfRows; + return window; } void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT* pQueryHandle, SDataBlockInfo* pDataBlockInfo) { @@ -2102,36 +2156,6 @@ SArray* tsdbRetrieveDataBlock(TsdbQueryHandleT* pQueryHandle, SArray* pIdList) { } } -static int32_t getAllTableList(STable* pSuperTable, SArray* list) { - SSkipListIterator* iter = tSkipListCreateIter(pSuperTable->pIndex); - while (tSkipListIterNext(iter)) { - SSkipListNode* pNode = tSkipListIterGet(iter); - - STable** pTable = (STable**) SL_GET_NODE_DATA((SSkipListNode*) pNode); - - STableKeyInfo info = {.pTable = *pTable, .lastKey = TSKEY_INITIAL_VAL}; - taosArrayPush(list, &info); - } - - tSkipListDestroyIter(iter); - return TSDB_CODE_SUCCESS; -} - -static void destroyHelper(void* param) { - if (param == NULL) { - return; - } - - - tQueryInfo* pInfo = (tQueryInfo*)param; - if (pInfo->optr != TSDB_RELATION_IN) { - taosTFree(pInfo->q); - } - -// tVariantDestroy(&(pInfo->q)); - free(param); -} - void filterPrepare(void* expr, void* param) { tExprNode* pExpr = (tExprNode*)expr; if (pExpr->_node.info != NULL) { @@ -2158,13 +2182,7 @@ void filterPrepare(void* expr, void* param) { } } -typedef struct STableGroupSupporter { - int32_t numOfCols; - SColIndex* pCols; - STSchema* pTagSchema; -} STableGroupSupporter; - -int32_t tableGroupComparFn(const void *p1, const void *p2, const void *param) { +static int32_t tableGroupComparFn(const void *p1, const void *p2, const void *param) { STableGroupSupporter* pTableGroupSupp = (STableGroupSupporter*) param; STable* pTable1 = ((STableKeyInfo*) p1)->pTable; STable* pTable2 = ((STableKeyInfo*) p2)->pTable; @@ -2217,8 +2235,19 @@ int32_t tableGroupComparFn(const void *p1, const void *p2, const void *param) { return 0; } -void createTableGroupImpl(SArray* pGroups, SArray* pTableList, size_t numOfTables, TSKEY skey, STableGroupSupporter* pSupp, - __ext_compar_fn_t compareFn) { +static int tsdbCheckInfoCompar(const void* key1, const void* key2) { + if (((STableCheckInfo*)key1)->tableId.tid < ((STableCheckInfo*)key2)->tableId.tid) { + return -1; + } else if (((STableCheckInfo*)key1)->tableId.tid > ((STableCheckInfo*)key2)->tableId.tid) { + return 1; + } else { + ASSERT(false); + return 0; + } +} + +void createTableGroupImpl(SArray* pGroups, SArray* pTableList, size_t numOfTables, TSKEY skey, + STableGroupSupporter* pSupp, __ext_compar_fn_t compareFn) { STable* pTable = taosArrayGetP(pTableList, 0); SArray* g = taosArrayInit(16, sizeof(STableKeyInfo)); @@ -2264,6 +2293,10 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC if (numOfOrderCols == 0 || size == 1) { // no group by tags clause or only one table SArray* sa = taosArrayInit(size, sizeof(STableKeyInfo)); + if (sa == NULL) { + taosArrayDestroy(pTableGroup); + return NULL; + } for(int32_t i = 0; i < size; ++i) { STableKeyInfo *pKeyInfo = taosArrayGet(pTableList, i); @@ -2278,20 +2311,19 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC taosArrayPush(pTableGroup, &sa); tsdbDebug("all %" PRIzu " tables belong to one group", size); } else { - STableGroupSupporter *pSupp = (STableGroupSupporter *) calloc(1, sizeof(STableGroupSupporter)); - pSupp->numOfCols = numOfOrderCols; - pSupp->pTagSchema = pTagSchema; - pSupp->pCols = pCols; + STableGroupSupporter sup = {0}; + sup.numOfCols = numOfOrderCols; + sup.pTagSchema = pTagSchema; + sup.pCols = pCols; - taosqsort(pTableList->pData, size, sizeof(STableKeyInfo), pSupp, tableGroupComparFn); - createTableGroupImpl(pTableGroup, pTableList, size, skey, pSupp, tableGroupComparFn); - taosTFree(pSupp); + taosqsort(pTableList->pData, size, sizeof(STableKeyInfo), &sup, tableGroupComparFn); + createTableGroupImpl(pTableGroup, pTableList, size, skey, &sup, tableGroupComparFn); } return pTableGroup; } -bool indexedNodeFilterFp(const void* pNode, void* param) { +static bool indexedNodeFilterFp(const void* pNode, void* param) { tQueryInfo* pInfo = (tQueryInfo*) param; STable* pTable = *(STable**)(SL_GET_NODE_DATA((SSkipListNode*)pNode)); @@ -2562,7 +2594,7 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) { tsdbDestroyHelper(&pQueryHandle->rhelper); SIOCostSummary* pCost = &pQueryHandle->cost; - tsdbDebug("%p :io-cost summary: statis-info:%"PRId64"us, datablock:%" PRId64"us, check data:%"PRId64"us, %p", + tsdbDebug("%p :io-cost summary: statis-info:%"PRId64" us, datablock:%" PRId64" us, check data:%"PRId64" us, %p", pQueryHandle, pCost->statisInfoLoadTime, pCost->blockLoadTime, pCost->checkForNextTime, pQueryHandle->qinfo); taosTFree(pQueryHandle); @@ -2589,14 +2621,3 @@ void tsdbDestroyTableGroup(STableGroupInfo *pGroupList) { taosArrayDestroy(pGroupList->pGroupList); } - -static int tsdbCheckInfoCompar(const void* key1, const void* key2) { - if (((STableCheckInfo*)key1)->tableId.tid < ((STableCheckInfo*)key2)->tableId.tid) { - return -1; - } else if (((STableCheckInfo*)key1)->tableId.tid > ((STableCheckInfo*)key2)->tableId.tid) { - return 1; - } else { - ASSERT(false); - return 0; - } -} diff --git a/src/util/inc/tcache.h b/src/util/inc/tcache.h index 11121fcf3b..3d6f9705ee 100644 --- a/src/util/inc/tcache.h +++ b/src/util/inc/tcache.h @@ -103,7 +103,7 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool ext * @param keepTime survival time in second * @return cached element */ -void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const void *pData, size_t dataSize, int keepTimeInSeconds); +void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const void *pData, size_t dataSize, int durationMS); /** * get data from cache @@ -113,16 +113,6 @@ void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const v */ void *taosCacheAcquireByKey(SCacheObj *pCacheObj, const void *key, size_t keyLen); -/** - * update the expire time of data in cache - * @param pCacheObj cache object - * @param key key - * @param keyLen keyLen - * @param expireTime new expire time of data - * @return - */ -//void* taosCacheUpdateExpireTimeByName(SCacheObj *pCacheObj, void *key, size_t keyLen, uint64_t expireTime); - /** * Add one reference count for the exist data, and assign this data for a new owner. * The new owner needs to invoke the taosCacheRelease when it does not need this data anymore. diff --git a/src/util/inc/tcoding.h b/src/util/inc/tcoding.h index 9300094e35..ff34c15607 100644 --- a/src/util/inc/tcoding.h +++ b/src/util/inc/tcoding.h @@ -355,9 +355,9 @@ static FORCE_INLINE void *taosDecodeString(void *buf, char **value) { uint64_t size = 0; buf = taosDecodeVariantU64(buf, &size); - *value = (char *)malloc(size + 1); + *value = (char *)malloc((size_t)size + 1); if (*value == NULL) return NULL; - memcpy(*value, buf, size); + memcpy(*value, buf, (size_t)size); (*value)[size] = '\0'; diff --git a/src/util/inc/tkvstore.h b/src/util/inc/tkvstore.h index 3b4e8e3757..b2b0ff05f5 100644 --- a/src/util/inc/tkvstore.h +++ b/src/util/inc/tkvstore.h @@ -58,7 +58,7 @@ int tdKVStoreStartCommit(SKVStore *pStore); int tdUpdateKVStoreRecord(SKVStore *pStore, uint64_t uid, void *cont, int contLen); int tdDropKVStoreRecord(SKVStore *pStore, uint64_t uid); int tdKVStoreEndCommit(SKVStore *pStore); -void tsdbGetStoreInfo(char *fname, uint32_t *magic, int32_t *size); +void tsdbGetStoreInfo(char *fname, uint32_t *magic, int64_t *size); #ifdef __cplusplus } diff --git a/src/util/inc/tlockfree.h b/src/util/inc/tlockfree.h index a81f597832..e960b601ca 100644 --- a/src/util/inc/tlockfree.h +++ b/src/util/inc/tlockfree.h @@ -36,6 +36,14 @@ typedef void (*_ref_fn_t)(const void* pObj); _ref_fn_t end; \ } _ref_func = {.begin = (s), .end = (e)}; +// set the initial reference count value +#define T_REF_INIT_VAL(x, _v) \ + do { \ + assert(_v >= 0); \ + atomic_store_32(&((x)->_ref.val), (_v)); \ + } while (0) + +// increase the reference count by 1 #define T_REF_INC(x) (atomic_add_fetch_32(&((x)->_ref.val), 1)) #define T_REF_INC_WITH_CB(x, p) \ diff --git a/src/util/inc/tstoken.h b/src/util/inc/tstoken.h index c1c6f2de7a..7aeb2da6b6 100644 --- a/src/util/inc/tstoken.h +++ b/src/util/inc/tstoken.h @@ -28,11 +28,11 @@ extern "C" { #define TSQL_TBNAME_L "tbname" // used to denote the minimum unite in sql parsing -typedef struct SSQLToken { +typedef struct SStrToken { uint32_t n; uint32_t type; char * z; -} SSQLToken; +} SStrToken; /** * tokenizer for sql string @@ -52,7 +52,7 @@ uint32_t tSQLGetToken(char *z, uint32_t *tokenType); * @param ignoreTokenTypes * @return */ -SSQLToken tStrGetToken(char *str, int32_t *i, bool isPrevOptr, uint32_t numOfIgnoreToken, uint32_t *ignoreTokenTypes); +SStrToken tStrGetToken(char *str, int32_t *i, bool isPrevOptr, uint32_t numOfIgnoreToken, uint32_t *ignoreTokenTypes); /** * check if it is a keyword or not @@ -76,7 +76,7 @@ bool isKeyWord(const char *z, int32_t len); * @param pToken * @return token type, if it is not a number, TK_ILLEGAL will return */ -static FORCE_INLINE int32_t isValidNumber(const SSQLToken* pToken) { +static FORCE_INLINE int32_t isValidNumber(const SStrToken* pToken) { const char* z = pToken->z; int32_t type = TK_ILLEGAL; diff --git a/src/util/src/hash.c b/src/util/src/hash.c index 714f36f1cb..7d10545ce7 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -657,7 +657,7 @@ void taosHashTableResize(SHashObj *pHashObj) { } int64_t st = taosGetTimestampUs(); - void *pNewEntryList = realloc(pHashObj->hashList, sizeof(void*) * newSize); + void *pNewEntryList = realloc(pHashObj->hashList, sizeof(void *) * newSize); if (pNewEntryList == NULL) { // todo handle error // uDebug("cache resize failed due to out of memory, capacity remain:%d", pHashObj->capacity); return; diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index bd903c8c23..dfa982b848 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -189,12 +189,12 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool ext return pCacheObj; } -void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const void *pData, size_t dataSize, int duration) { +void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const void *pData, size_t dataSize, int durationMS) { if (pCacheObj == NULL || pCacheObj->pHashTable == NULL || pCacheObj->deleting == 1) { return NULL; } - SCacheDataNode *pNode1 = taosCreateCacheNode(key, keyLen, pData, dataSize, duration); + SCacheDataNode *pNode1 = taosCreateCacheNode(key, keyLen, pData, dataSize, durationMS); if (pNode1 == NULL) { uError("cache:%s, key:%p, failed to added into cache, out of memory", pCacheObj->name, key); return NULL; diff --git a/src/util/src/tcompare.c b/src/util/src/tcompare.c index 9564588254..ba711ced8f 100644 --- a/src/util/src/tcompare.c +++ b/src/util/src/tcompare.c @@ -220,8 +220,14 @@ static int32_t compareStrPatternComp(const void* pLeft, const void* pRight) { char pattern[128] = {0}; memcpy(pattern, varDataVal(pRight), varDataLen(pRight)); assert(varDataLen(pRight) < 128); - - int32_t ret = patternMatch(pattern, varDataVal(pLeft), varDataLen(pLeft), &pInfo); + + size_t sz = varDataLen(pLeft); + char *buf = malloc(sz + 1); + memcpy(buf, varDataVal(pLeft), sz); + buf[sz] = 0; + + int32_t ret = patternMatch(pattern, buf, sz, &pInfo); + free(buf); return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; } diff --git a/src/util/src/tkvstore.c b/src/util/src/tkvstore.c index 6f7b2ffb88..6ba1d87d92 100644 --- a/src/util/src/tkvstore.c +++ b/src/util/src/tkvstore.c @@ -332,7 +332,7 @@ int tdKVStoreEndCommit(SKVStore *pStore) { return 0; } -void tsdbGetStoreInfo(char *fname, uint32_t *magic, int32_t *size) { +void tsdbGetStoreInfo(char *fname, uint32_t *magic, int64_t *size) { char buf[TD_KVSTORE_HEADER_SIZE] = "\0"; SStoreInfo info = {0}; @@ -349,7 +349,7 @@ void tsdbGetStoreInfo(char *fname, uint32_t *magic, int32_t *size) { close(fd); *magic = info.magic; - *size = (int32_t)offset; + *size = offset; return; @@ -575,7 +575,7 @@ static int tdRestoreKVStore(SKVStore *pStore) { } } - buf = malloc(maxBufSize); + buf = malloc((size_t)maxBufSize); if (buf == NULL) { uError("failed to allocate %" PRId64 " bytes in KV store %s", maxBufSize, pStore->fname); terrno = TAOS_SYSTEM_ERROR(errno); @@ -598,7 +598,7 @@ static int tdRestoreKVStore(SKVStore *pStore) { goto _err; } - if (taosTRead(pStore->fd, buf, pRecord->size) < pRecord->size) { + if (taosTRead(pStore->fd, buf, (size_t)pRecord->size) < pRecord->size) { uError("failed to read %" PRId64 " bytes from file %s since %s, offset %" PRId64, pRecord->size, pStore->fname, strerror(errno), pRecord->offset); terrno = TAOS_SYSTEM_ERROR(errno); diff --git a/src/util/src/tsched.c b/src/util/src/tsched.c index cf7f5c10d4..f014dd0fab 100644 --- a/src/util/src/tsched.c +++ b/src/util/src/tsched.c @@ -123,11 +123,6 @@ void *taosProcessSchedQueue(void *param) { while (1) { if (tsem_wait(&pSched->fullSem) != 0) { - if (errno == EINTR) { - /* sem_wait is interrupted by interrupt, ignore and continue */ - uDebug("wait %s fullSem was interrupted", pSched->label); - continue; - } uError("wait %s fullSem failed(%s)", pSched->label, strerror(errno)); } if (pSched->stop) { @@ -163,12 +158,8 @@ int taosScheduleTask(void *qhandle, SSchedMsg *pMsg) { return 0; } - while (tsem_wait(&pSched->emptySem) != 0) { - if (errno != EINTR) { - uError("wait %s emptySem failed(%s)", pSched->label, strerror(errno)); - break; - } - uDebug("wait %s emptySem was interrupted", pSched->label); + if (tsem_wait(&pSched->emptySem) != 0) { + uError("wait %s emptySem failed(%s)", pSched->label, strerror(errno)); } if (pthread_mutex_lock(&pSched->queueMutex) != 0) diff --git a/src/util/src/tsocket.c b/src/util/src/tsocket.c index f4778f50ce..61896a86df 100644 --- a/src/util/src/tsocket.c +++ b/src/util/src/tsocket.c @@ -19,26 +19,25 @@ #include "tutil.h" int taosGetFqdn(char *fqdn) { - int code = 0; char hostname[1024]; hostname[1023] = '\0'; - gethostname(hostname, 1023); + if (gethostname(hostname, 1023) == -1) { + uError("failed to get hostname, reason:%s", strerror(errno)); + return -1; + } struct addrinfo hints = {0}; struct addrinfo *result = NULL; - hints.ai_flags = AI_CANONNAME; - - int32_t ret = getaddrinfo(hostname, NULL, &hints, &result); - if (result) { - strcpy(fqdn, result->ai_canonname); - freeaddrinfo(result); - } else { + int ret = getaddrinfo(hostname, NULL, &hints, &result); + if (!result) { uError("failed to get fqdn, code:%d, reason:%s", ret, gai_strerror(ret)); - code = -1; + return -1; } - return code; + strcpy(fqdn, result->ai_canonname); + freeaddrinfo(result); + return 0; } uint32_t taosGetIpFromFqdn(const char *fqdn) { diff --git a/src/util/src/version.c b/src/util/src/version.c deleted file mode 100644 index ac5de90dc8..0000000000 --- a/src/util/src/version.c +++ /dev/null @@ -1,7 +0,0 @@ -char version[12] = "2.0.1.1"; -char compatible_version[12] = "2.0.0.0"; -char gitinfo[48] = "ae1966332948147bacce3d32f9ad539ab8721db2"; -char gitinfoOfInternal[48] = "bf53767db56cedb1c484df83a1f10536f12647ad"; -char buildinfo[64] = "Built by root at 2020-08-20 15:46"; - -void libtaos_2_0_1_1_Linux_x64() {}; diff --git a/src/util/src/version.c.in b/src/util/src/version.c.in new file mode 100644 index 0000000000..21c78a0eb4 --- /dev/null +++ b/src/util/src/version.c.in @@ -0,0 +1,7 @@ +char version[12] = "${TD_VER_NUMBER}"; +char compatible_version[12] = "${TD_VER_COMPATIBLE}"; +char gitinfo[48] = "${TD_VER_GIT}"; +char gitinfoOfInternal[48] = "${TD_VER_GIT_INTERNAL}"; +char buildinfo[64] = "Built at ${TD_VER_DATE}"; + +void libtaos_${TD_LIB_VER_NUMBER}_${TD_VER_OSTYPE}_${TD_VER_CPUTYPE}_${TD_VER_VERTYPE}() {}; diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 0a5e292f6d..c294e86839 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -30,6 +30,7 @@ #include "vnode.h" #include "vnodeInt.h" #include "query.h" +#include "dnode.h" #define TSDB_VNODE_VERSION_CONTENT_LEN 31 @@ -40,7 +41,7 @@ static int32_t vnodeReadCfg(SVnodeObj *pVnode); static int32_t vnodeSaveVersion(SVnodeObj *pVnode); static int32_t vnodeReadVersion(SVnodeObj *pVnode); static int vnodeProcessTsdbStatus(void *arg, int status); -static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int32_t *size, uint64_t *fversion); +static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion); static int vnodeGetWalInfo(void *ahandle, char *name, uint32_t *index); static void vnodeNotifyRole(void *ahandle, int8_t role); static void vnodeCtrlFlow(void *handle, int32_t mseconds); @@ -289,11 +290,18 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { pVnode->sync = syncStart(&syncInfo); if (pVnode->sync == NULL) { + vError("vgId:%d, failed to open sync module, replica:%d reason:%s", pVnode->vgId, pVnode->syncCfg.replica, + tstrerror(terrno)); vnodeCleanUp(pVnode); return terrno; } pVnode->qMgmt = qOpenQueryMgmt(pVnode->vgId); + if (pVnode->qMgmt == NULL) { + vnodeCleanUp(pVnode); + return terrno; + } + pVnode->events = NULL; pVnode->status = TAOS_VN_STATUS_READY; vDebug("vgId:%d, vnode is opened in %s, pVnode:%p", pVnode->vgId, rootDir, pVnode); @@ -357,10 +365,13 @@ void vnodeRelease(void *pVnodeRaw) { taosTFree(pVnode->rootDir); if (pVnode->dropped) { - char rootDir[TSDB_FILENAME_LEN] = {0}; + char rootDir[TSDB_FILENAME_LEN] = {0}; + char newDir[TSDB_FILENAME_LEN] = {0}; sprintf(rootDir, "%s/vnode%d", tsVnodeDir, vgId); - taosMvDir(tsVnodeBakDir, rootDir); + sprintf(newDir, "%s/vnode%d", tsVnodeBakDir, vgId); + taosRename(rootDir, newDir); taosRemoveDir(rootDir); + dnodeSendStatusMsgToMnode(); } tsem_destroy(&pVnode->sem); @@ -390,7 +401,7 @@ void *vnodeAcquireRqueue(int32_t vgId) { if (pVnode == NULL) return NULL; if (pVnode->status == TAOS_VN_STATUS_RESET) { - terrno = TSDB_CODE_VND_INVALID_STATUS; + terrno = TSDB_CODE_APP_NOT_READY; vInfo("vgId:%d, status is in reset", vgId); vnodeRelease(pVnode); return NULL; @@ -404,7 +415,7 @@ void *vnodeAcquireWqueue(int32_t vgId) { if (pVnode == NULL) return NULL; if (pVnode->status == TAOS_VN_STATUS_RESET) { - terrno = TSDB_CODE_VND_INVALID_STATUS; + terrno = TSDB_CODE_APP_NOT_READY; vInfo("vgId:%d, status is in reset", vgId); vnodeRelease(pVnode); return NULL; @@ -532,7 +543,7 @@ static int vnodeProcessTsdbStatus(void *arg, int status) { return 0; } -static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int32_t *size, uint64_t *fversion) { +static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion) { SVnodeObj *pVnode = ahandle; *fversion = pVnode->fversion; return tsdbGetFileInfo(pVnode->tsdb, name, index, eindex, size); @@ -545,8 +556,9 @@ static int vnodeGetWalInfo(void *ahandle, char *name, uint32_t *index) { static void vnodeNotifyRole(void *ahandle, int8_t role) { SVnodeObj *pVnode = ahandle; - vInfo("vgId:%d, sync role changed from %d to %d", pVnode->vgId, pVnode->role, role); + vInfo("vgId:%d, sync role changed from %s to %s", pVnode->vgId, syncRole[pVnode->role], syncRole[role]); pVnode->role = role; + dnodeSendStatusMsgToMnode(); if (pVnode->role == TAOS_SYNC_ROLE_MASTER) cqStart(pVnode->cq); diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index c41b245794..11315f5321 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -49,18 +49,18 @@ int32_t vnodeProcessRead(void *param, SReadMsg *pReadMsg) { if (pVnode->status != TAOS_VN_STATUS_READY) { vDebug("vgId:%d, msgType:%s not processed, vnode status is %d", pVnode->vgId, taosMsg[msgType], pVnode->status); - return TSDB_CODE_VND_INVALID_STATUS; + return TSDB_CODE_APP_NOT_READY; } // tsdb may be in reset state - if (pVnode->tsdb == NULL) return TSDB_CODE_RPC_NOT_READY; + if (pVnode->tsdb == NULL) return TSDB_CODE_APP_NOT_READY; if (pVnode->status == TAOS_VN_STATUS_CLOSING) - return TSDB_CODE_RPC_NOT_READY; + return TSDB_CODE_APP_NOT_READY; // TODO: Later, let slave to support query if (pVnode->syncCfg.replica > 1 && pVnode->role != TAOS_SYNC_ROLE_MASTER) { vDebug("vgId:%d, msgType:%s not processed, replica:%d role:%d", pVnode->vgId, taosMsg[msgType], pVnode->syncCfg.replica, pVnode->role); - return TSDB_CODE_RPC_NOT_READY; + return TSDB_CODE_APP_NOT_READY; } return (*vnodeProcessReadMsgFp[msgType])(pVnode, pReadMsg); diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index 900ff1fbba..1a9b05ed34 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -60,19 +60,19 @@ int32_t vnodeProcessWrite(void *param1, int qtype, void *param2, void *item) { } // tsdb may be in reset state - if (pVnode->tsdb == NULL) return TSDB_CODE_RPC_NOT_READY; + if (pVnode->tsdb == NULL) return TSDB_CODE_APP_NOT_READY; if (pVnode->status == TAOS_VN_STATUS_CLOSING) - return TSDB_CODE_RPC_NOT_READY; + return TSDB_CODE_APP_NOT_READY; if (pHead->version == 0) { // from client or CQ if (pVnode->status != TAOS_VN_STATUS_READY) { vDebug("vgId:%d, msgType:%s not processed, vnode status is %d", pVnode->vgId, taosMsg[pHead->msgType], pVnode->status); - return TSDB_CODE_VND_INVALID_STATUS; // it may be in deleting or closing state + return TSDB_CODE_APP_NOT_READY; // it may be in deleting or closing state } if (pVnode->role != TAOS_SYNC_ROLE_MASTER) { vDebug("vgId:%d, msgType:%s not processed, replica:%d role:%d", pVnode->vgId, taosMsg[pHead->msgType], pVnode->syncCfg.replica, pVnode->role); - return TSDB_CODE_RPC_NOT_READY; + return TSDB_CODE_APP_NOT_READY; } // assign version diff --git a/tests/examples/C#/TDengineDriver.cs b/tests/examples/C#/TDengineDriver.cs index 2797c362c1..b6f143e181 100644 --- a/tests/examples/C#/TDengineDriver.cs +++ b/tests/examples/C#/TDengineDriver.cs @@ -20,16 +20,17 @@ using System.Runtime.InteropServices; namespace TDengineDriver { enum TDengineDataType { - TSDB_DATA_TYPE_BOOL = 1, - TSDB_DATA_TYPE_TINYINT = 2, - TSDB_DATA_TYPE_SMALLINT = 3, - TSDB_DATA_TYPE_INT = 4, - TSDB_DATA_TYPE_BIGINT = 5, - TSDB_DATA_TYPE_FLOAT = 6, - TSDB_DATA_TYPE_DOUBLE = 7, - TSDB_DATA_TYPE_BINARY = 8, - TSDB_DATA_TYPE_TIMESTAMP = 9, - TSDB_DATA_TYPE_NCHAR = 10 + TSDB_DATA_TYPE_NULL = 0, // 1 bytes + TSDB_DATA_TYPE_BOOL = 1, // 1 bytes + TSDB_DATA_TYPE_TINYINT = 2, // 1 bytes + TSDB_DATA_TYPE_SMALLINT = 3, // 2 bytes + TSDB_DATA_TYPE_INT = 4, // 4 bytes + TSDB_DATA_TYPE_BIGINT = 5, // 8 bytes + TSDB_DATA_TYPE_FLOAT = 6, // 4 bytes + TSDB_DATA_TYPE_DOUBLE = 7, // 8 bytes + TSDB_DATA_TYPE_BINARY = 8, // string + TSDB_DATA_TYPE_TIMESTAMP = 9,// 8 bytes + TSDB_DATA_TYPE_NCHAR = 10 // unicode string } enum TDengineInitOption @@ -79,54 +80,53 @@ namespace TDengineDriver class TDengine { public const int TSDB_CODE_SUCCESS = 0; - - [DllImport("taos.dll", EntryPoint = "taos_init", CallingConvention = CallingConvention.StdCall)] + + [DllImport("taos.dll", EntryPoint = "taos_init", CallingConvention = CallingConvention.Cdecl)] static extern public void Init(); - [DllImport("taos.dll", EntryPoint = "taos_options", CallingConvention = CallingConvention.StdCall)] + [DllImport("taos.dll", EntryPoint = "taos_cleanup", CallingConvention = CallingConvention.Cdecl)] + static extern public void Cleanup(); + + [DllImport("taos.dll", EntryPoint = "taos_options", CallingConvention = CallingConvention.Cdecl)] static extern public void Options(int option, string value); - [DllImport("taos.dll", EntryPoint = "taos_connect", CallingConvention = CallingConvention.StdCall)] - static extern public long Connect(string ip, string user, string password, string db, int port); + [DllImport("taos.dll", EntryPoint = "taos_connect", CallingConvention = CallingConvention.Cdecl)] + static extern public IntPtr Connect(string ip, string user, string password, string db, short port); - [DllImport("taos.dll", EntryPoint = "taos_errstr", CallingConvention = CallingConvention.StdCall)] - static extern private IntPtr taos_errstr(long taos); - static public string Error(long conn) + [DllImport("taos.dll", EntryPoint = "taos_errstr", CallingConvention = CallingConvention.Cdecl)] + static extern private IntPtr taos_errstr(IntPtr res); + static public string Error(IntPtr res) { - IntPtr errPtr = taos_errstr(conn); + IntPtr errPtr = taos_errstr(res); return Marshal.PtrToStringAnsi(errPtr); } - [DllImport("taos.dll", EntryPoint = "taos_errno", CallingConvention = CallingConvention.StdCall)] - static extern public int ErrorNo(long taos); + [DllImport("taos.dll", EntryPoint = "taos_errno", CallingConvention = CallingConvention.Cdecl)] + static extern public int ErrorNo(IntPtr res); - [DllImport("taos.dll", EntryPoint = "taos_query", CallingConvention = CallingConvention.StdCall)] - static extern public int Query(long taos, string sqlstr); + [DllImport("taos.dll", EntryPoint = "taos_query", CallingConvention = CallingConvention.Cdecl)] + static extern public IntPtr Query(IntPtr conn, string sqlstr); - [DllImport("taos.dll", EntryPoint = "taos_affected_rows", CallingConvention = CallingConvention.StdCall)] - static extern public int AffectRows(long taos); + [DllImport("taos.dll", EntryPoint = "taos_affected_rows", CallingConvention = CallingConvention.Cdecl)] + static extern public int AffectRows(IntPtr res); - [DllImport("taos.dll", EntryPoint = "taos_use_result", CallingConvention = CallingConvention.StdCall)] - static extern public long UseResult(long taos); + [DllImport("taos.dll", EntryPoint = "taos_field_count", CallingConvention = CallingConvention.Cdecl)] + static extern public int FieldCount(IntPtr res); - [DllImport("taos.dll", EntryPoint = "taos_field_count", CallingConvention = CallingConvention.StdCall)] - static extern public int FieldCount(long taos); - - [DllImport("taos.dll", EntryPoint = "taos_fetch_fields", CallingConvention = CallingConvention.StdCall)] - static extern private IntPtr taos_fetch_fields(long res); - static public List FetchFields(long taos) + [DllImport("taos.dll", EntryPoint = "taos_fetch_fields", CallingConvention = CallingConvention.Cdecl)] + static extern private IntPtr taos_fetch_fields(IntPtr res); + static public List FetchFields(IntPtr res) { const int fieldSize = 68; List metas = new List(); - long result = TDengine.UseResult(taos); - if (result == 0) + if (res == IntPtr.Zero) { return metas; } - int fieldCount = FieldCount(taos); - IntPtr fieldsPtr = taos_fetch_fields(result); + int fieldCount = FieldCount(res); + IntPtr fieldsPtr = taos_fetch_fields(res); for (int i = 0; i < fieldCount; ++i) { @@ -134,21 +134,21 @@ namespace TDengineDriver TDengineMeta meta = new TDengineMeta(); meta.name = Marshal.PtrToStringAnsi(fieldsPtr + offset); - meta.size = Marshal.ReadInt16(fieldsPtr + offset + 64); - meta.type = Marshal.ReadByte(fieldsPtr + offset + 66); + meta.type = Marshal.ReadByte(fieldsPtr + offset + 65); + meta.size = Marshal.ReadInt16(fieldsPtr + offset + 66); metas.Add(meta); } return metas; } - [DllImport("taos.dll", EntryPoint = "taos_fetch_row", CallingConvention = CallingConvention.StdCall)] - static extern public IntPtr FetchRows(long res); + [DllImport("taos.dll", EntryPoint = "taos_fetch_row", CallingConvention = CallingConvention.Cdecl)] + static extern public IntPtr FetchRows(IntPtr res); - [DllImport("taos.dll", EntryPoint = "taos_free_result", CallingConvention = CallingConvention.StdCall)] - static extern public IntPtr FreeResult(long res); + [DllImport("taos.dll", EntryPoint = "taos_free_result", CallingConvention = CallingConvention.Cdecl)] + static extern public IntPtr FreeResult(IntPtr res); - [DllImport("taos.dll", EntryPoint = "taos_close", CallingConvention = CallingConvention.StdCall)] - static extern public int Close(long taos); + [DllImport("taos.dll", EntryPoint = "taos_close", CallingConvention = CallingConvention.Cdecl)] + static extern public int Close(IntPtr taos); } } \ No newline at end of file diff --git a/tests/examples/C#/TDengineTest.cs b/tests/examples/C#/TDengineTest.cs index 235775f68f..6b3f1160ad 100644 --- a/tests/examples/C#/TDengineTest.cs +++ b/tests/examples/C#/TDengineTest.cs @@ -28,7 +28,7 @@ namespace TDengineDriver private string configDir; private string user; private string password; - private int port = 0; + private short port = 0; //sql parameters private string dbName; @@ -43,7 +43,7 @@ namespace TDengineDriver private long batchRows; private long beginTimestamp = 1551369600000L; - private long conn = 0; + private IntPtr conn = IntPtr.Zero; private long rowsInserted = 0; static void Main(string[] args) @@ -191,7 +191,7 @@ namespace TDengineDriver { string db = ""; this.conn = TDengine.Connect(this.host, this.user, this.password, db, this.port); - if (this.conn == 0) + if (this.conn == IntPtr.Zero) { Console.WriteLine("Connect to TDengine failed"); ExitProgram(); @@ -211,58 +211,62 @@ namespace TDengineDriver StringBuilder sql = new StringBuilder(); sql.Append("create database if not exists ").Append(this.dbName); - int code = TDengine.Query(this.conn, sql.ToString()); - if (code == TDengine.TSDB_CODE_SUCCESS) + IntPtr res = TDengine.Query(this.conn, sql.ToString()); + if (res != IntPtr.Zero) { Console.WriteLine(sql.ToString() + " success"); } else { - Console.WriteLine(sql.ToString() + " failure, reason: " + TDengine.Error(conn)); + Console.WriteLine(sql.ToString() + " failure, reason: " + TDengine.Error(res)); ExitProgram(); } + TDengine.FreeResult(res); sql.Clear(); sql.Append("use ").Append(this.dbName); - code = TDengine.Query(this.conn, sql.ToString()); - if (code == TDengine.TSDB_CODE_SUCCESS) + res = TDengine.Query(this.conn, sql.ToString()); + if (res != IntPtr.Zero) { Console.WriteLine(sql.ToString() + " success"); } else { - Console.WriteLine(sql.ToString() + " failure, reason: " + TDengine.Error(this.conn)); + Console.WriteLine(sql.ToString() + " failure, reason: " + TDengine.Error(res)); ExitProgram(); } + TDengine.FreeResult(res); sql.Clear(); - sql.Append("create table if not exists ").Append(this.stableName).Append("(ts timestamp, v1 int) tags(t1 int)"); - code = TDengine.Query(this.conn, sql.ToString()); - if (code == TDengine.TSDB_CODE_SUCCESS) + sql.Append("create table if not exists ").Append(this.stableName).Append("(ts timestamp, v1 bool, v2 tinyint, v3 smallint, v4 int, v5 bigint, v6 float, v7 double, v8 binary(10), v9 nchar(10)) tags(t1 int)"); + res = TDengine.Query(this.conn, sql.ToString()); + if (res != IntPtr.Zero) { Console.WriteLine(sql.ToString() + " success"); } else { - Console.WriteLine(sql.ToString() + " failure, reason: " + TDengine.Error(this.conn)); + Console.WriteLine(sql.ToString() + " failure, reason: " + TDengine.Error(res)); ExitProgram(); } + TDengine.FreeResult(res); for (int i = 0; i < this.tableCount; i++) { sql.Clear(); sql = sql.Append("create table if not exists ").Append(this.tablePrefix).Append(i) .Append(" using ").Append(this.stableName).Append(" tags(").Append(i).Append(")"); - code = TDengine.Query(this.conn, sql.ToString()); - if (code == TDengine.TSDB_CODE_SUCCESS) + res = TDengine.Query(this.conn, sql.ToString()); + if (res != IntPtr.Zero) { Console.WriteLine(sql.ToString() + " success"); } else { - Console.WriteLine(sql.ToString() + " failure, reason: " + TDengine.Error(this.conn)); + Console.WriteLine(sql.ToString() + " failure, reason: " + TDengine.Error(res)); ExitProgram(); } + TDengine.FreeResult(res); } Console.WriteLine("create db and table success"); @@ -287,16 +291,22 @@ namespace TDengineDriver for (int batch = 0; batch < this.batchRows; ++batch) { long rows = loop * this.batchRows + batch; - sql.Append("(").Append(this.beginTimestamp + rows).Append(",").Append(rows).Append(")"); + sql.Append("(") + .Append(this.beginTimestamp + rows) + .Append(", 1, 2, 3,") + .Append(rows) + .Append(", 5, 6, 7, 'abc', 'def')"); } - int code = TDengine.Query(conn, sql.ToString()); - if (code != TDengine.TSDB_CODE_SUCCESS) + IntPtr res = TDengine.Query(this.conn, sql.ToString()); + if (res == IntPtr.Zero) { - Console.WriteLine(sql.ToString() + " failure, reason: " + TDengine.Error(conn)); + Console.WriteLine(sql.ToString() + " failure, reason: " + TDengine.Error(res)); } - int affectRows = TDengine.AffectRows(conn); + int affectRows = TDengine.AffectRows(res); this.rowsInserted += affectRows; + + TDengine.FreeResult(res); } } @@ -316,51 +326,45 @@ namespace TDengineDriver System.DateTime start = new System.DateTime(); long queryRows = 0; - - for (int i = 0; i < this.tableCount; ++i) + + for (int i = 0; i < 1/*this.tableCount*/; ++i) { String sql = "select * from " + this.dbName + "." + tablePrefix + i; Console.WriteLine(sql); - int code = TDengine.Query(conn, sql); - if (code != TDengine.TSDB_CODE_SUCCESS) + IntPtr res = TDengine.Query(conn, sql); + if (res == IntPtr.Zero) { - Console.WriteLine(sql + " failure, reason: " + TDengine.Error(conn)); + Console.WriteLine(sql + " failure, reason: " + TDengine.Error(res)); ExitProgram(); } - int fieldCount = TDengine.FieldCount(conn); - //Console.WriteLine("field count: " + fieldCount); + int fieldCount = TDengine.FieldCount(res); + Console.WriteLine("field count: " + fieldCount); - List metas = TDengine.FetchFields(conn); + List metas = TDengine.FetchFields(res); for (int j = 0; j < metas.Count; j++) { TDengineMeta meta = (TDengineMeta)metas[j]; - //Console.WriteLine("index:" + j + ", type:" + meta.type + ", typename:" + meta.TypeName() + ", name:" + meta.name + ", size:" + meta.size); - } - - long result = TDengine.UseResult(conn); - if (result == 0) - { - Console.WriteLine(sql + " result set is null"); - return; + Console.WriteLine("index:" + j + ", type:" + meta.type + ", typename:" + meta.TypeName() + ", name:" + meta.name + ", size:" + meta.size); } IntPtr rowdata; - while ((rowdata = TDengine.FetchRows(result)) != IntPtr.Zero) + StringBuilder builder = new StringBuilder(); + while ((rowdata = TDengine.FetchRows(res)) != IntPtr.Zero) { queryRows++; for (int fields = 0; fields < fieldCount; ++fields) { TDengineMeta meta = metas[fields]; - int offset = 8 * fields; + int offset = IntPtr.Size * fields; IntPtr data = Marshal.ReadIntPtr(rowdata, offset); - //Console.Write("---"); + builder.Append("---"); if (data == IntPtr.Zero) { - //Console.Write("NULL"); + builder.Append("NULL"); continue; } @@ -368,55 +372,61 @@ namespace TDengineDriver { case TDengineDataType.TSDB_DATA_TYPE_BOOL: bool v1 = Marshal.ReadByte(data) == 0 ? false : true; - //Console.Write(v1); + builder.Append(v1); break; case TDengineDataType.TSDB_DATA_TYPE_TINYINT: byte v2 = Marshal.ReadByte(data); - //Console.Write(v2); + builder.Append(v2); break; case TDengineDataType.TSDB_DATA_TYPE_SMALLINT: short v3 = Marshal.ReadInt16(data); - //Console.Write(v3); + builder.Append(v3); break; case TDengineDataType.TSDB_DATA_TYPE_INT: int v4 = Marshal.ReadInt32(data); - //Console.Write(v4); + builder.Append(v4); break; case TDengineDataType.TSDB_DATA_TYPE_BIGINT: long v5 = Marshal.ReadInt64(data); - //Console.Write(v5); + builder.Append(v5); break; case TDengineDataType.TSDB_DATA_TYPE_FLOAT: float v6 = (float)Marshal.PtrToStructure(data, typeof(float)); - //Console.Write(v6); + builder.Append(v6); break; case TDengineDataType.TSDB_DATA_TYPE_DOUBLE: double v7 = (double)Marshal.PtrToStructure(data, typeof(double)); - //Console.Write(v7); + builder.Append(v7); break; case TDengineDataType.TSDB_DATA_TYPE_BINARY: string v8 = Marshal.PtrToStringAnsi(data); - //Console.Write(v8); + builder.Append(v8); break; case TDengineDataType.TSDB_DATA_TYPE_TIMESTAMP: long v9 = Marshal.ReadInt64(data); - //Console.Write(v9); + builder.Append(v9); break; case TDengineDataType.TSDB_DATA_TYPE_NCHAR: string v10 = Marshal.PtrToStringAnsi(data); - //Console.Write(v10); + builder.Append(v10); break; } } - //Console.WriteLine("---"); + builder.Append("---"); + + if (queryRows <= 10) + { + Console.WriteLine(builder.ToString()); + } + builder.Clear(); } - if (TDengine.ErrorNo(conn) != 0) + if (TDengine.ErrorNo(res) != 0) { - Console.Write("Query is not complete， Error {0:G}", TDengine.ErrorNo(conn), TDengine.Error(conn)); + Console.Write("Query is not complete, Error {0:G}", TDengine.ErrorNo(res), TDengine.Error(res)); } - TDengine.FreeResult(result); + TDengine.FreeResult(res); } System.DateTime end = new System.DateTime(); @@ -428,14 +438,15 @@ namespace TDengineDriver public void CloseConnection() { - if (conn != 0) + if (this.conn != IntPtr.Zero) { - TDengine.Close(conn); + TDengine.Close(this.conn); } } static void ExitProgram() { + TDengine.Cleanup(); System.Environment.Exit(0); } } diff --git a/tests/perftest-scripts/tdengineTestWriteLoop.sh b/tests/perftest-scripts/tdengineTestWriteLoop.sh index e1ee4418dc..cb9d87d0c1 100755 --- a/tests/perftest-scripts/tdengineTestWriteLoop.sh +++ b/tests/perftest-scripts/tdengineTestWriteLoop.sh @@ -24,22 +24,46 @@ function runTest { for r in ${!rowsPerRequest[@]}; do for c in `seq 1 $clients`; do totalRPR=0 - OUTPUT_FILE=tdengineTestWrite-RPR${rowsPerRequest[$r]}-clients$c.out + if $v16 ; then + OUTPUT_FILE=tdengineTestWrite-v16-RPR${rowsPerRequest[$r]}-clients$c.out + else + OUTPUT_FILE=tdengineTestWrite-v20-RPR${rowsPerRequest[$r]}-clients$c.out + fi for i in `seq 1 $NUM_LOOP`; do - restartTaosd - $TAOSD_DIR/taos -s "drop database db" > /dev/null 2>&1 - printTo "loop i:$i, $TDTEST_DIR/tdengineTest \ + if ! $printresultonly ; then + restartTaosd + $TAOSD_DIR/taos -s "drop database db" > /dev/null 2>&1 + + if $v16 ; then + printTo "loop i:$i, $TDTEST_DIR/tdengineTest \ -dataDir $DATA_DIR \ -numOfFiles $NUM_OF_FILES \ - -w -clients $c \ - -rowsPerRequest ${rowsPerRequest[$r]}" - $TDTEST_DIR/tdengineTest \ - -dataDir $DATA_DIR \ - -numOfFiles $NUM_OF_FILES \ - -w -clients $c \ - -rowsPerRequest ${rowsPerRequest[$r]} \ - | tee $OUTPUT_FILE + -writeClients $c \ + -rowsPerRequest ${rowsPerRequest[$r]} \ + | tee $OUTPUT_FILE" + $TDTEST_DIR/tdengineTest \ + -dataDir $DATA_DIR \ + -numOfFiles $NUM_OF_FILES \ + -writeClients $c \ + -rowsPerRequest ${rowsPerRequest[$r]} \ + | tee $OUTPUT_FILE + else + printTo "loop i:$i, $TDTEST_DIR/tdengineTest \ + -dataDir $DATA_DIR \ + -numOfFiles $NUM_OF_FILES \ + -w -clients $c \ + -rowsPerRequest ${rowsPerRequest[$r]} \ + | tee $OUTPUT_FILE" + $TDTEST_DIR/tdengineTest \ + -dataDir $DATA_DIR \ + -numOfFiles $NUM_OF_FILES \ + -w -clients $c \ + -rowsPerRequest ${rowsPerRequest[$r]} \ + | tee $OUTPUT_FILE + fi + fi + RPR=`cat $OUTPUT_FILE | grep speed | awk '{print $(NF-1)}'` totalRPR=`echo "scale=4; $totalRPR + $RPR" | bc` printTo "rows:${rowsPerRequest[$r]}, clients:$c, i:$i RPR:$RPR" @@ -86,25 +110,30 @@ function restartTaosd { ################ Main ################ -master=false -develop=true +v16=false +v20=true verbose=false clients=1 +printresultonly=false while : ; do case $1 in + printresultonly) + printresultonly=true + shift ;; + -v) verbose=true shift ;; - master) - master=true - develop=false + v16) + v16=true + v20=false shift ;; - develop) - master=false - develop=true + v20) + v16=false + v20=true shift ;; -c) @@ -120,19 +149,24 @@ while : ; do esac done -if $master ; then - echo "Test master branch.." - cp /mnt/root/cfg/master/taos.cfg /etc/taos/taos.cfg - WORK_DIR=/mnt/root/TDengine.master +if $v16 ; then + echo "Test v16 branch.." + WORK_DIR=/mnt/root/TDengine.v16 + cp /mnt/root/cfg/v16/taos.cfg /etc/taos/taos.cfg else - echo "Test develop branch.." - cp /mnt/root/cfg/develop/taos.cfg /etc/taos/taos.cfg + echo "Test v20 branch.." + cp /mnt/root/cfg/v20/taos.cfg /etc/taos/taos.cfg WORK_DIR=/mnt/root/TDengine fi TAOSD_DIR=$WORK_DIR/debug/build/bin TDTEST_DIR=$WORK_DIR/tests/comparisonTest/tdengine +if [ ! -f $TDTEST_DIR/tdengineTest ]; then + echo "Please build tdengineTest first!" + exit 1 +fi + runTest echo "Test done!" diff --git a/tests/pytest/crash_gen.py b/tests/pytest/crash_gen.py index 8edba3903d..7d3eb959c0 100755 --- a/tests/pytest/crash_gen.py +++ b/tests/pytest/crash_gen.py @@ -1492,6 +1492,7 @@ class Task(): 0x386, # DB is being dropped?! 0x503, 0x510, # vnode not in ready state + 0x14, # db not ready, errno changed 0x600, 1000 # REST catch-all error ]: diff --git a/tests/pytest/insert/nchar.py b/tests/pytest/insert/nchar.py index b8e365f143..3319aa3c56 100644 --- a/tests/pytest/insert/nchar.py +++ b/tests/pytest/insert/nchar.py @@ -35,6 +35,8 @@ class TDTestCase: tdSql.checkRows(2) tdSql.checkData(1, 1, '涛思数据') + tdSql.error("insert into tb values (now, 'taosdata001')") + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/tag_lite/bigint.py b/tests/pytest/tag_lite/bigint.py index ecc6fad44d..f83961aaad 100644 --- a/tests/pytest/tag_lite/bigint.py +++ b/tests/pytest/tag_lite/bigint.py @@ -575,6 +575,20 @@ class TDTestCase: # TSIM: system sh/exec.sh -n dnode1 -s stop -x SIGINT # convert end + tdSql.execute("create database db") + tdSql.execute("use db") + tdSql.execute( + "create table if not exists st (ts timestamp, tagtype int) tags(dev bigint)") + + tdSql.execute( + 'CREATE TABLE if not exists dev_001 using st tags(%d)' % (pow(2, 63) - 1)) + tdSql.execute( + 'CREATE TABLE if not exists dev_002 using st tags(%d)' % (-1 * pow(2, 63) + 1)) + + print("==============step2") + tdSql.query("show tables") + tdSql.checkRows(2) + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/tag_lite/binary.py b/tests/pytest/tag_lite/binary.py index a5757dc5cd..4cbae63bec 100644 --- a/tests/pytest/tag_lite/binary.py +++ b/tests/pytest/tag_lite/binary.py @@ -579,6 +579,20 @@ class TDTestCase: # TSIM: system sh/exec.sh -n dnode1 -s stop -x SIGINT # convert end + tdSql.execute("create database db") + tdSql.execute("use db") + tdSql.execute( + "create table if not exists st (ts timestamp, tagtype int) tags(dev binary(5))") + tdSql.error( + 'CREATE TABLE if not exists dev_001 using st tags("dev_001")') + + tdSql.execute( + 'CREATE TABLE if not exists dev_002 using st tags("dev")') + + print("==============step2") + tdSql.query("show tables") + tdSql.checkRows(1) + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/tag_lite/float.py b/tests/pytest/tag_lite/float.py index 7b93cb0ac9..9c4d20e82b 100644 --- a/tests/pytest/tag_lite/float.py +++ b/tests/pytest/tag_lite/float.py @@ -575,6 +575,23 @@ class TDTestCase: # TSIM: system sh/exec.sh -n dnode1 -s stop -x SIGINT # convert end + tdSql.execute("create database db") + tdSql.execute("use db") + tdSql.execute( + "create table if not exists st (ts timestamp, tagtype int) tags(dev float)") + tdSql.error( + 'CREATE TABLE if not exists dev_001 using st tags(%f)' % -3.4E38 - 1) + tdSql.error( + 'CREATE TABLE if not exists dev_001 using st tags(%f)' % 3.4E38 + 1) + + tdSql.execute( + 'CREATE TABLE if not exists dev_001 using st tags(%f)' % 3.4E38) + tdSql.execute( + 'CREATE TABLE if not exists dev_002 using st tags(%f)' % -3.4E38) + + tdSql.query("show tables") + tdSql.checkRows(2) + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/tag_lite/int.py b/tests/pytest/tag_lite/int.py index 99d4a69624..d5a6917389 100644 --- a/tests/pytest/tag_lite/int.py +++ b/tests/pytest/tag_lite/int.py @@ -574,6 +574,24 @@ class TDTestCase: # TSIM: system sh/exec.sh -n dnode1 -s stop -x SIGINT # convert end + tdSql.execute("create database db") + tdSql.execute("use db") + tdSql.execute( + "create table if not exists st (ts timestamp, tagtype int) tags(dev int)") + tdSql.error( + 'CREATE TABLE if not exists dev_001 using st tags(%d)' % pow(2, 31)) + tdSql.error( + 'CREATE TABLE if not exists dev_001 using st tags(%d)' % (-1 * pow(2, 31))) + + tdSql.execute( + 'CREATE TABLE if not exists dev_001 using st tags(%d)' % (pow(2, 31) - 1)) + tdSql.execute( + 'CREATE TABLE if not exists dev_002 using st tags(%d)' % (-1 * pow(2, 31) + 1)) + + print("==============step2") + tdSql.query("show tables") + tdSql.checkRows(2) + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/tag_lite/nchar.py b/tests/pytest/tag_lite/nchar.py new file mode 100644 index 0000000000..851cc32b56 --- /dev/null +++ b/tests/pytest/tag_lite/nchar.py @@ -0,0 +1,48 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + + print("==============step1") + tdSql.execute( + "create table if not exists st (ts timestamp, tagtype int) tags(dev nchar(5))") + tdSql.error( + 'CREATE TABLE if not exists dev_001 using st tags("dev_001")') + tdSql.execute( + 'CREATE TABLE if not exists dev_002 using st tags("dev")') + + + print("==============step2") + tdSql.query("show tables") + tdSql.checkRows(1) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/tag_lite/smallint.py b/tests/pytest/tag_lite/smallint.py index 089af55a34..c060e3f82b 100644 --- a/tests/pytest/tag_lite/smallint.py +++ b/tests/pytest/tag_lite/smallint.py @@ -574,6 +574,23 @@ class TDTestCase: # TSIM: # TSIM: system sh/exec.sh -n dnode1 -s stop -x SIGINT # convert end + tdSql.execute("create database db") + tdSql.execute("use db") + tdSql.execute( + "create table if not exists st (ts timestamp, tagtype int) tags(dev smallint)") + tdSql.error( + 'CREATE TABLE if not exists dev_001 using st tags(%d)' % pow(2, 15)) + tdSql.error( + 'CREATE TABLE if not exists dev_001 using st tags(%d)' % (-1 * pow(2, 15))) + + tdSql.execute( + 'CREATE TABLE if not exists dev_001 using st tags(%d)' % (pow(2, 15) - 1)) + tdSql.execute( + 'CREATE TABLE if not exists dev_002 using st tags(%d)' % (-1 * pow(2, 15) + 1)) + + print("==============step2") + tdSql.query("show tables") + tdSql.checkRows(2) def stop(self): tdSql.close() diff --git a/tests/pytest/tag_lite/tinyint.py b/tests/pytest/tag_lite/tinyint.py index 55e33d013b..089dd46569 100644 --- a/tests/pytest/tag_lite/tinyint.py +++ b/tests/pytest/tag_lite/tinyint.py @@ -575,6 +575,24 @@ class TDTestCase: # TSIM: system sh/exec.sh -n dnode1 -s stop -x SIGINT # convert end + tdSql.execute("create database db") + tdSql.execute("use db") + tdSql.execute( + "create table if not exists st (ts timestamp, tagtype int) tags(dev tinyint)") + tdSql.error( + 'CREATE TABLE if not exists dev_001 using st tags(%d)' % pow(2, 7)) + tdSql.error( + 'CREATE TABLE if not exists dev_001 using st tags(%d)' % (-1 * pow(2, 7))) + + tdSql.execute( + 'CREATE TABLE if not exists dev_001 using st tags(%d)' % (pow(2, 7) - 1)) + tdSql.execute( + 'CREATE TABLE if not exists dev_002 using st tags(%d)' % (-1 * pow(2, 7) + 1)) + + print("==============step2") + tdSql.query("show tables") + tdSql.checkRows(2) + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/script/general/parser/bug.sim b/tests/script/general/parser/bug.sim deleted file mode 100644 index 2a46ad1fd6..0000000000 --- a/tests/script/general/parser/bug.sim +++ /dev/null @@ -1,43 +0,0 @@ -system sh/stop_dnodes.sh - -system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 - -system sh/cfg.sh -n dnode1 -c dDebugFlag -v 135 -system sh/cfg.sh -n dnode1 -c mDebugFlag -v 135 -system sh/cfg.sh -n dnode1 -c sdbDebugFlag -v 135 -system sh/cfg.sh -n dnode1 -c rpcDebugFlag -v 135 -system sh/cfg.sh -n dnode1 -c cDebugFlag -v 135 -system sh/cfg.sh -n dnode1 -c monitorDebugflag -v 135 -system sh/cfg.sh -n dnode1 -c httpDebugFlag -v 135 -system sh/cfg.sh -n dnode1 -c uDebugFlag -v 135 - -system sh/cfg.sh -n dnode1 -c httpCacheSessions -v 10 -system sh/cfg.sh -n dnode1 -c httpMaxThreads -v 10 -system sh/cfg.sh -n dnode1 -c httpEnableCompress -v 0 - -system sh/cfg.sh -n dnode1 -c maxVnodeConnections -v 30000 -system sh/cfg.sh -n dnode1 -c maxMgmtConnections -v 30000 -system sh/cfg.sh -n dnode1 -c maxMeterConnections -v 30000 -system sh/cfg.sh -n dnode1 -c maxShellConns -v 30000 - -system sh/exec.sh -n dnode1 -s start -sql connect - -sql create database db; -sql use db; -sql create table t (ts timestamp, i int); -sql create table st1 (ts timestamp, f1 int) tags(t1 int); -sql create table st2 (ts timestamp, f2 int) tags(t2 int); - -sql create table t1 using st1 tags(1); -sql create table t2 using st2 tags(1); - -sql insert into t1 values(1575880055000, 1); -sql insert into t2 values(1575880055000, 2); - -sql select st1.ts, st1.f1, st2.f2 from db.st1, db.st2 where st1.t1=st2.t2 and st1.ts=st2.ts - -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'select st1.ts, st1.f1, st2.f2 from db.st1, db.st2 where st1.t1=st2.t2 and st1.ts=st2.ts' 127.0.0.1:7111/restful/sql - -system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/general/parser/constCol.sim b/tests/script/general/parser/constCol.sim new file mode 100644 index 0000000000..a196ba2b50 --- /dev/null +++ b/tests/script/general/parser/constCol.sim @@ -0,0 +1,354 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c walLevel -v 0 + +system sh/cfg.sh -n dnode1 -c dDebugFlag -v 135 +system sh/cfg.sh -n dnode1 -c mDebugFlag -v 135 +system sh/cfg.sh -n dnode1 -c sdbDebugFlag -v 135 +system sh/cfg.sh -n dnode1 -c rpcDebugFlag -v 135 +system sh/cfg.sh -n dnode1 -c cDebugFlag -v 135 +system sh/cfg.sh -n dnode1 -c monitorDebugflag -v 135 +system sh/cfg.sh -n dnode1 -c httpDebugFlag -v 135 +system sh/cfg.sh -n dnode1 -c uDebugFlag -v 135 + +system sh/cfg.sh -n dnode1 -c httpCacheSessions -v 10 +system sh/cfg.sh -n dnode1 -c httpMaxThreads -v 10 +system sh/cfg.sh -n dnode1 -c httpEnableCompress -v 0 + +system sh/cfg.sh -n dnode1 -c maxVnodeConnections -v 30000 +system sh/cfg.sh -n dnode1 -c maxMgmtConnections -v 30000 +system sh/cfg.sh -n dnode1 -c maxMeterConnections -v 30000 +system sh/cfg.sh -n dnode1 -c maxShellConns -v 30000 + +system sh/exec.sh -n dnode1 -s start +sql connect + +sql create database db; +sql use db; +sql create table t (ts timestamp, i int); +sql create table st1 (ts timestamp, f1 int) tags(t1 int); +sql create table st2 (ts timestamp, f2 int) tags(t2 int); + +sql create table t1 using st1 tags(1); +sql create table t2 using st2 tags(1); + +sql insert into t1 values(1575880055000, 1); +sql insert into t1 values(1575880059000, 1); +sql insert into t1 values(1575880069000, 1); + +sql insert into t2 values(1575880055000, 2); + +sql select st1.ts, st1.f1, st2.f2 from db.st1, db.st2 where st1.t1=st2.t2 and st1.ts=st2.ts + +system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'select st1.ts, st1.f1, st2.f2 from db.st1, db.st2 where st1.t1=st2.t2 and st1.ts=st2.ts' 127.0.0.1:7111/restful/sql + +print ==============select with user-defined columns +sql select 'abc' as f, ts,f1 from t1 +if $rows != 3 then + return -1 +endi + +if $data00 != @abc@ then + return -1 +endi + +if $data01 != @19-12-09 16:27:35.000@ then + return -1 +endi + +if $data02 != 1 then + return -1 +endi + +sql select 'abc', ts, f1 from t1 +if $rows != 3 then + return -1 +endi + +if $data01 != @19-12-09 16:27:35.000@ then + return -1 +endi + +if $data02 != 1 then + return -1 +endi + +if $data10 != @abc@ then + return -1 +endi + +sql select 'abc' from t1 +if $rows != 3 then + return -1 +endi + +if $data00 != @abc@ then + return -1 +endi + +sql select 'abc' as f1 from t1 +if $rows != 3 then + return -1 +endi + +if $data00 != @abc@ then + return -1 +endi + +sql select 1 from t1 +if $rows != 3 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +if $data10 != 1 then + return -1 +endi + +if $data20 != 1 then + return -1 +endi + +sql select 1 as f1 from t1 +if $rows != 3 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +sql select 1 as f, f1 from t1 +if $rows != 3 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +if $data01 != 1 then + return -1 +endi + +sql select 1.123 as f, f1 from t1 +if $rows != 3 then + return -1 +endi + +if $data00 != 1.123000000 then + print expect 1.123000000 , actual:$data00 + return -1 +endi + +if $data10 != 1.123000000 then + print expect 1.123000000 , actual:$data10 + return -1 +endi + +sql select 1, f1 from t1 +if $rows != 3 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +if $data01 != 1 then + return -1 +endi + +if $data10 != 1 then + return -1 +endi + +sql select 1.2391, f1 from t1 +if $rows != 3 then + return -1 +endi + +if $data00 != 1.239100000 then + print expect 1.239100000 actual: $data00 + return -1 +endi + +if $data10 != 1.239100000 then + print expect 1.239100000 actual: $data00 + return -1 +endi + +if $data01 != 1 then + return -1 +endi + +print ===================== user-defined columns with agg functions +sql select 1 as t, count(*) from t1 +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +if $data01 != 3 then + return -1 +endi + +sql select 1, sum(f1) from t1 +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +if $data01 != 3 then + return -1 +endi + +sql select 1,2,3, sum(f1)*99, 4,5,6,7,8,9,10 from t1 +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +if $data01 != 2 then + return -1 +endi + +if $data02 != 3 then + return -1 +endi + +if $data03 != 297.000000000 then + print expect 297.000000000, actual:$data03 + return -1 +endi + +sql select sum(f1)*avg(f1)+12, 1,2,3,4,5,6,7,8,9,10 from t1 +if $rows != 1 then + return -1 +endi + +if $data00 != 15.000000000 then + print expect 15.000000000 actual:$data00 + return -1 +endi + +if $data01 != 1 then + return -1 +endi + +if $data02 != 2 then + return -1 +endi + +sql select 1.2987, f1, 'k' from t1 where f1=1 +if $rows != 3 then + return -1 +endi + +if $data00 != 1.298700000 then + print expect 1.298700000 actual:$data00 + return -1 +endi + +if $data01 != 1 then + return -1 +endi + +if $data02 != @k@ then + return -1 +endi + +print ====================user-defined columns with union +sql select f1, 'f1' from t1 union all select f1, 'f1' from t1; +if $rows != 6 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +if $data01 != @f1@ then + return -1 +endi + +if $data10 != 1 then + return -1 +endi + +if $data11 != @f1@ then + return -1 +endi + +print =====================udc with join +sql select st1.ts, st1.f1, st2.f2, 'abc', 1.9827 from db.st1, db.st2 where st1.t1=st2.t2 and st1.ts=st2.ts +if $rows != 1 then + return -1 +endi + +if $data00 != @19-12-09 16:27:35.000@ then + print expect @19-12-09 16:27:35.000@ actual:$data00 + return -1 +endi + +if $data01 != 1 then + return -1 +endi + +if $data02 != 2 then + return -1 +endi + +if $data03 != @abc@ then + return -1 +endi + +if $data04 != 1.982700000 then + print expect 1.982700000 actual:$data04 + return -1 +endi + +print ======================udc with interval +sql select count(*), 'uuu' from t1 interval(1s) order by ts desc; +if $rows != 3 then + return -1 +endi + +print ======================udc with tags +sql select t1,'abc',tbname from st1 +if $rows != 1 then + return -1 +endi + +if $data01 != @abc@ then + return -1 +endi + +if $data02 != @t1@ then + return -1 +endi + +print ======================udc with arithmetic +sql select 1+1 from t1 +if $rows != 3 then + return -1 +endi + +sql_error select from t1 +sql_error select abc from t1 +sql_error select abc as tu from t1 + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/general/parser/fill_stb.sim b/tests/script/general/parser/fill_stb.sim index 402b79d6b5..f0cd058352 100644 --- a/tests/script/general/parser/fill_stb.sim +++ b/tests/script/general/parser/fill_stb.sim @@ -133,12 +133,18 @@ endi if $data74 != -4.000000000 then return -1 endi + ## fill(value) + group by sql select max(c1), max(c2), max(c3), max(c4), max(c5) from $stb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, -1, -2, -3, -4, -5, -6, -7, -8) group by t1 $val = $rowNum * 2 +print $rowNum, $val + $val = $val - 1 $val = $val * $tbNum -if $rows != $val then +print ==================== $val + +if $rows != 190 then + print expect 190, actual:$rows return -1 endi if $data06 != 0 then diff --git a/tests/script/general/parser/join.sim b/tests/script/general/parser/join.sim index 882f561ae1..1bb0ff5448 100644 --- a/tests/script/general/parser/join.sim +++ b/tests/script/general/parser/join.sim @@ -330,7 +330,10 @@ sql_error select join_tb1.* from $tb1 , $tb2 where join_tb1.ts != join_tb0.ts an sql_error select join_tb1.* from $tb1 , $tb1 where join_tb1.ts = join_tb1.ts and join_tb1.ts >= 100000; sql_error select join_tb1.* from $tb1 , $tb1 where join_tb1.ts = join_tb1.ts order by ts; sql_error select join_tb1.* from $tb1 , $tb1 where join_tb1.ts = join_tb1.ts order by join_tb1.c7; - +sql_error select * from join_tb0, join_tb1 +sql_error select last_row(*) from join_tb0, join_tb1 +sql_error select last_row(*) from $tb1, $tb2 where join_tb1.ts < now +sql_error select last_row(*) from $tb1, $tb2 where join_tb1.ts = join_tb2.ts print ==================================super table join ============================== # select duplicate columns diff --git a/tests/script/general/parser/lastrow_query.sim b/tests/script/general/parser/lastrow_query.sim index 98eb5a8d6d..7954a8d228 100644 --- a/tests/script/general/parser/lastrow_query.sim +++ b/tests/script/general/parser/lastrow_query.sim @@ -128,3 +128,27 @@ if $rows != 86399 then return -1 endi +sql select t1,t1,count(*),t1,t1 from lr_stb0 where ts>'2018-09-24 00:00:00.000' and ts<'2018-09-25 00:00:00.000' interval(1h) fill(NULL) group by t1 order by ts DESC limit 30 +if $rows != 48 then + return -1 +endi + +sql select t1,t1,count(*),t1,t1 from lr_stb0 where ts>'2018-09-24 00:00:00.000' and ts<'2018-09-25 00:00:00.000' interval(1h) fill(NULL) group by t1 order by ts DESC limit 2 +if $rows != 4 then + return -1 +endi + +sql select t1,t1,count(*),tbname,t1,t1,tbname from lr_stb0 where ts>'2018-09-24 00:00:00.000' and ts<'2018-09-25 00:00:00.000' interval(1s) fill(NULL) group by tbname, t1 order by ts desc slimit 1 soffset 1 limit 250000 offset 1 +if $rows != 86399 then + return -1 +endi + +sql select t1,t1,count(*),t1,t1 from lr_stb0 where ts>'2018-09-24 00:00:00.000' and ts<'2018-09-25 00:00:00.000' interval(1h) fill(NULL) group by t1 order by ts desc limit 1 +if $rows != 2 then + return -1 +endi + +sql select t1,t1,count(*),t1,t1 from lr_stb0 where ts>'2018-09-24 00:00:00.000' and ts<'2018-09-25 00:00:00.000' interval(1h) fill(NULL) group by t1 order by ts desc limit 25 offset 1 +if $rows != 46 then + return -1 +endi diff --git a/tests/script/general/parser/nchar.sim b/tests/script/general/parser/nchar.sim index 8d4e468f0e..bdac5ace55 100644 --- a/tests/script/general/parser/nchar.sim +++ b/tests/script/general/parser/nchar.sim @@ -223,6 +223,7 @@ if $rows != 2 then endi print data00 = $data00 if $data00 != 5 then + print expect 5, actual: $data00 return -1 endi if $data10 != 5 then diff --git a/tests/script/general/parser/testSuite.sim b/tests/script/general/parser/testSuite.sim index aafba2d328..4e26d14cfd 100644 --- a/tests/script/general/parser/testSuite.sim +++ b/tests/script/general/parser/testSuite.sim @@ -97,7 +97,7 @@ run general/parser/topbot.sim sleep 2000 run general/parser/union.sim sleep 2000 -run general/parser/bug.sim +run general/parser/constCol.sim sleep 2000 run general/parser/sliding.sim diff --git a/tests/script/general/table/delete_writing.sim b/tests/script/general/table/delete_writing.sim index 868936dda7..b55d55eb8a 100644 --- a/tests/script/general/table/delete_writing.sim +++ b/tests/script/general/table/delete_writing.sim @@ -35,7 +35,7 @@ sleep 1000 print ======== step1 $x = 1 -while $x < 20 +while $x < 15 print drop table times $x sql drop table db.tb -x step1 diff --git a/tests/script/general/user/monitor.sim b/tests/script/general/user/monitor.sim index 208b85f16c..eb543612f5 100644 --- a/tests/script/general/user/monitor.sim +++ b/tests/script/general/user/monitor.sim @@ -11,9 +11,9 @@ sleep 3000 sql connect print ========== step2 -sql drop database log -x step21 - return -1 -step21: +#sql drop database log -x step21 +# return -1 +#step21: sql drop table log.dn -x step22 return -1 step22: diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index a48584b0ed..40598332d9 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -147,7 +147,6 @@ cd ../../../debug; make ./test.sh -f general/parser/join.sim ./test.sh -f general/parser/join_multivnode.sim ./test.sh -f general/parser/binary_escapeCharacter.sim -./test.sh -f general/parser/bug.sim ./test.sh -f general/parser/repeatAlter.sim ./test.sh -f general/parser/union.sim ./test.sh -f general/parser/topbot.sim @@ -294,6 +293,7 @@ cd ../../../debug; make ./test.sh -f unique/stable/replica3_dnode6.sim ./test.sh -f unique/stable/replica3_vnode3.sim +./test.sh -f unique/mnode/mgmt21.sim ./test.sh -f unique/mnode/mgmt22.sim ./test.sh -f unique/mnode/mgmt23.sim ./test.sh -f unique/mnode/mgmt24.sim diff --git a/tests/script/jenkins/unique.txt b/tests/script/jenkins/unique.txt index 06edb8890a..b271f5b726 100644 --- a/tests/script/jenkins/unique.txt +++ b/tests/script/jenkins/unique.txt @@ -1,22 +1,11 @@ cd ../../../debug; cmake .. cd ../../../debug; make -./test.sh -f unique/account/account_create.sim -./test.sh -f unique/account/account_delete.sim -./test.sh -f unique/account/account_len.sim -./test.sh -f unique/account/authority.sim -./test.sh -f unique/account/basic.sim -./test.sh -f unique/account/paras.sim -./test.sh -f unique/account/pass_alter.sim -./test.sh -f unique/account/pass_len.sim -./test.sh -f unique/account/usage.sim -./test.sh -f unique/account/user_create.sim -./test.sh -f unique/account/user_len.sim - ./test.sh -f unique/big/balance.sim ./test.sh -f unique/big/maxvnodes.sim ./test.sh -f unique/big/tcp.sim +./test.sh -f unique/cluster/alter.sim ./test.sh -f unique/cluster/balance1.sim ./test.sh -f unique/cluster/balance2.sim ./test.sh -f unique/cluster/balance3.sim @@ -61,6 +50,7 @@ cd ../../../debug; make ./test.sh -f unique/stable/replica3_dnode6.sim ./test.sh -f unique/stable/replica3_vnode3.sim +./test.sh -f unique/mnode/mgmt21.sim ./test.sh -f unique/mnode/mgmt22.sim ./test.sh -f unique/mnode/mgmt23.sim ./test.sh -f unique/mnode/mgmt24.sim @@ -77,31 +67,16 @@ cd ../../../debug; make ./test.sh -f unique/vnode/replica3_repeat.sim ./test.sh -f unique/vnode/replica3_vgroup.sim -./test.sh -f general/stream/metrics_1.sim -./test.sh -f general/stream/metrics_del.sim -./test.sh -f general/stream/metrics_n.sim -./test.sh -f general/stream/metrics_replica1_vnoden.sim -#./test.sh -f general/stream/new_stream.sim -./test.sh -f general/stream/restart_stream.sim -./test.sh -f general/stream/stream_1.sim -./test.sh -f general/stream/stream_2.sim -./test.sh -f general/stream/stream_3.sim -./test.sh -f general/stream/stream_restart.sim -./test.sh -f general/stream/table_1.sim -./test.sh -f general/stream/table_del.sim -./test.sh -f general/stream/table_n.sim -./test.sh -f general/stream/table_replica1_vnoden.sim - ./test.sh -f unique/arbitrator/check_cluster_cfg_para.sim -./test.sh -f unique/arbitrator/dn2_mn1_cache_file_sync.sim +#./test.sh -f unique/arbitrator/dn2_mn1_cache_file_sync.sim ./test.sh -f unique/arbitrator/dn3_mn1_full_createTableFail.sim -./test.sh -f unique/arbitrator/dn3_mn1_full_dropDnodeFail.sim ./test.sh -f unique/arbitrator/dn3_mn1_multiCreateDropTable.sim -./test.sh -f unique/arbitrator/dn3_mn1_nw_disable_timeout_autoDropDnode.sim -./test.sh -f unique/arbitrator/dn3_mn1_replica2_wal1_AddDelDnode.sim +#./test.sh -f unique/arbitrator/dn3_mn1_nw_disable_timeout_autoDropDnode.sim +#./test.sh -f unique/arbitrator/dn3_mn1_replica2_wal1_AddDelDnode.sim ./test.sh -f unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim ./test.sh -f unique/arbitrator/dn3_mn1_replica_change.sim -./test.sh -f unique/arbitrator/dn3_mn1_stopDnode_timeout.sim +#./test.sh -f unique/arbitrator/dn3_mn1_stopDnode_timeout.sim +# lower the priority while file corruption #./test.sh -f unique/arbitrator/dn3_mn1_vnode_change.sim #./test.sh -f unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim #./test.sh -f unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim @@ -126,6 +101,7 @@ cd ../../../debug; make ./test.sh -f unique/arbitrator/replica_changeWithArbitrator.sim ./test.sh -f unique/arbitrator/sync_replica2_alterTable_add.sim ./test.sh -f unique/arbitrator/sync_replica2_alterTable_drop.sim + ./test.sh -f unique/arbitrator/sync_replica2_dropDb.sim ./test.sh -f unique/arbitrator/sync_replica2_dropTable.sim ./test.sh -f unique/arbitrator/sync_replica3_alterTable_add.sim diff --git a/tests/script/tmp/prepare.sim b/tests/script/tmp/prepare.sim index f59eebede0..8b8f206233 100644 --- a/tests/script/tmp/prepare.sim +++ b/tests/script/tmp/prepare.sim @@ -32,10 +32,17 @@ system sh/cfg.sh -n dnode2 -c http -v 1 system sh/cfg.sh -n dnode3 -c http -v 1 system sh/cfg.sh -n dnode4 -c http -v 1 - +return +# for crash_gen +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 2 +system sh/cfg.sh -n dnode1 -c rpcMaxTime -v 101 +system sh/cfg.sh -n dnode1 -c cache -v 2 +system sh/cfg.sh -n dnode1 -c keep -v 36500 system sh/cfg.sh -n dnode1 -c walLevel -v 2 +# for windows + system sh/cfg.sh -n dnode1 -c firstEp -v 152.136.17.116:6030 system sh/cfg.sh -n dnode1 -c secondEp -v 152.136.17.116:6030 system sh/cfg.sh -n dnode1 -c serverPort -v 6030 diff --git a/tests/script/unique/account/usage.sim b/tests/script/unique/account/usage.sim index 3f12c46137..8d611d6fb6 100644 --- a/tests/script/unique/account/usage.sim +++ b/tests/script/unique/account/usage.sim @@ -112,22 +112,23 @@ print =============== step4 sql insert into d1.t1 values(now + 1s, 1) sql insert into d1.t1 values(now + 2s, 2) -# no write auth -sleep 3000 +sleep 10000 +print no write auth sql_error insert into d1.t1 values(now + 3s, 2) sql_error insert into d1.t1 values(now + 4s, 2) sql alter account root pass "taosdata" tseries 10 storage 36 streams 10 dbs 5 users 5 -sleep 3000 +sleep 10000 +print has write auth sql insert into d1.t1 values(now + 5s, 1) sql insert into d1.t1 values(now + 6s, 2) # no write auth -sleep 3000 +sleep 10000 +print no write auth sql_error insert into d1.t1 values(now + 7s, 2) sql_error insert into d1.t1 values(now + 8s, 2) - print =============== check grant sql_error create database d6 diff --git a/tests/script/unique/mnode/mgmt21.sim b/tests/script/unique/mnode/mgmt21.sim new file mode 100644 index 0000000000..53ad0eebe7 --- /dev/null +++ b/tests/script/unique/mnode/mgmt21.sim @@ -0,0 +1,44 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 + +system sh/cfg.sh -n dnode1 -c numOfMnodes -v 2 +system sh/cfg.sh -n dnode2 -c numOfMnodes -v 2 + +print ============== step1 +system sh/exec.sh -n dnode2 -s start +sleep 10000 + +system sh/exec.sh -n dnode1 -s start +sql connect + +sql show mnodes +print dnode1 ==> $data2_1 +print dnode2 ==> $data2_2 +if $data2_1 != master then + return -1 +endi + +print ============== step2 +sql create dnode $hostname2 + +$x = 0 +show2: + $x = $x + 1 + sleep 2000 + if $x == 10 then + return -1 + endi + +sql show mnodes +print dnode1 ==> $data2_1 +print dnode2 ==> $data2_2 +if $data2_1 != master then + goto show2 +endi +if $data2_2 != slave then + goto show2 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/mnode/testSuite.sim b/tests/script/unique/mnode/testSuite.sim index 33df24b860..b9adbe06a2 100644 --- a/tests/script/unique/mnode/testSuite.sim +++ b/tests/script/unique/mnode/testSuite.sim @@ -1,3 +1,4 @@ +run unique/mnode/mgmt21.sim run unique/mnode/mgmt22.sim run unique/mnode/mgmt23.sim run unique/mnode/mgmt24.sim diff --git a/tests/script/wtest.bat b/tests/script/wtest.bat index 1574b5013e..6cdd63b42d 100644 --- a/tests/script/wtest.bat +++ b/tests/script/wtest.bat @@ -6,8 +6,8 @@ echo Start TDengine Testing Case ... set "SCRIPT_DIR=%~dp0" echo SCRIPT_DIR: %SCRIPT_DIR% -set "BUILD_DIR=%~dp0..\..\debug\build\bin" -set "TSIM=%~dp0..\..\debug\build\bin\tsim" +set "BUILD_DIR=%~dp0..\..\debug\32\build\bin" +set "TSIM=%~dp0..\..\debug\32\build\bin\tsim" echo BUILD_DIR: %BUILD_DIR% set "SIM_DIR=%~dp0..\..\sim" @@ -47,7 +47,7 @@ echo qdebugFlag 143 >> %TAOS_CFG% echo udebugFlag 143 >> %TAOS_CFG% set "FILE_NAME=windows\testSuite.sim" -set "FIRSTEP=localhost" +set "FIRSTEP=192.168.1.182" if "%1" == "-f" set "FILE_NAME=%2" if "%1" == "-h" set "FIRSTEP=%2" if "%3" == "-f" set "FILE_NAME=%4" diff --git a/tests/test/c/CMakeLists.txt b/tests/test/c/CMakeLists.txt index cc8bc78e33..9d11895104 100644 --- a/tests/test/c/CMakeLists.txt +++ b/tests/test/c/CMakeLists.txt @@ -10,8 +10,8 @@ IF (TD_LINUX) #add_executable(insertPerTable insertPerTable.c) #target_link_libraries(insertPerTable taos_static pthread) - add_executable(insertPerRow insertPerRow.c) - target_link_libraries(insertPerRow taos_static pthread) + #add_executable(insertPerRow insertPerRow.c) + #target_link_libraries(insertPerRow taos_static pthread) #add_executable(importOneRow importOneRow.c) #target_link_libraries(importOneRow taos_static pthread) @@ -22,6 +22,9 @@ IF (TD_LINUX) #add_executable(hashPerformance hashPerformance.c) #target_link_libraries(hashPerformance taos_static tutil common pthread) - add_executable(createTablePerformance createTablePerformance.c) - target_link_libraries(createTablePerformance taos_static tutil common pthread) + #add_executable(createTablePerformance createTablePerformance.c) + #target_link_libraries(createTablePerformance taos_static tutil common pthread) + + add_executable(createNormalTable createNormalTable.c) + target_link_libraries(createNormalTable taos_static tutil common pthread) ENDIF() diff --git a/tests/test/c/createNormalTable.c b/tests/test/c/createNormalTable.c new file mode 100644 index 0000000000..18a648b9e1 --- /dev/null +++ b/tests/test/c/createNormalTable.c @@ -0,0 +1,218 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "taoserror.h" +#include "taos.h" +#include "tulog.h" +#include "tutil.h" +#include "tglobal.h" +#include "hash.h" + +#define MAX_RANDOM_POINTS 20000 +#define GREEN "\033[1;32m" +#define NC "\033[0m" + +char dbName[32] = "db"; +char stableName[64] = "st"; +int32_t numOfThreads = 30; +int32_t numOfTables = 100000; +int32_t replica = 1; +int32_t numOfColumns = 2; + +typedef struct { + int32_t tableBeginIndex; + int32_t tableEndIndex; + int32_t threadIndex; + char dbName[32]; + char stableName[64]; + float createTableSpeed; + pthread_t thread; +} SThreadInfo; + +void shellParseArgument(int argc, char *argv[]); +void *threadFunc(void *param); +void createDbAndSTable(); + +int main(int argc, char *argv[]) { + shellParseArgument(argc, argv); + taos_init(); + createDbAndSTable(); + + pPrint("%d threads are spawned to create table", numOfThreads); + + pthread_attr_t thattr; + pthread_attr_init(&thattr); + pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + SThreadInfo *pInfo = (SThreadInfo *)calloc(numOfThreads, sizeof(SThreadInfo)); + + int32_t numOfTablesPerThread = numOfTables / numOfThreads; + numOfTables = numOfTablesPerThread * numOfThreads; + for (int i = 0; i < numOfThreads; ++i) { + pInfo[i].tableBeginIndex = i * numOfTablesPerThread; + pInfo[i].tableEndIndex = (i + 1) * numOfTablesPerThread; + pInfo[i].threadIndex = i; + strcpy(pInfo[i].dbName, dbName); + strcpy(pInfo[i].stableName, stableName); + pthread_create(&(pInfo[i].thread), &thattr, threadFunc, (void *)(pInfo + i)); + } + + taosMsleep(300); + for (int i = 0; i < numOfThreads; i++) { + pthread_join(pInfo[i].thread, NULL); + } + + float createTableSpeed = 0; + for (int i = 0; i < numOfThreads; ++i) { + createTableSpeed += pInfo[i].createTableSpeed; + } + + pPrint("%s total speed:%.1f tables/second, threads:%d %s", GREEN, createTableSpeed, numOfThreads, NC); + + pthread_attr_destroy(&thattr); + free(pInfo); +} + +void createDbAndSTable() { + pPrint("start to create db and stable"); + char qstr[64000]; + + TAOS *con = taos_connect(NULL, "root", "taosdata", NULL, 0); + if (con == NULL) { + pError("failed to connect to DB, reason:%s", taos_errstr(con)); + exit(1); + } + + sprintf(qstr, "create database if not exists %s replica %d", dbName, replica); + TAOS_RES *pSql = taos_query(con, qstr); + int32_t code = taos_errno(pSql); + if (code != 0) { + pError("failed to create database:%s, sql:%s, code:%d reason:%s", dbName, qstr, taos_errno(con), taos_errstr(con)); + exit(0); + } + taos_free_result(pSql); + + sprintf(qstr, "use %s", dbName); + pSql = taos_query(con, qstr); + code = taos_errno(pSql); + if (code != 0) { + pError("failed to use db, code:%d reason:%s", taos_errno(con), taos_errstr(con)); + exit(0); + } + taos_free_result(pSql); + + taos_close(con); +} + +void *threadFunc(void *param) { + SThreadInfo *pInfo = (SThreadInfo *)param; + char qstr[65000]; + int code; + + TAOS *con = taos_connect(NULL, "root", "taosdata", NULL, 0); + if (con == NULL) { + pError("index:%d, failed to connect to DB, reason:%s", pInfo->threadIndex, taos_errstr(con)); + exit(1); + } + + sprintf(qstr, "use %s", pInfo->dbName); + TAOS_RES *pSql = taos_query(con, qstr); + taos_free_result(pSql); + + int64_t startMs = taosGetTimestampMs(); + + for (int32_t t = pInfo->tableBeginIndex; t < pInfo->tableEndIndex; ++t) { + sprintf(qstr, "create table %s%d (ts timestamp, i int)", stableName, t); + TAOS_RES *pSql = taos_query(con, qstr); + code = taos_errno(pSql); + if (code != 0) { + pError("failed to create table %s%d, reason:%s", stableName, t, tstrerror(code)); + } + taos_free_result(pSql); + } + + float createTableSpeed = 0; + for (int i = 0; i < numOfThreads; ++i) { + createTableSpeed += pInfo[i].createTableSpeed; + } + + int64_t endMs = taosGetTimestampMs(); + int32_t totalTables = pInfo->tableEndIndex - pInfo->tableBeginIndex; + float seconds = (endMs - startMs) / 1000.0; + float speed = totalTables / seconds; + pInfo->createTableSpeed = speed; + + pPrint("thread:%d, time:%.2f sec, speed:%.1f tables/second, ", pInfo->threadIndex, seconds, speed); + taos_close(con); + + return 0; +} + +void printHelp() { + char indent[10] = " "; + printf("Used to test the performance while create table\n"); + + printf("%s%s\n", indent, "-c"); + printf("%s%s%s%s\n", indent, indent, "Configuration directory, default is ", configDir); + printf("%s%s\n", indent, "-d"); + printf("%s%s%s%s\n", indent, indent, "The name of the database to be created, default is ", dbName); + printf("%s%s\n", indent, "-s"); + printf("%s%s%s%s\n", indent, indent, "The name of the super table to be created, default is ", stableName); + printf("%s%s\n", indent, "-t"); + printf("%s%s%s%d\n", indent, indent, "numOfThreads, default is ", numOfThreads); + printf("%s%s\n", indent, "-n"); + printf("%s%s%s%d\n", indent, indent, "numOfTables, default is ", numOfTables); + printf("%s%s\n", indent, "-r"); + printf("%s%s%s%d\n", indent, indent, "replica, default is ", replica); + printf("%s%s\n", indent, "-columns"); + printf("%s%s%s%d\n", indent, indent, "numOfColumns, default is ", numOfColumns); + + exit(EXIT_SUCCESS); +} + +void shellParseArgument(int argc, char *argv[]) { + for (int i = 1; i < argc; i++) { + if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) { + printHelp(); + exit(0); + } else if (strcmp(argv[i], "-d") == 0) { + strcpy(dbName, argv[++i]); + } else if (strcmp(argv[i], "-c") == 0) { + strcpy(configDir, argv[++i]); + } else if (strcmp(argv[i], "-s") == 0) { + strcpy(stableName, argv[++i]); + } else if (strcmp(argv[i], "-t") == 0) { + numOfThreads = atoi(argv[++i]); + } else if (strcmp(argv[i], "-n") == 0) { + numOfTables = atoi(argv[++i]); + } else if (strcmp(argv[i], "-r") == 0) { + replica = atoi(argv[++i]); + } else if (strcmp(argv[i], "-columns") == 0) { + numOfColumns = atoi(argv[++i]); + } else { + } + } + + pPrint("%s dbName:%s %s", GREEN, dbName, NC); + pPrint("%s stableName:%s %s", GREEN, stableName, NC); + pPrint("%s configDir:%s %s", GREEN, configDir, NC); + pPrint("%s numOfTables:%d %s", GREEN, numOfTables, NC); + pPrint("%s numOfThreads:%d %s", GREEN, numOfThreads, NC); + pPrint("%s numOfColumns:%d %s", GREEN, numOfColumns, NC); + pPrint("%s replica:%d %s", GREEN, replica, NC); + + pPrint("%s start create table performace test %s", GREEN, NC); +} diff --git a/tests/tsim/inc/sim.h b/tests/tsim/inc/sim.h index 18af21a506..6f3bc7099d 100644 --- a/tests/tsim/inc/sim.h +++ b/tests/tsim/inc/sim.h @@ -96,20 +96,20 @@ enum { struct _script_t; typedef struct _cmd_t { - short cmdno; - short nlen; - char name[MAX_SIM_CMD_NAME_LEN]; - bool (*parseCmd)(char *, struct _cmd_t *, int); - bool (*executeCmd)(struct _script_t *script, char *option); + int16_t cmdno; + int16_t nlen; + char name[MAX_SIM_CMD_NAME_LEN]; + bool (*parseCmd)(char *, struct _cmd_t *, int); + bool (*executeCmd)(struct _script_t *script, char *option); struct _cmd_t *next; } SCommand; typedef struct { - short cmdno; - short jump; // jump position - short errorJump; // sql jump flag, while '-x' exist in sql cmd, this flag - // will be SQL_JUMP_TRUE, otherwise is SQL_JUMP_FALSE */ - short lineNum; // correspodning line number in original file + int16_t cmdno; + int16_t jump; // jump position + int16_t errorJump; // sql jump flag, while '-x' exist in sql cmd, this flag + // will be SQL_JUMP_TRUE, otherwise is SQL_JUMP_FALSE */ + int16_t lineNum; // correspodning line number in original file int optionOffset; // relative option offset } SCmdLine; @@ -120,7 +120,7 @@ typedef struct _var_t { } SVariable; typedef struct _script_t { - int type; + int type; bool killed; void *taos; @@ -130,10 +130,10 @@ typedef struct _script_t { char system_exit_code[12]; char system_ret_content[MAX_SYSTEM_RESULT_LEN]; - int varLen; - int linePos; // current cmd position - int numOfLines; // number of lines in the script - int bgScriptLen; + int varLen; + int linePos; // current cmd position + int numOfLines; // number of lines in the script + int bgScriptLen; char fileName[MAX_FILE_NAME_LEN]; // script file name char error[MAX_ERROR_LEN]; char *optionBuffer; diff --git a/tests/tsim/inc/simParse.h b/tests/tsim/inc/simParse.h index cff66e6c4f..d3f92add71 100644 --- a/tests/tsim/inc/simParse.h +++ b/tests/tsim/inc/simParse.h @@ -33,21 +33,21 @@ enum { /* label stack */ typedef struct { - char top; /* number of labels */ - short pos[MAX_NUM_LABLES]; /* the position of the label */ - char label[MAX_NUM_LABLES][MAX_LABEL_LEN]; /* name of the label */ + char top; /* number of labels */ + int16_t pos[MAX_NUM_LABLES]; /* the position of the label */ + char label[MAX_NUM_LABLES][MAX_LABEL_LEN]; /* name of the label */ } SLabel; /* block definition */ typedef struct { - char top; /* the number of blocks stacked */ - char type[MAX_NUM_BLOCK]; /* the block type */ - short *pos[MAX_NUM_BLOCK]; /* position of the jump for if/elif/case */ - short back[MAX_NUM_BLOCK]; /* go back, endw and continue */ - char numJump[MAX_NUM_BLOCK]; - short *jump[MAX_NUM_BLOCK][MAX_NUM_JUMP]; /* break or elif */ - char sexp[MAX_NUM_BLOCK][40]; /*switch expression */ - char sexpLen[MAX_NUM_BLOCK]; /*switch expression length */ + char top; /* the number of blocks stacked */ + char type[MAX_NUM_BLOCK]; /* the block type */ + int16_t *pos[MAX_NUM_BLOCK]; /* position of the jump for if/elif/case */ + int16_t back[MAX_NUM_BLOCK]; /* go back, endw and continue */ + char numJump[MAX_NUM_BLOCK]; + int16_t *jump[MAX_NUM_BLOCK][MAX_NUM_JUMP]; /* break or elif */ + char sexp[MAX_NUM_BLOCK][40]; /*switch expression */ + char sexpLen[MAX_NUM_BLOCK]; /*switch expression length */ } SBlock; bool simParseExpression(char *token, int lineNum);