diff --git a/.gitignore b/.gitignore index c5d90eea50..e6e327327c 100644 --- a/.gitignore +++ b/.gitignore @@ -66,6 +66,8 @@ CMakeError.log /test/cfg /src/.vs *.o +version.c +taos.rc src/connector/jdbc/.settings/ tests/comparisonTest/cassandra/cassandratest/.classpath tests/comparisonTest/cassandra/cassandratest/.project diff --git a/CMakeLists.txt b/CMakeLists.txt index bfb8e902a8..565ab32f00 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,6 +28,7 @@ INCLUDE(cmake/input.inc) INCLUDE(cmake/platform.inc) INCLUDE(cmake/define.inc) INCLUDE(cmake/env.inc) +INCLUDE(cmake/version.inc) INCLUDE(cmake/install.inc) ADD_SUBDIRECTORY(deps) diff --git a/README.md b/README.md index 776d7a154b..8c26143a12 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![Build status](https://ci.appveyor.com/api/projects/status/kf3pwh2or5afsgl9/branch/master?svg=true)](https://ci.appveyor.com/project/sangshuduo/tdengine-2n8ge/branch/master) [![Coverage Status](https://coveralls.io/repos/github/taosdata/TDengine/badge.svg?branch=develop)](https://coveralls.io/github/taosdata/TDengine?branch=develop) [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/4201/badge)](https://bestpractices.coreinfrastructure.org/projects/4201) -![Docker Pulls](https://img.shields.io/docker/pulls/tdengine/tdengine) +[![Docker Pulls](https://img.shields.io/docker/pulls/tdengine/tdengine)](https://hub.docker.com/repository/docker/tdengine/tdengine) [![TDengine](TDenginelogo.png)](https://www.taosdata.com) @@ -29,24 +29,69 @@ For user manual, system design and architecture, engineering blogs, refer to [TD # Building At the moment, TDengine only supports building and running on Linux systems. You can choose to [install from packages](https://www.taosdata.com/en/getting-started/#Install-from-Package) or from the source code. This quick guide is for installation from the source only. -To build TDengine, use [CMake](https://cmake.org/) 2.8 or higher versions in the project directory. Install CMake for example on Ubuntu: -``` -sudo apt-get install -y cmake build-essential +To build TDengine, use [CMake](https://cmake.org/) 3.5 or higher versions in the project directory. + +## Install tools + +### Ubuntu & Debian: +```bash +sudo apt-get install -y gcc cmake build-essential git ``` To compile and package the JDBC driver source code, you should have a Java jdk-8 or higher and Apache Maven 2.7 or higher installed. -To install openjdk-8 on Ubuntu: -``` -sudo apt-get install openjdk-8-jdk -``` -To install Apache Maven on Ubuntu: -``` -sudo apt-get install maven +To install openjdk-8: +```bash +sudo apt-get install -y openjdk-8-jdk ``` -Build TDengine: - +To install Apache Maven: +```bash +sudo apt-get install -y maven ``` + +### Centos 7: +```bash +sudo yum install -y gcc gcc-c++ make cmake3 epel-release git +sudo yum remove -y cmake +sudo ln -s /usr/bin/cmake3 /usr/bin/cmake +``` + +To install openjdk-8: +```bash +sudo yum install -y java-1.8.0-openjdk +``` + +To install Apache Maven: +```bash +sudo yum install -y maven +``` + +### Centos 8 & Fedora: +```bash +sudo dnf install -y gcc gcc-c++ make cmake epel-release git +``` + +To install openjdk-8: +```bash +sudo dnf install -y java-1.8.0-openjdk +``` + +To install Apache Maven: +```bash +sudo dnf install -y maven +``` + +## Get the source codes + +- github: +```bash +git clone https://github.com/taosdata/TDengine.git +cd TDengine +``` + +## Build TDengine + +```bash mkdir debug && cd debug cmake .. && cmake --build . ``` @@ -54,12 +99,12 @@ cmake .. && cmake --build . To compile on an ARM processor (aarch64 or aarch32), please add option CPUTYPE as below: aarch64: -```cmd +```bash cmake .. -DCPUTYPE=aarch64 && cmake --build . ``` aarch32: -```cmd +```bash cmake .. -DCPUTYPE=aarch32 && cmake --build . ``` diff --git a/cmake/version.inc b/cmake/version.inc new file mode 100644 index 0000000000..c620d753a6 --- /dev/null +++ b/cmake/version.inc @@ -0,0 +1,69 @@ +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +PROJECT(TDengine) + +IF (DEFINED VERNUMBER) + SET(TD_VER_NUMBER ${VERNUMBER}) +ELSE () + SET(TD_VER_NUMBER "2.0.2.0") +ENDIF () + +IF (DEFINED VERCOMPATIBLE) + SET(TD_VER_COMPATIBLE ${VERCOMPATIBLE}) +ELSE () + SET(TD_VER_COMPATIBLE "2.0.0.0") +ENDIF () + +IF (DEFINED GITINFO) + SET(TD_VER_GIT ${GITINFO}) +ELSE () + SET(TD_VER_GIT "community") +ENDIF () + +IF (DEFINED GITINFOI) + SET(TD_VER_GIT_INTERNAL ${GITINFOI}) +ELSE () + SET(TD_VER_GIT_INTERNAL "internal") +ENDIF () + +IF (DEFINED VERDATE) + SET(TD_VER_DATE ${VERDATE}) +ELSE () + STRING(TIMESTAMP TD_VER_DATE "%Y-%m-%d %H:%M:%S") +ENDIF () + +IF (DEFINED VERTYPE) + SET(TD_VER_VERTYPE ${VERTYPE}) +ELSE () + SET(TD_VER_VERTYPE "stable") +ENDIF () + +IF (DEFINED CPUTYPE) + SET(TD_VER_CPUTYPE ${CPUTYPE}) +ELSE () + IF (TD_WINDOWS_32) + SET(TD_VER_CPUTYPE "x86") + ELSE () + SET(TD_VER_CPUTYPE "x64") + ENDIF () +ENDIF () + +IF (DEFINED OSTYPE) + SET(TD_VER_OSTYPE ${OSTYPE}) +ELSE () + SET(TD_VER_OSTYPE "Linux") +ENDIF () + +MESSAGE(STATUS "============= compile version parameter information start ============= ") +MESSAGE(STATUS "ver number:" ${TD_VER_NUMBER}) +MESSAGE(STATUS "compatible ver number:" ${TD_VER_COMPATIBLE}) +MESSAGE(STATUS "communit commit id:" ${TD_VER_GIT}) +MESSAGE(STATUS "internal commit id:" ${TD_VER_GIT_INTERNAL}) +MESSAGE(STATUS "build date:" ${TD_VER_DATE}) +MESSAGE(STATUS "ver type:" ${TD_VER_VERTYPE}) +MESSAGE(STATUS "ver cpu:" ${TD_VER_CPUTYPE}) +MESSAGE(STATUS "os type:" ${TD_VER_OSTYPE}) +MESSAGE(STATUS "============= compile version parameter information end ============= ") + +STRING(REPLACE "." "_" TD_LIB_VER_NUMBER ${TD_VER_NUMBER}) + +CONFIGURE_FILE("${TD_COMMUNITY_DIR}/src/util/src/version.c.in" "${TD_COMMUNITY_DIR}/src/util/src/version.c") diff --git a/deps/MsvcLibX/CMakeLists.txt b/deps/MsvcLibX/CMakeLists.txt index fc77a3b447..4428579e1c 100644 --- a/deps/MsvcLibX/CMakeLists.txt +++ b/deps/MsvcLibX/CMakeLists.txt @@ -4,5 +4,5 @@ PROJECT(TDengine) IF (TD_WINDOWS) INCLUDE_DIRECTORIES(include) AUX_SOURCE_DIRECTORY(src SRC) - ADD_LIBRARY(MsvcLibXw64 ${SRC}) + ADD_LIBRARY(MsvcLibXw ${SRC}) ENDIF () diff --git a/documentation20/webdocs/markdowndocs/administrator-ch.md b/documentation20/webdocs/markdowndocs/administrator-ch.md index 97f9f9e839..64cadf69cd 100644 --- a/documentation20/webdocs/markdowndocs/administrator-ch.md +++ b/documentation20/webdocs/markdowndocs/administrator-ch.md @@ -94,7 +94,7 @@ TDengine系统后台服务由taosd提供，可以在配置文件taos.cfg里修 - maxSQLLength：单条SQL语句允许最长限制。默认值：65380字节。 - telemetryReporting: 是否允许 TDengine 采集和上报基本使用信息，0表示不允许，1表示允许。 默认值：1。 -**注意：**对于端口，TDengine会使用从serverPort起12个连续的TCP和UDP端口号，请务必在防火墙打开。因此如果是缺省配置，需要打开从6030都6041共12个端口，而且必须TCP和UDP都打开。 +**注意：**对于端口，TDengine会使用从serverPort起13个连续的TCP和UDP端口号，请务必在防火墙打开。因此如果是缺省配置，需要打开从6030都6042共13个端口，而且必须TCP和UDP都打开。 不同应用场景的数据往往具有不同的数据特征，比如保留天数、副本数、采集频次、记录大小、采集点的数量、压缩等都可完全不同。为获得在存储上的最高效率，TDengine提供如下存储相关的系统配置参数： @@ -153,10 +153,10 @@ TDengine系统的前台交互客户端应用程序为taos，它与taosd共享同 系统管理员可以在CLI界面里添加、删除用户，也可以修改密码。CLI里SQL语法如下： ``` -CREATE USER PASS <‘password’>; +CREATE USER PASS <'password'>; ``` -创建用户，并指定用户名和密码，密码需要用单引号引起来 +创建用户，并指定用户名和密码，密码需要用单引号引起来,单引号为英文半角 ``` DROP USER ; @@ -165,10 +165,10 @@ DROP USER ; 删除用户，限root用户使用 ``` -ALTER USER PASS <‘password’>; +ALTER USER PASS <'password'>; ``` -修改用户密码, 为避免被转换为小写，密码需要用单引号引用 +修改用户密码, 为避免被转换为小写，密码需要用单引号引用,单引号为英文半角 ``` SHOW USERS; diff --git a/documentation20/webdocs/markdowndocs/architecture-ch.md b/documentation20/webdocs/markdowndocs/architecture-ch.md index bfe3b55bd2..7ab4b5d096 100644 --- a/documentation20/webdocs/markdowndocs/architecture-ch.md +++ b/documentation20/webdocs/markdowndocs/architecture-ch.md @@ -82,7 +82,7 @@ TDengine 分布式架构的逻辑结构图如下： ### 节点之间的通讯 **通讯方式：**TDengine系统的各个节点之间的通讯是通过TCP/UDP进行的。因为考虑到物联网场景，数据写入的包一般不大，因此TDengine 除采用TCP做传输之外，还采用UDP方式，因为UDP 更加高效，而且不受连接数的限制。TDengine实现了自己的超时、重传、确认等机制，以确保UDP的可靠传输。对于数据量不到15K的数据包，采取UDP的方式进行传输，超过15K的，或者是查询类的操作，自动采取TCP的方式进行传输。同时，TDengine根据配置和数据包，会自动对数据进行压缩/解压缩，数字签名/认证等处理。对于数据节点之间的数据复制，只采用TCP方式进行数据传输。 -**FQDN配置**：一个数据节点有一个或多个FQDN，可以在系统配置文件taos.cfg通过选项“fqdn"进行指定，如果没有指定，系统将自动获取FQDN。如果节点没有配置FQDN，可以直接使用IP地址作为FQDN，但不建议使用，因为IP地址可变，一旦变化，将让集群无法正常工作。一个数据节点的EP(End Point)由FQDN + Port组成。 +**FQDN配置**：一个数据节点有一个或多个FQDN，可以在系统配置文件taos.cfg通过参数“fqdn"进行指定，如果没有指定，系统将自动获取FQDN。如果节点没有配置FQDN，可以直接将该节点的配置参数fqdn设置为它的IP地址。但不建议使用IP，因为IP地址可变，一旦变化，将让集群无法正常工作。一个数据节点的EP(End Point)由FQDN + Port组成。采用FQDN，需要保证DNS服务正常工作，或者在节点以及应用所在的节点配置好hosts文件。 **端口配置：**一个数据节点对外的端口由TDengine的系统配置参数serverPort决定，对集群内部通讯的端口是serverPort+5。集群内数据节点之间的数据复制操作还占有一个TCP端口，是serverPort+10. 为支持多线程高效的处理UDP数据，每个对内和对外的UDP链接，都需要占用5个连续的端口。因此一个数据节点总的端口范围为serverPort到serverPort + 10，总共11个TCP/UDP端口。使用时，需要确保防火墙将这些端口打开。每个数据节点可以配置不同的serverPort。 diff --git a/documentation20/webdocs/markdowndocs/cluster-ch.md b/documentation20/webdocs/markdowndocs/cluster-ch.md index 31d70e73e1..097433a18a 100644 --- a/documentation20/webdocs/markdowndocs/cluster-ch.md +++ b/documentation20/webdocs/markdowndocs/cluster-ch.md @@ -10,7 +10,7 @@ TDengine的集群管理极其简单，除添加和删除节点需要人工干预 **第一步**：如果搭建集群的节点中，存有之前的测试数据、装过1.X的版本，或者装过其他版本的TDengine，请先将其删除，并清空所有数据，具体步骤请参考博客[《TDengine多种安装包的安装和卸载》](https://www.taosdata.com/blog/2019/08/09/566.html ) -**第二步**：建议关闭防火墙，至少保证端口：6030 - 6041的TCP和UDP端口都是开放的。**强烈建议**先关闭防火墙，集群搭建完毕之后，再来配置端口； +**第二步**：建议关闭防火墙，至少保证端口：6030 - 6042的TCP和UDP端口都是开放的。**强烈建议**先关闭防火墙，集群搭建完毕之后，再来配置端口； **第三步**：在所有节点安装TDengine，且版本必须是一致的，**但不要启动taosd**； @@ -33,7 +33,7 @@ fqdn h1.taosdata.com serverPort 6030 // 副本数为偶数的时候，需要配置，请参考《Arbitrator的使用》的部分 -arbitrator ha.taosdata.com:6030 +arbitrator ha.taosdata.com:6042 ``` 一定要修改的参数是firstEp, 其他参数可不做任何修改，除非你很清楚为什么要修改。 @@ -169,5 +169,5 @@ SHOW MNODES; 如果副本数为偶数，当一个vnode group里一半或超过一半的vnode不工作时，是无法从中选出master的。同理，一半或超过一半的mnode不工作时，是无法选出mnode的master的，因为存在“split brain”问题。为解决这个问题，TDengine引入了arbitrator的概念。Arbitrator模拟一个vnode或mnode在工作，但只简单的负责网络连接，不处理任何数据插入或访问。只要包含arbitrator在内，超过半数的vnode或mnode工作，那么该vnode group或mnode组就可以正常的提供数据插入或查询服务。比如对于副本数为2的情形，如果一个节点A离线，但另外一个节点B正常，而且能连接到arbitrator, 那么节点B就能正常工作。 -TDengine安装包里带有一个执行程序tarbitrator, 找任何一台Linux服务器运行它即可。该程序对系统资源几乎没有要求，只需要保证有网络连接即可。该应用的命令行参数`-p`可以指定其对外服务的端口号，缺省是6030。配置每个taosd实例时，可以在配置文件taos.cfg里将参数arbitrator设置为arbitrator的End Point。如果该参数配置了，当副本数为偶数数，系统将自动连接配置的arbitrator。 +TDengine安装包里带有一个执行程序tarbitrator, 找任何一台Linux服务器运行它即可。该程序对系统资源几乎没有要求，只需要保证有网络连接即可。该应用的命令行参数`-p`可以指定其对外服务的端口号，缺省是6042。配置每个taosd实例时，可以在配置文件taos.cfg里将参数arbitrator设置为arbitrator的End Point。如果该参数配置了，当副本数为偶数数，系统将自动连接配置的arbitrator。 diff --git a/documentation20/webdocs/markdowndocs/faq-ch.md b/documentation20/webdocs/markdowndocs/faq-ch.md index de47f376ac..2b51458f27 100644 --- a/documentation20/webdocs/markdowndocs/faq-ch.md +++ b/documentation20/webdocs/markdowndocs/faq-ch.md @@ -23,38 +23,51 @@ 客户端遇到链接故障，请按照下面的步骤进行检查： -1. 确保客户端与服务端版本号是完全一致的，开源社区版和企业版也不能混用 -2. 在服务器，执行 `systemctl status taosd` 检查*taosd*运行状态。如果没有运行，启动*taosd* -3. 确认客户端连接时指定了正确的服务器FQDN (Fully Qualified Domain Name(可在服务器上执行Linux命令hostname -f获得） -4. ping服务器FQDN，如果没有反应，请检查你的网络，DNS设置，或客户端所在计算机的系统hosts文件 -5. 检查防火墙设置，确认TCP/UDP 端口6030-6039 是打开的 -6. 对于Linux上的JDBC（ODBC, Python, Go等接口类似）连接, 确保*libtaos.so*在目录*/usr/local/lib/taos*里, 并且*/usr/local/lib/taos*在系统库函数搜索路径*LD_LIBRARY_PATH*里 -7. 对于windows上的JDBC, ODBC, Python, Go等连接，确保*driver/c/taos.dll*在你的系统搜索目录里 (建议*taos.dll*放在目录 *C:\Windows\System32*) -8. 如果仍不能排除连接故障，请使用命令行工具nc来分别判断指定端口的TCP和UDP连接是否通畅 +1. 检查网络环境 + * 云服务器：检查云服务器的安全组是否打开TCP/UDP 端口6030-6039的访问权限 + * 本地虚拟机：检查网络能否ping通，尽量避免使用`localhost` 作为hostname + * 公司服务器：如果为NAT网络环境，请务必检查服务器能否将消息返回值客户端 +2. 确保客户端与服务端版本号是完全一致的，开源社区版和企业版也不能混用 +3. 在服务器，执行 `systemctl status taosd` 检查*taosd*运行状态。如果没有运行，启动*taosd* +4. 确认客户端连接时指定了正确的服务器FQDN (Fully Qualified Domain Name(可在服务器上执行Linux命令hostname -f获得） +5. ping服务器FQDN，如果没有反应，请检查你的网络，DNS设置，或客户端所在计算机的系统hosts文件 +6. 检查防火墙设置，确认TCP/UDP 端口6030-6039 是打开的 +7. 对于Linux上的JDBC（ODBC, Python, Go等接口类似）连接, 确保*libtaos.so*在目录*/usr/local/lib/taos*里, 并且*/usr/local/lib/taos*在系统库函数搜索路径*LD_LIBRARY_PATH*里 +8. 对于windows上的JDBC, ODBC, Python, Go等连接，确保*driver/c/taos.dll*在你的系统搜索目录里 (建议*taos.dll*放在目录 *C:\Windows\System32*) +9. 如果仍不能排除连接故障，请使用命令行工具nc来分别判断指定端口的TCP和UDP连接是否通畅 检查UDP端口连接是否工作：`nc -vuz {hostIP} {port} ` 检查服务器侧TCP端口连接是否工作：`nc -l {port}` 检查客户端侧TCP端口链接是否工作：`nc {hostIP} {port}` -## 6. 虽然语法正确，为什么我还是得到 "Invalid SQL" 错误 +## 6. 遇到错误“Unexpected generic error in RPC”， 我怎么办？ +产生这个错误，是由于客户端或数据节点无法解析FQDN(Fully Qualified Domain Name)导致。对于TAOS Shell或客户端应用，请做如下检查： + +1. 请检查连接的服务器的FQDN是否正确 +2. 如果网络配置有DNS server, 请检查是否正常工作 +3. 如果网络没有配置DNS server, 请检查客户端所在机器的hosts文件，查看该FQDN是否配置，并是否有正确的IP地址。 +4. 如果网络配置OK，从客户端所在机器，你需要能Ping该连接的FQDN，否则客户端是无法链接服务器的 + + +## 7. 虽然语法正确，为什么我还是得到 "Invalid SQL" 错误 如果你确认语法正确，2.0之前版本，请检查SQL语句长度是否超过64K。如果超过，也会返回这个错误。 -## 7. 是否支持validation queries? +## 8. 是否支持validation queries? TDengine还没有一组专用的validation queries。然而建议你使用系统监测的数据库”log"来做。 -## 8. 我可以删除或更新一条记录吗？ +## 9. 我可以删除或更新一条记录吗？ 不能。因为TDengine是为联网设备采集的数据设计的，不容许修改。但TDengine提供数据保留策略，只要数据记录超过保留时长，就会被自动删除。 -## 10. 我怎么创建超过250列的表？ +## 10. 我怎么创建超过1024列的表？ 使用2.0及其以上版本，默认支持1024列；2.0之前的版本，TDengine最大允许创建250列的表。但是如果确实超过限值，建议按照数据特性，逻辑地将这个宽表分解成几个小表。 ## 10. 最有效的写入数据的方法是什么？ -批量插入。每条写入语句可以一张表同时插入多条记录，也可以同时插入多张表的记录。 +批量插入。每条写入语句可以一张表同时插入多条记录，也可以同时插入多张表的多条记录。 ## 11. 最有效的写入数据的方法是什么？windows系统下插入的nchar类数据中的汉字被解析成了乱码如何解决？ @@ -77,4 +90,11 @@ Connection = DriverManager.getConnection(url, properties); 2.0.4 ``` +## 14. 怎么报告问题？ +如果 FAQ 中的信息不能够帮到您，需要 TDengine 技术团队的技术支持与协助，请将以下两个目录中内容打包: +1. /var/log/taos +2. /etc/taos +附上必要的问题描述，以及发生该问题的执行操作，出现问题的表征及大概的时间，在 GitHub提交Issue。 + +为了保证有足够的debug信息，如果问题能够重复，请修改/etc/taos/taos.cfg文件，最后面添加一行“debugFlag 135"(不带引号本身），然后重启taosd, 重复问题，然后再递交。但系统正常运行时，请一定将debugFlag设置为131，否则会产生大量的日志信息，降低系统效率。 diff --git a/packaging/cfg/taos.cfg b/packaging/cfg/taos.cfg index 93a6936da3..a91dde8796 100644 --- a/packaging/cfg/taos.cfg +++ b/packaging/cfg/taos.cfg @@ -24,7 +24,7 @@ # dataDir /var/lib/taos # the arbitrator's fully qualified domain name (FQDN) for TDengine system, for cluster only -# arbitrator arbitrator_hostname:6030 +# arbitrator arbitrator_hostname:6042 # number of threads per CPU core # numOfThreadsPerCore 1.0 diff --git a/packaging/docker/Dockerfile b/packaging/docker/Dockerfile index 3fb34e8286..668d5a49eb 100644 --- a/packaging/docker/Dockerfile +++ b/packaging/docker/Dockerfile @@ -12,6 +12,6 @@ ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib" ENV LANG=en_US.UTF-8 ENV LANGUAGE=en_US:en ENV LC_ALL=en_US.UTF-8 -EXPOSE 6030 6031 6032 6033 6034 6035 6036 6037 6038 6039 6040 6041 +EXPOSE 6030 6031 6032 6033 6034 6035 6036 6037 6038 6039 6040 6041 6042 CMD ["taosd"] VOLUME [ "/var/lib/taos", "/var/log/taos","/etc/taos/" ] diff --git a/packaging/release.sh b/packaging/release.sh index 3ba1ba72b2..3c0378042d 100755 --- a/packaging/release.sh +++ b/packaging/release.sh @@ -10,7 +10,9 @@ set -e # -o [Linux | Kylin | Alpine | Raspberrypi | Darwin | Windows | Ningsi60 | Ningsi80 |...] # -V [stable | beta] # -l [full | lite] +# -s [static | dynamic] # -n [2.0.0.3] +# -m [2.0.0.0] # set parameters by default value verMode=edge # [cluster, edge] @@ -18,9 +20,11 @@ verType=stable # [stable, beta] cpuType=x64 # [aarch32 | aarch64 | x64 | x86 | mips64 ...] osType=Linux # [Linux | Kylin | Alpine | Raspberrypi | Darwin | Windows | Ningsi60 | Ningsi80 |...] pagMode=full # [full | lite] +soMode=dynamic # [static | dynamic] verNumber="" +verNumberComp="2.0.0.0" -while getopts "hv:V:c:o:l:n:" arg +while getopts "hv:V:c:o:l:s:n:m:" arg do case $arg in v) @@ -39,10 +43,18 @@ do #echo "pagMode=$OPTARG" pagMode=$(echo $OPTARG) ;; + s) + #echo "soMode=$OPTARG" + soMode=$(echo $OPTARG) + ;; n) #echo "verNumber=$OPTARG" verNumber=$(echo $OPTARG) ;; + m) + #echo "verNumberComp=$OPTARG" + verNumberComp=$(echo $OPTARG) + ;; o) #echo "osType=$OPTARG" osType=$(echo $OPTARG) @@ -53,7 +65,9 @@ do echo " -o [Linux | Kylin | Alpine | Raspberrypi | Darwin | Windows | Ningsi60 | Ningsi80 |...] " echo " -V [stable | beta] " echo " -l [full | lite] " + echo " -s [static | dynamic] " echo " -n [version number] " + echo " -m [compatible version number] " exit 0 ;; ?) #unknow option @@ -63,215 +77,142 @@ do esac done -echo "verMode=${verMode} verType=${verType} cpuType=${cpuType} osType=${osType} pagMode=${pagMode} verNumber=${verNumber}" +echo "verMode=${verMode} verType=${verType} cpuType=${cpuType} osType=${osType} pagMode=${pagMode} soMode=${soMode} verNumber=${verNumber} verNumberComp=${verNumberComp}" curr_dir=$(pwd) if [ "$osType" != "Darwin" ]; then - script_dir="$(dirname $(readlink -f $0))" - top_dir="$(readlink -f ${script_dir}/..)" + script_dir="$(dirname $(readlink -f $0))" + top_dir="$(readlink -f ${script_dir}/..)" else - script_dir=`dirname $0` - cd ${script_dir} - script_dir="$(pwd)" - top_dir=${script_dir}/.. + script_dir=`dirname $0` + cd ${script_dir} + script_dir="$(pwd)" + top_dir=${script_dir}/.. fi -versioninfo="${top_dir}/src/util/src/version.c" - csudo="" -if command -v sudo > /dev/null; then - csudo="sudo" -fi +#if command -v sudo > /dev/null; then +# csudo="sudo" +#fi function is_valid_version() { - [ -z $1 ] && return 1 || : + [ -z $1 ] && return 1 || : - rx='^([0-9]+\.){3}(\*|[0-9]+)$' - if [[ $1 =~ $rx ]]; then - return 0 - fi - return 1 + rx='^([0-9]+\.){3}(\*|[0-9]+)$' + if [[ $1 =~ $rx ]]; then + return 0 + fi + return 1 } function vercomp () { - if [[ $1 == $2 ]]; then - echo 0 - exit 0 - fi - - local IFS=. - local i ver1=($1) ver2=($2) - - # fill empty fields in ver1 with zeros - for ((i=${#ver1[@]}; i<${#ver2[@]}; i++)); do - ver1[i]=0 - done - - for ((i=0; i<${#ver1[@]}; i++)); do - if [[ -z ${ver2[i]} ]]; then - # fill empty fields in ver2 with zeros - ver2[i]=0 - fi - if ((10#${ver1[i]} > 10#${ver2[i]})); then - echo 1 - exit 0 - fi - if ((10#${ver1[i]} < 10#${ver2[i]})); then - echo 2 - exit 0 - fi - done + if [[ $1 == $2 ]]; then echo 0 -} + exit 0 + fi + + local IFS=. + local i ver1=($1) ver2=($2) -# 1. Read version information -version=$(cat ${versioninfo} | grep " version" | cut -d '"' -f2) -compatible_version=$(cat ${versioninfo} | grep " compatible_version" | cut -d '"' -f2) + # fill empty fields in ver1 with zeros + for ((i=${#ver1[@]}; i<${#ver2[@]}; i++)); do + ver1[i]=0 + done -if [ -z ${verNumber} ]; then - while true; do - read -p "Do you want to release a new version? [y/N]: " is_version_change - - if [[ ( "${is_version_change}" == "y") || ( "${is_version_change}" == "Y") ]]; then - read -p "Please enter the new version: " tversion - while true; do - if (! is_valid_version $tversion) || [ "$(vercomp $tversion $version)" = '2' ]; then - read -p "Please enter a correct version: " tversion - continue - fi - version=${tversion} - break - done - - echo - - read -p "Enter the oldest compatible version: " tversion - while true; do - - if [ -z $tversion ]; then - break - fi - - if (! is_valid_version $tversion) || [ "$(vercomp $version $tversion)" = '2' ]; then - read -p "enter correct compatible version: " tversion - else - compatible_version=$tversion - break - fi - done - - break - elif [[ ( "${is_version_change}" == "n") || ( "${is_version_change}" == "N") ]]; then - echo "Use old version: ${version} compatible version: ${compatible_version}." - break - else - continue + for ((i=0; i<${#ver1[@]}; i++)); do + if [[ -z ${ver2[i]} ]]; then + # fill empty fields in ver2 with zeros + ver2[i]=0 + fi + if ((10#${ver1[i]} > 10#${ver2[i]})); then + echo 1 + exit 0 + fi + if ((10#${ver1[i]} < 10#${ver2[i]})); then + echo 2 + exit 0 fi done -else - echo "old version: $version, new version: $verNumber" - #if ( ! is_valid_version $verNumber ) || [[ "$(vercomp $version $verNumber)" == '2' ]]; then - # echo "please enter correct version" - # exit 0 - #else - version=${verNumber} - #fi -fi + echo 0 +} -echo "=======================new version number: ${version}======================================" +# 1. check version information +if (( ! is_valid_version $verNumber ) || ( ! is_valid_version $verNumberComp ) || [[ "$(vercomp $verNumber $verNumberComp)" == '2' ]]); then + echo "please enter correct version" + exit 0 +fi + +echo "=======================new version number: ${verNumber}, compatible version: ${verNumberComp}======================================" -# output the version info to the buildinfo file. build_time=$(date +"%F %R") -echo "char version[12] = \"${version}\";" > ${versioninfo} -echo "char compatible_version[12] = \"${compatible_version}\";" >> ${versioninfo} -echo "char gitinfo[48] = \"$(git rev-parse --verify HEAD)\";" >> ${versioninfo} -if [ "$verMode" != "cluster" ]; then - echo "char gitinfoOfInternal[48] = \"\";" >> ${versioninfo} -else - enterprise_dir="${top_dir}/../enterprise" - cd ${enterprise_dir} - echo "char gitinfoOfInternal[48] = \"$(git rev-parse --verify HEAD)\";" >> ${versioninfo} - cd ${curr_dir} -fi -echo "char buildinfo[64] = \"Built by ${USER} at ${build_time}\";" >> ${versioninfo} -echo "" >> ${versioninfo} -tmp_version=$(echo $version | tr -s "." "_") -if [ "$verMode" == "cluster" ]; then - libtaos_info=${tmp_version}_${osType}_${cpuType} -else - libtaos_info=edge_${tmp_version}_${osType}_${cpuType} -fi -if [ "$verType" == "beta" ]; then - libtaos_info=${libtaos_info}_${verType} -fi -echo "void libtaos_${libtaos_info}() {};" >> ${versioninfo} + +# get commint id from git +gitinfo=$(git rev-parse --verify HEAD) +enterprise_dir="${top_dir}/../enterprise" +cd ${enterprise_dir} +gitinfoOfInternal=$(git rev-parse --verify HEAD) +cd ${curr_dir} # 2. cmake executable file compile_dir="${top_dir}/debug" if [ -d ${compile_dir} ]; then - ${csudo} rm -rf ${compile_dir} + ${csudo} rm -rf ${compile_dir} fi if [ "$osType" != "Darwin" ]; then - ${csudo} mkdir -p ${compile_dir} + ${csudo} mkdir -p ${compile_dir} else - mkdir -p ${compile_dir} + mkdir -p ${compile_dir} fi cd ${compile_dir} # check support cpu type if [[ "$cpuType" == "x64" ]] || [[ "$cpuType" == "aarch64" ]] || [[ "$cpuType" == "aarch32" ]] || [[ "$cpuType" == "mips64" ]] ; then - if [ "$verMode" != "cluster" ]; then - cmake ../ -DCPUTYPE=${cpuType} -DPAGMODE=${pagMode} -DOSTYPE=${osType} - else - cmake ../../ -DCPUTYPE=${cpuType} -DOSTYPE=${osType} - fi + if [ "$verMode" != "cluster" ]; then + cmake ../ -DCPUTYPE=${cpuType} -DOSTYPE=${osType} -DSOMODE=${soMode} -DVERTYPE=${verType} -DVERDATE="${build_time}" -DGITINFO=${gitinfo} -DGITINFOI=${gitinfoOfInternal} -DVERNUMBER=${verNumber} -DVERCOMPATIBLE=${verNumberComp} -DPAGMODE=${pagMode} + else + cmake ../../ -DCPUTYPE=${cpuType} -DOSTYPE=${osType} -DSOMODE=${soMode} -DVERTYPE=${verType} -DVERDATE="${build_time}" -DGITINFO=${gitinfo} -DGITINFOI=${gitinfoOfInternal} -DVERNUMBER=${verNumber} -DVERCOMPATIBLE=${verNumberComp} + fi else - echo "input cpuType=${cpuType} error!!!" - exit 1 + echo "input cpuType=${cpuType} error!!!" + exit 1 fi make cd ${curr_dir} -# 3. judge the operating system type, then Call the corresponding script for packaging -#osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release) -#osinfo=$(cat /etc/os-release | grep "NAME" | cut -d '"' -f2) -#echo "osinfo: ${osinfo}" - +# 3. Call the corresponding script for packaging if [ "$osType" != "Darwin" ]; then - if [[ "$verMode" != "cluster" ]] && [[ "$cpuType" == "x64" ]]; then - echo "====do deb package for the ubuntu system====" - output_dir="${top_dir}/debs" - if [ -d ${output_dir} ]; then - ${csudo} rm -rf ${output_dir} - fi - ${csudo} mkdir -p ${output_dir} - cd ${script_dir}/deb - ${csudo} ./makedeb.sh ${compile_dir} ${output_dir} ${version} ${cpuType} ${osType} ${verMode} ${verType} - - echo "====do rpm package for the centos system====" - output_dir="${top_dir}/rpms" - if [ -d ${output_dir} ]; then - ${csudo} rm -rf ${output_dir} - fi - ${csudo} mkdir -p ${output_dir} - cd ${script_dir}/rpm - ${csudo} ./makerpm.sh ${compile_dir} ${output_dir} ${version} ${cpuType} ${osType} ${verMode} ${verType} + if [[ "$verMode" != "cluster" ]] && [[ "$cpuType" == "x64" ]]; then + echo "====do deb package for the ubuntu system====" + output_dir="${top_dir}/debs" + if [ -d ${output_dir} ]; then + ${csudo} rm -rf ${output_dir} fi + ${csudo} mkdir -p ${output_dir} + cd ${script_dir}/deb + ${csudo} ./makedeb.sh ${compile_dir} ${output_dir} ${verNumber} ${cpuType} ${osType} ${verMode} ${verType} + + echo "====do rpm package for the centos system====" + output_dir="${top_dir}/rpms" + if [ -d ${output_dir} ]; then + ${csudo} rm -rf ${output_dir} + fi + ${csudo} mkdir -p ${output_dir} + cd ${script_dir}/rpm + ${csudo} ./makerpm.sh ${compile_dir} ${output_dir} ${verNumber} ${cpuType} ${osType} ${verMode} ${verType} + fi - echo "====do tar.gz package for all systems====" - cd ${script_dir}/tools - - ${csudo} ./makepkg.sh ${compile_dir} ${version} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} - ${csudo} ./makeclient.sh ${compile_dir} ${version} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} + echo "====do tar.gz package for all systems====" + cd ${script_dir}/tools + + ${csudo} ./makepkg.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} + ${csudo} ./makeclient.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} + ${csudo} ./makearbi.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} else - cd ${script_dir}/tools - ./makeclient.sh ${compile_dir} ${version} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} + cd ${script_dir}/tools + ./makeclient.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} fi -# 4. Clean up temporary compile directories -#${csudo} rm -rf ${compile_dir} - diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh old mode 100644 new mode 100755 index 64de09df6d..1cbb3ead9c --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -76,7 +76,7 @@ fi # get the operating system type for using the corresponding init file # ubuntu/debian(deb), centos/fedora(rpm), others: opensuse, redhat, ..., no verification #osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release) -if [[ -d /etc/os-release ]]; then +if [[ -e /etc/os-release ]]; then osinfo=$(cat /etc/os-release | grep "NAME" | cut -d '"' -f2) ||: else osinfo="" @@ -381,34 +381,29 @@ function install_service_on_sysvinit() { function clean_service_on_systemd() { taosd_service_config="${service_config_dir}/taosd.service" - if systemctl is-active --quiet taosd; then echo "TDengine is running, stopping it..." ${csudo} systemctl stop taosd &> /dev/null || echo &> /dev/null fi ${csudo} systemctl disable taosd &> /dev/null || echo &> /dev/null - ${csudo} rm -f ${taosd_service_config} - + + tarbitratord_service_config="${service_config_dir}/tarbitratord.service" + if systemctl is-active --quiet tarbitratord; then + echo "tarbitrator is running, stopping it..." + ${csudo} systemctl stop tarbitratord &> /dev/null || echo &> /dev/null + fi + ${csudo} systemctl disable tarbitratord &> /dev/null || echo &> /dev/null + ${csudo} rm -f ${tarbitratord_service_config} + if [ "$verMode" == "cluster" ]; then nginx_service_config="${service_config_dir}/nginxd.service" - if systemctl is-active --quiet nginxd; then echo "Nginx for TDengine is running, stopping it..." ${csudo} systemctl stop nginxd &> /dev/null || echo &> /dev/null fi ${csudo} systemctl disable nginxd &> /dev/null || echo &> /dev/null - ${csudo} rm -f ${nginx_service_config} - - tarbitratord_service_config="${service_config_dir}/tarbitratord.service" - if systemctl is-active --quiet tarbitratord; then - echo "tarbitrator is running, stopping it..." - ${csudo} systemctl stop tarbitratord &> /dev/null || echo &> /dev/null - fi - ${csudo} systemctl disable tarbitratord &> /dev/null || echo &> /dev/null - - ${csudo} rm -f ${tarbitratord_service_config} fi } @@ -418,7 +413,6 @@ function install_service_on_systemd() { clean_service_on_systemd taosd_service_config="${service_config_dir}/taosd.service" - ${csudo} bash -c "echo '[Unit]' >> ${taosd_service_config}" ${csudo} bash -c "echo 'Description=TDengine server service' >> ${taosd_service_config}" ${csudo} bash -c "echo 'After=network-online.target' >> ${taosd_service_config}" @@ -439,32 +433,30 @@ function install_service_on_systemd() { ${csudo} bash -c "echo '[Install]' >> ${taosd_service_config}" ${csudo} bash -c "echo 'WantedBy=multi-user.target' >> ${taosd_service_config}" ${csudo} systemctl enable taosd - + + tarbitratord_service_config="${service_config_dir}/tarbitratord.service" + ${csudo} bash -c "echo '[Unit]' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Description=TDengine arbitrator service' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'After=network-online.target' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Wants=network-online.target' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo '[Service]' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Type=simple' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'ExecStart=/usr/bin/tarbitrator' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'LimitNOFILE=infinity' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'LimitNPROC=infinity' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'LimitCORE=infinity' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'TimeoutStartSec=0' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'StandardOutput=null' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Restart=always' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'StartLimitBurst=3' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'StartLimitInterval=60s' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo '[Install]' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'WantedBy=multi-user.target' >> ${tarbitratord_service_config}" + #${csudo} systemctl enable tarbitratord + if [ "$verMode" == "cluster" ]; then - - tarbitratord_service_config="${service_config_dir}/tarbitratord.service" - - ${csudo} bash -c "echo '[Unit]' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo 'Description=TDengine arbitrator service' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo 'After=network-online.target' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo 'Wants=network-online.target' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo '[Service]' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo 'Type=simple' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo 'ExecStart=/usr/bin/tarbitrator' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo 'LimitNOFILE=infinity' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo 'LimitNPROC=infinity' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo 'LimitCORE=infinity' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo 'TimeoutStartSec=0' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo 'StandardOutput=null' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo 'Restart=always' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo 'StartLimitBurst=3' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo 'StartLimitInterval=60s' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo '[Install]' >> ${tarbitratord_service_config}" - ${csudo} bash -c "echo 'WantedBy=multi-user.target' >> ${tarbitratord_service_config}" - #${csudo} systemctl enable tarbitratord - nginx_service_config="${service_config_dir}/nginxd.service" ${csudo} bash -c "echo '[Unit]' >> ${nginx_service_config}" ${csudo} bash -c "echo 'Description=Nginx For TDengine Service' >> ${nginx_service_config}" @@ -713,6 +705,7 @@ function install_TDengine() { echo echo -e "\033[44;32;1mTDengine client is installed successfully!${NC}" fi + touch ~/.taos_history rm -rf $(tar -tf taos.tar.gz) } diff --git a/packaging/tools/install_arbi.sh b/packaging/tools/install_arbi.sh new file mode 100755 index 0000000000..a89d2257dc --- /dev/null +++ b/packaging/tools/install_arbi.sh @@ -0,0 +1,297 @@ +#!/bin/bash +# +# This file is used to install database on linux systems. The operating system +# is required to use systemd to manage services at boot + +set -e +#set -x + +# -----------------------Variables definition--------------------- +script_dir=$(dirname $(readlink -f "$0")) + +bin_link_dir="/usr/bin" +#inc_link_dir="/usr/include" + +#install main path +install_main_dir="/usr/local/tarbitrator" + +# old bin dir +bin_dir="/usr/local/tarbitrator/bin" + +service_config_dir="/etc/systemd/system" + +# Color setting +RED='\033[0;31m' +GREEN='\033[1;32m' +GREEN_DARK='\033[0;32m' +GREEN_UNDERLINE='\033[4;32m' +NC='\033[0m' + +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo" +fi + +update_flag=0 + +initd_mod=0 +service_mod=2 +if pidof systemd &> /dev/null; then + service_mod=0 +elif $(which service &> /dev/null); then + service_mod=1 + service_config_dir="/etc/init.d" + if $(which chkconfig &> /dev/null); then + initd_mod=1 + elif $(which insserv &> /dev/null); then + initd_mod=2 + elif $(which update-rc.d &> /dev/null); then + initd_mod=3 + else + service_mod=2 + fi +else + service_mod=2 +fi + + +# get the operating system type for using the corresponding init file +# ubuntu/debian(deb), centos/fedora(rpm), others: opensuse, redhat, ..., no verification +#osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release) +if [[ -e /etc/os-release ]]; then + osinfo=$(cat /etc/os-release | grep "NAME" | cut -d '"' -f2) ||: +else + osinfo="" +fi +#echo "osinfo: ${osinfo}" +os_type=0 +if echo $osinfo | grep -qwi "ubuntu" ; then +# echo "This is ubuntu system" + os_type=1 +elif echo $osinfo | grep -qwi "debian" ; then +# echo "This is debian system" + os_type=1 +elif echo $osinfo | grep -qwi "Kylin" ; then +# echo "This is Kylin system" + os_type=1 +elif echo $osinfo | grep -qwi "centos" ; then +# echo "This is centos system" + os_type=2 +elif echo $osinfo | grep -qwi "fedora" ; then +# echo "This is fedora system" + os_type=2 +else + echo " osinfo: ${osinfo}" + echo " This is an officially unverified linux system," + echo " if there are any problems with the installation and operation, " + echo " please feel free to contact taosdata.com for support." + os_type=1 +fi + +function kill_tarbitrator() { + pid=$(ps -ef | grep "tarbitrator" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo} kill -9 $pid || : + fi +} + +function install_main_path() { + #create install main dir and all sub dir + ${csudo} rm -rf ${install_main_dir} || : + ${csudo} mkdir -p ${install_main_dir} + ${csudo} mkdir -p ${install_main_dir}/bin + #${csudo} mkdir -p ${install_main_dir}/include + ${csudo} mkdir -p ${install_main_dir}/init.d +} + +function install_bin() { + # Remove links + ${csudo} rm -f ${bin_link_dir}/rmtarbitrator || : + ${csudo} rm -f ${bin_link_dir}/tarbitrator || : + ${csudo} cp -r ${script_dir}/bin/* ${install_main_dir}/bin && ${csudo} chmod 0555 ${install_main_dir}/bin/* + + #Make link + [ -x ${install_main_dir}/bin/remove_arbi.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove_arbi.sh ${bin_link_dir}/rmtarbitrator || : + [ -x ${install_main_dir}/bin/tarbitrator ] && ${csudo} ln -s ${install_main_dir}/bin/tarbitrator ${bin_link_dir}/tarbitrator || : +} + +function install_header() { + ${csudo} rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taoserror.h || : + ${csudo} cp -f ${script_dir}/inc/* ${install_main_dir}/include && ${csudo} chmod 644 ${install_main_dir}/include/* + ${csudo} ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h + ${csudo} ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h +} + +function clean_service_on_sysvinit() { + #restart_config_str="taos:2345:respawn:${service_config_dir}/taosd start" + #${csudo} sed -i "\|${restart_config_str}|d" /etc/inittab || : + + if pidof tarbitrator &> /dev/null; then + ${csudo} service tarbitratord stop || : + fi + + if ((${initd_mod}==1)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} chkconfig --del tarbitratord || : + fi + elif ((${initd_mod}==2)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} insserv -r tarbitratord || : + fi + elif ((${initd_mod}==3)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} update-rc.d -f tarbitratord remove || : + fi + fi + + ${csudo} rm -f ${service_config_dir}/tarbitratord || : + + if $(which init &> /dev/null); then + ${csudo} init q || : + fi +} + +function install_service_on_sysvinit() { + clean_service_on_sysvinit + sleep 1 + + # Install taosd service + + if ((${os_type}==1)); then + ${csudo} cp -f ${script_dir}/init.d/tarbitratord.deb ${install_main_dir}/init.d/tarbitratord + ${csudo} cp ${script_dir}/init.d/tarbitratord.deb ${service_config_dir}/tarbitratord && ${csudo} chmod a+x ${service_config_dir}/tarbitratord + elif ((${os_type}==2)); then + ${csudo} cp -f ${script_dir}/init.d/tarbitratord.rpm ${install_main_dir}/init.d/tarbitratord + ${csudo} cp ${script_dir}/init.d/tarbitratord.rpm ${service_config_dir}/tarbitratord && ${csudo} chmod a+x ${service_config_dir}/tarbitratord + fi + + #restart_config_str="taos:2345:respawn:${service_config_dir}/taosd start" + #${csudo} grep -q -F "$restart_config_str" /etc/inittab || ${csudo} bash -c "echo '${restart_config_str}' >> /etc/inittab" + + if ((${initd_mod}==1)); then + ${csudo} chkconfig --add tarbitratord || : + ${csudo} chkconfig --level 2345 tarbitratord on || : + elif ((${initd_mod}==2)); then + ${csudo} insserv tarbitratord || : + ${csudo} insserv -d tarbitratord || : + elif ((${initd_mod}==3)); then + ${csudo} update-rc.d tarbitratord defaults || : + fi +} + +function clean_service_on_systemd() { + tarbitratord_service_config="${service_config_dir}/tarbitratord.service" + if systemctl is-active --quiet tarbitratord; then + echo "tarbitrator is running, stopping it..." + ${csudo} systemctl stop tarbitratord &> /dev/null || echo &> /dev/null + fi + ${csudo} systemctl disable tarbitratord &> /dev/null || echo &> /dev/null + + ${csudo} rm -f ${tarbitratord_service_config} +} + +# taos:2345:respawn:/etc/init.d/tarbitratord start + +function install_service_on_systemd() { + clean_service_on_systemd + + tarbitratord_service_config="${service_config_dir}/tarbitratord.service" + + ${csudo} bash -c "echo '[Unit]' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Description=TDengine arbitrator service' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'After=network-online.target' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Wants=network-online.target' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo '[Service]' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Type=simple' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'ExecStart=/usr/bin/tarbitrator' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'LimitNOFILE=infinity' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'LimitNPROC=infinity' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'LimitCORE=infinity' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'TimeoutStartSec=0' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'StandardOutput=null' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'Restart=always' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'StartLimitBurst=3' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'StartLimitInterval=60s' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo '[Install]' >> ${tarbitratord_service_config}" + ${csudo} bash -c "echo 'WantedBy=multi-user.target' >> ${tarbitratord_service_config}" + ${csudo} systemctl enable tarbitratord +} + +function install_service() { + if ((${service_mod}==0)); then + install_service_on_systemd + elif ((${service_mod}==1)); then + install_service_on_sysvinit + else + # must manual stop taosd + kill_tarbitrator + fi +} + +function update_TDengine() { + # Start to update + echo -e "${GREEN}Start to update TDengine's arbitrator ...${NC}" + # Stop the service if running + if pidof tarbitrator &> /dev/null; then + if ((${service_mod}==0)); then + ${csudo} systemctl stop tarbitratord || : + elif ((${service_mod}==1)); then + ${csudo} service tarbitratord stop || : + else + kill_tarbitrator + fi + sleep 1 + fi + + install_main_path + #install_header + install_bin + install_service + + echo + #echo -e "${GREEN_DARK}To configure TDengine ${NC}: edit /etc/taos/taos.cfg" + if ((${service_mod}==0)); then + echo -e "${GREEN_DARK}To start arbitrator ${NC}: ${csudo} systemctl start tarbitratord${NC}" + elif ((${service_mod}==1)); then + echo -e "${GREEN_DARK}To start arbitrator ${NC}: ${csudo} service tarbitratord start${NC}" + else + echo -e "${GREEN_DARK}To start arbitrator ${NC}: ./tarbitrator${NC}" + fi + echo + echo -e "\033[44;32;1mTDengine's arbitrator is updated successfully!${NC}" +} + +function install_TDengine() { + # Start to install + echo -e "${GREEN}Start to install TDengine's arbitrator ...${NC}" + + install_main_path + #install_header + install_bin + install_service + echo + #echo -e "${GREEN_DARK}To configure TDengine ${NC}: edit /etc/taos/taos.cfg" + if ((${service_mod}==0)); then + echo -e "${GREEN_DARK}To start arbitrator ${NC}: ${csudo} systemctl start tarbitratord${NC}" + elif ((${service_mod}==1)); then + echo -e "${GREEN_DARK}To start arbitrator ${NC}: ${csudo} service tarbitratord start${NC}" + else + echo -e "${GREEN_DARK}To start arbitrator ${NC}: tarbitrator${NC}" + fi + + echo -e "\033[44;32;1mTDengine's arbitrator is installed successfully!${NC}" + echo +} + + +## ==============================Main program starts from here============================ +# Install server and client +if [ -x ${bin_dir}/tarbitrator ]; then + update_flag=1 + update_TDengine +else + install_TDengine +fi + diff --git a/packaging/tools/makearbi.sh b/packaging/tools/makearbi.sh new file mode 100755 index 0000000000..bc6179eff2 --- /dev/null +++ b/packaging/tools/makearbi.sh @@ -0,0 +1,75 @@ +#!/bin/bash +# +# Generate arbitrator's tar.gz setup package for all os system + +set -e +#set -x + +curr_dir=$(pwd) +compile_dir=$1 +version=$2 +build_time=$3 +cpuType=$4 +osType=$5 +verMode=$6 +verType=$7 +pagMode=$8 + +script_dir="$(dirname $(readlink -f $0))" +top_dir="$(readlink -f ${script_dir}/../..)" + +# create compressed install file. +build_dir="${compile_dir}/build" +code_dir="${top_dir}/src" +release_dir="${top_dir}/release" + +#package_name='linux' +if [ "$verMode" == "cluster" ]; then + install_dir="${release_dir}/TDengine-enterprise-arbitrator" +else + install_dir="${release_dir}/TDengine-arbitrator" +fi + +# Directories and files. +bin_files="${build_dir}/bin/tarbitrator ${script_dir}/remove_arbi.sh" +install_files="${script_dir}/install_arbi.sh" + +#header_files="${code_dir}/inc/taos.h ${code_dir}/inc/taoserror.h" +init_file_tarbitrator_deb=${script_dir}/../deb/tarbitratord +init_file_tarbitrator_rpm=${script_dir}/../rpm/tarbitratord + +# make directories. +mkdir -p ${install_dir} && cp ${install_files} ${install_dir} && chmod a+x ${install_dir}/install_arbi.sh || : +#mkdir -p ${install_dir}/inc && cp ${header_files} ${install_dir}/inc || : +mkdir -p ${install_dir}/bin && cp ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* || : +mkdir -p ${install_dir}/init.d && cp ${init_file_tarbitrator_deb} ${install_dir}/init.d/tarbitratord.deb || : +mkdir -p ${install_dir}/init.d && cp ${init_file_tarbitrator_rpm} ${install_dir}/init.d/tarbitratord.rpm || : + +cd ${release_dir} + +if [ "$verMode" == "cluster" ]; then + pkg_name=${install_dir}-${version}-${osType}-${cpuType} +elif [ "$verMode" == "edge" ]; then + pkg_name=${install_dir}-${version}-${osType}-${cpuType} +else + echo "unknow verMode, nor cluster or edge" + exit 1 +fi + +if [ "$verType" == "beta" ]; then + pkg_name=${pkg_name}-${verType} +elif [ "$verType" == "stable" ]; then + pkg_name=${pkg_name} +else + echo "unknow verType, nor stabel or beta" + exit 1 +fi + +tar -zcv -f "$(basename ${pkg_name}).tar.gz" $(basename ${install_dir}) --remove-files || : +exitcode=$? +if [ "$exitcode" != "0" ]; then + echo "tar ${pkg_name}.tar.gz error !!!" + exit $exitcode +fi + +cd ${curr_dir} diff --git a/packaging/tools/remove.sh b/packaging/tools/remove.sh index c775f98b83..dd6fe65eb7 100755 --- a/packaging/tools/remove.sh +++ b/packaging/tools/remove.sh @@ -69,10 +69,12 @@ function kill_tarbitrator() { } function clean_bin() { # Remove link - ${csudo} rm -f ${bin_link_dir}/taos || : - ${csudo} rm -f ${bin_link_dir}/taosd || : - ${csudo} rm -f ${bin_link_dir}/taosdemo || : - ${csudo} rm -f ${bin_link_dir}/rmtaos || : + ${csudo} rm -f ${bin_link_dir}/taos || : + ${csudo} rm -f ${bin_link_dir}/taosd || : + ${csudo} rm -f ${bin_link_dir}/taosdemo || : + ${csudo} rm -f ${bin_link_dir}/rmtaos || : + ${csudo} rm -f ${bin_link_dir}/tarbitrator || : + ${csudo} rm -f ${bin_link_dir}/set_core || : } function clean_lib() { @@ -207,7 +209,7 @@ ${csudo} rm -rf ${data_link_dir} || : ${csudo} rm -rf ${install_main_dir} ${csudo} rm -rf ${install_nginxd_dir} -if [[ -d /etc/os-release ]]; then +if [[ -e /etc/os-release ]]; then osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release) else osinfo="" diff --git a/packaging/tools/remove_arbi.sh b/packaging/tools/remove_arbi.sh new file mode 100755 index 0000000000..68fd9275fb --- /dev/null +++ b/packaging/tools/remove_arbi.sh @@ -0,0 +1,129 @@ +#!/bin/bash +# +# Script to stop the service and uninstall TDengine's arbitrator + +set -e +#set -x + +verMode=edge + +RED='\033[0;31m' +GREEN='\033[1;32m' +NC='\033[0m' + +#install main path +install_main_dir="/usr/local/tarbitrator" +bin_link_dir="/usr/bin" +#inc_link_dir="/usr/include" + +service_config_dir="/etc/systemd/system" +tarbitrator_service_name="tarbitratord" +csudo="" +if command -v sudo > /dev/null; then + csudo="sudo" +fi + +initd_mod=0 +service_mod=2 +if pidof systemd &> /dev/null; then + service_mod=0 +elif $(which service &> /dev/null); then + service_mod=1 + service_config_dir="/etc/init.d" + if $(which chkconfig &> /dev/null); then + initd_mod=1 + elif $(which insserv &> /dev/null); then + initd_mod=2 + elif $(which update-rc.d &> /dev/null); then + initd_mod=3 + else + service_mod=2 + fi +else + service_mod=2 +fi + +function kill_tarbitrator() { + pid=$(ps -ef | grep "tarbitrator" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo} kill -9 $pid || : + fi +} +function clean_bin() { + # Remove link + ${csudo} rm -f ${bin_link_dir}/tarbitrator || : +} + +function clean_header() { + # Remove link + ${csudo} rm -f ${inc_link_dir}/taos.h || : + ${csudo} rm -f ${inc_link_dir}/taoserror.h || : +} + +function clean_log() { + # Remove link + ${csudo} rm -rf /arbitrator.log || : +} + +function clean_service_on_systemd() { + tarbitratord_service_config="${service_config_dir}/${tarbitrator_service_name}.service" + + if systemctl is-active --quiet ${tarbitrator_service_name}; then + echo "TDengine tarbitrator is running, stopping it..." + ${csudo} systemctl stop ${tarbitrator_service_name} &> /dev/null || echo &> /dev/null + fi + ${csudo} systemctl disable ${tarbitrator_service_name} &> /dev/null || echo &> /dev/null + + ${csudo} rm -f ${tarbitratord_service_config} +} + +function clean_service_on_sysvinit() { + if pidof tarbitrator &> /dev/null; then + echo "TDengine's tarbitrator is running, stopping it..." + ${csudo} service tarbitratord stop || : + fi + + if ((${initd_mod}==1)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} chkconfig --del tarbitratord || : + fi + elif ((${initd_mod}==2)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} insserv -r tarbitratord || : + fi + elif ((${initd_mod}==3)); then + if [ -e ${service_config_dir}/tarbitratord ]; then + ${csudo} update-rc.d -f tarbitratord remove || : + fi + fi + + ${csudo} rm -f ${service_config_dir}/tarbitratord || : + + if $(which init &> /dev/null); then + ${csudo} init q || : + fi +} + +function clean_service() { + if ((${service_mod}==0)); then + clean_service_on_systemd + elif ((${service_mod}==1)); then + clean_service_on_sysvinit + else + # must manual stop + kill_tarbitrator + fi +} + +# Stop service and disable booting start. +clean_service +# Remove binary file and links +clean_bin +# Remove header file. +##clean_header +# Remove log file +clean_log + +${csudo} rm -rf ${install_main_dir} + +echo -e "${GREEN}TDengine's arbitrator is removed successfully!${NC}" diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index d5cbf3cab6..5b5fb3435d 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -35,12 +35,14 @@ IF (TD_LINUX) ELSEIF (TD_WINDOWS) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/jni/windows) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/jni/windows/win32) - + + CONFIGURE_FILE("${TD_COMMUNITY_DIR}/src/client/src/taos.rc.in" "${TD_COMMUNITY_DIR}/src/client/src/taos.rc") + ADD_LIBRARY(taos_static STATIC ${SRC}) TARGET_LINK_LIBRARIES(taos_static trpc tutil query) # generate dynamic library (*.dll) - ADD_LIBRARY(taos SHARED ${SRC}) + ADD_LIBRARY(taos SHARED ${SRC} ${TD_COMMUNITY_DIR}/src/client/src/taos.rc) IF (NOT TD_GODLL) SET_TARGET_PROPERTIES(taos PROPERTIES LINK_FLAGS /DEF:${TD_COMMUNITY_DIR}/src/client/src/taos.def) ENDIF () diff --git a/src/client/inc/tscLocalMerge.h b/src/client/inc/tscLocalMerge.h index 397a60d140..4e579b0729 100644 --- a/src/client/inc/tscLocalMerge.h +++ b/src/client/inc/tscLocalMerge.h @@ -87,7 +87,6 @@ typedef struct SRetrieveSupport { SSqlObj * pParentSql; tFilePage * localBuffer; // temp buffer, there is a buffer for each vnode to uint32_t numOfRetry; // record the number of retry times - pthread_mutex_t queryMutex; } SRetrieveSupport; int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOrderDescriptor **pDesc, diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 2ca6ba6691..f77897a74b 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -108,7 +108,7 @@ int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOff void tscDestroyDataBlock(STableDataBlocks* pDataBlock); void tscSortRemoveDataBlockDupRows(STableDataBlocks* dataBuf); -SParamInfo* tscAddParamToDataBlock(STableDataBlocks* pDataBlock, char type, uint8_t timePrec, short bytes, +SParamInfo* tscAddParamToDataBlock(STableDataBlocks* pDataBlock, char type, uint8_t timePrec, int16_t bytes, uint32_t offset); void* tscDestroyBlockArrayList(SArray* pDataBlockList); @@ -138,10 +138,10 @@ bool tscIsProjectionQuery(SQueryInfo* pQueryInfo); bool tscIsTwoStageSTableQuery(SQueryInfo* pQueryInfo, int32_t tableIndex); bool tscQueryTags(SQueryInfo* pQueryInfo); -void tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId, SColumnIndex* pIndex, - SSchema* pColSchema, int16_t colType); +SSqlExpr* tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId, + SColumnIndex* pIndex, SSchema* pColSchema, int16_t colType); -int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SSQLToken* pzTableName, SSqlObj* pSql); +int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SStrToken* pzTableName, SSqlObj* pSql); void tscClearInterpInfo(SQueryInfo* pQueryInfo); bool tscIsInsertData(char* sqlstr); @@ -194,11 +194,11 @@ SColumn* tscColumnListInsert(SArray* pColList, SColumnIndex* colIndex); SArray* tscColumnListClone(const SArray* src, int16_t tableIndex); void tscColumnListDestroy(SArray* pColList); -int32_t tscValidateName(SSQLToken* pToken); +int32_t tscValidateName(SStrToken* pToken); void tscIncStreamExecutionCount(void* pStream); -bool tscValidateColumnId(STableMetaInfo* pTableMetaInfo, int32_t colId); +bool tscValidateColumnId(STableMetaInfo* pTableMetaInfo, int32_t colId, int32_t numOfParams); // get starter position of metric query condition (query on tags) in SSqlCmd.payload SCond* tsGetSTableQueryCond(STagCond* pCond, uint64_t uid); @@ -217,7 +217,7 @@ STableMetaInfo* tscGetTableMetaInfoFromCmd(SSqlCmd *pCmd, int32_t subClauseIndex STableMetaInfo* tscGetMetaInfo(SQueryInfo *pQueryInfo, int32_t tableIndex); SQueryInfo *tscGetQueryInfoDetail(SSqlCmd* pCmd, int32_t subClauseIndex); -int32_t tscGetQueryInfoDetailSafely(SSqlCmd *pCmd, int32_t subClauseIndex, SQueryInfo** pQueryInfo); +SQueryInfo *tscGetQueryInfoDetailSafely(SSqlCmd *pCmd, int32_t subClauseIndex); void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache); diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 6d02bc7fbd..d2c52e972a 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -35,6 +35,40 @@ extern "C" { #include "qTsbuf.h" #include "tcmdtype.h" +#if 0 +static UNUSED_FUNC void *u_malloc (size_t __size) { + uint32_t v = rand(); + + if (v % 5000 <= 0) { + return NULL; + } else { + return malloc(__size); + } +} + +static UNUSED_FUNC void* u_calloc(size_t num, size_t __size) { + uint32_t v = rand(); + if (v % 5000 <= 0) { + return NULL; + } else { + return calloc(num, __size); + } +} + +static UNUSED_FUNC void* u_realloc(void* p, size_t __size) { + uint32_t v = rand(); + if (v % 5000 <= 0) { + return NULL; + } else { + return realloc(p, __size); + } +} + +#define calloc u_calloc +#define malloc u_malloc +#define realloc u_realloc +#endif + // forward declaration struct SSqlInfo; struct SLocalReducer; @@ -195,9 +229,9 @@ typedef struct STableDataBlocks { typedef struct SQueryInfo { int16_t command; // the command may be different for each subclause, so keep it seperately. - uint32_t type; // query/insert/import type + uint32_t type; // query/insert type char slidingTimeUnit; - STimeWindow window; + STimeWindow window; // query time window int64_t intervalTime; // aggregation time interval int64_t slidingTime; // sliding window in mseconds SSqlGroupbyExpr groupbyExpr; // group by tags info @@ -216,6 +250,7 @@ typedef struct SQueryInfo { char * msg; // pointer to the pCmd->payload to keep error message temporarily int64_t clauseLimit; // limit for current sub clause int64_t prjOffset; // offset value in the original sql expression, only applied at client side + int32_t udColumnId; // current user-defined constant output field column id, monotonically decreases from TSDB_UD_COLUMN_INDEX } SQueryInfo; typedef struct { @@ -234,7 +269,7 @@ typedef struct { char * curSql; // current sql, resume position of sql after parsing paused int8_t parseFinished; - short numOfCols; + int16_t numOfCols; uint32_t allocSize; char * payload; int32_t payloadLen; @@ -431,31 +466,36 @@ static FORCE_INLINE void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pField int32_t bytes = pInfo->pSqlExpr->resBytes; char* pData = pRes->data + pInfo->pSqlExpr->offset * pRes->numOfRows + bytes * pRes->row; - if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BINARY) { - int32_t realLen = varDataLen(pData); - assert(realLen <= bytes - VARSTR_HEADER_SIZE); - if (isNull(pData, type)) { - pRes->tsrow[columnIndex] = NULL; + // user defined constant value output columns + if (pInfo->pSqlExpr->colInfo.flag == TSDB_COL_UDC) { + if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BINARY) { + pData = pInfo->pSqlExpr->param[1].pz; + pRes->length[columnIndex] = pInfo->pSqlExpr->param[1].nLen; + pRes->tsrow[columnIndex] = (pInfo->pSqlExpr->param[1].nType == TSDB_DATA_TYPE_NULL) ? NULL : pData; } else { - pRes->tsrow[columnIndex] = ((tstr*)pData)->data; - } + assert(bytes == tDataTypeDesc[type].nSize); - if (realLen < pInfo->pSqlExpr->resBytes - VARSTR_HEADER_SIZE) { // todo refactor - *(pData + realLen + VARSTR_HEADER_SIZE) = 0; + pRes->tsrow[columnIndex] = isNull(pData, type) ? NULL : &pInfo->pSqlExpr->param[1].i64Key; + pRes->length[columnIndex] = bytes; } - - pRes->length[columnIndex] = realLen; } else { - assert(bytes == tDataTypeDesc[type].nSize); + if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BINARY) { + int32_t realLen = varDataLen(pData); + assert(realLen <= bytes - VARSTR_HEADER_SIZE); - if (isNull(pData, type)) { - pRes->tsrow[columnIndex] = NULL; + pRes->tsrow[columnIndex] = (isNull(pData, type)) ? NULL : ((tstr *)pData)->data; + if (realLen < pInfo->pSqlExpr->resBytes - VARSTR_HEADER_SIZE) { // todo refactor + *(pData + realLen + VARSTR_HEADER_SIZE) = 0; + } + + pRes->length[columnIndex] = realLen; } else { - pRes->tsrow[columnIndex] = pData; - } + assert(bytes == tDataTypeDesc[type].nSize); - pRes->length[columnIndex] = bytes; + pRes->tsrow[columnIndex] = isNull(pData, type) ? NULL : pData; + pRes->length[columnIndex] = bytes; + } } } diff --git a/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h b/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h index d81bbd9b6d..eaea91d1bf 100644 --- a/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h +++ b/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h @@ -89,6 +89,14 @@ JNIEXPORT jstring JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getErrMsgImp JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getResultSetImp (JNIEnv *env, jobject jobj, jlong con, jlong tres); +/* + * Class: com_taosdata_jdbc_TSDBJNIConnector + * Method: isUpdateQueryImp + * Signature: (J)J + */ +JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_isUpdateQueryImp + (JNIEnv *env, jobject jobj, jlong con, jlong tres); + /* * Class: com_taosdata_jdbc_TSDBJNIConnector * Method: freeResultSetImp diff --git a/src/client/src/taos.rc.in b/src/client/src/taos.rc.in new file mode 100644 index 0000000000..751be85fd0 --- /dev/null +++ b/src/client/src/taos.rc.in @@ -0,0 +1,31 @@ +1 VERSIONINFO + FILEVERSION ${TD_VER_NUMBER} + PRODUCTVERSION ${TD_VER_NUMBER} + FILEFLAGSMASK 0x17L +#ifdef _DEBUG + FILEFLAGS 0x1L +#else + FILEFLAGS 0x0L +#endif + FILEOS 0x4L + FILETYPE 0x0L + FILESUBTYPE 0x0L +BEGIN + BLOCK "StringFileInfo" + BEGIN + BLOCK "040904b0" + BEGIN + VALUE "FileDescription", "Native C Driver for TDengine" + VALUE "FileVersion", "${TD_VER_NUMBER}" + VALUE "InternalName", "taos.dll(${TD_VER_CPUTYPE})" + VALUE "LegalCopyright", "Copyright (C) 2020 TAOS Data" + VALUE "OriginalFilename", "" + VALUE "ProductName", "taos.dll(${TD_VER_CPUTYPE})" + VALUE "ProductVersion", "${TD_VER_NUMBER}" + END + END + BLOCK "VarFileInfo" + BEGIN + VALUE "Translation", 0x409, 1200 + END +END \ No newline at end of file diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index b05aef76eb..4643d255dc 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -433,7 +433,8 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { tscError("%p get tableMeta failed, code:%s", pSql, tstrerror(code)); goto _error; } else { - tscDebug("%p get tableMeta successfully", pSql); + const char* msg = (pCmd->command == TSDB_SQL_STABLEVGROUP)? "vgroup-list":"table-meta"; + tscDebug("%p get %s successfully", pSql, msg); } if (pSql->pStream == NULL) { diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index 04c6ef47a5..e74fcba246 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -1813,23 +1813,19 @@ static void last_dist_func_second_merge(SQLFunctionCtx *pCtx) { * NOTE: last_row does not use the interResultBuf to keep the result */ static void last_row_function(SQLFunctionCtx *pCtx) { - assert(pCtx->size == 1); - + assert(pCtx->size >= 1); char *pData = GET_INPUT_CHAR(pCtx); - assignVal(pCtx->aOutputBuf, pData, pCtx->inputBytes, pCtx->inputType); + + // assign the last element in current data block + assignVal(pCtx->aOutputBuf, pData + (pCtx->size - 1) * pCtx->inputBytes, pCtx->inputBytes, pCtx->inputType); SResultInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; - SLastrowInfo *pInfo = (SLastrowInfo *)pResInfo->interResultBuf; - pInfo->ts = pCtx->ptsList[0]; - - pInfo->hasResult = DATA_SET_FLAG; - - // set the result to final result buffer + // set the result to final result buffer in case of super table query if (pResInfo->superTableQ) { SLastrowInfo *pInfo1 = (SLastrowInfo *)(pCtx->aOutputBuf + pCtx->inputBytes); - pInfo1->ts = pCtx->ptsList[0]; + pInfo1->ts = pCtx->ptsList[pCtx->size - 1]; pInfo1->hasResult = DATA_SET_FLAG; DO_UPDATE_TAG_COLUMNS(pCtx, pInfo1->ts); @@ -2038,7 +2034,7 @@ static void copyTopBotRes(SQLFunctionCtx *pCtx, int32_t type) { tValuePair **tvp = pRes->res; int32_t step = QUERY_ASC_FORWARD_STEP; - int32_t len = GET_RES_INFO(pCtx)->numOfRes; + int32_t len = (int32_t)(GET_RES_INFO(pCtx)->numOfRes); switch (type) { case TSDB_DATA_TYPE_INT: { @@ -2412,10 +2408,10 @@ static void top_bottom_func_finalizer(SQLFunctionCtx *pCtx) { // user specify the order of output by sort the result according to timestamp if (pCtx->param[1].i64Key == PRIMARYKEY_TIMESTAMP_COL_INDEX) { __compar_fn_t comparator = (pCtx->param[2].i64Key == TSDB_ORDER_ASC) ? resAscComparFn : resDescComparFn; - qsort(tvp, pResInfo->numOfRes, POINTER_BYTES, comparator); + qsort(tvp, (size_t)pResInfo->numOfRes, POINTER_BYTES, comparator); } else if (pCtx->param[1].i64Key > PRIMARYKEY_TIMESTAMP_COL_INDEX) { __compar_fn_t comparator = (pCtx->param[2].i64Key == TSDB_ORDER_ASC) ? resDataAscComparFn : resDataDescComparFn; - qsort(tvp, pResInfo->numOfRes, POINTER_BYTES, comparator); + qsort(tvp, (size_t)pResInfo->numOfRes, POINTER_BYTES, comparator); } GET_TRUE_DATA_TYPE(); @@ -2909,33 +2905,41 @@ static FORCE_INLINE void date_col_output_function_f(SQLFunctionCtx *pCtx, int32_ } static void col_project_function(SQLFunctionCtx *pCtx) { + // the number of output rows should not affect the final number of rows, so set it to be 0 + if (pCtx->numOfParams == 2) { + return; + } + INC_INIT_VAL(pCtx, pCtx->size); - + char *pData = GET_INPUT_CHAR(pCtx); if (pCtx->order == TSDB_ORDER_ASC) { - memcpy(pCtx->aOutputBuf, pData, (size_t)pCtx->size * pCtx->inputBytes); + memcpy(pCtx->aOutputBuf, pData, (size_t) pCtx->size * pCtx->inputBytes); } else { for(int32_t i = 0; i < pCtx->size; ++i) { memcpy(pCtx->aOutputBuf + (pCtx->size - 1 - i) * pCtx->inputBytes, pData + i * pCtx->inputBytes, pCtx->inputBytes); } } - + pCtx->aOutputBuf += pCtx->size * pCtx->outputBytes; } static void col_project_function_f(SQLFunctionCtx *pCtx, int32_t index) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); - + if (pCtx->numOfParams == 2) { // the number of output rows should not affect the final number of rows, so set it to be 0 + return; + } + // only one output if (pCtx->param[0].i64Key == 1 && pResInfo->numOfRes >= 1) { return; } - + INC_INIT_VAL(pCtx, 1); char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); - + pCtx->aOutputBuf += pCtx->inputBytes; } @@ -3903,11 +3907,11 @@ static void ts_comp_function(SQLFunctionCtx *pCtx) { // primary ts must be existed, so no need to check its existance if (pCtx->order == TSDB_ORDER_ASC) { - tsBufAppend(pTSbuf, 0, pCtx->tag.i64Key, input, pCtx->size * TSDB_KEYSIZE); + tsBufAppend(pTSbuf, 0, &pCtx->tag, input, pCtx->size * TSDB_KEYSIZE); } else { for (int32_t i = pCtx->size - 1; i >= 0; --i) { char *d = GET_INPUT_CHAR_INDEX(pCtx, i); - tsBufAppend(pTSbuf, 0, pCtx->tag.i64Key, d, TSDB_KEYSIZE); + tsBufAppend(pTSbuf, 0, &pCtx->tag, d, TSDB_KEYSIZE); } } @@ -3926,7 +3930,7 @@ static void ts_comp_function_f(SQLFunctionCtx *pCtx, int32_t index) { STSBuf *pTSbuf = pInfo->pTSBuf; - tsBufAppend(pTSbuf, 0, pCtx->tag.i64Key, pData, TSDB_KEYSIZE); + tsBufAppend(pTSbuf, 0, &pCtx->tag, pData, TSDB_KEYSIZE); SET_VAL(pCtx, pCtx->size, 1); pResInfo->hasResult = DATA_SET_FLAG; diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c index 9dc2210ab6..563c9fa84e 100644 --- a/src/client/src/tscLocalMerge.c +++ b/src/client/src/tscLocalMerge.c @@ -370,7 +370,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); - TSKEY stime = MIN(pQueryInfo->window.skey, pQueryInfo->window.ekey); + TSKEY stime = (pQueryInfo->order.order == TSDB_ORDER_ASC)? pQueryInfo->window.skey : pQueryInfo->window.ekey; int64_t revisedSTime = taosGetIntervalStartTimestamp(stime, pQueryInfo->slidingTime, pQueryInfo->intervalTime, pQueryInfo->slidingTimeUnit, tinfo.precision); @@ -843,28 +843,6 @@ void savePrevRecordAndSetupFillInfo(SLocalReducer *pLocalReducer, SQueryInfo *pQ tColModelAppend(pModel, pLocalReducer->discardData, pLocalReducer->prevRowOfInput, 0, 1, 1); } -static void reversedCopyFromInterpolationToDstBuf(SQueryInfo *pQueryInfo, SSqlRes *pRes, tFilePage **pResPages, - SLocalReducer *pLocalReducer) { - assert(0); - size_t size = tscSqlExprNumOfExprs(pQueryInfo); - - for (int32_t i = 0; i < size; ++i) { - TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); - - int32_t offset = tscFieldInfoGetOffset(pQueryInfo, i); - assert(offset == getColumnModelOffset(pLocalReducer->resColModel, i)); - - char *src = pResPages[i]->data + (pRes->numOfRows - 1) * pField->bytes; - char *dst = pRes->data + pRes->numOfRows * offset; - - for (int32_t j = 0; j < pRes->numOfRows; ++j) { - memcpy(dst, src, (size_t)pField->bytes); - dst += pField->bytes; - src -= pField->bytes; - } - } -} - static void genFinalResWithoutFill(SSqlRes* pRes, SLocalReducer *pLocalReducer, SQueryInfo* pQueryInfo) { assert(pQueryInfo->intervalTime == 0 || pQueryInfo->fillType == TSDB_FILL_NONE); @@ -925,7 +903,8 @@ static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneO SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); SFillInfo *pFillInfo = pLocalReducer->pFillInfo; - int64_t actualETime = MAX(pQueryInfo->window.skey, pQueryInfo->window.ekey); + // todo extract function + int64_t actualETime = (pQueryInfo->order.order == TSDB_ORDER_ASC)? pQueryInfo->window.ekey: pQueryInfo->window.skey; tFilePage **pResPages = malloc(POINTER_BYTES * pQueryInfo->fieldsInfo.numOfOutput); for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { @@ -984,14 +963,10 @@ static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneO savePrevRecordAndSetupFillInfo(pLocalReducer, pQueryInfo, pFillInfo); } - if (pQueryInfo->order.order == TSDB_ORDER_ASC) { - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { - TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); - int16_t offset = getColumnModelOffset(pLocalReducer->resColModel, i); - memcpy(pRes->data + offset * pRes->numOfRows, pResPages[i]->data, (size_t)(pField->bytes * pRes->numOfRows)); - } - } else { // todo bug?? - reversedCopyFromInterpolationToDstBuf(pQueryInfo, pRes, pResPages, pLocalReducer); + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); + int16_t offset = getColumnModelOffset(pLocalReducer->resColModel, i); + memcpy(pRes->data + offset * pRes->numOfRows, pResPages[i]->data, (size_t)(pField->bytes * pRes->numOfRows)); } pRes->numOfRowsGroup += pRes->numOfRows; @@ -1248,8 +1223,6 @@ bool genFinalResults(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool noMoreCur printf("final result before interpo:\n"); // tColModelDisplay(pLocalReducer->resColModel, pLocalReducer->pBufForInterpo, pResBuf->num, pResBuf->num); #endif - - // no interval query, no fill operation if (pQueryInfo->intervalTime == 0 || pQueryInfo->fillType == TSDB_FILL_NONE) { @@ -1257,7 +1230,9 @@ bool genFinalResults(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool noMoreCur } else { SFillInfo* pFillInfo = pLocalReducer->pFillInfo; if (pFillInfo != NULL) { - taosFillSetStartInfo(pFillInfo, (int32_t)pResBuf->num, pQueryInfo->window.ekey); + TSKEY ekey = (pQueryInfo->order.order == TSDB_ORDER_ASC)? pQueryInfo->window.ekey: pQueryInfo->window.skey; + + taosFillSetStartInfo(pFillInfo, (int32_t)pResBuf->num, ekey); taosFillCopyInputDataFromOneFilePage(pFillInfo, pResBuf); } @@ -1292,7 +1267,7 @@ static void resetEnvForNewResultset(SSqlRes *pRes, SSqlCmd *pCmd, SLocalReducer // for group result interpolation, do not return if not data is generated if (pQueryInfo->fillType != TSDB_FILL_NONE) { - TSKEY skey = MIN(pQueryInfo->window.skey, pQueryInfo->window.ekey); + TSKEY skey = (pQueryInfo->order.order == TSDB_ORDER_ASC)? pQueryInfo->window.skey:pQueryInfo->window.ekey;//MIN(pQueryInfo->window.skey, pQueryInfo->window.ekey); int64_t newTime = taosGetIntervalStartTimestamp(skey, pQueryInfo->slidingTime, pQueryInfo->intervalTime, pQueryInfo->slidingTimeUnit, precision); taosResetFillInfo(pLocalReducer->pFillInfo, newTime); @@ -1345,7 +1320,7 @@ static bool doHandleLastRemainData(SSqlObj *pSql) { // if fillType == TSDB_FILL_NONE, return directly if (pQueryInfo->fillType != TSDB_FILL_NONE && ((pRes->numOfRowsGroup < pQueryInfo->limit.limit && pQueryInfo->limit.limit > 0) || (pQueryInfo->limit.limit < 0))) { - int64_t etime = (pQueryInfo->window.skey < pQueryInfo->window.ekey) ? pQueryInfo->window.ekey : pQueryInfo->window.skey; + int64_t etime = (pQueryInfo->order.order == TSDB_ORDER_ASC)? pQueryInfo->window.ekey : pQueryInfo->window.skey; int32_t rows = (int32_t)getFilledNumOfRes(pFillInfo, etime, pLocalReducer->resColModel->capacity); if (rows > 0) { @@ -1402,13 +1377,12 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) { tscResetForNextRetrieve(pRes); if (pSql->signature != pSql || pRes == NULL || pRes->pLocalReducer == NULL) { // all data has been processed - tscDebug("%p %s call the drop local reducer", pSql, __FUNCTION__); - tscDestroyLocalReducer(pSql); - return 0; + tscError("%p local merge abort due to error occurs, code:%s", pSql, tstrerror(pRes->code)); + return pRes->code; } SLocalReducer *pLocalReducer = pRes->pLocalReducer; - SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); + SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); // set the data merge in progress int32_t prevStatus = @@ -1503,8 +1477,8 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) { * so the processing of previous group is completed. */ int32_t numOfRes = finalizeRes(pQueryInfo, pLocalReducer); + bool sameGroup = isSameGroup(pCmd, pLocalReducer, pLocalReducer->prevRowOfInput, tmpBuffer); - bool sameGroup = isSameGroup(pCmd, pLocalReducer, pLocalReducer->prevRowOfInput, tmpBuffer); tFilePage *pResBuf = pLocalReducer->pResultBuf; /* diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index cb49bd80b7..47bfe0fcdc 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -40,7 +40,7 @@ enum { static int32_t tscAllocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t * numOfRows); -static int32_t tscToInteger(SSQLToken *pToken, int64_t *value, char **endPtr) { +static int32_t tscToInteger(SStrToken *pToken, int64_t *value, char **endPtr) { if (pToken->n == 0) { return TK_ILLEGAL; } @@ -73,7 +73,7 @@ static int32_t tscToInteger(SSQLToken *pToken, int64_t *value, char **endPtr) { return pToken->type; } -static int32_t tscToDouble(SSQLToken *pToken, double *value, char **endPtr) { +static int32_t tscToDouble(SStrToken *pToken, double *value, char **endPtr) { if (pToken->n == 0) { return TK_ILLEGAL; } @@ -89,9 +89,9 @@ static int32_t tscToDouble(SSQLToken *pToken, double *value, char **endPtr) { return pToken->type; } -int tsParseTime(SSQLToken *pToken, int64_t *time, char **next, char *error, int16_t timePrec) { +int tsParseTime(SStrToken *pToken, int64_t *time, char **next, char *error, int16_t timePrec) { int32_t index = 0; - SSQLToken sToken; + SStrToken sToken; int64_t interval; int64_t useconds = 0; char * pTokenEnd = *next; @@ -128,7 +128,7 @@ int tsParseTime(SSQLToken *pToken, int64_t *time, char **next, char *error, int1 * time expression: * e.g., now+12a, now-5h */ - SSQLToken valueToken; + SStrToken valueToken; index = 0; sToken = tStrGetToken(pTokenEnd, &index, false, 0, NULL); pTokenEnd += index; @@ -163,7 +163,7 @@ int tsParseTime(SSQLToken *pToken, int64_t *time, char **next, char *error, int1 return TSDB_CODE_SUCCESS; } -int32_t tsParseOneColumnData(SSchema *pSchema, SSQLToken *pToken, char *payload, char *msg, char **str, bool primaryKey, +int32_t tsParseOneColumnData(SSchema *pSchema, SStrToken *pToken, char *payload, char *msg, char **str, bool primaryKey, int16_t timePrec) { int64_t iv; int32_t numType; @@ -409,7 +409,7 @@ static int32_t tsCheckTimestamp(STableDataBlocks *pDataBlocks, const char *start int tsParseOneRowData(char **str, STableDataBlocks *pDataBlocks, SSchema schema[], SParsedDataColInfo *spd, char *error, int16_t timePrec, int32_t *code, char *tmpTokenBuf) { int32_t index = 0; - SSQLToken sToken = {0}; + SStrToken sToken = {0}; char * payload = pDataBlocks->pData + pDataBlocks->size; // 1. set the parsed value from sql string @@ -524,7 +524,7 @@ static int32_t rowDataCompar(const void *lhs, const void *rhs) { int tsParseValues(char **str, STableDataBlocks *pDataBlock, STableMeta *pTableMeta, int maxRows, SParsedDataColInfo *spd, char *error, int32_t *code, char *tmpTokenBuf) { int32_t index = 0; - SSQLToken sToken; + SStrToken sToken; int16_t numOfRows = 0; @@ -734,8 +734,8 @@ static int32_t doParseInsertStatement(SSqlObj *pSql, void *pTableList, char **st static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { int32_t index = 0; - SSQLToken sToken = {0}; - SSQLToken tableToken = {0}; + SStrToken sToken = {0}; + SStrToken tableToken = {0}; int32_t code = TSDB_CODE_SUCCESS; const int32_t TABLE_INDEX = 0; @@ -993,7 +993,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { return code; } -int validateTableName(char *tblName, int len, SSQLToken* psTblToken) { +int validateTableName(char *tblName, int len, SStrToken* psTblToken) { tstrncpy(psTblToken->z, tblName, TSDB_TABLE_FNAME_LEN); psTblToken->n = len; @@ -1031,11 +1031,11 @@ int tsParseInsertSql(SSqlObj *pSql) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); assert(pQueryInfo != NULL); - STableMetaInfo *pTableMetaInfo = NULL; - if (pQueryInfo->numOfTables == 0) { - pTableMetaInfo = tscAddEmptyMetaInfo(pQueryInfo); - } else { - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + STableMetaInfo *pTableMetaInfo = (pQueryInfo->numOfTables == 0)? tscAddEmptyMetaInfo(pQueryInfo):tscGetMetaInfo(pQueryInfo, 0); + if (pTableMetaInfo == NULL) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + code = terrno; + return code; } if ((code = tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE)) != TSDB_CODE_SUCCESS) { @@ -1057,7 +1057,7 @@ int tsParseInsertSql(SSqlObj *pSql) { while (1) { int32_t index = 0; - SSQLToken sToken = tStrGetToken(str, &index, false, 0, NULL); + SStrToken sToken = tStrGetToken(str, &index, false, 0, NULL); // no data in the sql string anymore. if (sToken.n == 0) { @@ -1083,7 +1083,7 @@ int tsParseInsertSql(SSqlObj *pSql) { pCmd->curSql = sToken.z; char buf[TSDB_TABLE_FNAME_LEN]; - SSQLToken sTblToken; + SStrToken sTblToken; sTblToken.z = buf; // Check if the table name available or not if (validateTableName(sToken.z, sToken.n, &sTblToken) != TSDB_CODE_SUCCESS) { @@ -1285,15 +1285,14 @@ int tsInsertInitialCheck(SSqlObj *pSql) { int32_t index = 0; SSqlCmd *pCmd = &pSql->cmd; - SSQLToken sToken = tStrGetToken(pSql->sqlstr, &index, false, 0, NULL); + SStrToken sToken = tStrGetToken(pSql->sqlstr, &index, false, 0, NULL); assert(sToken.type == TK_INSERT || sToken.type == TK_IMPORT); pCmd->count = 0; pCmd->command = TSDB_SQL_INSERT; pSql->res.numOfRows = 0; - SQueryInfo *pQueryInfo = NULL; - tscGetQueryInfoDetailSafely(pCmd, pCmd->clauseIndex, &pQueryInfo); + SQueryInfo *pQueryInfo = tscGetQueryInfoDetailSafely(pCmd, pCmd->clauseIndex); TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_INSERT | pCmd->insertType); diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index b996dd958a..620e8ea57a 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -155,7 +155,7 @@ static int normalStmtPrepare(STscStmt* stmt) { uint32_t i = 0, start = 0; while (sql[i] != 0) { - SSQLToken token = {0}; + SStrToken token = {0}; token.n = tSQLGetToken(sql + i, &token.type); if (token.type == TK_QUESTION) { diff --git a/src/client/src/tscProfile.c b/src/client/src/tscProfile.c index f48e7b7691..6ff97e9d00 100644 --- a/src/client/src/tscProfile.c +++ b/src/client/src/tscProfile.c @@ -285,9 +285,9 @@ void tscKillConnection(STscObj *pObj) { SSqlObj *pSql = pObj->sqlList; while (pSql) { - //taosStopRpcConn(pSql->thandle); pSql = pSql->next; } + SSqlStream *pStream = pObj->streamList; while (pStream) { diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index d108349085..5ce4c7125f 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -33,8 +33,8 @@ #define DEFAULT_PRIMARY_TIMESTAMP_COL_NAME "_c0" -// -1 is tbname column index, so here use the -2 as the initial value -#define COLUMN_INDEX_INITIAL_VAL (-2) +// -1 is tbname column index, so here use the -3 as the initial value +#define COLUMN_INDEX_INITIAL_VAL (-3) #define COLUMN_INDEX_INITIALIZER \ { COLUMN_INDEX_INITIAL_VAL, COLUMN_INDEX_INITIAL_VAL } #define COLUMN_INDEX_VALIDE(index) (((index).tableIndex >= 0) && ((index).columnIndex >= TSDB_TBNAME_COLUMN_INDEX)) @@ -51,12 +51,12 @@ static int32_t setShowInfo(SSqlObj* pSql, SSqlInfo* pInfo); static char* getAccountId(SSqlObj* pSql); static bool has(tFieldList* pFieldList, int32_t startIdx, const char* name); -static void getCurrentDBName(SSqlObj* pSql, SSQLToken* pDBToken); -static bool hasSpecifyDB(SSQLToken* pTableName); +static void getCurrentDBName(SSqlObj* pSql, SStrToken* pDBToken); +static bool hasSpecifyDB(SStrToken* pTableName); static bool validateTableColumnInfo(tFieldList* pFieldList, SSqlCmd* pCmd); static bool validateTagParams(tFieldList* pTagsList, tFieldList* pFieldList, SSqlCmd* pCmd); -static int32_t setObjFullName(char* fullName, const char* account, SSQLToken* pDB, SSQLToken* tableName, int32_t* len); +static int32_t setObjFullName(char* fullName, const char* account, SStrToken* pDB, SStrToken* tableName, int32_t* len); static void getColumnName(tSQLExprItem* pItem, char* resultFieldName, int32_t nameLength); static void getRevisedName(char* resultFieldName, int32_t functionId, int32_t maxLen, char* columnName); @@ -65,11 +65,11 @@ static int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int3 static int32_t insertResultField(SQueryInfo* pQueryInfo, int32_t outputIndex, SColumnList* pIdList, int16_t bytes, int8_t type, char* fieldName, SSqlExpr* pSqlExpr); static int32_t changeFunctionID(int32_t optr, int16_t* functionId); -static int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSelection, bool isSTable); +static int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSelection, bool isSTable, bool joinQuery); static bool validateIpAddress(const char* ip, size_t size); static bool hasUnsupportFunctionsForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo); -static bool functionCompatibleCheck(SQueryInfo* pQueryInfo); +static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery); static void setColumnOffsetValueInResultset(SQueryInfo* pQueryInfo); static int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* pCmd); @@ -100,11 +100,11 @@ static bool hasTimestampForPointInterpQuery(SQueryInfo* pQueryInfo); static int32_t parseLimitClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t index, SQuerySQL* pQuerySql, SSqlObj* pSql); static int32_t parseCreateDBOptions(SSqlCmd* pCmd, SCreateDBInfo* pCreateDbSql); -static int32_t getColumnIndexByName(SSqlCmd* pCmd, const SSQLToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex); -static int32_t getTableIndexByName(SSQLToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex); +static int32_t getColumnIndexByName(SSqlCmd* pCmd, const SStrToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex); +static int32_t getTableIndexByName(SStrToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex); static int32_t optrToString(tSQLExpr* pExpr, char** exprString); -static int32_t getTableIndexImpl(SSQLToken* pTableToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex); +static int32_t getTableIndexImpl(SStrToken* pTableToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex); static int32_t doFunctionsCompatibleCheck(SSqlCmd* pCmd, SQueryInfo* pQueryInfo); static int32_t doLocalQueryProcess(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql); static int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCMCreateDbMsg* pCreate); @@ -151,7 +151,7 @@ static int setColumnFilterInfoForTimestamp(SSqlCmd* pCmd, SQueryInfo* pQueryInfo return TSDB_CODE_SUCCESS; } -static int32_t handlePassword(SSqlCmd* pCmd, SSQLToken* pPwd) { +static int32_t handlePassword(SSqlCmd* pCmd, SStrToken* pPwd) { const char* msg1 = "password can not be empty"; const char* msg2 = "name or password too long"; const char* msg3 = "password needs single quote marks enclosed"; @@ -179,20 +179,24 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { return TSDB_CODE_TSC_APP_ERROR; } - SSqlCmd* pCmd = &(pSql->cmd); - SQueryInfo* pQueryInfo = NULL; + SSqlCmd* pCmd = &pSql->cmd; + SSqlRes* pRes = &pSql->res; + int32_t code = TSDB_CODE_SUCCESS; if (!pInfo->valid) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), pInfo->pzErrMsg); } - int32_t code = tscGetQueryInfoDetailSafely(pCmd, pCmd->clauseIndex, &pQueryInfo); + SQueryInfo* pQueryInfo = tscGetQueryInfoDetailSafely(pCmd, pCmd->clauseIndex); + if (pQueryInfo == NULL) { + pRes->code = terrno; + return pRes->code; + } - STableMetaInfo* pTableMetaInfo = NULL; - if (pQueryInfo->numOfTables == 0) { - pTableMetaInfo = tscAddEmptyMetaInfo(pQueryInfo); - } else { - pTableMetaInfo = pQueryInfo->pTableMetaInfo[0]; + STableMetaInfo* pTableMetaInfo = (pQueryInfo->numOfTables == 0)? tscAddEmptyMetaInfo(pQueryInfo) : pQueryInfo->pTableMetaInfo[0]; + if (pTableMetaInfo == NULL) { + pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; + return pRes->code; } pCmd->command = pInfo->type; @@ -206,7 +210,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { const char* msg2 = "invalid name"; const char* msg3 = "param name too long"; - SSQLToken* pzName = &pInfo->pDCLInfo->a[0]; + SStrToken* pzName = &pInfo->pDCLInfo->a[0]; if ((pInfo->type != TSDB_SQL_DROP_DNODE) && (tscValidateName(pzName) != TSDB_CODE_SUCCESS)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } @@ -241,7 +245,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { case TSDB_SQL_USE_DB: { const char* msg = "invalid db name"; - SSQLToken* pToken = &pInfo->pDCLInfo->a[0]; + SStrToken* pToken = &pInfo->pDCLInfo->a[0]; if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); @@ -296,7 +300,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); } - SSQLToken* pIpAddr = &pInfo->pDCLInfo->a[0]; + SStrToken* pIpAddr = &pInfo->pDCLInfo->a[0]; pIpAddr->n = strdequote(pIpAddr->z); break; } @@ -307,8 +311,8 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { const char* msg2 = "invalid user/account name"; const char* msg3 = "name too long"; - SSQLToken* pName = &pInfo->pDCLInfo->user.user; - SSQLToken* pPwd = &pInfo->pDCLInfo->user.passwd; + SStrToken* pName = &pInfo->pDCLInfo->user.user; + SStrToken* pPwd = &pInfo->pDCLInfo->user.passwd; if (handlePassword(pCmd, pPwd) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; @@ -337,7 +341,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { } case TSDB_SQL_DESCRIBE_TABLE: { - SSQLToken* pToken = &pInfo->pDCLInfo->a[0]; + SStrToken* pToken = &pInfo->pDCLInfo->a[0]; const char* msg2 = "table name is too long"; const char* msg1 = "invalid table name"; @@ -400,8 +404,8 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { // tDCLSQL* pDCL = pInfo->pDCLInfo; SUserInfo* pUser = &pInfo->pDCLInfo->user; - SSQLToken* pName = &pUser->user; - SSQLToken* pPwd = &pUser->passwd; + SStrToken* pName = &pUser->user; + SStrToken* pPwd = &pUser->passwd; if (pName->n >= TSDB_USER_LEN) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); @@ -423,7 +427,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { } else if (pUser->type == TSDB_ALTER_USER_PRIVILEGES) { assert(pPwd->type == TSDB_DATA_TYPE_NULL); - SSQLToken* pPrivilege = &pUser->privilege; + SStrToken* pPrivilege = &pUser->privilege; if (strncasecmp(pPrivilege->z, "super", 5) == 0 && pPrivilege->n == 5) { pCmd->count = 1; @@ -487,9 +491,10 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { const char* msg1 = "columns in select clause not identical"; for (int32_t i = pCmd->numOfClause; i < pInfo->subclauseInfo.numOfClause; ++i) { - SQueryInfo* pqi = NULL; - if ((code = tscGetQueryInfoDetailSafely(pCmd, i, &pqi)) != TSDB_CODE_SUCCESS) { - return code; + SQueryInfo* pqi = tscGetQueryInfoDetailSafely(pCmd, i); + if (pqi == NULL) { + pRes->code = terrno; + return pRes->code; } } @@ -581,7 +586,7 @@ int32_t parseIntervalClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQ } // interval is not null - SSQLToken* t = &pQuerySql->interval; + SStrToken* t = &pQuerySql->interval; if (getTimestampInUsFromStr(t->z, t->n, &pQueryInfo->intervalTime) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -667,7 +672,7 @@ int32_t parseSlidingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); - SSQLToken* pSliding = &pQuerySql->sliding; + SStrToken* pSliding = &pQuerySql->sliding; if (pSliding->n != 0) { getTimestampInUsFromStr(pSliding->z, pSliding->n, &pQueryInfo->slidingTime); if (tinfo.precision == TSDB_TIME_PRECISION_MILLI) { @@ -692,7 +697,7 @@ int32_t parseSlidingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu return TSDB_CODE_SUCCESS; } -int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SSQLToken* pzTableName, SSqlObj* pSql) { +int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SStrToken* pzTableName, SSqlObj* pSql) { const char* msg = "name too long"; SSqlCmd* pCmd = &pSql->cmd; @@ -709,7 +714,7 @@ int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SSQLToken* pzTableNa // db has been specified in sql string so we ignore current db path code = setObjFullName(pTableMetaInfo->name, getAccountId(pSql), NULL, pzTableName, NULL); } else { // get current DB name first, then set it into path - SSQLToken t = {0}; + SStrToken t = {0}; getCurrentDBName(pSql, &t); code = setObjFullName(pTableMetaInfo->name, NULL, &t, pzTableName, NULL); @@ -1027,13 +1032,13 @@ static bool has(tFieldList* pFieldList, int32_t startIdx, const char* name) { static char* getAccountId(SSqlObj* pSql) { return pSql->pTscObj->acctId; } -static void getCurrentDBName(SSqlObj* pSql, SSQLToken* pDBToken) { +static void getCurrentDBName(SSqlObj* pSql, SStrToken* pDBToken) { pDBToken->z = pSql->pTscObj->db; pDBToken->n = (uint32_t)strlen(pSql->pTscObj->db); } /* length limitation, strstr cannot be applied */ -static bool hasSpecifyDB(SSQLToken* pTableName) { +static bool hasSpecifyDB(SStrToken* pTableName) { for (uint32_t i = 0; i < pTableName->n; ++i) { if (pTableName->z[i] == TS_PATH_DELIMITER[0]) { return true; @@ -1043,7 +1048,7 @@ static bool hasSpecifyDB(SSQLToken* pTableName) { return false; } -int32_t setObjFullName(char* fullName, const char* account, SSQLToken* pDB, SSQLToken* tableName, int32_t* xlen) { +int32_t setObjFullName(char* fullName, const char* account, SStrToken* pDB, SStrToken* tableName, int32_t* xlen) { int32_t totalLen = 0; if (account != NULL) { @@ -1094,18 +1099,6 @@ int32_t setObjFullName(char* fullName, const char* account, SSQLToken* pDB, SSQL return (totalLen < TSDB_TABLE_FNAME_LEN) ? TSDB_CODE_SUCCESS : TSDB_CODE_TSC_INVALID_SQL; } -static void extractColumnNameFromString(tSQLExprItem* pItem) { - if (pItem->pNode->nSQLOptr == TK_STRING) { - pItem->pNode->val.nLen = strdequote(pItem->pNode->val.pz); - pItem->pNode->nSQLOptr = TK_ID; - - SSQLToken* pIdToken = &pItem->pNode->colInfo; - pIdToken->type = TK_ID; - pIdToken->z = pItem->pNode->val.pz; - pIdToken->n = pItem->pNode->val.nLen; - } -} - static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t exprIndex, tSQLExprItem* pItem) { const char* msg1 = "invalid column name, or illegal column type"; const char* msg2 = "invalid arithmetic expression in select clause"; @@ -1123,42 +1116,31 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t } int32_t tableIndex = columnList.ids[0].tableIndex; - - // todo potential data overflow - char* arithmeticExprStr = malloc(1024*1024); - char* p = arithmeticExprStr; - if (arithmeticType == NORMAL_ARITHMETIC) { pQueryInfo->type |= TSDB_QUERY_TYPE_PROJECTION_QUERY; // all columns in arithmetic expression must belong to the same table for (int32_t f = 1; f < columnList.num; ++f) { if (columnList.ids[f].tableIndex != tableIndex) { - taosTFree(arithmeticExprStr); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); } } - if (arithmeticExprToString(pItem->pNode, &p) != TSDB_CODE_SUCCESS) { - taosTFree(arithmeticExprStr); - return TSDB_CODE_TSC_INVALID_SQL; - } - // expr string is set as the parameter of function SColumnIndex index = {.tableIndex = tableIndex}; SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_ARITHM, &index, TSDB_DATA_TYPE_DOUBLE, sizeof(double), sizeof(double), false); - char* name = (pItem->aliasName != NULL)? pItem->aliasName:arithmeticExprStr; - tstrncpy(pExpr->aliasName, name, sizeof(pExpr->aliasName)); + char* name = (pItem->aliasName != NULL)? pItem->aliasName:pItem->pNode->token.z; + size_t len = MIN(sizeof(pExpr->aliasName), pItem->pNode->token.n + 1); + tstrncpy(pExpr->aliasName, name, len); tExprNode* pNode = NULL; SArray* colList = taosArrayInit(10, sizeof(SColIndex)); int32_t ret = exprTreeFromSqlExpr(pCmd, &pNode, pItem->pNode, pQueryInfo->exprList, pQueryInfo, colList); if (ret != TSDB_CODE_SUCCESS) { - taosTFree(arithmeticExprStr); taosArrayDestroy(colList); tExprTreeDestroy(&pNode, NULL); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); @@ -1167,8 +1149,8 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t size_t numOfNode = taosArrayGetSize(colList); for(int32_t k = 0; k < numOfNode; ++k) { SColIndex* pIndex = taosArrayGet(colList, k); - if (pIndex->flag == 1) { - taosTFree(arithmeticExprStr); + if (TSDB_COL_IS_TAG(pIndex->flag)) { + tExprTreeDestroy(&pNode, NULL); taosArrayDestroy(colList); tExprTreeDestroy(&pNode, NULL); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); @@ -1185,7 +1167,7 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t // TODO: other error handling } END_TRY - size_t len = tbufTell(&bw); + len = tbufTell(&bw); char* c = tbufGetData(&bw, true); // set the serialized binary string as the parameter of arithmetic expression @@ -1196,16 +1178,18 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t taosArrayDestroy(colList); tExprTreeDestroy(&pNode, NULL); } else { - if (arithmeticExprToString(pItem->pNode, &p) != TSDB_CODE_SUCCESS) { - taosTFree(arithmeticExprStr); - return TSDB_CODE_TSC_INVALID_SQL; - } - columnList.num = 0; columnList.ids[0] = (SColumnIndex) {0, 0}; - char* name = (pItem->aliasName != NULL)? pItem->aliasName:arithmeticExprStr; - insertResultField(pQueryInfo, exprIndex, &columnList, sizeof(double), TSDB_DATA_TYPE_DOUBLE, name, NULL); + char aliasName[TSDB_COL_NAME_LEN] = {0}; + if (pItem->aliasName != NULL) { + tstrncpy(aliasName, pItem->aliasName, TSDB_COL_NAME_LEN); + } else { + int32_t nameLen = MIN(TSDB_COL_NAME_LEN, pItem->pNode->token.n + 1); + tstrncpy(aliasName, pItem->pNode->token.z, nameLen); + } + + insertResultField(pQueryInfo, exprIndex, &columnList, sizeof(double), TSDB_DATA_TYPE_DOUBLE, aliasName, NULL); int32_t slot = tscNumOfFields(pQueryInfo) - 1; SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, slot); @@ -1221,7 +1205,6 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t int32_t ret = exprTreeFromSqlExpr(pCmd, &pArithExprInfo->pExpr, pItem->pNode, pQueryInfo->exprList, pQueryInfo, NULL); if (ret != TSDB_CODE_SUCCESS) { tExprTreeDestroy(&pArithExprInfo->pExpr, NULL); - taosTFree(arithmeticExprStr); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "invalid expression in select clause"); } @@ -1229,11 +1212,38 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t } } - taosTFree(arithmeticExprStr); return TSDB_CODE_SUCCESS; } -int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSelection, bool isSTable) { +static void tscInsertPrimaryTSSourceColumn(SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { + SColumnIndex tsCol = {.tableIndex = pIndex->tableIndex, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX}; + tscColumnListInsert(pQueryInfo->colList, &tsCol); +} + +static void addProjectQueryCol(SQueryInfo* pQueryInfo, int32_t startPos, SColumnIndex* pIndex, tSQLExprItem* pItem) { + SSqlExpr* pExpr = doAddProjectCol(pQueryInfo, startPos, pIndex->columnIndex, pIndex->tableIndex); + + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pIndex->tableIndex); + STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; + + SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, pIndex->columnIndex); + + char* colName = (pItem->aliasName == NULL) ? pSchema->name : pItem->aliasName; + tstrncpy(pExpr->aliasName, colName, sizeof(pExpr->aliasName)); + + SColumnList ids = {0}; + ids.num = 1; + ids.ids[0] = *pIndex; + + if (pIndex->columnIndex == TSDB_TBNAME_COLUMN_INDEX || pIndex->columnIndex == TSDB_UD_COLUMN_INDEX || + pIndex->columnIndex >= tscGetNumOfColumns(pTableMeta)) { + ids.num = 0; + } + + insertResultField(pQueryInfo, startPos, &ids, pExpr->resBytes, (int8_t)pExpr->resType, pExpr->aliasName, pExpr); +} + +int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSelection, bool isSTable, bool joinQuery) { assert(pSelection != NULL && pCmd != NULL); const char* msg2 = "functions can not be mixed up"; @@ -1251,16 +1261,14 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel tSQLExprItem* pItem = &pSelection->a[i]; // project on all fields - if (pItem->pNode->nSQLOptr == TK_ALL || pItem->pNode->nSQLOptr == TK_ID || pItem->pNode->nSQLOptr == TK_STRING) { + int32_t optr = pItem->pNode->nSQLOptr; + + if (optr == TK_ALL || optr == TK_ID || optr == TK_STRING || optr == TK_INTEGER || optr == TK_FLOAT) { // it is actually a function, but the function name is invalid if (pItem->pNode->nSQLOptr == TK_ID && (pItem->pNode->colInfo.z == NULL && pItem->pNode->colInfo.n == 0)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); } - // if the name of column is quoted, remove it and set the right information for later process - extractColumnNameFromString(pItem); - TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_PROJECTION_QUERY); - // select table_name1.field_name1, table_name2.field_name2 from table_name1, table_name2 if (addProjectionExprAndResultField(pCmd, pQueryInfo, pItem) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; @@ -1276,12 +1284,7 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel if (code != TSDB_CODE_SUCCESS) { return code; } - } else { - /* - * not support such expression - * e.g., select 12+5 from table_name - */ return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } @@ -1290,7 +1293,26 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel } } - if (!functionCompatibleCheck(pQueryInfo)) { + // there is only one user-defined column in the final result field, add the timestamp column. + size_t numOfSrcCols = taosArrayGetSize(pQueryInfo->colList); + if (numOfSrcCols <= 0 && !tscQueryTags(pQueryInfo)) { + SColumnIndex index = {0}; + + // set the constant column value always attached to first table. + STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, clauseIndex, 0); + SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, PRIMARYKEY_TIMESTAMP_COL_INDEX); + + // add the timestamp column into the output columns + int32_t numOfCols = (int32_t)tscSqlExprNumOfExprs(pQueryInfo); + tscAddSpecialColumnForSelect(pQueryInfo, numOfCols, TSDB_FUNC_PRJ, &index, pSchema, TSDB_COL_NORMAL); + + SFieldSupInfo* pSupInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, numOfCols); + pSupInfo->visible = false; + + pQueryInfo->type |= TSDB_QUERY_TYPE_PROJECTION_QUERY; + } + + if (!functionCompatibleCheck(pQueryInfo, joinQuery)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } @@ -1353,32 +1375,10 @@ SSqlExpr* doAddProjectCol(SQueryInfo* pQueryInfo, int32_t outputIndex, int32_t c pSchema->bytes, functionId == TSDB_FUNC_TAGPRJ); } -static void addProjectQueryCol(SQueryInfo* pQueryInfo, int32_t startPos, SColumnIndex* pIndex, tSQLExprItem* pItem) { - SSqlExpr* pExpr = doAddProjectCol(pQueryInfo, startPos, pIndex->columnIndex, pIndex->tableIndex); - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pIndex->tableIndex); - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - - SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, pIndex->columnIndex); - - char* colName = (pItem->aliasName == NULL) ? pSchema->name : pItem->aliasName; - tstrncpy(pExpr->aliasName, colName, sizeof(pExpr->aliasName)); - - SColumnList ids = {0}; - ids.num = 1; - ids.ids[0] = *pIndex; - - if (pIndex->columnIndex >= tscGetNumOfColumns(pTableMeta) || pIndex->columnIndex == TSDB_TBNAME_COLUMN_INDEX) { - ids.num = 0; - } - - insertResultField(pQueryInfo, startPos, &ids, pExpr->resBytes, (int8_t)pExpr->resType, pExpr->aliasName, pExpr); -} - -void tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId, +SSqlExpr* tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId, SColumnIndex* pIndex, SSchema* pColSchema, int16_t flag) { SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, outputColIndex, functionId, pIndex, pColSchema->type, - pColSchema->bytes, pColSchema->bytes, flag); + pColSchema->bytes, pColSchema->bytes, TSDB_COL_IS_TAG(flag)); tstrncpy(pExpr->aliasName, pColSchema->name, sizeof(pExpr->aliasName)); SColumnList ids = getColumnList(1, pIndex->tableIndex, pIndex->columnIndex); @@ -1394,6 +1394,8 @@ void tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex if (TSDB_COL_IS_TAG(flag)) { tscColumnListInsert(pTableMetaInfo->tagColList, pIndex); } + + return pExpr; } static int32_t doAddProjectionExprAndResultFields(SQueryInfo* pQueryInfo, SColumnIndex* pIndex, int32_t startPos) { @@ -1426,18 +1428,16 @@ static int32_t doAddProjectionExprAndResultFields(SQueryInfo* pQueryInfo, SColum return numOfTotalColumns; } -static void tscInsertPrimaryTSSourceColumn(SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { - SColumnIndex tsCol = {.tableIndex = pIndex->tableIndex, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX}; - tscColumnListInsert(pQueryInfo->colList, &tsCol); -} - int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExprItem* pItem) { const char* msg0 = "invalid column name"; const char* msg1 = "tag for normal table query is not allowed"; - - int32_t startPos = (int32_t)tscSqlExprNumOfExprs(pQueryInfo); - if (pItem->pNode->nSQLOptr == TK_ALL) { // project on all fields + int32_t startPos = (int32_t)tscSqlExprNumOfExprs(pQueryInfo); + int32_t optr = pItem->pNode->nSQLOptr; + + if (optr == TK_ALL) { // project on all fields + TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_PROJECTION_QUERY); + SColumnIndex index = COLUMN_INDEX_INITIALIZER; if (getTableIndexByName(&pItem->pNode->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg0); @@ -1453,7 +1453,24 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t } else { doAddProjectionExprAndResultFields(pQueryInfo, &index, startPos); } - } else if (pItem->pNode->nSQLOptr == TK_ID) { // simple column projection query + + // add the primary timestamp column even though it is not required by user + tscInsertPrimaryTSSourceColumn(pQueryInfo, &index); + } else if (optr == TK_STRING || optr == TK_INTEGER || optr == TK_FLOAT) { // simple column projection query + SColumnIndex index = COLUMN_INDEX_INITIALIZER; + + // user-specified constant value as a new result column + index.columnIndex = (pQueryInfo->udColumnId--); + index.tableIndex = 0; + + SSchema colSchema = tGetUserSpecifiedColumnSchema(&pItem->pNode->val, &pItem->pNode->token, pItem->aliasName); + SSqlExpr* pExpr = + tscAddSpecialColumnForSelect(pQueryInfo, startPos, TSDB_FUNC_PRJ, &index, &colSchema, TSDB_COL_UDC); + + // NOTE: the first parameter is reserved for the tag column id during join query process. + pExpr->numOfParams = 2; + tVariantAssign(&pExpr->param[1], &pItem->pNode->val); + } else if (optr == TK_ID) { SColumnIndex index = COLUMN_INDEX_INITIALIZER; if (getColumnIndexByName(pCmd, &pItem->pNode->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { @@ -1472,8 +1489,10 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t } addProjectQueryCol(pQueryInfo, startPos, &index, pItem); + pQueryInfo->type |= TSDB_QUERY_TYPE_PROJECTION_QUERY; } + // add the primary timestamp column even though it is not required by user tscInsertPrimaryTSSourceColumn(pQueryInfo, &index); } else { return TSDB_CODE_TSC_INVALID_SQL; @@ -1561,7 +1580,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col SColumnIndex index = COLUMN_INDEX_INITIALIZER; if (pItem->pNode->pParam != NULL) { - SSQLToken* pToken = &pItem->pNode->pParam->a[0].pNode->colInfo; + SStrToken* pToken = &pItem->pNode->pParam->a[0].pNode->colInfo; if (pToken->z == NULL || pToken->n == 0) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } @@ -1570,7 +1589,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col if (pParamElem->pNode->nSQLOptr == TK_ALL) { // select table.* // check if the table name is valid or not - SSQLToken tmpToken = pParamElem->pNode->colInfo; + SStrToken tmpToken = pParamElem->pNode->colInfo; if (getTableIndexByName(&tmpToken, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); @@ -1768,7 +1787,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col if (pParamElem->pNode->nSQLOptr == TK_ALL) { // select table.* - SSQLToken tmpToken = pParamElem->pNode->colInfo; + SStrToken tmpToken = pParamElem->pNode->colInfo; if (getTableIndexByName(&tmpToken, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); @@ -1834,10 +1853,11 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col for (int32_t i = 0; i < tscGetNumOfColumns(pTableMetaInfo->pTableMeta); ++i) { SColumnIndex index = {.tableIndex = j, .columnIndex = i}; - if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, functionID, pItem->aliasName, colIndex + i + j, &index) != - 0) { + if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, functionID, pItem->aliasName, colIndex, &index) != 0) { return TSDB_CODE_TSC_INVALID_SQL; } + + colIndex++; } numOfFields += tscGetNumOfColumns(pTableMetaInfo->pTableMeta); @@ -2045,7 +2065,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col // todo refactor static SColumnList getColumnList(int32_t num, int16_t tableIndex, int32_t columnIndex) { - assert(num == 1 && columnIndex >= -1 && tableIndex >= 0); + assert(num == 1 && tableIndex >= 0); SColumnList columnList = {0}; columnList.num = num; @@ -2070,16 +2090,16 @@ void getRevisedName(char* resultFieldName, int32_t functionId, int32_t maxLen, c snprintf(resultFieldName, maxLen, "%s(%s)", aAggs[functionId].aName, columnName); } -static bool isTablenameToken(SSQLToken* token) { - SSQLToken tmpToken = *token; - SSQLToken tableToken = {0}; +static bool isTablenameToken(SStrToken* token) { + SStrToken tmpToken = *token; + SStrToken tableToken = {0}; extractTableNameFromToken(&tmpToken, &tableToken); return (strncasecmp(TSQL_TBNAME_L, tmpToken.z, tmpToken.n) == 0 && tmpToken.n == strlen(TSQL_TBNAME_L)); } -static int16_t doGetColumnIndex(SQueryInfo* pQueryInfo, int32_t index, SSQLToken* pToken) { +static int16_t doGetColumnIndex(SQueryInfo* pQueryInfo, int32_t index, SStrToken* pToken) { STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, index)->pTableMeta; int32_t numOfCols = tscGetNumOfColumns(pTableMeta) + tscGetNumOfTags(pTableMeta); @@ -2101,7 +2121,7 @@ static int16_t doGetColumnIndex(SQueryInfo* pQueryInfo, int32_t index, SSQLToken return columnIndex; } -int32_t doGetColumnIndexByName(SSqlCmd* pCmd, SSQLToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { +int32_t doGetColumnIndexByName(SSqlCmd* pCmd, SStrToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { const char* msg0 = "ambiguous column name"; const char* msg1 = "invalid column name"; @@ -2143,7 +2163,7 @@ int32_t doGetColumnIndexByName(SSqlCmd* pCmd, SSQLToken* pToken, SQueryInfo* pQu } } -int32_t getTableIndexImpl(SSQLToken* pTableToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { +int32_t getTableIndexImpl(SStrToken* pTableToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { if (pTableToken->n == 0) { // only one table and no table name prefix in column name if (pQueryInfo->numOfTables == 1) { pIndex->tableIndex = 0; @@ -2169,8 +2189,8 @@ int32_t getTableIndexImpl(SSQLToken* pTableToken, SQueryInfo* pQueryInfo, SColum return TSDB_CODE_SUCCESS; } -int32_t getTableIndexByName(SSQLToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { - SSQLToken tableToken = {0}; +int32_t getTableIndexByName(SStrToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { + SStrToken tableToken = {0}; extractTableNameFromToken(pToken, &tableToken); if (getTableIndexImpl(&tableToken, pQueryInfo, pIndex) != TSDB_CODE_SUCCESS) { @@ -2180,12 +2200,12 @@ int32_t getTableIndexByName(SSQLToken* pToken, SQueryInfo* pQueryInfo, SColumnIn return TSDB_CODE_SUCCESS; } -int32_t getColumnIndexByName(SSqlCmd* pCmd, const SSQLToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { +int32_t getColumnIndexByName(SSqlCmd* pCmd, const SStrToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { if (pQueryInfo->pTableMetaInfo == NULL || pQueryInfo->numOfTables == 0) { return TSDB_CODE_TSC_INVALID_SQL; } - SSQLToken tmpToken = *pToken; + SStrToken tmpToken = *pToken; if (getTableIndexByName(&tmpToken, pQueryInfo, pIndex) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; @@ -2297,7 +2317,7 @@ int32_t setShowInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { int16_t showType = pShowInfo->showType; if (showType == TSDB_MGMT_TABLE_TABLE || showType == TSDB_MGMT_TABLE_METRIC || showType == TSDB_MGMT_TABLE_VGROUP) { // db prefix in tagCond, show table conds in payload - SSQLToken* pDbPrefixToken = &pShowInfo->prefix; + SStrToken* pDbPrefixToken = &pShowInfo->prefix; if (pDbPrefixToken->type != 0) { if (pDbPrefixToken->n >= TSDB_DB_NAME_LEN) { // db name is too long @@ -2319,7 +2339,7 @@ int32_t setShowInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { } // show table/stable like 'xxxx', set the like pattern for show tables - SSQLToken* pPattern = &pShowInfo->pattern; + SStrToken* pPattern = &pShowInfo->pattern; if (pPattern->type != 0) { pPattern->n = strdequote(pPattern->z); @@ -2337,7 +2357,7 @@ int32_t setShowInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { } // show vnodes may be ip addr of dnode in payload - SSQLToken* pDnodeIp = &pShowInfo->prefix; + SStrToken* pDnodeIp = &pShowInfo->prefix; if (pDnodeIp->n >= TSDB_IPv4ADDR_LEN) { // ip addr is too long return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } @@ -2358,7 +2378,7 @@ int32_t setKillInfo(SSqlObj* pSql, struct SSqlInfo* pInfo, int32_t killType) { SSqlCmd* pCmd = &pSql->cmd; pCmd->command = pInfo->type; - SSQLToken* idStr = &(pInfo->pDCLInfo->ip); + SStrToken* idStr = &(pInfo->pDCLInfo->ip); if (idStr->n > TSDB_KILL_MSG_LEN) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -2510,7 +2530,7 @@ bool hasUnsupportFunctionsForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) return false; } -static bool functionCompatibleCheck(SQueryInfo* pQueryInfo) { +static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery) { int32_t startIdx = 0; SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, startIdx); @@ -2542,6 +2562,10 @@ static bool functionCompatibleCheck(SQueryInfo* pQueryInfo) { if (functionCompatList[functionId] != factor) { return false; } + + if (functionId == TSDB_FUNC_LAST_ROW && joinQuery) { + return false; + } } return true; @@ -2579,7 +2603,7 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* for (int32_t i = 0; i < pList->nExpr; ++i) { tVariant* pVar = &pList->a[i].pVar; - SSQLToken token = {pVar->nLen, pVar->nType, pVar->pz}; + SStrToken token = {pVar->nLen, pVar->nType, pVar->pz}; SColumnIndex index = COLUMN_INDEX_INITIALIZER; if (getColumnIndexByName(pCmd, &token, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { @@ -2659,9 +2683,12 @@ static SColumnFilterInfo* addColumnFilterInfo(SColumn* pColumn) { } int32_t size = pColumn->numOfFilters + 1; - char* tmp = (char*)realloc((void*)(pColumn->filterInfo), sizeof(SColumnFilterInfo) * (size)); + + char* tmp = (char*) realloc((void*)(pColumn->filterInfo), sizeof(SColumnFilterInfo) * (size)); if (tmp != NULL) { pColumn->filterInfo = (SColumnFilterInfo*)tmp; + } else { + return NULL; } pColumn->numOfFilters++; @@ -2945,9 +2972,16 @@ static int32_t extractColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SC } else { // update the existed column filter information, find the filter info here pColFilter = &pColumn->filterInfo[0]; } + + if (pColFilter == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } } else if (sqlOptr == TK_OR) { // TODO fixme: failed to invalid the filter expression: "col1 = 1 OR col2 = 2" pColFilter = addColumnFilterInfo(pColumn); + if (pColFilter == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } } else { // error; return TSDB_CODE_TSC_INVALID_SQL; } @@ -3072,7 +3106,6 @@ static int32_t getColumnQueryCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQ static int32_t getJoinCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr* pExpr) { const char* msg1 = "invalid join query condition"; - const char* msg2 = "join on binary/nchar not supported"; const char* msg3 = "type of join columns must be identical"; const char* msg4 = "invalid column name in join condition"; @@ -3116,10 +3149,6 @@ static int32_t getJoinCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr* return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } - if (pTagSchema1->type == TSDB_DATA_TYPE_BINARY || pTagSchema1->type == TSDB_DATA_TYPE_NCHAR) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - pTagCond->joinInfo.hasJoin = true; return TSDB_CODE_SUCCESS; } @@ -3156,7 +3185,7 @@ int32_t doArithmeticExprToString(tSQLExpr* pExpr, char** exprString) { return TSDB_CODE_SUCCESS; } -static int32_t arithmeticExprToString(tSQLExpr* pExpr, char** str) { +static UNUSED_FUNC int32_t arithmeticExprToString(tSQLExpr* pExpr, char** str) { char* start = *str; int32_t code = doArithmeticExprToString(pExpr, str); @@ -3610,7 +3639,7 @@ static void doExtractExprForSTable(SSqlCmd* pCmd, tSQLExpr** pExpr, SQueryInfo* return; } - SSQLToken t = {0}; + SStrToken t = {0}; extractTableNameFromToken(&pLeft->colInfo, &t); *pOut = *pExpr; @@ -3690,7 +3719,7 @@ static int32_t setTableCondForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, num = j; char* name = extractDBName(pTableMetaInfo->name, db); - SSQLToken dbToken = { .type = TK_STRING, .z = name, .n = (uint32_t)strlen(name) }; + SStrToken dbToken = { .type = TK_STRING, .z = name, .n = (uint32_t)strlen(name) }; for (int32_t i = 0; i < num; ++i) { if (i >= 1) { @@ -3699,7 +3728,7 @@ static int32_t setTableCondForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, char idBuf[TSDB_TABLE_FNAME_LEN] = {0}; int32_t xlen = (int32_t)strlen(segments[i]); - SSQLToken t = {.z = segments[i], .n = xlen, .type = TK_STRING}; + SStrToken t = {.z = segments[i], .n = xlen, .type = TK_STRING}; int32_t ret = setObjFullName(idBuf, account, &dbToken, &t, &xlen); if (ret != TSDB_CODE_SUCCESS) { @@ -4020,7 +4049,7 @@ int32_t getTimeRange(STimeWindow* win, tSQLExpr* pRight, int32_t optr, int16_t t return TSDB_CODE_TSC_INVALID_SQL; } } else { - SSQLToken token = {.z = pRight->val.pz, .n = pRight->val.nLen, .type = TK_ID}; + SStrToken token = {.z = pRight->val.pz, .n = pRight->val.nLen, .type = TK_ID}; int32_t len = tSQLGetToken(pRight->val.pz, &token.type); if ((token.type != TK_INTEGER && token.type != TK_FLOAT) || len != pRight->val.nLen) { @@ -4266,7 +4295,7 @@ int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu return TSDB_CODE_SUCCESS; } - SSQLToken columnName = {pVar->nLen, pVar->nType, pVar->pz}; + SStrToken columnName = {pVar->nLen, pVar->nType, pVar->pz}; SColumnIndex index = {0}; if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { // super table query @@ -4335,7 +4364,7 @@ int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu } tVariant* pVar2 = &pSortorder->a[1].pVar; - SSQLToken cname = {pVar2->nLen, pVar2->nType, pVar2->pz}; + SStrToken cname = {pVar2->nLen, pVar2->nType, pVar2->pz}; if (getColumnIndexByName(pCmd, &cname, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } @@ -4464,7 +4493,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { } SColumnIndex index = COLUMN_INDEX_INITIALIZER; - SSQLToken name = {.z = pItem->pVar.pz, .n = pItem->pVar.nLen, .type = TK_STRING}; + SStrToken name = {.z = pItem->pVar.pz, .n = pItem->pVar.nLen, .type = TK_STRING}; if (getColumnIndexByName(pCmd, &name, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; @@ -4502,12 +4531,12 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { SColumnIndex srcIndex = COLUMN_INDEX_INITIALIZER; SColumnIndex destIndex = COLUMN_INDEX_INITIALIZER; - SSQLToken srcToken = {.z = pSrcItem->pVar.pz, .n = pSrcItem->pVar.nLen, .type = TK_STRING}; + SStrToken srcToken = {.z = pSrcItem->pVar.pz, .n = pSrcItem->pVar.nLen, .type = TK_STRING}; if (getColumnIndexByName(pCmd, &srcToken, pQueryInfo, &srcIndex) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg17); } - SSQLToken destToken = {.z = pDstItem->pVar.pz, .n = pDstItem->pVar.nLen, .type = TK_STRING}; + SStrToken destToken = {.z = pDstItem->pVar.pz, .n = pDstItem->pVar.nLen, .type = TK_STRING}; if (getColumnIndexByName(pCmd, &destToken, pQueryInfo, &destIndex) == TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg19); } @@ -4531,7 +4560,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { int16_t numOfTags = tscGetNumOfTags(pTableMeta); SColumnIndex columnIndex = COLUMN_INDEX_INITIALIZER; - SSQLToken name = {.type = TK_STRING, .z = pTagName->pz, .n = pTagName->nLen}; + SStrToken name = {.type = TK_STRING, .z = pTagName->pz, .n = pTagName->nLen}; if (getColumnIndexByName(pCmd, &name, pQueryInfo, &columnIndex) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -4624,7 +4653,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { tVariantListItem* pItem = &pAlterSQL->varList->a[0]; SColumnIndex columnIndex = COLUMN_INDEX_INITIALIZER; - SSQLToken name = {.type = TK_STRING, .z = pItem->pVar.pz, .n = pItem->pVar.nLen}; + SStrToken name = {.type = TK_STRING, .z = pItem->pVar.pz, .n = pItem->pVar.nLen}; if (getColumnIndexByName(pCmd, &name, pQueryInfo, &columnIndex) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg17); } @@ -4687,7 +4716,7 @@ int32_t validateFunctionsInIntervalOrGroupbyQuery(SSqlCmd* pCmd, SQueryInfo* pQu } } - if (pExpr->functionId == TSDB_FUNC_PRJ || pExpr->functionId == TSDB_FUNC_DIFF || + if ((pExpr->functionId == TSDB_FUNC_PRJ && pExpr->numOfParams == 0) || pExpr->functionId == TSDB_FUNC_DIFF || pExpr->functionId == TSDB_FUNC_ARITHM) { isProjectionFunction = true; } @@ -4744,7 +4773,7 @@ int32_t validateDNodeConfig(tDCLSQL* pOptions) { {"dDebugFlag", 10}, {"mqttDebugFlag", 13}, {"wDebugFlag", 10}, {"tmrDebugFlag", 12}, }; - SSQLToken* pOptionToken = &pOptions->a[1]; + SStrToken* pOptionToken = &pOptions->a[1]; if (pOptions->nTokens == 2) { // reset log and reset query cache does not need value @@ -4756,7 +4785,7 @@ int32_t validateDNodeConfig(tDCLSQL* pOptions) { } } else if ((strncasecmp(cfgOptions[tokenBalance].name, pOptionToken->z, pOptionToken->n) == 0) && (cfgOptions[tokenBalance].len == pOptionToken->n)) { - SSQLToken* pValToken = &pOptions->a[2]; + SStrToken* pValToken = &pOptions->a[2]; int32_t vnodeId = 0; int32_t dnodeId = 0; strdequote(pValToken->z); @@ -4767,14 +4796,14 @@ int32_t validateDNodeConfig(tDCLSQL* pOptions) { return TSDB_CODE_SUCCESS; } else if ((strncasecmp(cfgOptions[tokenMonitor].name, pOptionToken->z, pOptionToken->n) == 0) && (cfgOptions[tokenMonitor].len == pOptionToken->n)) { - SSQLToken* pValToken = &pOptions->a[2]; + SStrToken* pValToken = &pOptions->a[2]; int32_t val = strtol(pValToken->z, NULL, 10); if (val != 0 && val != 1) { return TSDB_CODE_TSC_INVALID_SQL; // options value is invalid } return TSDB_CODE_SUCCESS; } else { - SSQLToken* pValToken = &pOptions->a[2]; + SStrToken* pValToken = &pOptions->a[2]; int32_t val = strtol(pValToken->z, NULL, 10); if (val < 0 || val > 256) { @@ -4803,7 +4832,7 @@ int32_t validateLocalConfig(tDCLSQL* pOptions) { SDNodeDynConfOption LOCAL_DYNAMIC_CFG_OPTIONS[6] = {{"resetLog", 8}, {"rpcDebugFlag", 12}, {"tmrDebugFlag", 12}, {"cDebugFlag", 10}, {"uDebugFlag", 10}, {"debugFlag", 9}}; - SSQLToken* pOptionToken = &pOptions->a[0]; + SStrToken* pOptionToken = &pOptions->a[0]; if (pOptions->nTokens == 1) { // reset log does not need value @@ -4814,7 +4843,7 @@ int32_t validateLocalConfig(tDCLSQL* pOptions) { } } } else { - SSQLToken* pValToken = &pOptions->a[1]; + SStrToken* pValToken = &pOptions->a[1]; int32_t val = strtol(pValToken->z, NULL, 10); if (val < 131 || val > 199) { @@ -4839,7 +4868,7 @@ int32_t validateColumnName(char* name) { return TSDB_CODE_TSC_INVALID_SQL; } - SSQLToken token = {.z = name}; + SStrToken token = {.z = name}; token.n = tSQLGetToken(name, &token.type); if (token.type != TK_STRING && token.type != TK_ID) { @@ -5018,7 +5047,7 @@ static int32_t setTimePrecision(SSqlCmd* pCmd, SCMCreateDbMsg* pMsg, SCreateDBIn pMsg->precision = TSDB_TIME_PRECISION_MILLI; // millisecond by default - SSQLToken* pToken = &pCreateDbInfo->precision; + SStrToken* pToken = &pCreateDbInfo->precision; if (pToken->n > 0) { pToken->n = strdequote(pToken->z); @@ -5689,7 +5718,7 @@ int32_t doCheckForCreateTable(SSqlObj* pSql, int32_t subClauseIndex, SSqlInfo* p assert(pFieldList != NULL); // if sql specifies db, use it, otherwise use default db - SSQLToken* pzTableName = &(pCreateTable->name); + SStrToken* pzTableName = &(pCreateTable->name); if (tscValidateName(pzTableName) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); @@ -5744,7 +5773,7 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { STableMetaInfo* pStableMeterMetaInfo = tscGetMetaInfo(pQueryInfo, STABLE_INDEX); // super table name, create table by using dst - SSQLToken* pToken = &(pCreateTable->usingInfo.stableName); + SStrToken* pToken = &(pCreateTable->usingInfo.stableName); if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); @@ -5837,7 +5866,7 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); // if sql specifies db, use it, otherwise use default db - SSQLToken* pzTableName = &(pCreateTable->name); + SStrToken* pzTableName = &(pCreateTable->name); SQuerySQL* pQuerySql = pCreateTable->pSelect; if (tscValidateName(pzTableName) != TSDB_CODE_SUCCESS) { @@ -5850,7 +5879,7 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { } tVariant* pVar = &pSrcMeterName->a[0].pVar; - SSQLToken srcToken = {.z = pVar->pz, .n = pVar->nLen, .type = TK_STRING}; + SStrToken srcToken = {.z = pVar->pz, .n = pVar->nLen, .type = TK_STRING}; if (tscValidateName(&srcToken) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } @@ -5865,7 +5894,7 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { } bool isSTable = UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo); - if (parseSelectClause(&pSql->cmd, 0, pQuerySql->pSelection, isSTable) != TSDB_CODE_SUCCESS) { + if (parseSelectClause(&pSql->cmd, 0, pQuerySql->pSelection, isSTable, false) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -5991,7 +6020,7 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { pTableItem->nLen = strdequote(pTableItem->pz); - SSQLToken tableName = {.z = pTableItem->pz, .n = pTableItem->nLen, .type = TK_STRING}; + SStrToken tableName = {.z = pTableItem->pz, .n = pTableItem->nLen, .type = TK_STRING}; if (tscValidateName(&tableName) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg0); } @@ -6002,13 +6031,13 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { STableMetaInfo* pTableMetaInfo1 = tscGetMetaInfo(pQueryInfo, i/2); - SSQLToken t = {.type = TSDB_DATA_TYPE_BINARY, .n = pTableItem->nLen, .z = pTableItem->pz}; + SStrToken t = {.type = TSDB_DATA_TYPE_BINARY, .n = pTableItem->nLen, .z = pTableItem->pz}; if (tscSetTableFullName(pTableMetaInfo1, &t, pSql) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } tVariant* pTableItem1 = &pQuerySql->from->a[i + 1].pVar; - SSQLToken aliasName = {.z = pTableItem1->pz, .n = pTableItem1->nLen, .type = TK_STRING}; + SStrToken aliasName = {.z = pTableItem1->pz, .n = pTableItem1->nLen, .type = TK_STRING}; if (tscValidateName(&aliasName) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg11); } @@ -6048,7 +6077,8 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { return TSDB_CODE_TSC_INVALID_SQL; } - if (parseSelectClause(pCmd, index, pQuerySql->pSelection, isSTable) != TSDB_CODE_SUCCESS) { + int32_t joinQuery = (pQuerySql->from != NULL && pQuerySql->from->nExpr > 2); + if (parseSelectClause(pCmd, index, pQuerySql->pSelection, isSTable, joinQuery) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -6082,6 +6112,9 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { } } else { // set the time rang pQueryInfo->window = TSWINDOW_INITIALIZER; + if (pQuerySql->from->nExpr > 2) { // it is a join query, no wher clause is not allowed. + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "condition missing for join query "); + } } // user does not specified the query time window, twa is not allowed in such case. @@ -6131,7 +6164,7 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { if (pQueryInfo->intervalTime > 0) { int64_t timeRange = ABS(pQueryInfo->window.skey - pQueryInfo->window.ekey); // number of result is not greater than 10,000,000 - if ((timeRange == 0) || (timeRange / pQueryInfo->intervalTime) > MAX_RETRIEVE_ROWS_IN_INTERVAL_QUERY) { + if ((timeRange == 0) || (timeRange / pQueryInfo->intervalTime) > MAX_INTERVAL_TIME_WINDOW) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6); } } @@ -6230,7 +6263,7 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pS (*pExpr)->_node.pLeft = pLeft; (*pExpr)->_node.pRight = pRight; - SSQLToken t = {.type = pSqlExpr->nSQLOptr}; + SStrToken t = {.type = pSqlExpr->nSQLOptr}; (*pExpr)->_node.optr = getBinaryExprOptr(&t); assert((*pExpr)->_node.optr != 0); diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 9282fa74fb..ecb85472fc 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -226,13 +226,17 @@ int tscSendMsgToServer(SSqlObj *pSql) { .handle = &pSql->pRpcCtx, .code = 0 }; - // NOTE: the rpc context should be acquired before sending data to server. // Otherwise, the pSql object may have been released already during the response function, which is // processMsgFromServer function. In the meanwhile, the assignment of the rpc context to sql object will absolutely // cause crash. - rpcSendRequest(pObj->pDnodeConn, &pSql->epSet, &rpcMsg); - return TSDB_CODE_SUCCESS; + if (pObj != NULL && pObj->signature == pObj) { + rpcSendRequest(pObj->pDnodeConn, &pSql->epSet, &rpcMsg); + return TSDB_CODE_SUCCESS; + } else { + //pObj->signature has been reset by other thread, ignore concurrency problem + return TSDB_CODE_TSC_CONN_KILLED; + } } void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { @@ -314,10 +318,10 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { pRes->rspLen = 0; - if (pRes->code != TSDB_CODE_TSC_QUERY_CANCELLED) { - pRes->code = (rpcMsg->code != TSDB_CODE_SUCCESS) ? rpcMsg->code : TSDB_CODE_RPC_NETWORK_UNAVAIL; - } else { + if (pRes->code == TSDB_CODE_TSC_QUERY_CANCELLED) { tscDebug("%p query is cancelled, code:%s", pSql, tstrerror(pRes->code)); + } else { + pRes->code = rpcMsg->code; } if (pRes->code == TSDB_CODE_SUCCESS) { @@ -454,35 +458,21 @@ void tscKillSTableQuery(SSqlObj *pSql) { return; } + pSql->res.code = TSDB_CODE_TSC_QUERY_CANCELLED; + for (int i = 0; i < pSql->numOfSubs; ++i) { + // NOTE: pSub may have been released already here SSqlObj *pSub = pSql->pSubs[i]; if (pSub == NULL) { continue; } - /* - * here, we cannot set the command = TSDB_SQL_KILL_QUERY. Otherwise, it may cause - * sub-queries not correctly released and master sql object of super table query reaches an abnormal state. - */ - rpcCancelRequest(pSub->pRpcCtx); pSub->res.code = TSDB_CODE_TSC_QUERY_CANCELLED; - tscQueueAsyncRes(pSub); - } - - /* - * 1. if the subqueries are not launched or partially launched, we need to waiting the launched - * query return to successfully free allocated resources. - * 2. if no any subqueries are launched yet, which means the super table query only in parse sql stage, - * set the res.code, and return. - */ - const int64_t MAX_WAITING_TIME = 10000; // 10 Sec. - int64_t stime = taosGetTimestampMs(); - - while (pCmd->command != TSDB_SQL_RETRIEVE_LOCALMERGE && pCmd->command != TSDB_SQL_RETRIEVE_EMPTY_RESULT) { - taosMsleep(100); - if (taosGetTimestampMs() - stime > MAX_WAITING_TIME) { - break; + if (pSub->pRpcCtx != NULL) { + rpcCancelRequest(pSub->pRpcCtx); } + + tscQueueAsyncRes(pSub); // async res? not other functions? } tscDebug("%p super table query cancelled", pSql); @@ -638,26 +628,29 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { tscError("%p failed to malloc for query msg", pSql); - return -1; // todo add test for this + return TSDB_CODE_TSC_INVALID_SQL; // todo add test for this } SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; - - if (taosArrayGetSize(pQueryInfo->colList) <= 0 && !tscQueryTags(pQueryInfo)) { - tscError("%p illegal value of numOfCols in query msg: %d", pSql, tscGetNumOfColumns(pTableMeta)); - return -1; + + size_t numOfSrcCols = taosArrayGetSize(pQueryInfo->colList); + if (numOfSrcCols <= 0 && !tscQueryTags(pQueryInfo)) { + tscError("%p illegal value of numOfCols in query msg: %"PRIu64", table cols:%d", pSql, numOfSrcCols, + tscGetNumOfColumns(pTableMeta)); + + return TSDB_CODE_TSC_INVALID_SQL; } if (pQueryInfo->intervalTime < 0) { tscError("%p illegal value of aggregation time interval in query msg: %ld", pSql, pQueryInfo->intervalTime); - return -1; + return TSDB_CODE_TSC_INVALID_SQL; } if (pQueryInfo->groupbyExpr.numOfGroupCols < 0) { tscError("%p illegal value of numOfGroupCols in query msg: %d", pSql, pQueryInfo->groupbyExpr.numOfGroupCols); - return -1; + return TSDB_CODE_TSC_INVALID_SQL; } SQueryTableMsg *pQueryMsg = (SQueryTableMsg *)pCmd->payload; @@ -735,7 +728,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { if (pColFilter->lowerRelOptr == TSDB_RELATION_INVALID && pColFilter->upperRelOptr == TSDB_RELATION_INVALID) { tscError("invalid filter info"); - return -1; + return TSDB_CODE_TSC_INVALID_SQL; } } } @@ -744,10 +737,10 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { for (int32_t i = 0; i < tscSqlExprNumOfExprs(pQueryInfo); ++i) { SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, i); - if (!tscValidateColumnId(pTableMetaInfo, pExpr->colInfo.colId)) { + if (!tscValidateColumnId(pTableMetaInfo, pExpr->colInfo.colId, pExpr->numOfParams)) { /* column id is not valid according to the cached table meta, the table meta is expired */ tscError("%p table schema is not matched with parsed sql", pSql); - return -1; + return TSDB_CODE_TSC_INVALID_SQL; } pSqlFuncExpr->colInfo.colId = htons(pExpr->colInfo.colId); @@ -944,8 +937,8 @@ int32_t tscBuildAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SCMCreateAcctMsg *pAlterMsg = (SCMCreateAcctMsg *)pCmd->payload; - SSQLToken *pName = &pInfo->pDCLInfo->user.user; - SSQLToken *pPwd = &pInfo->pDCLInfo->user.passwd; + SStrToken *pName = &pInfo->pDCLInfo->user.user; + SStrToken *pPwd = &pInfo->pDCLInfo->user.passwd; strncpy(pAlterMsg->user, pName->z, pName->n); strncpy(pAlterMsg->pass, pPwd->z, pPwd->n); @@ -1147,13 +1140,13 @@ int32_t tscBuildShowMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pShowMsg->type = pShowInfo->showType; if (pShowInfo->showType != TSDB_MGMT_TABLE_VNODES) { - SSQLToken *pPattern = &pShowInfo->pattern; + SStrToken *pPattern = &pShowInfo->pattern; if (pPattern->type > 0) { // only show tables support wildcard query strncpy(pShowMsg->payload, pPattern->z, pPattern->n); pShowMsg->payloadLen = htons(pPattern->n); } } else { - SSQLToken *pEpAddr = &pShowInfo->prefix; + SStrToken *pEpAddr = &pShowInfo->prefix; assert(pEpAddr->n > 0 && pEpAddr->type > 0); strncpy(pShowMsg->payload, pEpAddr->z, pEpAddr->n); @@ -1295,7 +1288,7 @@ int tscBuildAlterTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { int size = tscEstimateAlterTableMsgLength(pCmd); if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { tscError("%p failed to malloc for alter table msg", pSql); - return -1; + return TSDB_CODE_TSC_OUT_OF_MEMORY; } SCMAlterTableMsg *pAlterTableMsg = (SCMAlterTableMsg *)pCmd->payload; @@ -1443,6 +1436,12 @@ int tscProcessRetrieveLocalMergeRsp(SSqlObj *pSql) { SSqlRes *pRes = &pSql->res; SSqlCmd *pCmd = &pSql->cmd; + int32_t code = pRes->code; + if (pRes->code != TSDB_CODE_SUCCESS) { + tscQueueAsyncRes(pSql); + return code; + } + pRes->code = tscDoLocalMerge(pSql); SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); @@ -1453,7 +1452,7 @@ int tscProcessRetrieveLocalMergeRsp(SSqlObj *pSql) { pRes->row = 0; pRes->completed = (pRes->numOfRows == 0); - int32_t code = pRes->code; + code = pRes->code; if (pRes->code == TSDB_CODE_SUCCESS) { (*pSql->fp)(pSql->param, pSql, pRes->numOfRows); } else { @@ -1647,7 +1646,7 @@ int tscBuildHeartBeatMsg(SSqlObj *pSql, SSqlInfo *pInfo) { if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { pthread_mutex_unlock(&pObj->mutex); tscError("%p failed to malloc for heartbeat msg", pSql); - return -1; + return TSDB_CODE_TSC_OUT_OF_MEMORY; } SCMHeartBeatMsg *pHeartbeat = (SCMHeartBeatMsg *)pCmd->payload; @@ -1717,7 +1716,7 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { assert(pTableMetaInfo->pTableMeta == NULL); pTableMetaInfo->pTableMeta = (STableMeta *) taosCachePut(tscCacheHandle, pTableMetaInfo->name, - strlen(pTableMetaInfo->name), pTableMeta, size, tsTableMetaKeepTimer); + strlen(pTableMetaInfo->name), pTableMeta, size, tsTableMetaKeepTimer * 1000); // todo handle out of memory case if (pTableMetaInfo->pTableMeta == NULL) { @@ -1921,7 +1920,7 @@ int tscProcessShowRsp(SSqlObj *pSql) { STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg, &size); pTableMetaInfo->pTableMeta = taosCachePut(tscCacheHandle, key, strlen(key), (char *)pTableMeta, size, - tsTableMetaKeepTimer); + tsTableMetaKeepTimer * 1000); SSchema *pTableSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); if (pQueryInfo->colList == NULL) { @@ -1961,8 +1960,12 @@ static void createHBObj(STscObj* pObj) { pSql->fp = tscProcessHeartBeatRsp; - SQueryInfo *pQueryInfo = NULL; - tscGetQueryInfoDetailSafely(&pSql->cmd, 0, &pQueryInfo); + SQueryInfo *pQueryInfo = tscGetQueryInfoDetailSafely(&pSql->cmd, 0); + if (pQueryInfo == NULL) { + pSql->res.code = terrno; + return; + } + pQueryInfo->command = TSDB_SQL_HB; pSql->cmd.command = pQueryInfo->command; @@ -2147,8 +2150,7 @@ static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInf tscAddSubqueryInfo(&pNew->cmd); - SQueryInfo *pNewQueryInfo = NULL; - tscGetQueryInfoDetailSafely(&pNew->cmd, 0, &pNewQueryInfo); + SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetailSafely(&pNew->cmd, 0); pNew->cmd.autoCreated = pSql->cmd.autoCreated; // create table if not exists if (TSDB_CODE_SUCCESS != tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE + pSql->cmd.payloadLen)) { @@ -2251,8 +2253,8 @@ int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) { pNew->cmd.command = TSDB_SQL_STABLEVGROUP; - SQueryInfo *pNewQueryInfo = NULL; - if ((code = tscGetQueryInfoDetailSafely(&pNew->cmd, 0, &pNewQueryInfo)) != TSDB_CODE_SUCCESS) { + SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetailSafely(&pNew->cmd, 0); + if (pNewQueryInfo == NULL) { tscFreeSqlObj(pNew); return code; } diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 1bd885466c..f63923e046 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -201,7 +201,7 @@ TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, } TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port) { - tscDebug("try to create a connection to %s:%u, user:%s db:%s", ip, port, user, db); + tscDebug("try to create a connection to %s:%u, user:%s db:%s", ip, port != 0 ? port : tsServerPort , user, db); if (user == NULL) user = TSDB_DEFAULT_USER; if (pass == NULL) pass = TSDB_DEFAULT_PASS; @@ -655,27 +655,30 @@ int* taos_fetch_lengths(TAOS_RES *res) { char *taos_get_client_info() { return version; } void taos_stop_query(TAOS_RES *res) { - if (res == NULL) { + SSqlObj *pSql = (SSqlObj *)res; + if (pSql == NULL || pSql->signature != pSql) { return; } - SSqlObj *pSql = (SSqlObj *)res; + tscDebug("%p start to cancel query", res); SSqlCmd *pCmd = &pSql->cmd; - if (pSql->signature != pSql) return; - tscDebug("%p start to cancel query", res); - - + // TODO there are multi-thread problem. + // It may have been released by the other thread already. + // The ref count may fix this problem. SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) { - tscKillSTableQuery(pSql); - } - if (pSql->cmd.command < TSDB_SQL_LOCAL) { - rpcCancelRequest(pSql->pRpcCtx); - } + // set the error code for master pSqlObj firstly pSql->res.code = TSDB_CODE_TSC_QUERY_CANCELLED; - tscQueueAsyncRes(pSql); + + if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) { + assert(pSql->pRpcCtx == NULL); + tscKillSTableQuery(pSql); + } else { + if (pSql->cmd.command < TSDB_SQL_LOCAL) { + rpcCancelRequest(pSql->pRpcCtx); + } + } tscDebug("%p query is cancelled", res); } @@ -824,8 +827,11 @@ static int tscParseTblNameList(SSqlObj *pSql, const char *tblNameList, int32_t t int code = TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; char *str = (char *)tblNameList; - SQueryInfo *pQueryInfo = NULL; - tscGetQueryInfoDetailSafely(pCmd, pCmd->clauseIndex, &pQueryInfo); + SQueryInfo *pQueryInfo = tscGetQueryInfoDetailSafely(pCmd, pCmd->clauseIndex); + if (pQueryInfo == NULL) { + pSql->res.code = terrno; + return terrno; + } STableMetaInfo *pTableMetaInfo = tscAddEmptyMetaInfo(pQueryInfo); @@ -850,7 +856,7 @@ static int tscParseTblNameList(SSqlObj *pSql, const char *tblNameList, int32_t t str = nextStr + 1; len = (int32_t)strtrim(tblName); - SSQLToken sToken = {.n = len, .type = TK_ID, .z = tblName}; + SStrToken sToken = {.n = len, .type = TK_ID, .z = tblName}; tSQLGetToken(tblName, &sToken.type); // Check if the table name available or not diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index c022e320a3..2fb264c756 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -95,13 +95,14 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, SJ tscInfo("%" PRId64 ", tags:%d \t %" PRId64 ", tags:%d", elem1.ts, elem1.tag, elem2.ts, elem2.tag); #endif - if (elem1.tag < elem2.tag || (elem1.tag == elem2.tag && tsCompare(order, elem1.ts, elem2.ts))) { + int32_t res = tVariantCompare(&elem1.tag, &elem2.tag); + if (res == -1 || (res == 0 && tsCompare(order, elem1.ts, elem2.ts))) { if (!tsBufNextPos(pSupporter1->pTSBuf)) { break; } numOfInput1++; - } else if (elem1.tag > elem2.tag || (elem1.tag == elem2.tag && tsCompare(order, elem2.ts, elem1.ts))) { + } else if ((res > 0) || (res == 0 && tsCompare(order, elem2.ts, elem1.ts))) { if (!tsBufNextPos(pSupporter2->pTSBuf)) { break; } @@ -121,8 +122,8 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, SJ win->ekey = elem1.ts; } - tsBufAppend(output1, elem1.vnode, elem1.tag, (const char*)&elem1.ts, sizeof(elem1.ts)); - tsBufAppend(output2, elem2.vnode, elem2.tag, (const char*)&elem2.ts, sizeof(elem2.ts)); + tsBufAppend(output1, elem1.vnode, &elem1.tag, (const char*)&elem1.ts, sizeof(elem1.ts)); + tsBufAppend(output2, elem2.vnode, &elem2.tag, (const char*)&elem2.ts, sizeof(elem2.ts)); } else { pLimit->offset -= 1; } @@ -354,11 +355,12 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) { pExpr = tscSqlExprGet(pQueryInfo, 0); } - // set the join condition tag column info, to do extract method + // set the join condition tag column info, todo extract method if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { assert(pQueryInfo->tagCond.joinInfo.hasJoin); int16_t colId = tscGetJoinTagColIdByUid(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->id.uid); + // set the tag column id for executor to extract correct tag value pExpr->param[0].i64Key = colId; pExpr->numOfParams = 1; } @@ -435,6 +437,7 @@ int32_t tscCompareTidTags(const void* p1, const void* p2) { if (t1->vgId != t2->vgId) { return (t1->vgId > t2->vgId) ? 1 : -1; } + if (t1->tid != t2->tid) { return (t1->tid > t2->tid) ? 1 : -1; } @@ -541,6 +544,7 @@ static bool checkForDuplicateTagVal(SQueryInfo* pQueryInfo, SJoinSupporter* p1, for(int32_t i = 1; i < p1->num; ++i) { STidTags* prev = (STidTags*) varDataVal(p1->pIdTagList + (i - 1) * p1->tagSize); STidTags* p = (STidTags*) varDataVal(p1->pIdTagList + i * p1->tagSize); + assert(prev->vgId >= 1 && p->vgId >= 1); if (doCompare(prev->tag, p->tag, pColSchema->type, pColSchema->bytes) == 0) { tscError("%p join tags have same value for different table, free all sub SqlObj and quit", pPSqlObj); @@ -577,6 +581,7 @@ static int32_t getIntersectionOfTableTuple(SQueryInfo* pQueryInfo, SSqlObj* pPar while(i < p1->num && j < p2->num) { STidTags* pp1 = (STidTags*) varDataVal(p1->pIdTagList + i * p1->tagSize); STidTags* pp2 = (STidTags*) varDataVal(p2->pIdTagList + j * p2->tagSize); + assert(pp1->tid != 0 && pp2->tid != 0); int32_t ret = doCompare(pp1->tag, pp2->tag, pColSchema->type, pColSchema->bytes); if (ret == 0) { @@ -686,6 +691,7 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow freeJoinSubqueryObj(pParentSql); pParentSql->res.code = code; tscQueueAsyncRes(pParentSql); + taosArrayDestroy(s1); taosArrayDestroy(s2); return; @@ -1145,7 +1151,7 @@ void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) { ///////////////////////////////////////////////////////////////////////////////////////// static void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code); -static SSqlObj *tscCreateSqlObjForSubquery(SSqlObj *pSql, SRetrieveSupport *trsupport, SSqlObj *prevSqlObj); +static SSqlObj *tscCreateSTableSubquery(SSqlObj *pSql, SRetrieveSupport *trsupport, SSqlObj *prevSqlObj); int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter *pSupporter) { SSqlCmd * pCmd = &pSql->cmd; @@ -1218,6 +1224,16 @@ int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter int32_t tagColId = tscGetJoinTagColIdByUid(pTagCond, pTableMetaInfo->pTableMeta->id.uid); SSchema* s = tscGetTableColumnSchemaById(pTableMetaInfo->pTableMeta, tagColId); + // get the tag colId column index + int32_t numOfTags = tscGetNumOfTags(pTableMetaInfo->pTableMeta); + SSchema* pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); + for(int32_t i = 0; i < numOfTags; ++i) { + if (pSchema[i].colId == tagColId) { + index.columnIndex = i; + break; + } + } + int16_t bytes = 0; int16_t type = 0; int32_t inter = 0; @@ -1287,8 +1303,14 @@ int32_t tscHandleMasterJoinQuery(SSqlObj* pSql) { SSqlCmd* pCmd = &pSql->cmd; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); assert((pQueryInfo->type & TSDB_QUERY_TYPE_SUBQUERY) == 0); - + + // todo add test SSubqueryState *pState = calloc(1, sizeof(SSubqueryState)); + if (pState == NULL) { + pSql->res.code = TSDB_CODE_TSC_OUT_OF_MEMORY; + return pSql->res.code; + } + pState->numOfTotal = pQueryInfo->numOfTables; pState->numOfRemain = pState->numOfTotal; @@ -1302,7 +1324,7 @@ int32_t tscHandleMasterJoinQuery(SSqlObj* pSql) { pSql->res.code = TSDB_CODE_TSC_OUT_OF_MEMORY; if (0 == i) { taosTFree(pState); - } + } return pSql->res.code; } @@ -1332,10 +1354,6 @@ static void doCleanupSubqueries(SSqlObj *pSql, int32_t numOfSubs, SSubqueryState SRetrieveSupport* pSupport = pSub->param; taosTFree(pSupport->localBuffer); - - pthread_mutex_unlock(&pSupport->queryMutex); - pthread_mutex_destroy(&pSupport->queryMutex); - taosTFree(pSupport); tscFreeSqlObj(pSub); @@ -1408,14 +1426,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { trs->pParentSql = pSql; trs->pFinalColModel = pModel; - pthread_mutexattr_t mutexattr; - memset(&mutexattr, 0, sizeof(pthread_mutexattr_t)); - - pthread_mutexattr_settype(&mutexattr, PTHREAD_MUTEX_RECURSIVE_NP); - pthread_mutex_init(&trs->queryMutex, &mutexattr); - pthread_mutexattr_destroy(&mutexattr); - - SSqlObj *pNew = tscCreateSqlObjForSubquery(pSql, trs, NULL); + SSqlObj *pNew = tscCreateSTableSubquery(pSql, trs, NULL); if (pNew == NULL) { tscError("%p failed to malloc buffer for subObj, orderOfSub:%d, reason:%s", pSql, i, strerror(errno)); taosTFree(trs->localBuffer); @@ -1460,15 +1471,16 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { } static void tscFreeSubSqlObj(SRetrieveSupport *trsupport, SSqlObj *pSql) { - tscDebug("%p start to free subquery result", pSql); - + tscDebug("%p start to free subquery obj", pSql); + + int32_t index = trsupport->subqueryIndex; + SSqlObj *pParentSql = trsupport->pParentSql; + + assert(pSql == pParentSql->pSubs[index]); + pParentSql->pSubs[index] = NULL; + taos_free_result(pSql); - taosTFree(trsupport->localBuffer); - - pthread_mutex_unlock(&trsupport->queryMutex); - pthread_mutex_destroy(&trsupport->queryMutex); - taosTFree(trsupport); } @@ -1477,23 +1489,11 @@ static void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, i static void tscAbortFurtherRetryRetrieval(SRetrieveSupport *trsupport, TAOS_RES *tres, int32_t code) { // set no disk space error info -#ifdef WINDOWS - LPVOID lpMsgBuf; - FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, NULL, - GetLastError(), MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), // Default language - (LPTSTR)&lpMsgBuf, 0, NULL); - tscError("sub:%p failed to flush data to disk:reason:%s", tres, lpMsgBuf); - LocalFree(lpMsgBuf); -#else tscError("sub:%p failed to flush data to disk, reason:%s", tres, tstrerror(code)); -#endif - SSqlObj* pParentSql = trsupport->pParentSql; pParentSql->res.code = code; trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; - - pthread_mutex_unlock(&trsupport->queryMutex); tscHandleSubqueryError(trsupport, tres, pParentSql->res.code); } @@ -1512,13 +1512,10 @@ static int32_t tscReissueSubquery(SRetrieveSupport *trsupport, SSqlObj *pSql, in // clear local saved number of results trsupport->localBuffer->num = 0; - pthread_mutex_unlock(&trsupport->queryMutex); - - tscTrace("%p sub:%p retrieve failed, code:%s, orderOfSub:%d, retry:%d", trsupport->pParentSql, pSql, + tscError("%p sub:%p retrieve/query failed, code:%s, orderOfSub:%d, retry:%d", trsupport->pParentSql, pSql, tstrerror(code), subqueryIndex, trsupport->numOfRetry); - SSqlObj *pNew = tscCreateSqlObjForSubquery(trsupport->pParentSql, trsupport, pSql); - + SSqlObj *pNew = tscCreateSTableSubquery(trsupport->pParentSql, trsupport, pSql); if (pNew == NULL) { tscError("%p sub:%p failed to create new subquery due to error:%s, abort retry, vgId:%d, orderOfSub:%d", trsupport->pParentSql, pSql, tstrerror(terrno), pVgroup->vgId, trsupport->subqueryIndex); @@ -1529,8 +1526,15 @@ static int32_t tscReissueSubquery(SRetrieveSupport *trsupport, SSqlObj *pSql, in return pParentSql->res.code; } - taos_free_result(pSql); - return tscProcessSql(pNew); + int32_t ret = tscProcessSql(pNew); + + // if failed to process sql, let following code handle the pSql + if (ret == TSDB_CODE_SUCCESS) { + taos_free_result(pSql); + return ret; + } else { + return ret; + } } void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numOfRows) { @@ -1550,14 +1554,14 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO */ pSql->res.numOfRows = 0; trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; // disable retry efforts - tscDebug("%p query is cancelled, sub:%p, orderOfSub:%d abort retrieve, code:%d", pParentSql, pSql, - subqueryIndex, pParentSql->res.code); + tscDebug("%p query is cancelled, sub:%p, orderOfSub:%d abort retrieve, code:%s", pParentSql, pSql, + subqueryIndex, tstrerror(pParentSql->res.code)); } if (numOfRows >= 0) { // current query is successful, but other sub query failed, still abort current query. tscDebug("%p sub:%p retrieve numOfRows:%d,orderOfSub:%d", pParentSql, pSql, numOfRows, subqueryIndex); - tscError("%p sub:%p abort further retrieval due to other queries failure,orderOfSub:%d,code:%d", pParentSql, pSql, - subqueryIndex, pParentSql->res.code); + tscError("%p sub:%p abort further retrieval due to other queries failure,orderOfSub:%d,code:%s", pParentSql, pSql, + subqueryIndex, tstrerror(pParentSql->res.code)); } else { if (trsupport->numOfRetry++ < MAX_NUM_OF_SUBQUERY_RETRY && pParentSql->res.code == TSDB_CODE_SUCCESS) { if (tscReissueSubquery(trsupport, pSql, numOfRows) == TSDB_CODE_SUCCESS) { @@ -1589,10 +1593,10 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO taosTFree(trsupport->pState); tscFreeSubSqlObj(trsupport, pSql); - + // in case of second stage join subquery, invoke its callback function instead of regular QueueAsyncRes SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pParentSql->cmd, 0); - + if (!TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_SEC_STAGE)) { (*pParentSql->fp)(pParentSql->param, pParentSql, pParentSql->res.code); } else { // regular super table query @@ -1671,7 +1675,7 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p // only free once taosTFree(trsupport->pState); tscFreeSubSqlObj(trsupport, pSql); - + // set the command flag must be after the semaphore been correctly set. pParentSql->cmd.command = TSDB_SQL_RETRIEVE_LOCALMERGE; if (pParentSql->res.code == TSDB_CODE_SUCCESS) { @@ -1687,24 +1691,22 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR int32_t idx = trsupport->subqueryIndex; SSqlObj * pParentSql = trsupport->pParentSql; + assert(tres != NULL); SSqlObj *pSql = (SSqlObj *)tres; - if (pSql == NULL) { // sql object has been released in error process, return immediately - tscDebug("%p subquery has been released, idx:%d, abort", pParentSql, idx); - return; - } - +// if (pSql == NULL) { // sql object has been released in error process, return immediately +// tscDebug("%p subquery has been released, idx:%d, abort", pParentSql, idx); +// return; +// } + SSubqueryState* pState = trsupport->pState; assert(pState->numOfRemain <= pState->numOfTotal && pState->numOfRemain >= 0 && pParentSql->numOfSubs == pState->numOfTotal); - // query process and cancel query process may execute at the same time - pthread_mutex_lock(&trsupport->queryMutex); - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); - SCMVgroupInfo* pVgroup = &pTableMetaInfo->vgroupList->vgroups[0]; + SCMVgroupInfo *pVgroup = &pTableMetaInfo->vgroupList->vgroups[0]; if (pParentSql->res.code != TSDB_CODE_SUCCESS) { trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; - tscTrace("%p query cancelled or failed, sub:%p, vgId:%d, orderOfSub:%d, code:%s, global code:%s", + tscDebug("%p query cancelled/failed, sub:%p, vgId:%d, orderOfSub:%d, code:%s, global code:%s", pParentSql, pSql, pVgroup->vgId, trsupport->subqueryIndex, tstrerror(numOfRows), tstrerror(pParentSql->res.code)); tscHandleSubqueryError(param, tres, numOfRows); @@ -1715,13 +1717,13 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR assert(numOfRows == taos_errno(pSql)); if (trsupport->numOfRetry++ < MAX_NUM_OF_SUBQUERY_RETRY) { - tscTrace("%p sub:%p failed code:%s, retry:%d", pParentSql, pSql, tstrerror(numOfRows), trsupport->numOfRetry); + tscError("%p sub:%p failed code:%s, retry:%d", pParentSql, pSql, tstrerror(numOfRows), trsupport->numOfRetry); if (tscReissueSubquery(trsupport, pSql, numOfRows) == TSDB_CODE_SUCCESS) { return; } } else { - tscTrace("%p sub:%p reach the max retry times, set global code:%s", pParentSql, pSql, tstrerror(numOfRows)); + tscDebug("%p sub:%p reach the max retry times, set global code:%s", pParentSql, pSql, tstrerror(numOfRows)); atomic_val_compare_exchange_32(&pParentSql->res.code, TSDB_CODE_SUCCESS, numOfRows); // set global code and abort } @@ -1766,13 +1768,9 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR (int32_t)pRes->numOfRows, pQueryInfo->groupbyExpr.orderType); if (ret != 0) { // set no disk space error info, and abort retry tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_TSC_NO_DISKSPACE); - } else if (pRes->completed) { tscAllDataRetrievedFromDnode(trsupport, pSql); - return; - } else { // continue fetch data from dnode - pthread_mutex_unlock(&trsupport->queryMutex); taos_fetch_rows_a(tres, tscRetrieveFromDnodeCallBack, param); } @@ -1781,15 +1779,15 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR } } -static SSqlObj *tscCreateSqlObjForSubquery(SSqlObj *pSql, SRetrieveSupport *trsupport, SSqlObj *prevSqlObj) { +static SSqlObj *tscCreateSTableSubquery(SSqlObj *pSql, SRetrieveSupport *trsupport, SSqlObj *prevSqlObj) { const int32_t table_index = 0; SSqlObj *pNew = createSubqueryObj(pSql, table_index, tscRetrieveDataRes, trsupport, TSDB_SQL_SELECT, prevSqlObj); if (pNew != NULL) { // the sub query of two-stage super table query SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0); + pQueryInfo->type |= TSDB_QUERY_TYPE_STABLE_SUBQUERY; - - assert(pQueryInfo->numOfTables == 1 && pNew->cmd.numOfClause == 1); + assert(pQueryInfo->numOfTables == 1 && pNew->cmd.numOfClause == 1 && trsupport->subqueryIndex < pSql->numOfSubs); // launch subquery for each vnode, so the subquery index equals to the vgroupIndex. STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, table_index); @@ -1806,7 +1804,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { SSqlObj* pParentSql = trsupport->pParentSql; SSqlObj* pSql = (SSqlObj *) tres; - + SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); assert(pSql->cmd.numOfClause == 1 && pQueryInfo->numOfTables == 1); @@ -1816,7 +1814,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { // stable query killed or other subquery failed, all query stopped if (pParentSql->res.code != TSDB_CODE_SUCCESS) { trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; - tscTrace("%p query cancelled or failed, sub:%p, vgId:%d, orderOfSub:%d, code:%s, global code:%s", + tscError("%p query cancelled or failed, sub:%p, vgId:%d, orderOfSub:%d, code:%s, global code:%s", pParentSql, pSql, pVgroup->vgId, trsupport->subqueryIndex, tstrerror(code), tstrerror(pParentSql->res.code)); tscHandleSubqueryError(param, tres, code); @@ -1834,12 +1832,12 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { assert(code == taos_errno(pSql)); if (trsupport->numOfRetry++ < MAX_NUM_OF_SUBQUERY_RETRY) { - tscTrace("%p sub:%p failed code:%s, retry:%d", pParentSql, pSql, tstrerror(code), trsupport->numOfRetry); + tscError("%p sub:%p failed code:%s, retry:%d", pParentSql, pSql, tstrerror(code), trsupport->numOfRetry); if (tscReissueSubquery(trsupport, pSql, code) == TSDB_CODE_SUCCESS) { return; } } else { - tscTrace("%p sub:%p reach the max retry times, set global code:%s", pParentSql, pSql, tstrerror(code)); + tscError("%p sub:%p reach the max retry times, set global code:%s", pParentSql, pSql, tstrerror(code)); atomic_val_compare_exchange_32(&pParentSql->res.code, TSDB_CODE_SUCCESS, code); // set global code and abort } @@ -1847,7 +1845,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { return; } - tscTrace("%p sub:%p query complete, ep:%s, vgId:%d, orderOfSub:%d, retrieve data", trsupport->pParentSql, pSql, + tscDebug("%p sub:%p query complete, ep:%s, vgId:%d, orderOfSub:%d, retrieve data", trsupport->pParentSql, pSql, pVgroup->epAddr[0].fqdn, pVgroup->vgId, trsupport->subqueryIndex); if (pSql->res.qhandle == 0) { // qhandle is NULL, code is TSDB_CODE_SUCCESS means no results generated from this vnode @@ -1926,8 +1924,14 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) { size_t size = taosArrayGetSize(pCmd->pDataBlocks); assert(size > 0); - pSql->pSubs = calloc(size, POINTER_BYTES); + // the number of already initialized subqueries + int32_t numOfSub = 0; + pSql->numOfSubs = (uint16_t)size; + pSql->pSubs = calloc(size, POINTER_BYTES); + if (pSql->pSubs == NULL) { + goto _error; + } tscDebug("%p submit data to %" PRIzu " vnode(s)", pSql, size); @@ -1936,10 +1940,13 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) { pState->numOfRemain = pSql->numOfSubs; pRes->code = TSDB_CODE_SUCCESS; - int32_t numOfSub = 0; while(numOfSub < pSql->numOfSubs) { SInsertSupporter* pSupporter = calloc(1, sizeof(SInsertSupporter)); + if (pSupporter == NULL) { + goto _error; + } + pSupporter->pSql = pSql; pSupporter->pState = pState; pSupporter->index = numOfSub; @@ -1972,7 +1979,7 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) { if (numOfSub < pSql->numOfSubs) { tscError("%p failed to prepare subObj structure and launch sub-insertion", pSql); pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; - return pRes->code; // free all allocated resource + goto _error; } pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); @@ -2070,46 +2077,8 @@ void tscBuildResFromSubqueries(SSqlObj *pSql) { doBuildResFromSubqueries(pSql); tsem_post(&pSql->rspSem); - return; - - // continue retrieve data from vnode -// if (!tscHasRemainDataInSubqueryResultSet(pSql)) { -// tscDebug("%p at least one subquery exhausted, free all other %d subqueries", pSql, pSql->numOfSubs - 1); -// SSubqueryState* pState = NULL; -// -// // free all sub sqlobj -// for (int32_t i = 0; i < pSql->numOfSubs; ++i) { -// SSqlObj* pChildObj = pSql->pSubs[i]; -// if (pChildObj == NULL) { -// continue; -// } -// -// SJoinSupporter* pSupporter = (SJoinSupporter*)pChildObj->param; -// pState = pSupporter->pState; -// -// tscDestroyJoinSupporter(pChildObj->param); -// taos_free_result(pChildObj); -// } -// -// free(pState); -// -// pRes->completed = true; // set query completed -// tsem_post(&pSql->rspSem); -// return; -// } - -// tscFetchDatablockFromSubquery(pSql); -// if (pRes->code != TSDB_CODE_SUCCESS) { -// return; -// } } - -// if (pSql->res.code == TSDB_CODE_SUCCESS) { -// (*pSql->fp)(pSql->param, pSql, pRes->numOfRows); -// } else { -// tscQueueAsyncRes(pSql); -// } } static void transferNcharData(SSqlObj *pSql, int32_t columnIndex, TAOS_FIELD *pField) { @@ -2159,7 +2128,6 @@ void **doSetResultRowData(SSqlObj *pSql, bool finalResult) { SSqlRes *pRes = &pSql->res; assert(pRes->row >= 0 && pRes->row <= pRes->numOfRows); - if (pRes->row >= pRes->numOfRows) { // all the results has returned to invoker taosTFree(pRes->tsrow); return pRes->tsrow; diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 64a871ff74..b61fd7e8c9 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -373,7 +373,7 @@ void tscFreeSqlObj(SSqlObj* pSql) { if (pSql == NULL || pSql->signature != pSql) { return; } - + tscDebug("%p start to free sql object", pSql); tscPartiallyFreeSqlObj(pSql); @@ -388,6 +388,7 @@ void tscFreeSqlObj(SSqlObj* pSql) { taosTFree(pSql->sqlstr); tsem_destroy(&pSql->rspSem); + free(pSql); } @@ -404,7 +405,7 @@ void tscDestroyDataBlock(STableDataBlocks* pDataBlock) { taosTFree(pDataBlock); } -SParamInfo* tscAddParamToDataBlock(STableDataBlocks* pDataBlock, char type, uint8_t timePrec, short bytes, +SParamInfo* tscAddParamToDataBlock(STableDataBlocks* pDataBlock, char type, uint8_t timePrec, int16_t bytes, uint32_t offset) { uint32_t needed = pDataBlock->numOfParams + 1; if (needed > pDataBlock->numOfAllocedParams) { @@ -485,15 +486,6 @@ int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock) { return TSDB_CODE_SUCCESS; } -//void tscFreeUnusedDataBlocks(SDataBlockList* pList) { -// /* release additional memory consumption */ -// for (int32_t i = 0; i < pList->nSize; ++i) { -// STableDataBlocks* pDataBlock = pList->pData[i]; -// pDataBlock->pData = realloc(pDataBlock->pData, pDataBlock->size); -// pDataBlock->nAllocSize = (uint32_t)pDataBlock->size; -// } -//} - /** * create the in-memory buffer for each table to keep the submitted data block * @param initialSize @@ -518,6 +510,11 @@ int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOff } dataBuf->pData = calloc(1, dataBuf->nAllocSize); + if (dataBuf->pData == NULL) { + tscError("failed to allocated memory, reason:%s", strerror(errno)); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + dataBuf->ordered = true; dataBuf->prevTS = INT64_MIN; @@ -742,7 +739,7 @@ bool tscIsInsertData(char* sqlstr) { int32_t index = 0; do { - SSQLToken t0 = tStrGetToken(sqlstr, &index, false, 0, NULL); + SStrToken t0 = tStrGetToken(sqlstr, &index, false, 0, NULL); if (t0.type != TK_LP) { return t0.type == TK_INSERT || t0.type == TK_IMPORT; } @@ -926,17 +923,23 @@ void tscFieldInfoClear(SFieldInfo* pFieldInfo) { } static SSqlExpr* doBuildSqlExpr(SQueryInfo* pQueryInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type, - int16_t size, int16_t interSize, bool isTagCol) { + int16_t size, int16_t interSize, int32_t colType) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pColIndex->tableIndex); SSqlExpr* pExpr = calloc(1, sizeof(SSqlExpr)); + if (pExpr == NULL) { + return NULL; + } + pExpr->functionId = functionId; - + // set the correct columnIndex index if (pColIndex->columnIndex == TSDB_TBNAME_COLUMN_INDEX) { pExpr->colInfo.colId = TSDB_TBNAME_COLUMN_INDEX; + } else if (pColIndex->columnIndex <= TSDB_UD_COLUMN_INDEX) { + pExpr->colInfo.colId = pColIndex->columnIndex; } else { - if (isTagCol) { + if (TSDB_COL_IS_TAG(colType)) { SSchema* pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); pExpr->colInfo.colId = pSchema[pColIndex->columnIndex].colId; tstrncpy(pExpr->colInfo.name, pSchema[pColIndex->columnIndex].name, sizeof(pExpr->colInfo.name)); @@ -948,9 +951,9 @@ static SSqlExpr* doBuildSqlExpr(SQueryInfo* pQueryInfo, int16_t functionId, SCol } } - pExpr->colInfo.flag = isTagCol? TSDB_COL_TAG:TSDB_COL_NORMAL; - + pExpr->colInfo.flag = colType; pExpr->colInfo.colIndex = pColIndex->columnIndex; + pExpr->resType = type; pExpr->resBytes = size; pExpr->interBytes = interSize; @@ -1060,8 +1063,11 @@ void tscSqlExprCopy(SArray* dst, const SArray* src, uint64_t uid, bool deepcopy) if (deepcopy) { SSqlExpr* p1 = calloc(1, sizeof(SSqlExpr)); + if (p1 == NULL) { + assert(0); + } + *p1 = *pExpr; - for (int32_t j = 0; j < pExpr->numOfParams; ++j) { tVariantAssign(&p1->param[j], &pExpr->param[j]); } @@ -1097,16 +1103,22 @@ SColumn* tscColumnListInsert(SArray* pColumnList, SColumnIndex* pColIndex) { if (i >= numOfCols || numOfCols == 0) { SColumn* b = calloc(1, sizeof(SColumn)); + if (b == NULL) { + return NULL; + } + b->colIndex = *pColIndex; - taosArrayInsert(pColumnList, i, &b); } else { SColumn* pCol = taosArrayGetP(pColumnList, i); if (i < numOfCols && (pCol->colIndex.columnIndex > col || pCol->colIndex.tableIndex != pColIndex->tableIndex)) { SColumn* b = calloc(1, sizeof(SColumn)); + if (b == NULL) { + return NULL; + } + b->colIndex = *pColIndex; - taosArrayInsert(pColumnList, i, &b); } } @@ -1128,7 +1140,10 @@ SColumn* tscColumnClone(const SColumn* src) { assert(src != NULL); SColumn* dst = calloc(1, sizeof(SColumn)); - + if (dst == NULL) { + return NULL; + } + dst->colIndex = src->colIndex; dst->numOfFilters = src->numOfFilters; dst->filterInfo = tscFilterInfoClone(src->filterInfo, src->numOfFilters); @@ -1183,7 +1198,7 @@ void tscColumnListDestroy(SArray* pColumnList) { * 'first_part.second_part' * */ -static int32_t validateQuoteToken(SSQLToken* pToken) { +static int32_t validateQuoteToken(SStrToken* pToken) { strdequote(pToken->z); pToken->n = (uint32_t)strtrim(pToken->z); @@ -1199,7 +1214,7 @@ static int32_t validateQuoteToken(SSQLToken* pToken) { return TSDB_CODE_SUCCESS; } -int32_t tscValidateName(SSQLToken* pToken) { +int32_t tscValidateName(SStrToken* pToken) { if (pToken->type != TK_STRING && pToken->type != TK_ID) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -1286,12 +1301,12 @@ void tscIncStreamExecutionCount(void* pStream) { ps->num += 1; } -bool tscValidateColumnId(STableMetaInfo* pTableMetaInfo, int32_t colId) { +bool tscValidateColumnId(STableMetaInfo* pTableMetaInfo, int32_t colId, int32_t numOfParams) { if (pTableMetaInfo->pTableMeta == NULL) { return false; } - if (colId == TSDB_TBNAME_COLUMN_INDEX) { + if (colId == TSDB_TBNAME_COLUMN_INDEX || (colId <= TSDB_UD_COLUMN_INDEX && numOfParams == 2)) { return true; } @@ -1338,6 +1353,10 @@ void tscTagCondCopy(STagCond* dest, const STagCond* src) { if (pCond->len > 0) { assert(pCond->cond != NULL); c.cond = malloc(c.len); + if (c.cond == NULL) { + assert(0); + } + memcpy(c.cond, pCond->cond, c.len); } @@ -1463,20 +1482,20 @@ STableMetaInfo* tscGetMetaInfo(SQueryInfo* pQueryInfo, int32_t tableIndex) { return pQueryInfo->pTableMetaInfo[tableIndex]; } -int32_t tscGetQueryInfoDetailSafely(SSqlCmd* pCmd, int32_t subClauseIndex, SQueryInfo** pQueryInfo) { +SQueryInfo* tscGetQueryInfoDetailSafely(SSqlCmd* pCmd, int32_t subClauseIndex) { + SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, subClauseIndex); int32_t ret = TSDB_CODE_SUCCESS; - *pQueryInfo = tscGetQueryInfoDetail(pCmd, subClauseIndex); - - while ((*pQueryInfo) == NULL) { + while ((pQueryInfo) == NULL) { if ((ret = tscAddSubqueryInfo(pCmd)) != TSDB_CODE_SUCCESS) { - return ret; + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + return NULL; } - (*pQueryInfo) = tscGetQueryInfoDetail(pCmd, subClauseIndex); + pQueryInfo = tscGetQueryInfoDetail(pCmd, subClauseIndex); } - return TSDB_CODE_SUCCESS; + return pQueryInfo; } STableMetaInfo* tscGetTableMetaInfoByUid(SQueryInfo* pQueryInfo, uint64_t uid, int32_t* index) { @@ -1507,6 +1526,7 @@ void tscInitQueryInfo(SQueryInfo* pQueryInfo) { assert(pQueryInfo->exprList == NULL); pQueryInfo->exprList = taosArrayInit(4, POINTER_BYTES); pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); + pQueryInfo->udColumnId = TSDB_UD_COLUMN_INDEX; } int32_t tscAddSubqueryInfo(SSqlCmd* pCmd) { @@ -1522,8 +1542,11 @@ int32_t tscAddSubqueryInfo(SSqlCmd* pCmd) { pCmd->pQueryInfo = (SQueryInfo**)tmp; SQueryInfo* pQueryInfo = calloc(1, sizeof(SQueryInfo)); + if (pQueryInfo == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + tscInitQueryInfo(pQueryInfo); - pQueryInfo->msg = pCmd->payload; // pointer to the parent error message buffer pCmd->pQueryInfo[pCmd->numOfClause++] = pQueryInfo; @@ -1584,14 +1607,18 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST SVgroupsInfo* vgroupList, SArray* pTagCols) { void* pAlloc = realloc(pQueryInfo->pTableMetaInfo, (pQueryInfo->numOfTables + 1) * POINTER_BYTES); if (pAlloc == NULL) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; return NULL; } pQueryInfo->pTableMetaInfo = pAlloc; - pQueryInfo->pTableMetaInfo[pQueryInfo->numOfTables] = calloc(1, sizeof(STableMetaInfo)); + STableMetaInfo* pTableMetaInfo = calloc(1, sizeof(STableMetaInfo)); + if (pTableMetaInfo == NULL) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + return NULL; + } - STableMetaInfo* pTableMetaInfo = pQueryInfo->pTableMetaInfo[pQueryInfo->numOfTables]; - assert(pTableMetaInfo != NULL); + pQueryInfo->pTableMetaInfo[pQueryInfo->numOfTables] = pTableMetaInfo; if (name != NULL) { tstrncpy(pTableMetaInfo->name, name, sizeof(pTableMetaInfo->name)); @@ -1602,10 +1629,18 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST if (vgroupList != NULL) { size_t size = sizeof(SVgroupsInfo) + sizeof(SCMVgroupInfo) * vgroupList->numOfVgroups; pTableMetaInfo->vgroupList = malloc(size); + if (pTableMetaInfo->vgroupList == NULL) { + return NULL; + } + memcpy(pTableMetaInfo->vgroupList, vgroupList, size); } pTableMetaInfo->tagColList = taosArrayInit(4, POINTER_BYTES); + if (pTableMetaInfo->tagColList == NULL) { + return NULL; + } + if (pTagCols != NULL) { tscColumnListCopy(pTableMetaInfo->tagColList, pTagCols, -1); } @@ -1671,8 +1706,7 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cm return NULL; } - SQueryInfo* pQueryInfo = NULL; - tscGetQueryInfoDetailSafely(pCmd, 0, &pQueryInfo); + SQueryInfo* pQueryInfo = tscGetQueryInfoDetailSafely(pCmd, 0); assert(pSql->cmd.clauseIndex == 0); STableMetaInfo* pMasterTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, pSql->cmd.clauseIndex, 0); @@ -1754,6 +1788,7 @@ static void doSetSqlExprAndResultFieldInfo(SQueryInfo* pQueryInfo, SQueryInfo* p SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void* param, int32_t cmd, SSqlObj* pPrevSql) { SSqlCmd* pCmd = &pSql->cmd; + SSqlObj* pNew = (SSqlObj*)calloc(1, sizeof(SSqlObj)); if (pNew == NULL) { tscError("%p new subquery failed, tableIndex:%d", pSql, tableIndex); @@ -1769,10 +1804,8 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void pNew->sqlstr = strdup(pSql->sqlstr); if (pNew->sqlstr == NULL) { tscError("%p new subquery failed, tableIndex:%d, vgroupIndex:%d", pSql, tableIndex, pTableMetaInfo->vgroupIndex); - - free(pNew); terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - return NULL; + goto _error; } SSqlCmd* pnCmd = &pNew->cmd; @@ -1789,9 +1822,8 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void pnCmd->parseFinished = 1; if (tscAddSubqueryInfo(pnCmd) != TSDB_CODE_SUCCESS) { - tscFreeSqlObj(pNew); terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - return NULL; + goto _error; } SQueryInfo* pNewQueryInfo = tscGetQueryInfoDetail(pnCmd, 0); @@ -1816,20 +1848,28 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void pNewQueryInfo->groupbyExpr = pQueryInfo->groupbyExpr; if (pQueryInfo->groupbyExpr.columnInfo != NULL) { pNewQueryInfo->groupbyExpr.columnInfo = taosArrayClone(pQueryInfo->groupbyExpr.columnInfo); + if (pNewQueryInfo->groupbyExpr.columnInfo == NULL) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto _error; + } } tscTagCondCopy(&pNewQueryInfo->tagCond, &pQueryInfo->tagCond); if (pQueryInfo->fillType != TSDB_FILL_NONE) { pNewQueryInfo->fillVal = malloc(pQueryInfo->fieldsInfo.numOfOutput * sizeof(int64_t)); + if (pNewQueryInfo->fillVal == NULL) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto _error; + } + memcpy(pNewQueryInfo->fillVal, pQueryInfo->fillVal, pQueryInfo->fieldsInfo.numOfOutput * sizeof(int64_t)); } if (tscAllocPayload(pnCmd, TSDB_DEFAULT_PAYLOAD_SIZE) != TSDB_CODE_SUCCESS) { tscError("%p new subquery failed, tableIndex:%d, vgroupIndex:%d", pSql, tableIndex, pTableMetaInfo->vgroupIndex); - tscFreeSqlObj(pNew); terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - return NULL; + goto _error; } tscColumnListCopy(pNewQueryInfo->colList, pQueryInfo->colList, (int16_t)tableIndex); @@ -1872,16 +1912,15 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void if (pFinalInfo->pTableMeta == NULL) { tscError("%p new subquery failed since no tableMeta in cache, name:%s", pSql, name); - tscFreeSqlObj(pNew); - if (pPrevSql != NULL) { + if (pPrevSql != NULL) { // pass the previous error to client assert(pPrevSql->res.code != TSDB_CODE_SUCCESS); terrno = pPrevSql->res.code; } else { terrno = TSDB_CODE_TSC_APP_ERROR; } - return NULL; + goto _error; } assert(pNewQueryInfo->numOfTables == 1); @@ -1906,6 +1945,10 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void } return pNew; + +_error: + tscFreeSqlObj(pNew); + return NULL; } /** diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index 4ffc631566..ef0713c415 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -29,6 +29,7 @@ extern uint16_t tsServerPort; extern uint16_t tsDnodeShellPort; extern uint16_t tsDnodeDnodePort; extern uint16_t tsSyncPort; +extern uint16_t tsArbitratorPort; extern int32_t tsStatusInterval; extern int32_t tsNumOfMnodes; extern int32_t tsEnableVnodeBak; diff --git a/src/common/inc/tname.h b/src/common/inc/tname.h index 2263a5dae1..2a4ac3fc40 100644 --- a/src/common/inc/tname.h +++ b/src/common/inc/tname.h @@ -4,6 +4,7 @@ #include "os.h" #include "taosmsg.h" #include "tstoken.h" +#include "tvariant.h" typedef struct SDataStatis { int16_t colId; @@ -24,10 +25,12 @@ void extractTableName(const char *tableId, char *name); char* extractDBName(const char *tableId, char *name); -void extractTableNameFromToken(SSQLToken *pToken, SSQLToken* pTable); +void extractTableNameFromToken(SStrToken *pToken, SStrToken* pTable); SSchema tGetTableNameColumnSchema(); +SSchema tGetUserSpecifiedColumnSchema(tVariant* pVal, SStrToken* exprStr, const char* name); + bool tscValidateTableNameLength(size_t len); SColumnFilterInfo* tscFilterInfoClone(const SColumnFilterInfo* src, int32_t numOfFilters); diff --git a/src/common/inc/tvariant.h b/src/common/inc/tvariant.h index 4fd6ea5541..e9973bcb95 100644 --- a/src/common/inc/tvariant.h +++ b/src/common/inc/tvariant.h @@ -36,7 +36,7 @@ typedef struct tVariant { }; } tVariant; -void tVariantCreate(tVariant *pVar, SSQLToken *token); +void tVariantCreate(tVariant *pVar, SStrToken *token); void tVariantCreateFromString(tVariant *pVar, char *pz, uint32_t len, uint32_t type); @@ -46,6 +46,8 @@ void tVariantDestroy(tVariant *pV); void tVariantAssign(tVariant *pDst, const tVariant *pSrc); +int32_t tVariantCompare(const tVariant* p1, const tVariant* p2); + int32_t tVariantToString(tVariant *pVar, char *dst); int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool includeLengthPrefix); diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index e9d7a71477..795585e5c9 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -37,6 +37,7 @@ uint16_t tsServerPort = 6030; uint16_t tsDnodeShellPort = 6030; // udp[6035-6039] tcp[6035] uint16_t tsDnodeDnodePort = 6035; // udp/tcp uint16_t tsSyncPort = 6040; +uint16_t tsArbitratorPort = 6042; int32_t tsStatusInterval = 1; // second int32_t tsNumOfMnodes = 3; int32_t tsEnableVnodeBak = 1; @@ -54,7 +55,7 @@ int8_t tsDaylight = 0; char tsTimezone[TSDB_TIMEZONE_LEN] = {0}; char tsLocale[TSDB_LOCALE_LEN] = {0}; char tsCharset[TSDB_LOCALE_LEN] = {0}; // default encode string -int32_t tsEnableCoreFile = 1; +int32_t tsEnableCoreFile = 0; int32_t tsMaxBinaryDisplayWidth = 30; /* @@ -1331,7 +1332,10 @@ int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port) { *port = atoi(temp+1); } - if (*port == 0) *port = tsServerPort; + if (*port == 0) { + *port = tsServerPort; + return -1; + } return 0; } diff --git a/src/common/src/tname.c b/src/common/src/tname.c index d80ddb0ee2..01945dbb00 100644 --- a/src/common/src/tname.c +++ b/src/common/src/tname.c @@ -4,6 +4,7 @@ #include "tname.h" #include "tstoken.h" #include "ttokendef.h" +#include "tvariant.h" // todo refactor UNUSED_FUNC static FORCE_INLINE const char* skipSegments(const char* input, char delim, int32_t num) { @@ -43,7 +44,30 @@ SSchema tGetTableNameColumnSchema() { s.bytes = TSDB_TABLE_NAME_LEN - 1 + VARSTR_HEADER_SIZE; s.type = TSDB_DATA_TYPE_BINARY; s.colId = TSDB_TBNAME_COLUMN_INDEX; - strncpy(s.name, TSQL_TBNAME_L, TSDB_COL_NAME_LEN); + tstrncpy(s.name, TSQL_TBNAME_L, TSDB_COL_NAME_LEN); + return s; +} + +SSchema tGetUserSpecifiedColumnSchema(tVariant* pVal, SStrToken* exprStr, const char* name) { + SSchema s = {0}; + + s.type = pVal->nType; + if (s.type == TSDB_DATA_TYPE_BINARY || s.type == TSDB_DATA_TYPE_NCHAR) { + s.bytes = (int16_t)(pVal->nLen + VARSTR_HEADER_SIZE); + } else { + s.bytes = tDataTypeDesc[pVal->nType].nSize; + } + + s.colId = TSDB_UD_COLUMN_INDEX; + if (name != NULL) { + tstrncpy(s.name, name, sizeof(s.name)); + } else { + size_t len = strdequote(exprStr->z); + size_t tlen = MIN(sizeof(s.name), len + 1); + + tstrncpy(s.name, exprStr->z, tlen); + } + return s; } @@ -110,7 +134,7 @@ int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, in * tablePrefix.columnName * extract table name and save it in pTable, with only column name in pToken */ -void extractTableNameFromToken(SSQLToken* pToken, SSQLToken* pTable) { +void extractTableNameFromToken(SStrToken* pToken, SStrToken* pTable) { const char sep = TS_PATH_DELIMITER[0]; if (pToken == pTable || pToken == NULL || pTable == NULL) { diff --git a/src/common/src/tvariant.c b/src/common/src/tvariant.c index 5460d21252..069d8eb251 100644 --- a/src/common/src/tvariant.c +++ b/src/common/src/tvariant.c @@ -12,12 +12,10 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ +#include "os.h" #include "tvariant.h" #include "hash.h" -#include "hashfunc.h" -#include "os.h" -#include "hash.h" #include "taos.h" #include "taosdef.h" #include "tstoken.h" @@ -25,7 +23,7 @@ #include "tutil.h" // todo support scientific expression number and oct number -void tVariantCreate(tVariant *pVar, SSQLToken *token) { tVariantCreateFromString(pVar, token->z, token->n, token->type); } +void tVariantCreate(tVariant *pVar, SStrToken *token) { tVariantCreateFromString(pVar, token->z, token->n, token->type); } void tVariantCreateFromString(tVariant *pVar, char *pz, uint32_t len, uint32_t type) { memset(pVar, 0, sizeof(tVariant)); @@ -102,10 +100,9 @@ void tVariantCreateFromBinary(tVariant *pVar, const char *pz, size_t len, uint32 } case TSDB_DATA_TYPE_NCHAR: { // here we get the nchar length from raw binary bits length size_t lenInwchar = len / TSDB_NCHAR_SIZE; + pVar->wpz = calloc(1, (lenInwchar + 1) * TSDB_NCHAR_SIZE); - - wcsncpy(pVar->wpz, (wchar_t *)pz, lenInwchar); - pVar->wpz[lenInwchar] = 0; + memcpy(pVar->wpz, pz, lenInwchar * TSDB_NCHAR_SIZE); pVar->nLen = (int32_t)len; break; @@ -169,6 +166,50 @@ void tVariantAssign(tVariant *pDst, const tVariant *pSrc) { char* n = strdup(p); taosArrayPush(pDst->arr, &n); } + + return; + } + + pDst->nLen = tDataTypeDesc[pDst->nType].nSize; +} + +int32_t tVariantCompare(const tVariant* p1, const tVariant* p2) { + if (p1->nType == TSDB_DATA_TYPE_NULL && p2->nType == TSDB_DATA_TYPE_NULL) { + return 0; + } + + if (p1->nType == TSDB_DATA_TYPE_NULL) { + return -1; + } + + if (p2->nType == TSDB_DATA_TYPE_NULL) { + return 1; + } + + switch (p1->nType) { + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: { + if (p1->nLen == p2->nLen) { + return memcmp(p1->pz, p2->pz, p1->nLen); + } else { + return p1->nLen > p2->nLen? 1:-1; + } + }; + + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_DOUBLE: + if (p1->dKey == p2->dKey) { + return 0; + } else { + return p1->dKey > p2->dKey? 1:-1; + } + + default: + if (p1->i64Key == p2->i64Key) { + return 0; + } else { + return p1->i64Key > p2->i64Key? 1:-1; + } } } @@ -228,7 +269,7 @@ static int32_t doConvertToInteger(tVariant *pVariant, char *pDest, int32_t type, errno = 0; char *endPtr = NULL; - SSQLToken token = {0}; + SStrToken token = {0}; token.n = tSQLGetToken(pVariant->pz, &token.type); if (token.type == TK_MINUS || token.type == TK_PLUS) { @@ -277,7 +318,7 @@ static int32_t doConvertToInteger(tVariant *pVariant, char *pDest, int32_t type, errno = 0; wchar_t *endPtr = NULL; - SSQLToken token = {0}; + SStrToken token = {0}; token.n = tSQLGetToken(pVariant->pz, &token.type); if (token.type == TK_MINUS || token.type == TK_PLUS) { @@ -436,7 +477,7 @@ static int32_t toNchar(tVariant *pVariant, char **pDest, int32_t *pDestSize) { } static FORCE_INLINE int32_t convertToDouble(char *pStr, int32_t len, double *value) { - SSQLToken stoken = {.z = pStr, .n = len}; + SStrToken stoken = {.z = pStr, .n = len}; if (TK_ILLEGAL == isValidNumber(&stoken)) { return -1; @@ -462,7 +503,7 @@ static FORCE_INLINE int32_t convertToInteger(tVariant *pVariant, int64_t *result errno = 0; char *endPtr = NULL; - SSQLToken token = {0}; + SStrToken token = {0}; token.n = tSQLGetToken(pVariant->pz, &token.type); if (token.type == TK_MINUS || token.type == TK_PLUS) { @@ -479,7 +520,7 @@ static FORCE_INLINE int32_t convertToInteger(tVariant *pVariant, int64_t *result return 0; } - SSQLToken sToken = {.z = pVariant->pz, .n = pVariant->nLen}; + SStrToken sToken = {.z = pVariant->pz, .n = pVariant->nLen}; if (TK_ILLEGAL == isValidNumber(&sToken)) { return -1; } @@ -515,7 +556,7 @@ static FORCE_INLINE int32_t convertToInteger(tVariant *pVariant, int64_t *result errno = 0; wchar_t *endPtr = NULL; - SSQLToken token = {0}; + SStrToken token = {0}; token.n = tSQLGetToken(pVariant->pz, &token.type); if (token.type == TK_MINUS || token.type == TK_PLUS) { diff --git a/src/connector/jdbc/.classpath b/src/connector/jdbc/.classpath index 39abf1c5e9..a5d95095cc 100644 --- a/src/connector/jdbc/.classpath +++ b/src/connector/jdbc/.classpath @@ -28,22 +28,5 @@ - - - - - - - - - - - - - - - - - diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java index a36f7c67a0..06ae449596 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java @@ -52,7 +52,7 @@ public class TSDBStatement implements Statement { if (isClosed) { throw new SQLException("Invalid method call on a closed statement."); } - + // TODO make sure it is not a update query pSql = this.connecter.executeQuery(sql); @@ -68,21 +68,21 @@ public class TSDBStatement implements Statement { this.connecter.freeResultSet(pSql); return null; } - + if (!this.connecter.isUpdateQuery(pSql)) { return new TSDBResultSet(this.connecter, resultSetPointer); } else { this.connecter.freeResultSet(pSql); return null; } - + } public int executeUpdate(String sql) throws SQLException { if (isClosed) { throw new SQLException("Invalid method call on a closed statement."); } - + // TODO check if current query is update query pSql = this.connecter.executeQuery(sql); long resultSetPointer = this.connecter.getResultSet(); @@ -94,7 +94,7 @@ public class TSDBStatement implements Statement { this.affectedRows = this.connecter.getAffectedRows(pSql); this.connecter.freeResultSet(pSql); - + return this.affectedRows; } @@ -192,7 +192,7 @@ public class TSDBStatement implements Statement { if (isClosed) { throw new SQLException("Invalid method call on a closed statement."); } - + return this.affectedRows; } diff --git a/src/connector/jdbc/src/test/java/TestPreparedStatement.java b/src/connector/jdbc/src/test/java/TestPreparedStatement.java index 2e387206a4..1edb957493 100644 --- a/src/connector/jdbc/src/test/java/TestPreparedStatement.java +++ b/src/connector/jdbc/src/test/java/TestPreparedStatement.java @@ -12,7 +12,7 @@ public class TestPreparedStatement { Class.forName("com.taosdata.jdbc.TSDBDriver"); Properties properties = new Properties(); properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, "localhost"); - connection = DriverManager.getConnection("jdbc:TAOS://localhost:0/?user=root&password=taosdata", properties); + connection = DriverManager.getConnection("jdbc:TAOS://localhost:0/", properties); String rawSql = "select * from test.log0601"; // String[] params = new String[]{"ts", "c1"}; PreparedStatement pstmt = (TSDBPreparedStatement) connection.prepareStatement(rawSql); diff --git a/src/connector/jdbc/src/test/java/TestTSDBDatabaseMetaData.java b/src/connector/jdbc/src/test/java/TestTSDBDatabaseMetaData.java index f7e0e78441..39a08f0fe9 100644 --- a/src/connector/jdbc/src/test/java/TestTSDBDatabaseMetaData.java +++ b/src/connector/jdbc/src/test/java/TestTSDBDatabaseMetaData.java @@ -13,7 +13,7 @@ public class TestTSDBDatabaseMetaData { Class.forName("com.taosdata.jdbc.TSDBDriver"); Properties properties = new Properties(); properties.setProperty(TSDBDriver.PROPERTY_KEY_HOST, "localhost"); - connection = DriverManager.getConnection("jdbc:TAOS://localhost:0/?user=root&password=taosdata", properties); + connection = DriverManager.getConnection("jdbc:TAOS://localhost:0/", properties); dbMetaData = connection.getMetaData(); resSet = dbMetaData.getCatalogs(); while(resSet.next()) { diff --git a/src/connector/jdbc/src/test/java/TestTSDBSubscribe.java b/src/connector/jdbc/src/test/java/TestTSDBSubscribe.java index df730efa69..47acb20064 100644 --- a/src/connector/jdbc/src/test/java/TestTSDBSubscribe.java +++ b/src/connector/jdbc/src/test/java/TestTSDBSubscribe.java @@ -3,7 +3,6 @@ import com.taosdata.jdbc.TSDBDriver; import com.taosdata.jdbc.TSDBResultSet; import com.taosdata.jdbc.TSDBSubscribe; -import java.sql.Connection; import java.sql.DriverManager; import java.util.Properties; @@ -17,12 +16,10 @@ public class TestTSDBSubscribe { properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - String cs = String.format("jdbc:TAOS://%s:0/%s?user=root&password=taosdata", host, database); + String cs = String.format("jdbc:TAOS://%s:0/%s", host, database); return (TSDBConnection)DriverManager.getConnection(cs, properties); } - - public static void main(String[] args) throws Exception { String usage = "java -Djava.ext.dirs=../ TestTSDBSubscribe [-host host] <-db database> <-topic topic> <-sql sql>"; if (args.length < 2) { diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/BatchInsertTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/BatchInsertTest.java index cb78a5ca0e..c49293c96b 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/BatchInsertTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/BatchInsertTest.java @@ -40,8 +40,7 @@ public class BatchInsertTest extends BaseTest { properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + "?user=root&password=taosdata" - , properties); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); statement = connection.createStatement(); statement.executeUpdate("drop database if exists " + dbName); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ConnectionTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ConnectionTest.java index 3fc8913ca3..a54ece4ead 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ConnectionTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ConnectionTest.java @@ -29,16 +29,14 @@ public class ConnectionTest extends BaseTest { properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + "?user=root&password=taosdata" - , properties); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); assertTrue(null != connection); statement = connection.createStatement(); assertTrue(null != statement); // try reconnect - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + "?user=root&password=taosdata" - , properties); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); try { statement.execute("create database if not exists " + dbName); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/DatabaseMetaDataTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/DatabaseMetaDataTest.java index 4c4d56020d..284af3dfe7 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/DatabaseMetaDataTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/DatabaseMetaDataTest.java @@ -26,8 +26,7 @@ public class DatabaseMetaDataTest extends BaseTest { properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + "?user=root&password=taosdata", - properties); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); String sql = "drop database if exists " + dbName; statement = (TSDBPreparedStatement) connection.prepareStatement(sql); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ImportTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ImportTest.java index 016eff1a35..dbe16d9fea 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ImportTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ImportTest.java @@ -28,8 +28,7 @@ public class ImportTest extends BaseTest { properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + "?user=root&password=taosdata" - , properties); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); statement = connection.createStatement(); statement.executeUpdate("drop database if exists " + dbName); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/PreparedStatementTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/PreparedStatementTest.java index d391771269..2cf88f6630 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/PreparedStatementTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/PreparedStatementTest.java @@ -33,8 +33,7 @@ public class PreparedStatementTest extends BaseTest { properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + "?user=root&password=taosdata", - properties); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/",properties); String sql = "drop database if exists " + dbName; statement = (TSDBPreparedStatement) connection.prepareStatement(sql); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ResultSetTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ResultSetTest.java index a0b9c051c6..8067c547df 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ResultSetTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/ResultSetTest.java @@ -35,8 +35,7 @@ public class ResultSetTest extends BaseTest { properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + "?user=root&password=taosdata" - , properties); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); statement = connection.createStatement(); statement.executeUpdate("drop database if exists " + dbName); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SelectTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SelectTest.java index de21cc6195..1844a92b47 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SelectTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SelectTest.java @@ -28,8 +28,7 @@ public class SelectTest extends BaseTest { properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + "?user=root&password=taosdata" - , properties); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); statement = connection.createStatement(); statement.executeUpdate("drop database if exists " + dbName); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StableTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StableTest.java index c1321d08fc..6e01fb7c34 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StableTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StableTest.java @@ -31,8 +31,7 @@ public class StableTest extends BaseTest { properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + "?user=root&password=taosdata" - , properties); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); statement = connection.createStatement(); statement.executeUpdate("create database if not exists " + dbName); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StatementTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StatementTest.java index 4fe5c39486..db7b8c8cb1 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StatementTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/StatementTest.java @@ -30,8 +30,7 @@ public class StatementTest extends BaseTest { properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + "?user=root&password=taosdata" - , properties); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); statement = connection.createStatement(); statement.executeUpdate("drop database if exists " + dbName); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java index 2dc27adab7..07b43d1227 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java @@ -32,8 +32,7 @@ public class SubscribeTest extends BaseTest { properties.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); properties.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); - connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/" + "?user=root&password=taosdata" - , properties); + connection = DriverManager.getConnection("jdbc:TAOS://" + host + ":0/", properties); statement = connection.createStatement(); statement.executeUpdate("create database if not exists " + dbName); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java index 9ea5a431a5..a0981063a5 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBDriverTest.java @@ -12,7 +12,7 @@ public class TSDBDriverTest { @Test public void urlParserTest() throws SQLException { TSDBDriver driver = new TSDBDriver(); - String url = "jdbc:TSDB://127.0.0.1:0/db?user=root&password=your_password"; + String url = "jdbc:TSDB://127.0.0.1:0/db"; Properties properties = new Properties(); driver.parseURL(url, properties); diff --git a/src/dnode/src/dnodeSystem.c b/src/dnode/src/dnodeSystem.c index 6f32bc0f7a..56316e9619 100644 --- a/src/dnode/src/dnodeSystem.c +++ b/src/dnode/src/dnodeSystem.c @@ -119,11 +119,8 @@ int32_t main(int32_t argc, char *argv[]) { syslog(LOG_INFO, "Started TDengine service successfully."); - for (int res = tsem_wait(&exitSem); res != 0; res = tsem_wait(&exitSem)) { - if (res != EINTR) { - syslog(LOG_ERR, "failed to wait exit semphore: %d", res); - break; - } + if (tsem_wait(&exitSem) != 0) { + syslog(LOG_ERR, "failed to wait exit semphore: %s", strerror(errno)); } dnodeCleanUpSystem(); diff --git a/src/inc/taos.h b/src/inc/taos.h index f3cc9bb4d7..7e8f174b7c 100644 --- a/src/inc/taos.h +++ b/src/inc/taos.h @@ -93,7 +93,7 @@ DLL_EXPORT int taos_result_precision(TAOS_RES *res); // get the time precision DLL_EXPORT void taos_free_result(TAOS_RES *res); DLL_EXPORT int taos_field_count(TAOS_RES *tres); DLL_EXPORT int taos_num_fields(TAOS_RES *res); -DLL_EXPORT int taos_affected_rows(TAOS_RES *taos); +DLL_EXPORT int taos_affected_rows(TAOS_RES *res); DLL_EXPORT TAOS_FIELD *taos_fetch_fields(TAOS_RES *res); DLL_EXPORT int taos_select_db(TAOS *taos, const char *db); DLL_EXPORT int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields); diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 3dea8da18a..b61890347f 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -63,7 +63,7 @@ typedef struct tstr { extern const int32_t TYPE_BYTES[11]; // TODO: replace and remove code below #define CHAR_BYTES sizeof(char) -#define SHORT_BYTES sizeof(short) +#define SHORT_BYTES sizeof(int16_t) #define INT_BYTES sizeof(int) #define LONG_BYTES sizeof(int64_t) #define FLOAT_BYTES sizeof(float) @@ -286,7 +286,8 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_MAX_REPLICA 5 -#define TSDB_TBNAME_COLUMN_INDEX (-1) +#define TSDB_TBNAME_COLUMN_INDEX (-1) +#define TSDB_UD_COLUMN_INDEX (-100) #define TSDB_MULTI_METERMETA_MAX_NUM 100000 // maximum batch size allowed to load metermeta #define TSDB_MIN_CACHE_BLOCK_SIZE 1 @@ -395,6 +396,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_PORT_DNODEDNODE 5 #define TSDB_PORT_SYNC 10 #define TSDB_PORT_HTTP 11 +#define TSDB_PORT_ARBITRATOR 12 #define TAOS_QTYPE_RPC 0 #define TAOS_QTYPE_FWD 1 diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 57c2b322fa..951c511022 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -66,6 +66,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_MSG_TYPE, 0, 0x0011, "Invalid me TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_RESPONSE_TYPE, 0, 0x0012, "Invalid response type") TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INVALID_TIME_STAMP, 0, 0x0013, "Invalid timestamp") TAOS_DEFINE_ERROR(TSDB_CODE_APP_NOT_READY, 0, 0x0014, "Database not ready") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_FQDN_ERROR, 0, 0x0015, "Unable to resolve FQDN") //common & util TAOS_DEFINE_ERROR(TSDB_CODE_COM_OPS_NOT_SUPPORT, 0, 0x0100, "Operation not supported") @@ -96,6 +97,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_APP_ERROR, 0, 0x0211, "Applicatio TAOS_DEFINE_ERROR(TSDB_CODE_TSC_ACTION_IN_PROGRESS, 0, 0x0212, "Action in progress") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_DISCONNECTED, 0, 0x0213, "Disconnected from service") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_NO_WRITE_AUTH, 0, 0x0214, "No write permission") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_CONN_KILLED, 0, 0x0215, "Connection killed") // mnode TAOS_DEFINE_ERROR(TSDB_CODE_MND_MSG_NOT_PROCESSED, 0, 0x0300, "Message not processed") @@ -138,6 +140,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_NOT_READY, 0, 0x033C, "Cluster no TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACCT_ALREADY_EXIST, 0, 0x0340, "Account already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_ACCT, 0, 0x0341, "Invalid account") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_ACCT_OPTION, 0, 0x0342, "Invalid account options") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACCT_EXPIRED, 0, 0x0343, "Account authorization has expired") TAOS_DEFINE_ERROR(TSDB_CODE_MND_USER_ALREADY_EXIST, 0, 0x0350, "User already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_USER, 0, 0x0351, "Invalid user") @@ -220,6 +223,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_EXCEED_TAGS_LIMIT, 0, 0x0706, "Tag condit TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_READY, 0, 0x0707, "Query not ready") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_HAS_RSP, 0, 0x0708, "Query should response") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_IN_EXEC, 0, 0x0709, "Multiple retrieval of this query") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW, 0, 0x070A, "Too many time window in query") // grant TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_EXPIRED, 0, 0x0800, "License expired") diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 1daa1da038..761a267ce5 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -133,7 +133,7 @@ enum _mgmt_table { TSDB_MGMT_TABLE_MODULE, TSDB_MGMT_TABLE_QUERIES, TSDB_MGMT_TABLE_STREAMS, - TSDB_MGMT_TABLE_CONFIGS, + TSDB_MGMT_TABLE_VARIABLES, TSDB_MGMT_TABLE_CONNS, TSDB_MGMT_TABLE_SCORES, TSDB_MGMT_TABLE_GRANTS, @@ -167,9 +167,9 @@ enum _mgmt_table { #define TSDB_VN_WRITE_ACCCESS ((char)0x2) #define TSDB_VN_ALL_ACCCESS (TSDB_VN_READ_ACCCESS | TSDB_VN_WRITE_ACCCESS) -#define TSDB_COL_NORMAL 0x0u -#define TSDB_COL_TAG 0x1u -#define TSDB_COL_JOIN 0x2u +#define TSDB_COL_NORMAL 0x0u // the normal column of the table +#define TSDB_COL_TAG 0x1u // the tag column type +#define TSDB_COL_UDC 0x2u // the user specified normal string column, it is a dummy column extern char *taosMsg[]; @@ -424,7 +424,10 @@ typedef struct SColumnInfo { int16_t type; int16_t bytes; int16_t numOfFilters; - SColumnFilterInfo *filters; + union{ + int64_t placeholder; + SColumnFilterInfo *filters; + }; } SColumnInfo; typedef struct STableIdInfo { diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index 4776d1cda7..be73d8d383 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -115,7 +115,7 @@ int tsdbDropTable(TSDB_REPO_T *pRepo, STableId tableId); int tsdbUpdateTableTagValue(TSDB_REPO_T *repo, SUpdateTableTagValMsg *pMsg); TSKEY tsdbGetTableLastKey(TSDB_REPO_T *repo, uint64_t uid); -uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_t eindex, int32_t *size); +uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_t eindex, int64_t *size); // the TSDB repository info typedef struct STsdbRepoInfo { diff --git a/src/inc/tsync.h b/src/inc/tsync.h index 2dfac7ec32..ff9c9901bd 100644 --- a/src/inc/tsync.h +++ b/src/inc/tsync.h @@ -64,7 +64,7 @@ typedef struct { if name is provided(name[0] is not zero), get the named file at the specified index. If not there, return zero. If it is there, set the size to file size, and return file magic number. Index shall not be updated. */ -typedef uint32_t (*FGetFileInfo)(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int32_t *size, uint64_t *fversion); +typedef uint32_t (*FGetFileInfo)(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion); // get the wal file from index or after // return value, -1: error, 1:more wal files, 0:last WAL. if name[0]==0, no WAL file diff --git a/src/inc/ttokendef.h b/src/inc/ttokendef.h index e77f5741d3..c1579fe9e7 100644 --- a/src/inc/ttokendef.h +++ b/src/inc/ttokendef.h @@ -69,7 +69,7 @@ #define TK_QUERIES 51 #define TK_CONNECTIONS 52 #define TK_STREAMS 53 -#define TK_CONFIGS 54 +#define TK_VARIABLES 54 #define TK_SCORES 55 #define TK_GRANTS 56 #define TK_VNODES 57 diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c index d82ddf8871..8f7f4d9324 100644 --- a/src/kit/shell/src/shellEngine.c +++ b/src/kit/shell/src/shellEngine.c @@ -25,6 +25,8 @@ #include "taosdef.h" #include "taoserror.h" #include "tglobal.h" +#include "tsclient.h" + #include /**************** Global variables ****************/ @@ -64,11 +66,6 @@ TAOS *shellInit(SShellArguments *args) { } taos_init(); - /* - * set tsTableMetaKeepTimer = 3000ms - * means not save cache in shell - */ - tsTableMetaKeepTimer = 3000; // Connect to the database. TAOS *con = NULL; @@ -287,7 +284,7 @@ void shellRunCommandOnServer(TAOS *con, char command[]) { st = taosGetTimestampUs(); TAOS_RES* pSql = taos_query(con, command); - result = pSql; // set it into the global variable + atomic_store_ptr(&result, pSql); // set the global TAOS_RES pointer if (taos_errno(pSql)) { taos_error(pSql); @@ -298,17 +295,16 @@ void shellRunCommandOnServer(TAOS *con, char command[]) { fprintf(stdout, "Database changed.\n\n"); fflush(stdout); - result = NULL; + atomic_store_ptr(&result, 0); taos_free_result(pSql); return; } - int num_fields = taos_field_count(pSql); - if (num_fields != 0) { // select and show kinds of commands + if (!tscIsUpdateQuery(pSql)) { // select and show kinds of commands int error_no = 0; int numOfRows = shellDumpResult(pSql, fname, &error_no, printMode); if (numOfRows < 0) { - result = NULL; + atomic_store_ptr(&result, 0); taos_free_result(pSql); return; } @@ -331,7 +327,7 @@ void shellRunCommandOnServer(TAOS *con, char command[]) { wordfree(&full_path); } - result = NULL; + atomic_store_ptr(&result, 0); taos_free_result(pSql); } @@ -497,7 +493,6 @@ static int dumpResultToFile(const char* fname, TAOS_RES* tres) { } while( row != NULL); result = NULL; - //taos_free_result(tres); fclose(fp); return numOfRows; @@ -802,8 +797,8 @@ void write_history() { } void taos_error(TAOS_RES *tres) { + atomic_store_ptr(&result, 0); fprintf(stderr, "\nDB error: %s\n", taos_errstr(tres)); - result = NULL; taos_free_result(tres); } diff --git a/src/kit/shell/src/shellMain.c b/src/kit/shell/src/shellMain.c index 8481f498dd..44de6641f6 100644 --- a/src/kit/shell/src/shellMain.c +++ b/src/kit/shell/src/shellMain.c @@ -18,11 +18,10 @@ pthread_t pid; -// TODO: IMPLEMENT INTERRUPT HANDLER. -void interruptHandler(int signum) { +void shellQueryInterruptHandler(int signum) { #ifdef LINUX - taos_stop_query(result); - result = NULL; + void* pResHandle = atomic_val_compare_exchange_64(&result, result, 0); + taos_stop_query(pResHandle); #else printf("\nReceive ctrl+c or other signal, quit shell.\n"); exit(0); @@ -86,7 +85,7 @@ int main(int argc, char* argv[]) { struct sigaction act; memset(&act, 0, sizeof(struct sigaction)); - act.sa_handler = interruptHandler; + act.sa_handler = shellQueryInterruptHandler; sigaction(SIGTERM, &act, NULL); sigaction(SIGINT, &act, NULL); diff --git a/src/kit/taosnetwork/client.c b/src/kit/taosnetwork/client.c index 102f9b9d89..b7db2ba0a2 100644 --- a/src/kit/taosnetwork/client.c +++ b/src/kit/taosnetwork/client.c @@ -27,35 +27,46 @@ #include #include #include +#include -#define MAX_PKG_LEN (64*1000) -#define BUFFER_SIZE (MAX_PKG_LEN + 1024) +#define MAX_PKG_LEN (64*1000) +#define BUFFER_SIZE (MAX_PKG_LEN + 1024) +#define TEST_FQDN_LEN 128 +#define TEST_IPv4ADDR_LEN 16 typedef struct { - int port; - char *host; + uint16_t port; + uint32_t hostIp; + char fqdn[TEST_FQDN_LEN]; uint16_t pktLen; } info_s; typedef struct Arguments { - char * host; + char host[TEST_IPv4ADDR_LEN]; + char fqdn[TEST_FQDN_LEN]; uint16_t port; uint16_t max_port; uint16_t pktLen; } SArguments; static struct argp_option options[] = { - {0, 'h', "host", 0, "The host to connect to TDEngine. Default is localhost.", 0}, + {0, 'h', "host ip", 0, "The host ip to connect to TDEngine. Default is localhost.", 0}, {0, 'p', "port", 0, "The TCP or UDP port number to use for the connection. Default is 6030.", 1}, - {0, 'm', "max port", 0, "The max TCP or UDP port number to use for the connection. Default is 6060.", 2}, + {0, 'm', "max port", 0, "The max TCP or UDP port number to use for the connection. Default is 6042.", 2}, + {0, 'f', "host fqdn", 0, "The host fqdn to connect to TDEngine.", 3}, {0, 'l', "test pkg len", 0, "The len of pkg for test. Default is 1000 Bytes, max not greater than 64k Bytes.\nNotes: This parameter must be consistent between the client and the server.", 3}}; static error_t parse_opt(int key, char *arg, struct argp_state *state) { - + wordexp_t full_path; SArguments *arguments = state->input; switch (key) { case 'h': - arguments->host = arg; + if (wordexp(arg, &full_path, 0) != 0) { + fprintf(stderr, "Invalid host ip %s\n", arg); + return -1; + } + strcpy(arguments->host, full_path.we_wordv[0]); + wordfree(&full_path); break; case 'p': arguments->port = atoi(arg); @@ -66,6 +77,14 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { case 'l': arguments->pktLen = atoi(arg); break; + case 'f': + if (wordexp(arg, &full_path, 0) != 0) { + fprintf(stderr, "Invalid host fqdn %s\n", arg); + return -1; + } + strcpy(arguments->fqdn, full_path.we_wordv[0]); + wordfree(&full_path); + break; default: return ARGP_ERR_UNKNOWN; @@ -76,8 +95,6 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { static struct argp argp = {options, parse_opt, 0, 0}; int checkTcpPort(info_s *info) { - int port = info->port; - char *host = info->host; int clientSocket; struct sockaddr_in serverAddr; @@ -88,21 +105,35 @@ int checkTcpPort(info_s *info) { printf("socket() fail: %s\n", strerror(errno)); return -1; } - serverAddr.sin_family = AF_INET; - serverAddr.sin_port = htons(port); - serverAddr.sin_addr.s_addr = inet_addr(host); + // set send and recv overtime + struct timeval timeout; + timeout.tv_sec = 2; //s + timeout.tv_usec = 0; //us + if (setsockopt(clientSocket, SOL_SOCKET,SO_SNDTIMEO, (char *)&timeout, sizeof(struct timeval)) == -1) { + perror("setsockopt send timer failed:"); + } + if (setsockopt(clientSocket, SOL_SOCKET,SO_RCVTIMEO, (char *)&timeout, sizeof(struct timeval)) == -1) { + perror("setsockopt recv timer failed:"); + } + + serverAddr.sin_family = AF_INET; + serverAddr.sin_port = htons(info->port); + + serverAddr.sin_addr.s_addr = info->hostIp; //printf("=================================\n"); if (connect(clientSocket, (struct sockaddr *)&serverAddr, sizeof(serverAddr)) < 0) { - printf("connect() fail: %s\n", strerror(errno)); + printf("connect() fail: %s\t", strerror(errno)); return -1; } //printf("Connect to: %s:%d...success\n", host, port); memset(sendbuf, 0, BUFFER_SIZE); memset(recvbuf, 0, BUFFER_SIZE); - sprintf(sendbuf, "client send tcp pkg to %s:%d, content: 1122334455", host, port); + struct in_addr ipStr; + memcpy(&ipStr, &info->hostIp, 4); + sprintf(sendbuf, "client send tcp pkg to %s:%d, content: 1122334455", inet_ntoa(ipStr), info->port); sprintf(sendbuf + info->pktLen - 16, "1122334455667788"); send(clientSocket, sendbuf, info->pktLen, 0); @@ -120,7 +151,7 @@ int checkTcpPort(info_s *info) { if (errno == EINTR) { continue; } else { - printf("recv ack pkg from TCP port: %d fail:%s.\n", port, strerror(errno)); + printf("recv ack pkg from TCP port: %d fail:%s.\n", info->port, strerror(errno)); close(clientSocket); return -1; } @@ -132,7 +163,7 @@ int checkTcpPort(info_s *info) { } if (iDataNum < info->pktLen) { - printf("recv ack pkg len: %d, less than req pkg len: %d from tcp port: %d\n", iDataNum, info->pktLen, port); + printf("recv ack pkg len: %d, less than req pkg len: %d from tcp port: %d\n", iDataNum, info->pktLen, info->port); return -1; } //printf("Read ack pkg len:%d from tcp port: %d, buffer: %s %s\n", info->pktLen, port, recvbuf, recvbuf+iDataNum-8); @@ -142,8 +173,6 @@ int checkTcpPort(info_s *info) { } int checkUdpPort(info_s *info) { - int port = info->port; - char *host = info->host; int clientSocket; struct sockaddr_in serverAddr; @@ -154,15 +183,28 @@ int checkUdpPort(info_s *info) { perror("socket"); return -1; } + + // set overtime + struct timeval timeout; + timeout.tv_sec = 2; //s + timeout.tv_usec = 0; //us + if (setsockopt(clientSocket, SOL_SOCKET,SO_SNDTIMEO, (char *)&timeout, sizeof(struct timeval)) == -1) { + perror("setsockopt send timer failed:"); + } + if (setsockopt(clientSocket, SOL_SOCKET,SO_RCVTIMEO, (char *)&timeout, sizeof(struct timeval)) == -1) { + perror("setsockopt recv timer failed:"); + } serverAddr.sin_family = AF_INET; - serverAddr.sin_port = htons(port); - serverAddr.sin_addr.s_addr = inet_addr(host); + serverAddr.sin_port = htons(info->port); + serverAddr.sin_addr.s_addr = info->hostIp; memset(sendbuf, 0, BUFFER_SIZE); memset(recvbuf, 0, BUFFER_SIZE); - sprintf(sendbuf, "client send udp pkg to %s:%d, content: 1122334455", host, port); + struct in_addr ipStr; + memcpy(&ipStr, &info->hostIp, 4); + sprintf(sendbuf, "client send udp pkg to %s:%d, content: 1122334455", inet_ntoa(ipStr), info->port); sprintf(sendbuf + info->pktLen - 16, "1122334455667788"); socklen_t sin_size = sizeof(*(struct sockaddr *)&serverAddr); @@ -176,7 +218,7 @@ int checkUdpPort(info_s *info) { iDataNum = recvfrom(clientSocket, recvbuf, BUFFER_SIZE, 0, (struct sockaddr *)&serverAddr, &sin_size); if (iDataNum < info->pktLen) { - printf("Read ack pkg len: %d, less than req pkg len: %d from udp port: %d\n", iDataNum, info->pktLen, port); + printf("Read ack pkg len: %d, less than req pkg len: %d from udp port: %d\t\t", iDataNum, info->pktLen, info->port); return -1; } @@ -185,10 +227,61 @@ int checkUdpPort(info_s *info) { return 0; } -int main(int argc, char *argv[]) { - SArguments arguments = {"127.0.0.1", 6030, 6060, 1000}; - info_s info; +int32_t getIpFromFqdn(const char *fqdn, uint32_t* ip) { + struct addrinfo hints = {0}; + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; + + struct addrinfo *result = NULL; + + int32_t ret = getaddrinfo(fqdn, NULL, &hints, &result); + if (result) { + struct sockaddr *sa = result->ai_addr; + struct sockaddr_in *si = (struct sockaddr_in*)sa; + struct in_addr ia = si->sin_addr; + *ip = ia.s_addr; + freeaddrinfo(result); + return 0; + } else { + printf("Failed get the ip address from fqdn:%s, code:%d, reason:%s", fqdn, ret, gai_strerror(ret)); + return -1; + } +} + +void checkPort(uint32_t hostIp, uint16_t startPort, uint16_t maxPort, uint16_t pktLen) { int ret; + info_s info; + memset(&info, 0, sizeof(info_s)); + info.hostIp = hostIp; + info.pktLen = pktLen; + + for (uint16_t port = startPort; port <= maxPort; port++) { + //printf("test: %s:%d\n", info.host, port); + printf("\n"); + + info.port = port; + ret = checkTcpPort(&info); + if (ret != 0) { + printf("tcp port:%d test fail.\t\n", port); + } else { + printf("tcp port:%d test ok.\t\t", port); + } + + ret = checkUdpPort(&info); + if (ret != 0) { + printf("udp port:%d test fail.\t\n", port); + } else { + printf("udp port:%d test ok.\t\t", port); + } + } + + printf("\n"); + return ; +} + +int main(int argc, char *argv[]) { + SArguments arguments = {"127.0.0.1", "", 6030, 6042, 1000}; + int ret; argp_parse(&argp, argc, argv, 0, 0, &arguments); if (arguments.pktLen > MAX_PKG_LEN) { @@ -196,32 +289,25 @@ int main(int argc, char *argv[]) { exit(0); } - printf("host: %s\tport: %d\tmax_port: %d\tpkgLen: %d\n", arguments.host, arguments.port, arguments.max_port, arguments.pktLen); + printf("host ip: %s\thost fqdn: %s\tport: %d\tmax_port: %d\tpkgLen: %d\n", arguments.host, arguments.fqdn, arguments.port, arguments.max_port, arguments.pktLen); - int port = arguments.port; - - info.host = arguments.host; - info.pktLen = arguments.pktLen; - - for (; port <= arguments.max_port; port++) { - //printf("test: %s:%d\n", info.host, port); + if (arguments.host[0] != 0) { + printf("\nstart connect to %s test:\n", arguments.host); + checkPort(inet_addr(arguments.host), arguments.port, arguments.max_port, arguments.pktLen); printf("\n"); - - info.port = port; - ret = checkTcpPort(&info); - if (ret != 0) { - printf("tcp port:%d test fail.\t\t", port); - } else { - printf("tcp port:%d test ok.\t\t", port); - } - - ret = checkUdpPort(&info); - if (ret != 0) { - printf("udp port:%d test fail.\t\t", port); - } else { - printf("udp port:%d test ok.\t\t", port); - } } - printf("\n"); + + if (arguments.fqdn[0] != 0) { + uint32_t hostIp = 0; + ret = getIpFromFqdn(arguments.fqdn, &hostIp); + if (ret) { + printf("\n"); + return 0; + } + printf("\nstart connetc to %s test:\n", arguments.fqdn); + checkPort(hostIp, arguments.port, arguments.max_port, arguments.pktLen); + printf("\n"); + } + return 0; } diff --git a/src/kit/taosnetwork/server.c b/src/kit/taosnetwork/server.c index 1c3bc6fa09..97be1d3b63 100644 --- a/src/kit/taosnetwork/server.c +++ b/src/kit/taosnetwork/server.c @@ -142,9 +142,9 @@ static void *bindTcpPort(void *sarg) { printf("recv Client: %s pkg from TCP port: %d, pkg len: %d\n", inet_ntoa(clientAddr.sin_addr), port, iDataNum); if (iDataNum > 0) { send(client, buffer, iDataNum, 0); - break; } } + close(serverSocket); return NULL; } @@ -201,7 +201,7 @@ static void *bindUdpPort(void *sarg) { int main(int argc, char *argv[]) { - SArguments arguments = {"127.0.0.1", 6030, 6060, 1000}; + SArguments arguments = {"127.0.0.1", 6030, 6042, 1000}; argp_parse(&argp, argc, argv, 0, 0, &arguments); if (arguments.pktLen > MAX_PKG_LEN) { printf("test pkg len overflow: %d, max len not greater than %d bytes\n", arguments.pktLen, MAX_PKG_LEN); diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index 26b4430777..cdea9eda60 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -354,7 +354,7 @@ static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate, SMnodeMsg mDebug("db:%s, already exist, ignore exist is set", pCreate->db); return TSDB_CODE_SUCCESS; } else { - mError("db:%s, is already exist, ignore exist not set", pCreate->db); + mError("db:%s, already exist, ignore exist not set", pCreate->db); return TSDB_CODE_MND_DB_ALREADY_EXIST; } } @@ -1059,10 +1059,12 @@ static int32_t mnodeProcessDropDbMsg(SMnodeMsg *pMsg) { } } +#if 0 if (mnodeCheckIsMonitorDB(pMsg->pDb->name, tsMonitorDbName)) { mError("db:%s, can't drop monitor database", pDrop->db); return TSDB_CODE_MND_MONITOR_DB_FORBIDDEN; } +#endif int32_t code = mnodeSetDbDropping(pMsg->pDb); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index 5da57a3093..d5b9d18e37 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -49,7 +49,7 @@ static int32_t mnodeProcessCreateDnodeMsg(SMnodeMsg *pMsg); static int32_t mnodeProcessDropDnodeMsg(SMnodeMsg *pMsg); static int32_t mnodeProcessCfgDnodeMsg(SMnodeMsg *pMsg); static void mnodeProcessCfgDnodeMsgRsp(SRpcMsg *rpcMsg) ; -static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *rpcMsg); +static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg); static int32_t mnodeGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mnodeRetrieveModules(SShowObj *pShow, char *data, int32_t rows, void *pConn); static int32_t mnodeGetConfigMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); @@ -161,8 +161,8 @@ int32_t mnodeInitDnodes() { mnodeAddPeerMsgHandle(TSDB_MSG_TYPE_DM_STATUS, mnodeProcessDnodeStatusMsg); mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_MODULE, mnodeGetModuleMeta); mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_MODULE, mnodeRetrieveModules); - mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_CONFIGS, mnodeGetConfigMeta); - mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_CONFIGS, mnodeRetrieveConfigs); + mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_VARIABLES, mnodeGetConfigMeta); + mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_VARIABLES, mnodeRetrieveConfigs); mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_VNODES, mnodeGetVnodeMeta); mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_VNODES, mnodeRetrieveVnodes); mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_DNODE, mnodeGetDnodeMeta); @@ -518,7 +518,7 @@ static int32_t mnodeCreateDnode(char *ep, SMnodeMsg *pMsg) { SDnodeObj *pDnode = mnodeGetDnodeByEp(ep); if (pDnode != NULL) { mnodeDecDnodeRef(pDnode); - mError("dnode:%d is already exist, %s:%d", pDnode->dnodeId, pDnode->dnodeFqdn, pDnode->dnodePort); + mError("dnode:%d, already exist, %s:%d", pDnode->dnodeId, pDnode->dnodeFqdn, pDnode->dnodePort); return TSDB_CODE_MND_DNODE_ALREADY_EXIST; } @@ -733,7 +733,7 @@ static int32_t mnodeRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, vo } static bool mnodeCheckModuleInDnode(SDnodeObj *pDnode, int32_t moduleType) { - uint32_t status = pDnode->moduleStatus & (1 << moduleType); + uint32_t status = pDnode->moduleStatus & (1u << moduleType); return status > 0; } @@ -758,7 +758,7 @@ static int32_t mnodeGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *p pShow->bytes[cols] = 40 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "end point"); + strcpy(pSchema[cols].name, "end_point"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -792,7 +792,9 @@ static int32_t mnodeGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *p int32_t mnodeRetrieveModules(SShowObj *pShow, char *data, int32_t rows, void *pConn) { int32_t numOfRows = 0; - char * pWrite; + + char* pWrite; + char* moduleName[5] = { "MNODE", "HTTP", "MONITOR", "MQTT", "UNKNOWN" }; while (numOfRows < rows) { SDnodeObj *pDnode = NULL; @@ -807,28 +809,18 @@ int32_t mnodeRetrieveModules(SShowObj *pShow, char *data, int32_t rows, void *pC cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strncpy(pWrite, pDnode->dnodeEp, pShow->bytes[cols]-1); + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pDnode->dnodeEp, pShow->bytes[cols] - 1); cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - switch (moduleType) { - case TSDB_MOD_MNODE: - strcpy(pWrite, "mnode"); - break; - case TSDB_MOD_HTTP: - strcpy(pWrite, "http"); - break; - case TSDB_MOD_MONITOR: - strcpy(pWrite, "monitor"); - break; - default: - strcpy(pWrite, "unknown"); - } + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, moduleName[moduleType], pShow->bytes[cols]); cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; bool enable = mnodeCheckModuleInDnode(pDnode, moduleType); - strcpy(pWrite, enable ? "enable" : "disable"); + + char* v = enable? "enable":"disable"; + STR_TO_VARSTR(pWrite, v); cols++; numOfRows++; @@ -862,13 +854,13 @@ static int32_t mnodeGetConfigMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *p pShow->bytes[cols] = TSDB_CFG_OPTION_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "config name"); + tstrncpy(pSchema[cols].name, "name", sizeof(pSchema[cols].name)); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; pShow->bytes[cols] = TSDB_CFG_VALUE_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "config value"); + tstrncpy(pSchema[cols].name, "value", sizeof(pSchema[cols].name)); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -903,27 +895,32 @@ static int32_t mnodeRetrieveConfigs(SShowObj *pShow, char *data, int32_t rows, v int32_t cols = 0; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - snprintf(pWrite, TSDB_CFG_OPTION_LEN, "%s", cfg->option); + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, cfg->option, TSDB_CFG_OPTION_LEN); + cols++; + int32_t t = 0; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; switch (cfg->valType) { case TAOS_CFG_VTYPE_INT16: - snprintf(pWrite, TSDB_CFG_VALUE_LEN, "%d", *((int16_t *)cfg->ptr)); + t = snprintf(varDataVal(pWrite), TSDB_CFG_VALUE_LEN, "%d", *((int16_t *)cfg->ptr)); + varDataSetLen(pWrite, t); numOfRows++; break; case TAOS_CFG_VTYPE_INT32: - snprintf(pWrite, TSDB_CFG_VALUE_LEN, "%d", *((int32_t *)cfg->ptr)); + t = snprintf(varDataVal(pWrite), TSDB_CFG_VALUE_LEN, "%d", *((int32_t *)cfg->ptr)); + varDataSetLen(pWrite, t); numOfRows++; break; case TAOS_CFG_VTYPE_FLOAT: - snprintf(pWrite, TSDB_CFG_VALUE_LEN, "%f", *((float *)cfg->ptr)); + t = snprintf(varDataVal(pWrite), TSDB_CFG_VALUE_LEN, "%f", *((float *)cfg->ptr)); + varDataSetLen(pWrite, t); numOfRows++; break; case TAOS_CFG_VTYPE_STRING: case TAOS_CFG_VTYPE_IPSTR: case TAOS_CFG_VTYPE_DIRECTORY: - snprintf(pWrite, TSDB_CFG_VALUE_LEN, "%s", (char *)cfg->ptr); + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, cfg->ptr, TSDB_CFG_VALUE_LEN); numOfRows++; break; default: diff --git a/src/mnode/src/mnodeMain.c b/src/mnode/src/mnodeMain.c index 53f3474c3c..d63a575868 100644 --- a/src/mnode/src/mnodeMain.c +++ b/src/mnode/src/mnodeMain.c @@ -21,6 +21,7 @@ #include "tgrant.h" #include "ttimer.h" #include "tglobal.h" +#include "mnode.h" #include "dnode.h" #include "mnodeDef.h" #include "mnodeInt.h" @@ -107,13 +108,18 @@ int32_t mnodeStartSystem() { tsMgmtIsRunning = true; mInfo("mnode is initialized successfully"); + + sdbUpdateSync(); + return 0; } int32_t mnodeInitSystem() { mnodeInitTimer(); - if (!mnodeNeedStart()) return 0; - return mnodeStartSystem(); + if (mnodeNeedStart()) { + return mnodeStartSystem(); + } + return 0; } void mnodeCleanupSystem() { @@ -159,14 +165,19 @@ static void mnodeCleanupTimer() { static bool mnodeNeedStart() { struct stat dirstat; - bool fileExist = (stat(tsMnodeDir, &dirstat) == 0); + char mnodeFileName[TSDB_FILENAME_LEN * 2] = {0}; + sprintf(mnodeFileName, "%s/wal/wal0", tsMnodeDir); + + bool fileExist = (stat(mnodeFileName, &dirstat) == 0); bool asMaster = (strcmp(tsFirst, tsLocalEp) == 0); if (asMaster || fileExist) { + mDebug("mnode module start, asMaster:%d fileExist:%d", asMaster, fileExist); return true; + } else { + mDebug("mnode module won't start, asMaster:%d fileExist:%d", asMaster, fileExist); + return false; } - - return false; } bool mnodeIsRunning() { diff --git a/src/mnode/src/mnodeProfile.c b/src/mnode/src/mnodeProfile.c index 01a824baa7..06f992c26a 100644 --- a/src/mnode/src/mnodeProfile.c +++ b/src/mnode/src/mnodeProfile.c @@ -35,8 +35,8 @@ #include "mnodeVgroup.h" #include "mnodeWrite.h" -#define CONN_KEEP_TIME (tsShellActivityTimer * 3000) -#define CONN_CHECK_TIME (tsShellActivityTimer * 2000) +#define CONN_KEEP_TIME (tsShellActivityTimer * 3) +#define CONN_CHECK_TIME (tsShellActivityTimer * 2) #define QUERY_ID_SIZE 20 #define QUERY_STREAM_SAVE_SIZE 20 @@ -100,7 +100,7 @@ SConnObj *mnodeCreateConn(char *user, uint32_t ip, uint16_t port) { }; tstrncpy(connObj.user, user, sizeof(connObj.user)); - SConnObj *pConn = taosCachePut(tsMnodeConnCache, &connId, sizeof(int32_t), &connObj, sizeof(connObj), CONN_KEEP_TIME); + SConnObj *pConn = taosCachePut(tsMnodeConnCache, &connId, sizeof(int32_t), &connObj, sizeof(connObj), CONN_KEEP_TIME * 1000); mDebug("connId:%d, is created, user:%s ip:%s:%u", connId, user, taosIpStr(ip), port); return pConn; diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 3651aa8aad..e3ed7daf8c 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -224,7 +224,7 @@ void sdbUpdateMnodeRoles() { mnodeUpdateMnodeEpSet(); } -static uint32_t sdbGetFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int32_t *size, uint64_t *fversion) { +static uint32_t sdbGetFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion) { sdbUpdateMnodeRoles(); return 0; } @@ -291,6 +291,11 @@ static void sdbConfirmForward(void *ahandle, void *param, int32_t code) { } void sdbUpdateSync() { + if (!mnodeIsRunning()) { + mDebug("mnode not start yet, update sync info later"); + return; + } + SSyncCfg syncCfg = {0}; int32_t index = 0; diff --git a/src/mnode/src/mnodeShow.c b/src/mnode/src/mnodeShow.c index 8a84b66a33..f66ef6b7a3 100644 --- a/src/mnode/src/mnodeShow.c +++ b/src/mnode/src/mnodeShow.c @@ -98,7 +98,7 @@ static char *mnodeGetShowType(int32_t showType) { case TSDB_MGMT_TABLE_MODULE: return "show modules"; case TSDB_MGMT_TABLE_QUERIES: return "show queries"; case TSDB_MGMT_TABLE_STREAMS: return "show streams"; - case TSDB_MGMT_TABLE_CONFIGS: return "show configs"; + case TSDB_MGMT_TABLE_VARIABLES: return "show configs"; case TSDB_MGMT_TABLE_CONNS: return "show connections"; case TSDB_MGMT_TABLE_SCORES: return "show scores"; case TSDB_MGMT_TABLE_GRANTS: return "show grants"; @@ -313,6 +313,7 @@ static int32_t mnodeProcessConnectMsg(SMnodeMsg *pMsg) { if (pDb->status != TSDB_DB_STATUS_READY) { mError("db:%s, status:%d, in dropping", pDb->name, pDb->status); code = TSDB_CODE_MND_DB_IN_DROPPING; + mnodeDecDbRef(pDb); goto connect_over; } mnodeDecDbRef(pDb); diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index dbc1bffa5c..03b1399ea7 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -294,6 +294,7 @@ static int32_t mnodeChildTableActionRestored() { SSdbOper desc = {.type = SDB_OPER_LOCAL, .pObj = pTable, .table = tsChildTableSdb}; sdbDeleteRow(&desc); mnodeDecTableRef(pTable); + mnodeDecDbRef(pDb); continue; } mnodeDecDbRef(pDb); @@ -1259,6 +1260,7 @@ static int32_t mnodeGetShowSuperTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, if (pDb->status != TSDB_DB_STATUS_READY) { mError("db:%s, status:%d, in dropping", pDb->name, pDb->status); + mnodeDecDbRef(pDb); return TSDB_CODE_MND_DB_IN_DROPPING; } @@ -1323,6 +1325,7 @@ int32_t mnodeRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows, if (pDb->status != TSDB_DB_STATUS_READY) { mError("db:%s, status:%d, in dropping", pDb->name, pDb->status); + mnodeDecDbRef(pDb); return 0; } @@ -1714,7 +1717,8 @@ static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) { (sdbGetVersion() & ((1ul << 16) - 1ul)); pTable->superTable = pMsg->pSTable; } else { - pTable->uid = (((uint64_t) pTable->createdTime) << 16) + (sdbGetVersion() & ((1ul << 16) - 1ul)); + pTable->uid = (((uint64_t)pTable->vgId) << 40) + ((((uint64_t)pTable->sid) & ((1ul << 24) - 1ul)) << 16) + + (sdbGetVersion() & ((1ul << 16) - 1ul)); pTable->sversion = 0; pTable->numOfColumns = htons(pCreate->numOfColumns); pTable->sqlLen = htons(pCreate->sqlLen); @@ -2494,6 +2498,7 @@ static int32_t mnodeGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void if (pDb->status != TSDB_DB_STATUS_READY) { mError("db:%s, status:%d, in dropping", pDb->name, pDb->status); + mnodeDecDbRef(pDb); return TSDB_CODE_MND_DB_IN_DROPPING; } @@ -2547,6 +2552,7 @@ static int32_t mnodeRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows if (pDb->status != TSDB_DB_STATUS_READY) { mError("db:%s, status:%d, in dropping", pDb->name, pDb->status); + mnodeDecDbRef(pDb); return 0; } @@ -2715,6 +2721,7 @@ static int32_t mnodeGetStreamTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, vo if (pDb->status != TSDB_DB_STATUS_READY) { mError("db:%s, status:%d, in dropping", pDb->name, pDb->status); + mnodeDecDbRef(pDb); return TSDB_CODE_MND_DB_IN_DROPPING; } @@ -2767,6 +2774,7 @@ static int32_t mnodeRetrieveStreamTables(SShowObj *pShow, char *data, int32_t ro if (pDb->status != TSDB_DB_STATUS_READY) { mError("db:%s, status:%d, in dropping", pDb->name, pDb->status); + mnodeDecDbRef(pDb); return 0; } diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index ff253c1935..aa6631ff83 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -89,6 +89,7 @@ static int32_t mnodeVgroupActionInsert(SSdbOper *pOper) { if (pDb->status != TSDB_DB_STATUS_READY) { mError("vgId:%d, db:%s status:%d, in dropping", pVgroup->vgId, pDb->name, pDb->status); + mnodeDecDbRef(pDb); return TSDB_CODE_MND_DB_IN_DROPPING; } @@ -617,6 +618,7 @@ static int32_t mnodeGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *p if (pDb->status != TSDB_DB_STATUS_READY) { mError("db:%s, status:%d, in dropping", pDb->name, pDb->status); + mnodeDecDbRef(pDb); return TSDB_CODE_MND_DB_IN_DROPPING; } @@ -708,6 +710,7 @@ static int32_t mnodeRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, v if (pDb->status != TSDB_DB_STATUS_READY) { mError("db:%s, status:%d, in dropping", pDb->name, pDb->status); + mnodeDecDbRef(pDb); return 0; } @@ -784,7 +787,10 @@ void mnodeAddTableIntoVgroup(SVgObj *pVgroup, SChildTableObj *pTable) { if (pTable->sid >= 1) { taosIdPoolMarkStatus(pVgroup->idPool, pTable->sid); pVgroup->numOfTables++; - mnodeIncVgroupRef(pVgroup); + // The create vgroup message may be received later than the create table message + // and the writing order in sdb is therefore uncertain + // which will cause the reference count of the vgroup to be incorrect when restarting + // mnodeIncVgroupRef(pVgroup); } } @@ -792,7 +798,10 @@ void mnodeRemoveTableFromVgroup(SVgObj *pVgroup, SChildTableObj *pTable) { if (pTable->sid >= 1) { taosFreeId(pVgroup->idPool, pTable->sid); pVgroup->numOfTables--; - mnodeDecVgroupRef(pVgroup); + // The create vgroup message may be received later than the create table message + // and the writing order in sdb is therefore uncertain + // which will cause the reference count of the vgroup to be incorrect when restarting + // mnodeDecVgroupRef(pVgroup); } } diff --git a/src/os/inc/osSemphone.h b/src/os/inc/osSemphone.h index fd88d2d798..4280b458a6 100644 --- a/src/os/inc/osSemphone.h +++ b/src/os/inc/osSemphone.h @@ -23,7 +23,7 @@ extern "C" { #ifndef TAOS_OS_FUNC_SEMPHONE #define tsem_t sem_t #define tsem_init sem_init - #define tsem_wait sem_wait + int tsem_wait(tsem_t* sem); #define tsem_post sem_post #define tsem_destroy sem_destroy #endif diff --git a/src/os/src/detail/osSemphone.c b/src/os/src/detail/osSemphone.c index 74f8859029..b91888845e 100644 --- a/src/os/src/detail/osSemphone.c +++ b/src/os/src/detail/osSemphone.c @@ -16,6 +16,18 @@ #define _DEFAULT_SOURCE #include "os.h" +#ifndef TAOS_OS_FUNC_SEMPHONE + +int tsem_wait(tsem_t* sem) { + int ret = 0; + do { + ret = sem_wait(sem); + } while (ret != 0 && errno == EINTR); + return ret; +} + +#endif + #ifndef TAOS_OS_FUNC_SEMPHONE_PTHREAD bool taosCheckPthreadValid(pthread_t thread) { return thread != 0; } diff --git a/src/os/src/windows/CMakeLists.txt b/src/os/src/windows/CMakeLists.txt index 588d3b7f68..9dcc9e7e6d 100644 --- a/src/os/src/windows/CMakeLists.txt +++ b/src/os/src/windows/CMakeLists.txt @@ -4,4 +4,4 @@ PROJECT(TDengine) AUX_SOURCE_DIRECTORY(. SRC) ADD_LIBRARY(os ${SRC}) -TARGET_LINK_LIBRARIES(os winmm IPHLPAPI ws2_32 MsvcLibXw64) +TARGET_LINK_LIBRARIES(os winmm IPHLPAPI ws2_32 MsvcLibXw) diff --git a/src/os/src/windows/w64Atomic.c b/src/os/src/windows/wAtomic.c similarity index 100% rename from src/os/src/windows/w64Atomic.c rename to src/os/src/windows/wAtomic.c diff --git a/src/os/src/windows/w64Dir.c b/src/os/src/windows/wDir.c similarity index 100% rename from src/os/src/windows/w64Dir.c rename to src/os/src/windows/wDir.c diff --git a/src/os/src/windows/w64Env.c b/src/os/src/windows/wEnv.c similarity index 100% rename from src/os/src/windows/w64Env.c rename to src/os/src/windows/wEnv.c diff --git a/src/os/src/windows/w64File.c b/src/os/src/windows/wFile.c similarity index 100% rename from src/os/src/windows/w64File.c rename to src/os/src/windows/wFile.c diff --git a/src/os/src/windows/w64Getline.c b/src/os/src/windows/wGetline.c similarity index 100% rename from src/os/src/windows/w64Getline.c rename to src/os/src/windows/wGetline.c diff --git a/src/os/src/windows/w64Godll.c b/src/os/src/windows/wGodll.c similarity index 100% rename from src/os/src/windows/w64Godll.c rename to src/os/src/windows/wGodll.c diff --git a/src/os/src/windows/w64Lz4.c b/src/os/src/windows/wLz4.c similarity index 100% rename from src/os/src/windows/w64Lz4.c rename to src/os/src/windows/wLz4.c diff --git a/src/os/src/windows/w64Semphone.c b/src/os/src/windows/wSemphone.c similarity index 100% rename from src/os/src/windows/w64Semphone.c rename to src/os/src/windows/wSemphone.c diff --git a/src/os/src/windows/w64Socket.c b/src/os/src/windows/wSocket.c similarity index 100% rename from src/os/src/windows/w64Socket.c rename to src/os/src/windows/wSocket.c diff --git a/src/os/src/windows/w64String.c b/src/os/src/windows/wString.c similarity index 100% rename from src/os/src/windows/w64String.c rename to src/os/src/windows/wString.c diff --git a/src/os/src/windows/w64Strptime.c b/src/os/src/windows/wStrptime.c similarity index 100% rename from src/os/src/windows/w64Strptime.c rename to src/os/src/windows/wStrptime.c diff --git a/src/os/src/windows/w64Sysinfo.c b/src/os/src/windows/wSysinfo.c similarity index 100% rename from src/os/src/windows/w64Sysinfo.c rename to src/os/src/windows/wSysinfo.c diff --git a/src/os/src/windows/w64Time.c b/src/os/src/windows/wTime.c similarity index 100% rename from src/os/src/windows/w64Time.c rename to src/os/src/windows/wTime.c diff --git a/src/os/src/windows/w64Timer.c b/src/os/src/windows/wTimer.c similarity index 100% rename from src/os/src/windows/w64Timer.c rename to src/os/src/windows/wTimer.c diff --git a/src/os/src/windows/w64Wordexp.c b/src/os/src/windows/wWordexp.c similarity index 100% rename from src/os/src/windows/w64Wordexp.c rename to src/os/src/windows/wWordexp.c diff --git a/src/query/inc/qAst.h b/src/query/inc/qAst.h index ec568a6cdb..547616dee6 100644 --- a/src/query/inc/qAst.h +++ b/src/query/inc/qAst.h @@ -80,7 +80,7 @@ void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, S void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order, char *(*cb)(void *, const char*, int32_t)); -uint8_t getBinaryExprOptr(SSQLToken *pToken); +uint8_t getBinaryExprOptr(SStrToken *pToken); void tExprNodeDestroy(tExprNode *pNode, void (*fp)(void *)); void exprTreeToBinary(SBufferWriter* bw, tExprNode* pExprTree); diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index b5487561b2..7093495763 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -33,6 +33,17 @@ struct SColumnFilterElem; typedef bool (*__filter_func_t)(struct SColumnFilterElem* pFilter, char* val1, char* val2); typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order); +typedef struct SPosInfo { + int32_t pageId:20; + int32_t rowId:12; +} SPosInfo; + +typedef struct SGroupResInfo { + int32_t groupId; + int32_t numOfDataPages; + SPosInfo pos; +} SGroupResInfo; + typedef struct SSqlGroupbyExpr { int16_t tableIndex; SArray* columnInfo; // SArray, group by columns information @@ -41,21 +52,12 @@ typedef struct SSqlGroupbyExpr { int16_t orderType; // order by type: asc/desc } SSqlGroupbyExpr; -typedef struct SPosInfo { - int32_t pageId; - int32_t rowId; -} SPosInfo; - -typedef struct SWindowStatus { - bool closed; -} SWindowStatus; - typedef struct SWindowResult { - uint16_t numOfRows; // number of rows of current time window - SWindowStatus status; // this result status: closed or opened SPosInfo pos; // Position of current result in disk-based output buffer + uint16_t numOfRows; // number of rows of current time window + bool closed; // this result status: closed or opened SResultInfo* resultInfo; // For each result column, there is a resultInfo - STimeWindow window; // The time window that current result covers. + TSKEY skey; // start key of current time window } SWindowResult; /** @@ -79,6 +81,7 @@ typedef struct SWindowResInfo { int64_t startTime; // start time of the first time window for sliding query int64_t prevSKey; // previous (not completed) sliding window start key int64_t threshold; // threshold to halt query and return the generated results. + int64_t interval; // time window interval } SWindowResInfo; typedef struct SColumnFilterElem { @@ -98,7 +101,7 @@ typedef struct STableQueryInfo { // todo merge with the STableQueryInfo struct TSKEY lastKey; int32_t groupIndex; // group id in table list int16_t queryRangeSet; // denote if the query range is set, only available for interval query - int64_t tag; + tVariant tag; STimeWindow win; STSCursor cur; void* pTable; // for retrieve the page id list @@ -121,8 +124,9 @@ typedef struct SQueryCostInfo { uint32_t loadBlockStatis; uint32_t discardBlocks; uint64_t elapsedTime; - uint64_t computTime; + uint64_t firstStageMergeTime; uint64_t internalSupSize; + uint64_t numOfTimeWindows; } SQueryCostInfo; typedef struct SQuery { @@ -189,21 +193,18 @@ typedef struct SQInfo { int64_t owner; // if it is in execution void* tsdb; int32_t vgId; - STableGroupInfo tableGroupInfo; // table id list < only includes the STable list> + STableGroupInfo tableGroupInfo; // table list SArray STableGroupInfo tableqinfoGroupInfo; // this is a group array list, including SArray structure SQueryRuntimeEnv runtimeEnv; - int32_t groupIndex; - int32_t offset; // offset in group result set of subgroup, todo refactor SArray* arrTableIdInfo; + int32_t groupIndex; /* * the query is executed position on which meter of the whole list. * when the index reaches the last one of the list, it means the query is completed. - * We later may refactor to remove this attribution by using another flag to denote - * whether a multimeter query is completed or not. */ int32_t tableIndex; - int32_t numOfGroupResultPages; + SGroupResInfo groupResInfo; void* pBuf; // allocated buffer for STableQueryInfo, sizeof(STableQueryInfo)*numOfTables; pthread_mutex_t lock; // used to synchronize the rsp/query threads diff --git a/src/query/inc/qSqlparser.h b/src/query/inc/qSqlparser.h index cd9618b223..69fc0bc7ef 100644 --- a/src/query/inc/qSqlparser.h +++ b/src/query/inc/qSqlparser.h @@ -26,7 +26,7 @@ extern "C" { #include "tstoken.h" #include "tvariant.h" -#define ParseTOKENTYPE SSQLToken +#define ParseTOKENTYPE SStrToken extern char tTokenTypeSwitcher[13]; #define toTSDBType(x) \ @@ -71,16 +71,16 @@ typedef struct SQuerySQL { struct tSQLExpr * pWhere; // where clause [optional] tVariantList * pGroupby; // groupby clause, only for tags[optional] tVariantList * pSortOrder; // orderby [optional] - SSQLToken interval; // interval [optional] - SSQLToken sliding; // sliding window [optional] + SStrToken interval; // interval [optional] + SStrToken sliding; // sliding window [optional] SLimitVal limit; // limit offset [optional] SLimitVal slimit; // group limit offset [optional] tVariantList * fillType; // fill type[optional] - SSQLToken selectToken; // sql string + SStrToken selectToken; // sql string } SQuerySQL; typedef struct SCreateTableSQL { - struct SSQLToken name; // meter name, create table [meterName] xxx + struct SStrToken name; // meter name, create table [meterName] xxx bool existCheck; int8_t type; // create normal table/from super table/ stream @@ -90,7 +90,7 @@ typedef struct SCreateTableSQL { } colInfo; struct { - SSQLToken stableName; // super table name, for using clause + SStrToken stableName; // super table name, for using clause tVariantList *pTagVals; // create by using metric, tag value STagData tagdata; } usingInfo; @@ -99,7 +99,7 @@ typedef struct SCreateTableSQL { } SCreateTableSQL; typedef struct SAlterTableSQL { - SSQLToken name; + SStrToken name; int16_t type; STagData tagData; @@ -108,7 +108,7 @@ typedef struct SAlterTableSQL { } SAlterTableSQL; typedef struct SCreateDBInfo { - SSQLToken dbname; + SStrToken dbname; int32_t replica; int32_t cacheBlockSize; int32_t maxTablesPerVnode; @@ -121,7 +121,7 @@ typedef struct SCreateDBInfo { int32_t walLevel; int32_t quorum; int32_t compressionLevel; - SSQLToken precision; + SStrToken precision; bool ignoreExists; tVariantList *keep; @@ -136,33 +136,33 @@ typedef struct SCreateAcctSQL { int64_t maxStorage; int64_t maxQueryTime; int32_t maxConnections; - SSQLToken stat; + SStrToken stat; } SCreateAcctSQL; typedef struct SShowInfo { uint8_t showType; - SSQLToken prefix; - SSQLToken pattern; + SStrToken prefix; + SStrToken pattern; } SShowInfo; typedef struct SUserInfo { - SSQLToken user; - SSQLToken passwd; - SSQLToken privilege; + SStrToken user; + SStrToken passwd; + SStrToken privilege; int16_t type; } SUserInfo; typedef struct tDCLSQL { int32_t nTokens; /* Number of expressions on the list */ int32_t nAlloc; /* Number of entries allocated below */ - SSQLToken *a; /* one entry for element */ + SStrToken *a; /* one entry for element */ bool existsCheck; union { SCreateDBInfo dbOpt; SCreateAcctSQL acctOpt; SShowInfo showOpt; - SSQLToken ip; + SStrToken ip; }; SUserInfo user; @@ -194,14 +194,14 @@ typedef struct tSQLExpr { // the full sql string of function(col, param), which is actually the raw // field name, since the function name is kept in nSQLOptr already - SSQLToken operand; - SSQLToken colInfo; // field id + SStrToken operand; + SStrToken colInfo; // field id tVariant val; // value only for string, float, int struct tSQLExpr *pLeft; // left child struct tSQLExpr *pRight; // right child - struct tSQLExprList *pParam; // function parameters + SStrToken token; // original sql expr string } tSQLExpr; // used in select clause. select from xxx @@ -210,6 +210,7 @@ typedef struct tSQLExprItem { char * aliasName; // alias name, null-terminated string } tSQLExprItem; +// todo refactor by using SArray typedef struct tSQLExprList { int32_t nExpr; /* Number of expressions on the list */ int32_t nAlloc; /* Number of entries allocated below */ @@ -242,7 +243,7 @@ tVariantList *tVariantListAppend(tVariantList *pList, tVariant *pVar, uint8_t so tVariantList *tVariantListInsert(tVariantList *pList, tVariant *pVar, uint8_t sortOrder, int32_t index); -tVariantList *tVariantListAppendToken(tVariantList *pList, SSQLToken *pAliasToken, uint8_t sortOrder); +tVariantList *tVariantListAppendToken(tVariantList *pList, SStrToken *pAliasToken, uint8_t sortOrder); void tVariantListDestroy(tVariantList *pList); tFieldList *tFieldListAppend(tFieldList *pList, TAOS_FIELD *pField); @@ -253,61 +254,61 @@ tSQLExpr *tSQLExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optType); void tSQLExprDestroy(tSQLExpr *); -tSQLExprList *tSQLExprListAppend(tSQLExprList *pList, tSQLExpr *pNode, SSQLToken *pToken); +tSQLExprList *tSQLExprListAppend(tSQLExprList *pList, tSQLExpr *pNode, SStrToken *pToken); void tSQLExprListDestroy(tSQLExprList *pList); -SQuerySQL *tSetQuerySQLElems(SSQLToken *pSelectToken, tSQLExprList *pSelection, tVariantList *pFrom, tSQLExpr *pWhere, - tVariantList *pGroupby, tVariantList *pSortOrder, SSQLToken *pInterval, - SSQLToken *pSliding, tVariantList *pFill, SLimitVal *pLimit, SLimitVal *pGLimit); +SQuerySQL *tSetQuerySQLElems(SStrToken *pSelectToken, tSQLExprList *pSelection, tVariantList *pFrom, tSQLExpr *pWhere, + tVariantList *pGroupby, tVariantList *pSortOrder, SStrToken *pInterval, + SStrToken *pSliding, tVariantList *pFill, SLimitVal *pLimit, SLimitVal *pGLimit); -SCreateTableSQL *tSetCreateSQLElems(tFieldList *pCols, tFieldList *pTags, SSQLToken *pMetricName, +SCreateTableSQL *tSetCreateSQLElems(tFieldList *pCols, tFieldList *pTags, SStrToken *pMetricName, tVariantList *pTagVals, SQuerySQL *pSelect, int32_t type); void tSQLExprNodeDestroy(tSQLExpr *pExpr); tSQLExpr *tSQLExprNodeClone(tSQLExpr *pExpr); -SAlterTableSQL *tAlterTableSQLElems(SSQLToken *pMeterName, tFieldList *pCols, tVariantList *pVals, int32_t type); +SAlterTableSQL *tAlterTableSQLElems(SStrToken *pMeterName, tFieldList *pCols, tVariantList *pVals, int32_t type); tSQLExprListList *tSQLListListAppend(tSQLExprListList *pList, tSQLExprList *pExprList); void destroyAllSelectClause(SSubclauseInfo *pSql); void doDestroyQuerySql(SQuerySQL *pSql); -SSqlInfo * setSQLInfo(SSqlInfo *pInfo, void *pSqlExprInfo, SSQLToken *pMeterName, int32_t type); +SSqlInfo * setSQLInfo(SSqlInfo *pInfo, void *pSqlExprInfo, SStrToken *pMeterName, int32_t type); SSubclauseInfo *setSubclause(SSubclauseInfo *pClause, void *pSqlExprInfo); SSubclauseInfo *appendSelectClause(SSubclauseInfo *pInfo, void *pSubclause); -void setCreatedTableName(SSqlInfo *pInfo, SSQLToken *pMeterName, SSQLToken *pIfNotExists); +void setCreatedTableName(SSqlInfo *pInfo, SStrToken *pMeterName, SStrToken *pIfNotExists); void SQLInfoDestroy(SSqlInfo *pInfo); void setDCLSQLElems(SSqlInfo *pInfo, int32_t type, int32_t nParams, ...); -void setDropDBTableInfo(SSqlInfo *pInfo, int32_t type, SSQLToken* pToken, SSQLToken* existsCheck); -void setShowOptions(SSqlInfo *pInfo, int32_t type, SSQLToken* prefix, SSQLToken* pPatterns); +void setDropDBTableInfo(SSqlInfo *pInfo, int32_t type, SStrToken* pToken, SStrToken* existsCheck); +void setShowOptions(SSqlInfo *pInfo, int32_t type, SStrToken* prefix, SStrToken* pPatterns); -tDCLSQL *tTokenListAppend(tDCLSQL *pTokenList, SSQLToken *pToken); +tDCLSQL *tTokenListAppend(tDCLSQL *pTokenList, SStrToken *pToken); -void setCreateDBSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *pToken, SCreateDBInfo *pDB, SSQLToken *pIgExists); +void setCreateDBSQL(SSqlInfo *pInfo, int32_t type, SStrToken *pToken, SCreateDBInfo *pDB, SStrToken *pIgExists); -void setCreateAcctSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *pName, SSQLToken *pPwd, SCreateAcctSQL *pAcctInfo); -void setCreateUserSQL(SSqlInfo *pInfo, SSQLToken *pName, SSQLToken *pPasswd); -void setKillSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *ip); -void setAlterUserSQL(SSqlInfo *pInfo, int16_t type, SSQLToken *pName, SSQLToken* pPwd, SSQLToken *pPrivilege); +void setCreateAcctSQL(SSqlInfo *pInfo, int32_t type, SStrToken *pName, SStrToken *pPwd, SCreateAcctSQL *pAcctInfo); +void setCreateUserSQL(SSqlInfo *pInfo, SStrToken *pName, SStrToken *pPasswd); +void setKillSQL(SSqlInfo *pInfo, int32_t type, SStrToken *ip); +void setAlterUserSQL(SSqlInfo *pInfo, int16_t type, SStrToken *pName, SStrToken* pPwd, SStrToken *pPrivilege); void setDefaultCreateDbOption(SCreateDBInfo *pDBInfo); // prefix show db.tables; -void setDBName(SSQLToken *pCpxName, SSQLToken *pDB); +void setDBName(SStrToken *pCpxName, SStrToken *pDB); -tSQLExpr *tSQLExprIdValueCreate(SSQLToken *pToken, int32_t optType); +tSQLExpr *tSQLExprIdValueCreate(SStrToken *pToken, int32_t optType); -tSQLExpr *tSQLExprCreateFunction(tSQLExprList *pList, SSQLToken *pFuncToken, SSQLToken *endToken, int32_t optType); +tSQLExpr *tSQLExprCreateFunction(tSQLExprList *pList, SStrToken *pFuncToken, SStrToken *endToken, int32_t optType); -void tSQLSetColumnInfo(TAOS_FIELD *pField, SSQLToken *pName, TAOS_FIELD *pType); +void tSQLSetColumnInfo(TAOS_FIELD *pField, SStrToken *pName, TAOS_FIELD *pType); -void tSQLSetColumnType(TAOS_FIELD *pField, SSQLToken *pToken); +void tSQLSetColumnType(TAOS_FIELD *pField, SStrToken *pToken); void *ParseAlloc(void *(*mallocProc)(size_t)); diff --git a/src/query/inc/qTsbuf.h b/src/query/inc/qTsbuf.h index 59b224e096..46e6f79014 100644 --- a/src/query/inc/qTsbuf.h +++ b/src/query/inc/qTsbuf.h @@ -22,6 +22,7 @@ extern "C" { #include "os.h" #include "taosdef.h" +#include "tvariant.h" #define MEM_BUF_SIZE (1 << 20) #define TS_COMP_FILE_MAGIC 0x87F5EC4C @@ -42,9 +43,9 @@ typedef struct STSRawBlock { } STSRawBlock; typedef struct STSElem { - TSKEY ts; - int64_t tag; - int32_t vnode; + TSKEY ts; + tVariant tag; + int32_t vnode; } STSElem; typedef struct STSCursor { @@ -55,11 +56,11 @@ typedef struct STSCursor { } STSCursor; typedef struct STSBlock { - int64_t tag; // tag value - int32_t numOfElem; // number of elements - int32_t compLen; // size after compressed - int32_t padding; // 0xFFFFFFFF by default, after the payload - char* payload; // actual data that is compressed + tVariant tag; // tag value + int32_t numOfElem; // number of elements + int32_t compLen; // size after compressed + int32_t padding; // 0xFFFFFFFF by default, after the payload + char* payload; // actual data that is compressed } STSBlock; /* @@ -84,8 +85,8 @@ typedef struct STSBuf { uint32_t fileSize; STSVnodeBlockInfoEx* pData; - int32_t numOfAlloc; - int32_t numOfVnodes; + uint32_t numOfAlloc; + uint32_t numOfVnodes; char* assistBuf; int32_t bufSize; @@ -109,7 +110,7 @@ STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_ void* tsBufDestroy(STSBuf* pTSBuf); -void tsBufAppend(STSBuf* pTSBuf, int32_t vnodeId, int64_t tag, const char* pData, int32_t len); +void tsBufAppend(STSBuf* pTSBuf, int32_t vnodeId, tVariant* tag, const char* pData, int32_t len); int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf, int32_t vnodeIdx); STSBuf* tsBufClone(STSBuf* pTSBuf); @@ -122,7 +123,7 @@ void tsBufResetPos(STSBuf* pTSBuf); STSElem tsBufGetElem(STSBuf* pTSBuf); bool tsBufNextPos(STSBuf* pTSBuf); -STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t vnodeId, int64_t tag); +STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t vnodeId, tVariant* tag); STSCursor tsBufGetCursor(STSBuf* pTSBuf); void tsBufSetTraverseOrder(STSBuf* pTSBuf, int32_t order); diff --git a/src/query/inc/qUtil.h b/src/query/inc/qUtil.h index b7f57bf1b5..6de3c7c0e5 100644 --- a/src/query/inc/qUtil.h +++ b/src/query/inc/qUtil.h @@ -38,7 +38,8 @@ static FORCE_INLINE SWindowResult *getWindowResult(SWindowResInfo *pWindowResInf return &pWindowResInfo->pResult[slot]; } -#define curTimeWindow(_winres) ((_winres)->curIndex) +#define curTimeWindowIndex(_winres) ((_winres)->curIndex) +#define GET_TIMEWINDOW(_winresInfo, _win) (STimeWindow) {(_win)->skey, ((_win)->skey + (_winresInfo)->interval - 1)} #define GET_ROW_PARAM_FOR_MULTIOUTPUT(_q, tbq, sq) (((tbq) && (!sq))? (_q)->pSelectExpr[1].base.arg->argValue.i64:1) bool isWindowResClosed(SWindowResInfo *pWindowResInfo, int32_t slot); diff --git a/src/query/inc/sql.y b/src/query/inc/sql.y index 2b2a967edf..7500dcba56 100644 --- a/src/query/inc/sql.y +++ b/src/query/inc/sql.y @@ -2,8 +2,8 @@ //usage: lemon sql.y %token_prefix TK_ -%token_type {SSQLToken} -%default_type {SSQLToken} +%token_type {SStrToken} +%default_type {SStrToken} %extra_argument {SSqlInfo* pInfo} %fallback ID BOOL TINYINT SMALLINT INTEGER BIGINT FLOAT DOUBLE STRING TIMESTAMP BINARY NCHAR. @@ -73,18 +73,18 @@ cmd ::= SHOW MODULES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_MODULE, 0, 0); cmd ::= SHOW QUERIES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_QUERIES, 0, 0); } cmd ::= SHOW CONNECTIONS.{ setShowOptions(pInfo, TSDB_MGMT_TABLE_CONNS, 0, 0);} cmd ::= SHOW STREAMS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_STREAMS, 0, 0); } -cmd ::= SHOW CONFIGS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_CONFIGS, 0, 0); } +cmd ::= SHOW VARIABLES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_VARIABLES, 0, 0); } cmd ::= SHOW SCORES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_SCORES, 0, 0); } cmd ::= SHOW GRANTS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_GRANTS, 0, 0); } cmd ::= SHOW VNODES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_VNODES, 0, 0); } cmd ::= SHOW VNODES IPTOKEN(X). { setShowOptions(pInfo, TSDB_MGMT_TABLE_VNODES, &X, 0); } -%type dbPrefix {SSQLToken} +%type dbPrefix {SStrToken} dbPrefix(A) ::=. {A.n = 0; A.type = 0;} dbPrefix(A) ::= ids(X) DOT. {A = X; } -%type cpxName {SSQLToken} +%type cpxName {SStrToken} cpxName(A) ::= . {A.n = 0; } cpxName(A) ::= DOT ids(Y). {A = Y; A.n += 1; } @@ -101,19 +101,19 @@ cmd ::= SHOW dbPrefix(X) STABLES. { } cmd ::= SHOW dbPrefix(X) STABLES LIKE ids(Y). { - SSQLToken token; + SStrToken token; setDBName(&token, &X); setShowOptions(pInfo, TSDB_MGMT_TABLE_METRIC, &token, &Y); } cmd ::= SHOW dbPrefix(X) VGROUPS. { - SSQLToken token; + SStrToken token; setDBName(&token, &X); setShowOptions(pInfo, TSDB_MGMT_TABLE_VGROUP, &token, 0); } cmd ::= SHOW dbPrefix(X) VGROUPS ids(Y). { - SSQLToken token; + SStrToken token; setDBName(&token, &X); setShowOptions(pInfo, TSDB_MGMT_TABLE_VGROUP, &token, &Y); } @@ -145,7 +145,7 @@ cmd ::= ALTER DNODE ids(X) ids(Y). { setDCLSQLElems(pInfo, TSDB_SQL cmd ::= ALTER DNODE ids(X) ids(Y) ids(Z). { setDCLSQLElems(pInfo, TSDB_SQL_CFG_DNODE, 3, &X, &Y, &Z); } cmd ::= ALTER LOCAL ids(X). { setDCLSQLElems(pInfo, TSDB_SQL_CFG_LOCAL, 1, &X); } cmd ::= ALTER LOCAL ids(X) ids(Y). { setDCLSQLElems(pInfo, TSDB_SQL_CFG_LOCAL, 2, &X, &Y); } -cmd ::= ALTER DATABASE ids(X) alter_db_optr(Y). { SSQLToken t = {0}; setCreateDBSQL(pInfo, TSDB_SQL_ALTER_DB, &X, &Y, &t);} +cmd ::= ALTER DATABASE ids(X) alter_db_optr(Y). { SStrToken t = {0}; setCreateDBSQL(pInfo, TSDB_SQL_ALTER_DB, &X, &Y, &t);} cmd ::= ALTER ACCOUNT ids(X) acct_optr(Z). { setCreateAcctSQL(pInfo, TSDB_SQL_ALTER_ACCT, &X, NULL, &Z);} cmd ::= ALTER ACCOUNT ids(X) PASS ids(Y) acct_optr(Z). { setCreateAcctSQL(pInfo, TSDB_SQL_ALTER_ACCT, &X, &Y, &Z);} @@ -153,15 +153,15 @@ cmd ::= ALTER ACCOUNT ids(X) PASS ids(Y) acct_optr(Z). { setCreateAcctSQL(p // An IDENTIFIER can be a generic identifier, or one of several keywords. // Any non-standard keyword can also be an identifier. // And "ids" is an identifer-or-string. -%type ids {SSQLToken} +%type ids {SStrToken} ids(A) ::= ID(X). {A = X; } ids(A) ::= STRING(X). {A = X; } -%type ifexists {SSQLToken} +%type ifexists {SStrToken} ifexists(X) ::= IF EXISTS. {X.n = 1;} ifexists(X) ::= . {X.n = 0;} -%type ifnotexists {SSQLToken} +%type ifnotexists {SStrToken} ifnotexists(X) ::= IF NOT EXISTS. {X.n = 1;} ifnotexists(X) ::= . {X.n = 0;} @@ -416,7 +416,7 @@ selcollist(A) ::= sclp(P) STAR. { // An option "AS " phrase that can follow one of the expressions that // define the result set, or one of the tables in the FROM clause. // -%type as {SSQLToken} +%type as {SStrToken} as(X) ::= AS ids(Y). { X = Y; } as(X) ::= ids(Y). { X = Y; } as(X) ::= . { X.n = 0; } @@ -458,10 +458,10 @@ tablelist(A) ::= tablelist(Y) COMMA ids(X) cpxName(Z) ids(F). { } // The value of interval should be the form of "number+[a,s,m,h,d,n,y]" or "now" -%type tmvar {SSQLToken} +%type tmvar {SStrToken} tmvar(A) ::= VARIABLE(X). {A = X;} -%type interval_opt {SSQLToken} +%type interval_opt {SStrToken} interval_opt(N) ::= INTERVAL LP tmvar(E) RP. {N = E; } interval_opt(N) ::= . {N.n = 0; N.z = NULL; N.type = 0; } @@ -482,7 +482,7 @@ fill_opt(N) ::= FILL LP ID(Y) RP. { N = tVariantListAppendToken(NULL, &Y, -1); } -%type sliding_opt {SSQLToken} +%type sliding_opt {SStrToken} sliding_opt(K) ::= SLIDING LP tmvar(E) RP. {K = E; } sliding_opt(K) ::= . {K.n = 0; K.z = NULL; K.type = 0; } diff --git a/src/query/inc/tsqlfunction.h b/src/query/inc/tsqlfunction.h index 58d2ff260e..65ab82883b 100644 --- a/src/query/inc/tsqlfunction.h +++ b/src/query/inc/tsqlfunction.h @@ -91,16 +91,16 @@ extern "C" { #define QUERY_COND_REL_PREFIX_IN "IN|" #define QUERY_COND_REL_PREFIX_LIKE "LIKE|" -#define QUERY_COND_REL_PREFIX_IN_LEN 3 +#define QUERY_COND_REL_PREFIX_IN_LEN 3 #define QUERY_COND_REL_PREFIX_LIKE_LEN 5 -#define QUERY_ASC_FORWARD_STEP 1 +#define QUERY_ASC_FORWARD_STEP 1 #define QUERY_DESC_FORWARD_STEP -1 #define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP) -#define MAX_RETRIEVE_ROWS_IN_INTERVAL_QUERY 10000000 -#define TOP_BOTTOM_QUERY_LIMIT 100 +#define MAX_INTERVAL_TIME_WINDOW 10000000 +#define TOP_BOTTOM_QUERY_LIMIT 100 enum { MASTER_SCAN = 0x0u, @@ -137,13 +137,13 @@ typedef struct SInterpInfoDetail { } SInterpInfoDetail; typedef struct SResultInfo { - int8_t hasResult; // result generated, not NULL value - bool initialized; // output buffer has been initialized - bool complete; // query has completed - bool superTableQ; // is super table query - int32_t numOfRes; // num of output result in current buffer - int32_t bufLen; // buffer size - void* interResultBuf; // output result buffer + int8_t hasResult; // result generated, not NULL value + bool initialized; // output buffer has been initialized + bool complete; // query has completed + bool superTableQ; // is super table query + uint32_t bufLen; // buffer size + uint64_t numOfRes; // num of output result in current buffer + void* interResultBuf; // output result buffer } SResultInfo; struct SQLFunctionCtx; diff --git a/src/query/src/qAst.c b/src/query/src/qAst.c index afa3618a96..634f014d97 100644 --- a/src/query/src/qAst.c +++ b/src/query/src/qAst.c @@ -44,7 +44,7 @@ * ver 0.3, pipeline filter in the form of: (a+2)/9 > 14 * */ -static tExprNode *tExprNodeCreate(SSchema *pSchema, int32_t numOfCols, SSQLToken *pToken); +static tExprNode *tExprNodeCreate(SSchema *pSchema, int32_t numOfCols, SStrToken *pToken); static tExprNode *createSyntaxTree(SSchema *pSchema, int32_t numOfCols, char *str, int32_t *i); static void destroySyntaxTree(tExprNode *); @@ -103,7 +103,7 @@ static void reviseBinaryExprIfNecessary(tExprNode **pLeft, tExprNode **pRight, u } } -static tExprNode *tExprNodeCreate(SSchema *pSchema, int32_t numOfCols, SSQLToken *pToken) { +static tExprNode *tExprNodeCreate(SSchema *pSchema, int32_t numOfCols, SStrToken *pToken) { /* if the token is not a value, return false */ if (pToken->type == TK_RP || (pToken->type != TK_INTEGER && pToken->type != TK_FLOAT && pToken->type != TK_ID && pToken->type != TK_TBNAME && pToken->type != TK_STRING && pToken->type != TK_BOOL)) { @@ -117,7 +117,7 @@ static tExprNode *tExprNodeCreate(SSchema *pSchema, int32_t numOfCols, SSQLToken int32_t i = 0; if (pToken->type == TK_ID) { do { - SSQLToken tableToken = {0}; + SStrToken tableToken = {0}; extractTableNameFromToken(pToken, &tableToken); size_t len = strlen(pSchema[i].name); @@ -157,7 +157,7 @@ static tExprNode *tExprNodeCreate(SSchema *pSchema, int32_t numOfCols, SSQLToken return pNode; } -uint8_t getBinaryExprOptr(SSQLToken *pToken) { +uint8_t getBinaryExprOptr(SStrToken *pToken) { switch (pToken->type) { case TK_LT: return TSDB_RELATION_LESS; @@ -234,7 +234,7 @@ uint8_t isQueryOnPrimaryKey(const char *primaryColumnName, const tExprNode *pLef } static tExprNode *createSyntaxTree(SSchema *pSchema, int32_t numOfCols, char *str, int32_t *i) { - SSQLToken t0 = tStrGetToken(str, i, false, 0, NULL); + SStrToken t0 = tStrGetToken(str, i, false, 0, NULL); if (t0.n == 0) { return NULL; } diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 03b699c1b8..4e2e31d269 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -27,7 +27,8 @@ #include "query.h" #include "queryLog.h" #include "tlosertree.h" -#include "tscompression.h" + +#define MAX_ROWS_PER_RESBUF_PAGE ((1u<<12) - 1) /** * check if the primary column is load by default, otherwise, the program will @@ -35,6 +36,9 @@ */ #define Q_STATUS_EQUAL(p, s) (((p) & (s)) != 0) #define TSDB_COL_IS_TAG(f) (((f)&TSDB_COL_TAG) != 0) +#define TSDB_COL_IS_NORMAL_COL(f) ((f) == TSDB_COL_NORMAL) +#define TSDB_COL_IS_UD_COL(f) ((f) == TSDB_COL_UDC) + #define QUERY_IS_ASC_QUERY(q) (GET_FORWARD_DIRECTION_FACTOR((q)->order.order) == QUERY_ASC_FORWARD_STEP) #define IS_MASTER_SCAN(runtime) ((runtime)->scanFlag == MASTER_SCAN) @@ -93,7 +97,8 @@ typedef struct { #if 0 static UNUSED_FUNC void *u_malloc (size_t __size) { uint32_t v = rand(); - if (v % 5 <= 1) { + + if (v % 1000 <= 0) { return NULL; } else { return malloc(__size); @@ -102,15 +107,25 @@ static UNUSED_FUNC void *u_malloc (size_t __size) { static UNUSED_FUNC void* u_calloc(size_t num, size_t __size) { uint32_t v = rand(); - if (v % 5 <= 1) { + if (v % 1000 <= 0) { return NULL; } else { return calloc(num, __size); } } +static UNUSED_FUNC void* u_realloc(void* p, size_t __size) { + uint32_t v = rand(); + if (v % 5 <= 1) { + return NULL; + } else { + return realloc(p, __size); + } +} + #define calloc u_calloc #define malloc u_malloc +#define realloc u_realloc #endif #define CLEAR_QUERY_STATUS(q, st) ((q)->status &= (~(st))) @@ -151,7 +166,7 @@ static bool hasMainOutput(SQuery *pQuery); static void buildTagQueryResult(SQInfo *pQInfo); static int32_t setAdditionalInfo(SQInfo *pQInfo, void *pTable, STableQueryInfo *pTableQueryInfo); -static int32_t flushFromResultBuf(SQInfo *pQInfo); +static int32_t flushFromResultBuf(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo); bool doFilterData(SQuery *pQuery, int32_t elemPos) { for (int32_t k = 0; k < pQuery->numOfFilterCols; ++k) { @@ -212,22 +227,22 @@ int64_t getNumOfResult(SQueryRuntimeEnv *pRuntimeEnv) { */ void updateNumOfResult(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOfRes) { SQuery *pQuery = pRuntimeEnv->pQuery; - + for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { SResultInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]); - + int16_t functionId = pRuntimeEnv->pCtx[j].functionId; if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TAGPRJ || functionId == TSDB_FUNC_TS_DUMMY) { continue; } - + assert(pResInfo->numOfRes > numOfRes); pResInfo->numOfRes = numOfRes; } } -static int32_t getGroupResultId(int32_t groupIndex) { +static UNUSED_FUNC int32_t getGroupResultId(int32_t groupIndex) { int32_t base = 20000000; return base + (groupIndex * 10000); } @@ -315,10 +330,10 @@ bool isTSCompQuery(SQuery *pQuery) { return pQuery->pSelectExpr[0].base.function static bool limitResults(SQueryRuntimeEnv* pRuntimeEnv) { SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv); SQuery* pQuery = pRuntimeEnv->pQuery; - + if ((pQuery->limit.limit > 0) && (pQuery->rec.total + pQuery->rec.rows > pQuery->limit.limit)) { pQuery->rec.rows = pQuery->limit.limit - pQuery->rec.total; - + qDebug("QInfo:%p discard remain data due to result limitation, limit:%"PRId64", current return:%" PRId64 ", total:%"PRId64, pQInfo, pQuery->limit.limit, pQuery->rec.rows, pQuery->rec.total + pQuery->rec.rows); assert(pQuery->rec.rows >= 0); @@ -371,14 +386,14 @@ static bool hasTagValOutput(SQuery* pQuery) { * @return */ static bool hasNullValue(SColIndex* pColIndex, SDataStatis *pStatis, SDataStatis **pColStatis) { - if (pStatis != NULL && !TSDB_COL_IS_TAG(pColIndex->flag)) { + if (pStatis != NULL && TSDB_COL_IS_NORMAL_COL(pColIndex->flag)) { *pColStatis = &pStatis[pColIndex->colIndex]; assert((*pColStatis)->colId == pColIndex->colId); } else { *pColStatis = NULL; } - if (TSDB_COL_IS_TAG(pColIndex->flag) || pColIndex->colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + if (TSDB_COL_IS_TAG(pColIndex->flag) || TSDB_COL_IS_UD_COL(pColIndex->flag) || pColIndex->colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { return false; } @@ -412,6 +427,7 @@ static SWindowResult *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWin char *t = realloc(pWindowResInfo->pResult, (size_t)(newCap * sizeof(SWindowResult))); pRuntimeEnv->summary.internalSupSize += (newCap - pWindowResInfo->capacity) * sizeof(SWindowResult); + pRuntimeEnv->summary.numOfTimeWindows += (newCap - pWindowResInfo->capacity); if (t == NULL) { longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -425,7 +441,10 @@ static SWindowResult *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWin pRuntimeEnv->summary.internalSupSize += (pQuery->numOfOutput * sizeof(SResultInfo) + pRuntimeEnv->interBufSize) * inc; for (int32_t i = pWindowResInfo->capacity; i < newCap; ++i) { - createQueryResultInfo(pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, pRuntimeEnv->interBufSize); + int32_t ret = createQueryResultInfo(pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, pRuntimeEnv->interBufSize); + if (ret != TSDB_CODE_SUCCESS) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } } pWindowResInfo->capacity = (int32_t)newCap; @@ -436,6 +455,11 @@ static SWindowResult *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWin taosHashPut(pWindowResInfo->hashList, pData, bytes, (char *)&pWindowResInfo->curIndex, sizeof(int32_t)); } + // too many time window in query + if (pWindowResInfo->size > MAX_INTERVAL_TIME_WINDOW) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW); + } + return getWindowResult(pWindowResInfo, pWindowResInfo->curIndex); } @@ -447,8 +471,9 @@ static STimeWindow getActiveTimeWindow(SWindowResInfo *pWindowResInfo, int64_t t w.skey = pWindowResInfo->prevSKey; w.ekey = w.skey + pQuery->intervalTime - 1; } else { - int32_t slot = curTimeWindow(pWindowResInfo); - w = getWindowResult(pWindowResInfo, slot)->window; + int32_t slot = curTimeWindowIndex(pWindowResInfo); + SWindowResult* pWindowRes = getWindowResult(pWindowResInfo, slot); + w = GET_TIMEWINDOW(pWindowResInfo, pWindowRes); } if (w.skey > ts || w.ekey < ts) { @@ -549,15 +574,15 @@ static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowRes } // set time window for current result - pWindowRes->window = *win; + pWindowRes->skey = win->skey; setWindowResOutputBufInitCtx(pRuntimeEnv, pWindowRes); return TSDB_CODE_SUCCESS; } -static SWindowStatus *getTimeWindowResStatus(SWindowResInfo *pWindowResInfo, int32_t slot) { +static bool getTimeWindowResStatus(SWindowResInfo *pWindowResInfo, int32_t slot) { assert(slot >= 0 && slot < pWindowResInfo->size); - return &pWindowResInfo->pResult[slot].status; + return pWindowResInfo->pResult[slot].closed; } static FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey, int16_t pos, @@ -599,7 +624,7 @@ static int32_t doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKe // no qualified results exist, abort check int32_t numOfClosed = 0; - + if (pWindowResInfo->size == 0) { return pWindowResInfo->size; } @@ -617,16 +642,17 @@ static int32_t doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKe for (i = 0; i < pWindowResInfo->size; ++i) { SWindowResult *pResult = &pWindowResInfo->pResult[i]; - if (pResult->status.closed) { + if (pResult->closed) { numOfClosed += 1; continue; } - if ((pResult->window.ekey <= lastKey && QUERY_IS_ASC_QUERY(pQuery)) || - (pResult->window.skey >= lastKey && !QUERY_IS_ASC_QUERY(pQuery))) { + TSKEY ekey = pResult->skey + pWindowResInfo->interval; + if ((ekey <= lastKey && QUERY_IS_ASC_QUERY(pQuery)) || + (pResult->skey >= lastKey && !QUERY_IS_ASC_QUERY(pQuery))) { closeTimeWindow(pWindowResInfo, i); } else { - skey = pResult->window.skey; + skey = pResult->skey; break; } } @@ -639,26 +665,26 @@ static int32_t doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKe pWindowResInfo->curIndex = i; } - pWindowResInfo->prevSKey = pWindowResInfo->pResult[pWindowResInfo->curIndex].window.skey; + pWindowResInfo->prevSKey = pWindowResInfo->pResult[pWindowResInfo->curIndex].skey; // the number of completed slots are larger than the threshold, return current generated results to client. if (numOfClosed > pWindowResInfo->threshold) { qDebug("QInfo:%p total result window:%d closed:%d, reached the output threshold %d, return", GET_QINFO_ADDR(pRuntimeEnv), pWindowResInfo->size, numOfClosed, pQuery->rec.threshold); - + setQueryStatus(pQuery, QUERY_RESBUF_FULL); } else { qDebug("QInfo:%p total result window:%d already closed:%d", GET_QINFO_ADDR(pRuntimeEnv), pWindowResInfo->size, numOfClosed); } } - + // output has reached the limitation, set query completed if (pQuery->limit.limit > 0 && (pQuery->limit.limit + pQuery->limit.offset) <= numOfClosed && pRuntimeEnv->scanFlag == MASTER_SCAN) { setQueryStatus(pQuery, QUERY_COMPLETED); } - + assert(pWindowResInfo->prevSKey != TSKEY_INITIAL_VAL); return numOfClosed; } @@ -672,7 +698,7 @@ static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SDataBlockInfo *pDataBlo int32_t step = GET_FORWARD_DIRECTION_FACTOR(order); STableQueryInfo* item = pQuery->current; - + if (QUERY_IS_ASC_QUERY(pQuery)) { if (ekey < pDataBlockInfo->window.ekey) { num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn); @@ -703,12 +729,12 @@ static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SDataBlockInfo *pDataBlo return num; } -static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SWindowStatus *pStatus, STimeWindow *pWin, +static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, bool closed, STimeWindow *pWin, int32_t offset, int32_t forwardStep, TSKEY *tsBuf, int32_t numOfTotal) { SQuery * pQuery = pRuntimeEnv->pQuery; SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; - if (IS_MASTER_SCAN(pRuntimeEnv) || pStatus->closed) { + if (IS_MASTER_SCAN(pRuntimeEnv) || closed) { for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { int32_t functionId = pQuery->pSelectExpr[k].base.functionId; @@ -732,12 +758,11 @@ static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SWindowStat } } -static void doRowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SWindowStatus *pStatus, STimeWindow *pWin, - int32_t offset) { +static void doRowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, bool closed, STimeWindow *pWin, int32_t offset) { SQuery * pQuery = pRuntimeEnv->pQuery; SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; - if (IS_MASTER_SCAN(pRuntimeEnv) || pStatus->closed) { + if (IS_MASTER_SCAN(pRuntimeEnv) || closed) { for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { pCtx[k].nStartQueryTimestamp = pWin->skey; @@ -822,14 +847,14 @@ static FORCE_INLINE TSKEY reviseWindowEkey(SQuery *pQuery, STimeWindow *pWindow) //todo binary search static void* getDataBlockImpl(SArray* pDataBlock, int32_t colId) { int32_t numOfCols = (int32_t)taosArrayGetSize(pDataBlock); - + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData *p = taosArrayGet(pDataBlock, i); if (colId == p->info.colId) { return p->pData; } } - + return NULL; } @@ -879,19 +904,19 @@ static char *getDataBlock(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas } assert(dataBlock != NULL); - sas->data[i] = dataBlock/* + pQuery->colList[i].bytes*/; // start from the offset + sas->data[i] = dataBlock; // start from the offset } } else { // other type of query function SColIndex *pCol = &pQuery->pSelectExpr[col].base.colInfo; - if (TSDB_COL_IS_TAG(pCol->flag)) { - dataBlock = NULL; - } else { + if (TSDB_COL_IS_NORMAL_COL(pCol->flag)) { SColIndex* pColIndex = &pQuery->pSelectExpr[col].base.colInfo; SColumnInfoData *p = taosArrayGet(pDataBlock, pColIndex->colIndex); assert(p->info.colId == pColIndex->colId); dataBlock = p->pData; + } else { + dataBlock = NULL; } } @@ -958,7 +983,7 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * TSKEY ekey = reviseWindowEkey(pQuery, &win); forwardStep = getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, tsCols, pQuery->pos, ekey, searchFn, true); - SWindowStatus *pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo)); + bool pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); doBlockwiseApplyFunctions(pRuntimeEnv, pStatus, &win, startPos, forwardStep, tsCols, pDataBlockInfo->rows); } @@ -986,8 +1011,8 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * TSKEY ekey = reviseWindowEkey(pQuery, &nextWin); forwardStep = getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, tsCols, startPos, ekey, searchFn, true); - SWindowStatus *pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo)); - doBlockwiseApplyFunctions(pRuntimeEnv, pStatus, &nextWin, startPos, forwardStep, tsCols, pDataBlockInfo->rows); + bool closed = getTimeWindowResStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); + doBlockwiseApplyFunctions(pRuntimeEnv, closed, &nextWin, startPos, forwardStep, tsCols, pDataBlockInfo->rows); } pWindowResInfo->curIndex = index; @@ -1040,8 +1065,8 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat return -1; } - pWindowRes->window.skey = v; - pWindowRes->window.ekey = v; + pWindowRes->skey = v; + assert(pRuntimeEnv->windowResInfo.interval == 0); if (pWindowRes->pos.pageId == -1) { int32_t ret = addNewWindowResultBuf(pWindowRes, pResultBuf, GROUPRESULTID, pRuntimeEnv->numOfRowsPerPage); @@ -1103,7 +1128,7 @@ static int32_t doTSJoinFilter(SQueryRuntimeEnv *pRuntimeEnv, int32_t offset) { SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; // compare tag first - if (pCtx[0].tag.i64Key != elem.tag) { + if (tVariantCompare(&pCtx[0].tag, &elem.tag) != 0) { return TS_JOIN_TAG_NOT_EQUALS; } @@ -1140,7 +1165,7 @@ static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx if (functionId == TSDB_FUNC_TS) { return true; } - + if (pResInfo->complete || functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TS_DUMMY) { return false; } @@ -1247,8 +1272,8 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS continue; } - SWindowStatus *pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo)); - doRowwiseApplyFunctions(pRuntimeEnv, pStatus, &win, offset); + bool closed = getTimeWindowResStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); + doRowwiseApplyFunctions(pRuntimeEnv, closed, &win, offset); STimeWindow nextWin = win; int32_t index = pWindowResInfo->curIndex; @@ -1271,8 +1296,8 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS } if (hasTimeWindow) { - pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo)); - doRowwiseApplyFunctions(pRuntimeEnv, pStatus, &nextWin, offset); + closed = getTimeWindowResStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); + doRowwiseApplyFunctions(pRuntimeEnv, closed, &nextWin, offset); } } @@ -1327,10 +1352,10 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pDataBlockInfo, SDataStatis *pStatis, __block_search_fn_t searchFn, SArray *pDataBlock) { SQuery *pQuery = pRuntimeEnv->pQuery; - + STableQueryInfo* pTableQInfo = pQuery->current; SWindowResInfo* pWindowResInfo = &pRuntimeEnv->windowResInfo; - + if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL || pRuntimeEnv->groupbyNormalCol) { rowwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, pDataBlock); } else { @@ -1373,10 +1398,10 @@ static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBl void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY *tsCol, SDataBlockInfo* pBlockInfo, SDataStatis *pStatis, void *param, int32_t colIndex) { - + int32_t functionId = pQuery->pSelectExpr[colIndex].base.functionId; int32_t colId = pQuery->pSelectExpr[colIndex].base.colInfo.colId; - + SDataStatis *tpField = NULL; pCtx->hasNull = hasNullValue(&pQuery->pSelectExpr[colIndex].base.colInfo, pStatis, &tpField); pCtx->aInputElemBuf = inputData; @@ -1432,7 +1457,7 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY pInterpInfo->type = (int8_t)pQuery->fillType; pInterpInfo->ts = pQuery->window.skey; pInterpInfo->primaryCol = (colId == PRIMARYKEY_TIMESTAMP_COL_INDEX); - + if (pQuery->fillVal != NULL) { if (isNull((const char*) &pQuery->fillVal[colIndex], pCtx->inputType)) { pCtx->param[1].nType = TSDB_DATA_TYPE_NULL; @@ -1459,19 +1484,22 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY } // set the output buffer for the selectivity + tag query -static void setCtxTagColumnInfo(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx) { +static int32_t setCtxTagColumnInfo(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx) { SQuery* pQuery = pRuntimeEnv->pQuery; if (isSelectivityWithTagsQuery(pQuery)) { int32_t num = 0; int16_t tagLen = 0; - + SQLFunctionCtx *p = NULL; SQLFunctionCtx **pTagCtx = calloc(pQuery->numOfOutput, POINTER_BYTES); + if (pTagCtx == NULL) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { SSqlFuncMsg *pSqlFuncMsg = &pQuery->pSelectExpr[i].base; - + if (pSqlFuncMsg->functionId == TSDB_FUNC_TAG_DUMMY || pSqlFuncMsg->functionId == TSDB_FUNC_TS_DUMMY) { tagLen += pCtx[i].outputBytes; pTagCtx[num++] = &pCtx[i]; @@ -1490,9 +1518,11 @@ static void setCtxTagColumnInfo(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *p p->tagInfo.numOfTagCols = num; p->tagInfo.tagsLen = tagLen; } else { - taosTFree(pTagCtx); + taosTFree(pTagCtx); } } + + return TSDB_CODE_SUCCESS; } static FORCE_INLINE void setWindowResultInfo(SResultInfo *pResultInfo, SQuery *pQuery, bool isStableQuery, char* buf) { @@ -1536,12 +1566,14 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order pCtx->inputBytes = pQuery->tagColList[index].bytes; pCtx->inputType = pQuery->tagColList[index].type; } - + } else if (TSDB_COL_IS_UD_COL(pIndex->flag)) { + pCtx->inputBytes = pSqlFuncMsg->arg[0].argBytes; + pCtx->inputType = pSqlFuncMsg->arg[0].argType; } else { pCtx->inputBytes = pQuery->colList[index].bytes; pCtx->inputType = pQuery->colList[index].type; } - + assert(isValidDataType(pCtx->inputType)); pCtx->ptsOutputBuf = NULL; @@ -1556,7 +1588,7 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order int16_t type = pSqlFuncMsg->arg[j].argType; int16_t bytes = pSqlFuncMsg->arg[j].argBytes; if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - tVariantCreateFromBinary(&pCtx->param[j], pSqlFuncMsg->arg->argValue.pz, bytes, type); + tVariantCreateFromBinary(&pCtx->param[j], pSqlFuncMsg->arg[j].argValue.pz, bytes, type); } else { tVariantCreateFromBinary(&pCtx->param[j], (char *)&pSqlFuncMsg->arg[j].argValue.i64, bytes, type); } @@ -1592,7 +1624,9 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order resetCtxOutputBuf(pRuntimeEnv); } - setCtxTagColumnInfo(pRuntimeEnv, pRuntimeEnv->pCtx); + if (setCtxTagColumnInfo(pRuntimeEnv, pRuntimeEnv->pCtx) != TSDB_CODE_SUCCESS) { + goto _clean; + } qDebug("QInfo:%p init runtime completed", GET_QINFO_ADDR(pRuntimeEnv)); return TSDB_CODE_SUCCESS; @@ -1736,13 +1770,20 @@ static bool needReverseScan(SQuery *pQuery) { return false; } +/** + * The following 4 kinds of query are treated as the tags query + * tagprj, tid_tag query, count(tbname), 'abc' (user defined constant value column) query + */ static bool onlyQueryTags(SQuery* pQuery) { for(int32_t i = 0; i < pQuery->numOfOutput; ++i) { SExprInfo* pExprInfo = &pQuery->pSelectExpr[i]; int32_t functionId = pExprInfo->base.functionId; - if (functionId != TSDB_FUNC_TAGPRJ && functionId != TSDB_FUNC_TID_TAG && - (!(functionId == TSDB_FUNC_COUNT && pExprInfo->base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX))) { + + if (functionId != TSDB_FUNC_TAGPRJ && + functionId != TSDB_FUNC_TID_TAG && + (!(functionId == TSDB_FUNC_COUNT && pExprInfo->base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX)) && + (!(functionId == TSDB_FUNC_PRJ && pExprInfo->base.colInfo.flag == TSDB_COL_UDC))) { return false; } } @@ -1803,7 +1844,7 @@ bool colIdCheck(SQuery *pQuery) { return false; } } - + return true; } @@ -1859,16 +1900,12 @@ static void changeExecuteScanOrder(SQInfo *pQInfo, bool stableQuery) { // descending order query for last_row query if (isFirstLastRowQuery(pQuery)) { qDebug("QInfo:%p scan order changed for last_row query, old:%d, new:%d", GET_QINFO_ADDR(pQuery), - pQuery->order.order, TSDB_ORDER_DESC); - - pQuery->order.order = TSDB_ORDER_DESC; - - int64_t skey = MIN(pQuery->window.skey, pQuery->window.ekey); - int64_t ekey = MAX(pQuery->window.skey, pQuery->window.ekey); - - pQuery->window.skey = ekey; - pQuery->window.ekey = skey; + pQuery->order.order, TSDB_ORDER_ASC); + pQuery->order.order = TSDB_ORDER_ASC; + if (pQuery->window.skey > pQuery->window.ekey) { + SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); + } return; } @@ -1963,7 +2000,7 @@ static void getIntermediateBufInfo(SQueryRuntimeEnv* pRuntimeEnv, int32_t* ps, i } pRuntimeEnv->numOfRowsPerPage = ((*ps) - sizeof(tFilePage)) / (*rowsize); - + assert(pRuntimeEnv->numOfRowsPerPage <= MAX_ROWS_PER_RESBUF_PAGE); } #define IS_PREFILTER_TYPE(_t) ((_t) != TSDB_DATA_TYPE_BINARY && (_t) != TSDB_DATA_TYPE_NCHAR) @@ -2117,21 +2154,21 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, void* pQueryHandle, if (tsdbRetrieveDataBlockStatisInfo(pQueryHandle, pStatis) != TSDB_CODE_SUCCESS) { // return DISK_DATA_LOAD_FAILED; } - + pRuntimeEnv->summary.loadBlockStatis += 1; - + if (*pStatis == NULL) { // data block statistics does not exist, load data block *pDataBlock = tsdbRetrieveDataBlock(pQueryHandle, NULL); pRuntimeEnv->summary.totalCheckedRows += pBlockInfo->rows; } } else { assert(status == BLK_DATA_ALL_NEEDED); - + // load the data block statistics to perform further filter pRuntimeEnv->summary.loadBlockStatis += 1; if (tsdbRetrieveDataBlockStatisInfo(pQueryHandle, pStatis) != TSDB_CODE_SUCCESS) { } - + if (!needToLoadDataBlock(pRuntimeEnv, *pStatis, pRuntimeEnv->pCtx, pBlockInfo->rows)) { // current block has been discard due to filter applied pRuntimeEnv->summary.discardBlocks += 1; @@ -2139,7 +2176,7 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, void* pQueryHandle, pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); return BLK_DATA_DISCARD; } - + pRuntimeEnv->summary.totalCheckedRows += pBlockInfo->rows; pRuntimeEnv->summary.loadBlocks += 1; *pDataBlock = tsdbRetrieveDataBlock(pQueryHandle, NULL); @@ -2224,7 +2261,7 @@ static void ensureOutputBufferSimple(SQueryRuntimeEnv* pRuntimeEnv, int32_t capa char *tmp = realloc(pQuery->sdata[i], bytes * capacity + sizeof(tFilePage)); if (tmp == NULL) { // todo handle the oom - assert(0); + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } else { pQuery->sdata[i] = (tFilePage *)tmp; } @@ -2244,18 +2281,18 @@ static void ensureOutputBuffer(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pB SQuery* pQuery = pRuntimeEnv->pQuery; if (!QUERY_IS_INTERVAL_QUERY(pQuery) && !pRuntimeEnv->groupbyNormalCol && !isFixedOutputQuery(pRuntimeEnv) && !isTSCompQuery(pQuery)) { SResultRec *pRec = &pQuery->rec; - + if (pQuery->rec.capacity - pQuery->rec.rows < pBlockInfo->rows) { int32_t remain = (int32_t)(pRec->capacity - pRec->rows); int32_t newSize = (int32_t)(pRec->capacity + (pBlockInfo->rows - remain)); - + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t bytes = pQuery->pSelectExpr[i].bytes; assert(bytes > 0 && newSize > 0); char *tmp = realloc(pQuery->sdata[i], bytes * newSize + sizeof(tFilePage)); if (tmp == NULL) { // todo handle the oom - assert(0); + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } else { memset(tmp + sizeof(tFilePage) + bytes * pRec->rows, 0, (size_t)((newSize - pRec->rows) * bytes)); pQuery->sdata[i] = (tFilePage *)tmp; @@ -2272,7 +2309,7 @@ static void ensureOutputBuffer(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pB qDebug("QInfo:%p realloc output buffer, new size: %d rows, old:%" PRId64 ", remain:%" PRId64, GET_QINFO_ADDR(pRuntimeEnv), newSize, pRec->capacity, newSize - pRec->rows); - + pRec->capacity = newSize; } } @@ -2382,7 +2419,7 @@ static void doSetTagValueInParam(void *tsdb, void* pTable, int32_t tagColId, tVa if (tagColId == TSDB_TBNAME_COLUMN_INDEX) { char* val = tsdbGetTableName(pTable); assert(val != NULL); - + tVariantCreateFromBinary(tag, varDataVal(val), varDataLen(val), TSDB_DATA_TYPE_BINARY); } else { char* val = tsdbGetTableTagVal(pTable, tagColId, type, bytes); @@ -2390,7 +2427,7 @@ static void doSetTagValueInParam(void *tsdb, void* pTable, int32_t tagColId, tVa tag->nType = TSDB_DATA_TYPE_NULL; return; } - + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { if (isNull(val, type)) { tag->nType = TSDB_DATA_TYPE_NULL; @@ -2437,7 +2474,7 @@ void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, void *pTable, void *tsdb) { // set tag value, by which the results are aggregated. for (int32_t idx = 0; idx < pQuery->numOfOutput; ++idx) { SExprInfo* pLocalExprInfo = &pQuery->pSelectExpr[idx]; - + // ts_comp column required the tag value for join filter if (!TSDB_COL_IS_TAG(pLocalExprInfo->base.colInfo.flag)) { continue; @@ -2487,14 +2524,14 @@ static void doMerge(SQueryRuntimeEnv *pRuntimeEnv, int64_t timestamp, SWindowRes // in case of tag column, the tag information should be extracted from input buffer if (functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TAG) { tVariantDestroy(&pCtx[i].tag); - + int32_t type = pCtx[i].outputType; if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { tVariantCreateFromBinary(&pCtx[i].tag, varDataVal(pCtx[i].aInputElemBuf), varDataLen(pCtx[i].aInputElemBuf), type); } else { tVariantCreateFromBinary(&pCtx[i].tag, pCtx[i].aInputElemBuf, pCtx[i].inputBytes, pCtx[i].inputType); } - + } } @@ -2584,7 +2621,7 @@ void UNUSED_FUNC displayInterResult(tFilePage **pdata, SQueryRuntimeEnv* pRuntim for (int32_t j = 0; j < numOfRows; ++j) { for (int32_t i = 0; i < numOfCols; ++i) { - + switch (pQuery->pSelectExpr[i].type) { case TSDB_DATA_TYPE_BINARY: { int32_t type = pQuery->pSelectExpr[i].type; @@ -2659,7 +2696,7 @@ int32_t tableResultComparFn(const void *pLeft, const void *pRight, void *param) } int32_t mergeIntoGroupResult(SQInfo *pQInfo) { - int64_t st = taosGetTimestampMs(); + int64_t st = taosGetTimestampUs(); int32_t ret = TSDB_CODE_SUCCESS; int32_t numOfGroups = (int32_t)(GET_NUM_OF_TABLEGROUP(pQInfo)); @@ -2678,23 +2715,30 @@ int32_t mergeIntoGroupResult(SQInfo *pQInfo) { break; } - assert(pQInfo->numOfGroupResultPages == 0); + assert(pQInfo->groupResInfo.numOfDataPages == 0); qDebug("QInfo:%p no result in group %d, continue", pQInfo, pQInfo->groupIndex - 1); } - if (pQInfo->groupIndex == numOfGroups && pQInfo->offset == pQInfo->numOfGroupResultPages) { + SGroupResInfo* info = &pQInfo->groupResInfo; + if (pQInfo->groupIndex == numOfGroups && info->pos.pageId == info->numOfDataPages) { SET_STABLE_QUERY_OVER(pQInfo); } - qDebug("QInfo:%p merge res data into group, index:%d, total group:%d, elapsed time:%" PRId64 "ms", pQInfo, - pQInfo->groupIndex - 1, numOfGroups, taosGetTimestampMs() - st); + int64_t elapsedTime = taosGetTimestampUs() - st; + qDebug("QInfo:%p merge res data into group, index:%d, total group:%d, elapsed time:%" PRId64 "us", pQInfo, + pQInfo->groupIndex - 1, numOfGroups, elapsedTime); + pQInfo->runtimeEnv.summary.firstStageMergeTime += elapsedTime; return TSDB_CODE_SUCCESS; } void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { - if (pQInfo->offset == pQInfo->numOfGroupResultPages) { - pQInfo->numOfGroupResultPages = 0; + SGroupResInfo* pGroupResInfo = &pQInfo->groupResInfo; + + // all results have been return to client, try next group + if (pGroupResInfo->pos.pageId == pGroupResInfo->numOfDataPages) { + pGroupResInfo->numOfDataPages = 0; + pGroupResInfo->pos.rowId = 0; // current results of group has been sent to client, try next group if (mergeIntoGroupResult(pQInfo) != TSDB_CODE_SUCCESS) { @@ -2703,7 +2747,7 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { // check if all results has been sent to client int32_t numOfGroup = (int32_t)(GET_NUM_OF_TABLEGROUP(pQInfo)); - if (pQInfo->numOfGroupResultPages == 0 && pQInfo->groupIndex == numOfGroup) { + if (pGroupResInfo->numOfDataPages == 0 && pQInfo->groupIndex == numOfGroup) { SET_STABLE_QUERY_OVER(pQInfo); return; } @@ -2712,29 +2756,50 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { SQueryRuntimeEnv * pRuntimeEnv = &pQInfo->runtimeEnv; SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; - int32_t id = getGroupResultId(pQInfo->groupIndex - 1); - SIDList list = getDataBufPagesIdList(pResultBuf, pQInfo->offset + id); - - int32_t size = (int32_t)(taosArrayGetSize(list)); + int32_t id = pQInfo->groupResInfo.groupId; + SIDList list = getDataBufPagesIdList(pResultBuf, id); int32_t offset = 0; - for (int32_t j = 0; j < size; ++j) { + int32_t numOfCopiedRows = 0; + + size_t size = taosArrayGetSize(list); + assert(size == pGroupResInfo->numOfDataPages); + + bool done = false; + for (int32_t j = pGroupResInfo->pos.pageId; j < size; ++j) { SPageInfo* pi = *(SPageInfo**) taosArrayGet(list, j); - tFilePage *pData = getResBufPage(pResultBuf, pi->pageId); + tFilePage* pData = getResBufPage(pResultBuf, pi->pageId); + + assert(pData->num > 0 && pData->num <= pRuntimeEnv->numOfRowsPerPage && pGroupResInfo->pos.rowId < pData->num); + int32_t numOfRes = (int32_t)(pData->num - pGroupResInfo->pos.rowId); + + if (numOfRes > pQuery->rec.capacity - offset) { + numOfCopiedRows = (int32_t)(pQuery->rec.capacity - offset); + pGroupResInfo->pos.rowId += numOfCopiedRows; + done = true; + } else { + numOfCopiedRows = (int32_t)pData->num; + + pGroupResInfo->pos.pageId += 1; + pGroupResInfo->pos.rowId = 0; + } for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; char * pDest = pQuery->sdata[i]->data; - memcpy(pDest + offset * bytes, pData->data + pRuntimeEnv->offset[i] * pData->num, (size_t)(bytes * pData->num)); + + memcpy(pDest + offset * bytes, pData->data + pRuntimeEnv->offset[i] * pRuntimeEnv->numOfRowsPerPage, + (size_t)bytes * numOfCopiedRows); } - offset += (int32_t)pData->num; + offset += numOfCopiedRows; + if (done) { + break; + } } assert(pQuery->rec.rows == 0); - pQuery->rec.rows += offset; - pQInfo->offset += 1; } int64_t getNumOfResultWindowRes(SQuery *pQuery, SWindowResult *pWindowRes) { @@ -2767,7 +2832,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { size_t size = taosArrayGetSize(pGroup); tFilePage **buffer = pQuery->sdata; - int32_t* posList = calloc(size, sizeof(int32_t)); + int32_t *posList = calloc(size, sizeof(int32_t)); STableQueryInfo **pTableList = malloc(POINTER_BYTES * size); if (pTableList == NULL || posList == NULL) { @@ -2780,23 +2845,38 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { // todo opt for the case of one table per group int32_t numOfTables = 0; + SIDList pageList = NULL; + int32_t tid = -1; + for (int32_t i = 0; i < size; ++i) { STableQueryInfo *item = taosArrayGetP(pGroup, i); SIDList list = getDataBufPagesIdList(pRuntimeEnv->pResultBuf, TSDB_TABLEID(item->pTable)->tid); + pageList = list; + tid = TSDB_TABLEID(item->pTable)->tid; + if (taosArrayGetSize(list) > 0 && item->windowResInfo.size > 0) { - pTableList[numOfTables] = item; - numOfTables += 1; + pTableList[numOfTables++] = item; } } + // there is no data in current group if (numOfTables == 0) { taosTFree(posList); taosTFree(pTableList); - - assert(pQInfo->numOfGroupResultPages == 0); return 0; } else if (numOfTables == 1) { // no need to merge results since only one table in each group + taosTFree(posList); + taosTFree(pTableList); + + SGroupResInfo* pGroupResInfo = &pQInfo->groupResInfo; + + pGroupResInfo->numOfDataPages = (int32_t)taosArrayGetSize(pageList); + pGroupResInfo->groupId = tid; + pGroupResInfo->pos.pageId = 0; + pGroupResInfo->pos.rowId = 0; + + return pGroupResInfo->numOfDataPages; } SCompSupporter cs = {pTableList, posList, pQInfo}; @@ -2810,9 +2890,15 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { } char* buf = calloc(1, pRuntimeEnv->interBufSize); + if (buf == NULL) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + setWindowResultInfo(pResultInfo, pQuery, pRuntimeEnv->stableQuery, buf); resetMergeResultBuf(pQuery, pRuntimeEnv->pCtx, pResultInfo); + pQInfo->groupResInfo.groupId = getGroupResultId(pQInfo->groupIndex); + // todo add windowRes iterator int64_t lastTimestamp = -1; int64_t startt = taosGetTimestampMs(); @@ -2820,6 +2906,13 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { while (1) { if (IS_QUERY_KILLED(pQInfo)) { qDebug("QInfo:%p it is already killed, abort", pQInfo); + + taosTFree(pTableList); + taosTFree(posList); + taosTFree(pTree); + taosTFree(pResultInfo); + taosTFree(buf); + longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); } @@ -2832,7 +2925,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { char *b = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes, page); TSKEY ts = GET_INT64_VAL(b); - assert(ts == pWindowRes->window.skey); + assert(ts == pWindowRes->skey); int64_t num = getNumOfResultWindowRes(pQuery, pWindowRes); if (num <= 0) { cs.position[pos] += 1; @@ -2850,7 +2943,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { doMerge(pRuntimeEnv, ts, pWindowRes, true); } else { // copy data to disk buffer if (buffer[0]->num == pQuery->rec.capacity) { - if (flushFromResultBuf(pQInfo) != TSDB_CODE_SUCCESS) { + if (flushFromResultBuf(pRuntimeEnv, &pQInfo->groupResInfo) != TSDB_CODE_SUCCESS) { return -1; } @@ -2887,7 +2980,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { } if (buffer[0]->num != 0) { // there are data in buffer - if (flushFromResultBuf(pQInfo) != TSDB_CODE_SUCCESS) { + if (flushFromResultBuf(pRuntimeEnv, &pQInfo->groupResInfo) != TSDB_CODE_SUCCESS) { qError("QInfo:%p failed to flush data into temp file, abort query", pQInfo); taosTFree(pTree); @@ -2911,16 +3004,14 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { taosTFree(posList); taosTFree(pTree); - pQInfo->offset = 0; - taosTFree(pResultInfo); taosTFree(buf); - return pQInfo->numOfGroupResultPages; + + return pQInfo->groupResInfo.numOfDataPages; } -int32_t flushFromResultBuf(SQInfo *pQInfo) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery * pQuery = pRuntimeEnv->pQuery; +int32_t flushFromResultBuf(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo) { + SQuery *pQuery = pRuntimeEnv->pQuery; SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; @@ -2928,32 +3019,32 @@ int32_t flushFromResultBuf(SQInfo *pQInfo) { int32_t pageId = -1; int32_t capacity = pResultBuf->numOfRowsPerPage; - int32_t remain = (int32_t)pQuery->sdata[0]->num; + int32_t remain = (int32_t) pQuery->sdata[0]->num; int32_t offset = 0; while (remain > 0) { - int32_t r = remain; - if (r > capacity) { - r = capacity; - } + int32_t rows = (remain > capacity)? capacity:remain; + assert(rows > 0); - int32_t id = getGroupResultId(pQInfo->groupIndex) + pQInfo->numOfGroupResultPages; - tFilePage *buf = getNewDataBuf(pResultBuf, id, &pageId); + // get the output buffer page + tFilePage *buf = getNewDataBuf(pResultBuf, pGroupResInfo->groupId, &pageId); + buf->num = rows; - // pagewise copy to dest buffer + // pagewisely copy to dest buffer for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; - buf->num = r; - memcpy(buf->data + pRuntimeEnv->offset[i] * buf->num, ((char *)pQuery->sdata[i]->data) + offset * bytes, - (size_t)(buf->num * bytes)); + char* output = buf->data + pRuntimeEnv->offset[i] * pRuntimeEnv->numOfRowsPerPage; + char* src = ((char *) pQuery->sdata[i]->data) + offset * bytes; + memcpy(output, src, (size_t)(buf->num * bytes)); } - offset += r; - remain -= r; + offset += rows; + remain -= rows; + + pGroupResInfo->numOfDataPages += 1; } - pQInfo->numOfGroupResultPages += 1; return TSDB_CODE_SUCCESS; } @@ -2972,10 +3063,10 @@ static void updateTableQueryInfoForReverseScan(SQuery *pQuery, STableQueryInfo * if (pTableQueryInfo == NULL) { return; } - + // order has changed already int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); - + // TODO validate the assertion // if (!QUERY_IS_ASC_QUERY(pQuery)) { // assert(pTableQueryInfo->win.ekey >= pTableQueryInfo->lastKey + step); @@ -2991,7 +3082,7 @@ static void updateTableQueryInfoForReverseScan(SQuery *pQuery, STableQueryInfo * SWAP(pTableQueryInfo->win.skey, pTableQueryInfo->win.ekey, TSKEY); pTableQueryInfo->lastKey = pTableQueryInfo->win.skey; - + SWITCH_ORDER(pTableQueryInfo->cur.order); pTableQueryInfo->cur.vgroupIndex = -1; @@ -3001,10 +3092,10 @@ static void updateTableQueryInfoForReverseScan(SQuery *pQuery, STableQueryInfo * static void disableFuncInReverseScanImpl(SQInfo* pQInfo, SWindowResInfo *pWindowResInfo, int32_t order) { SQuery* pQuery = pQInfo->runtimeEnv.pQuery; - + for (int32_t i = 0; i < pWindowResInfo->size; ++i) { - SWindowStatus *pStatus = getTimeWindowResStatus(pWindowResInfo, i); - if (!pStatus->closed) { + bool closed = getTimeWindowResStatus(pWindowResInfo, i); + if (!closed) { continue; } @@ -3065,7 +3156,8 @@ static void setupQueryRangeForReverseScan(SQInfo* pQInfo) { STableQueryInfo *pCheckInfo = taosArrayGetP(group, j); updateTableQueryInfoForReverseScan(pQuery, pCheckInfo); - // update the last key in tableKeyInfo list + // update the last key in tableKeyInfo list, the tableKeyInfo is used to build the tsdbQueryHandle and decide + // the start check timestamp of tsdbQueryHandle STableKeyInfo *pTableKeyInfo = taosArrayGet(tableKeyGroup, j); pTableKeyInfo->lastKey = pCheckInfo->lastKey; @@ -3178,7 +3270,7 @@ void skipResults(SQueryRuntimeEnv *pRuntimeEnv) { if (pQuery->rec.rows <= pQuery->limit.offset) { qDebug("QInfo:%p skip rows:%" PRId64 ", new offset:%" PRIu64, GET_QINFO_ADDR(pRuntimeEnv), pQuery->rec.rows, pQuery->limit.offset - pQuery->rec.rows); - + pQuery->limit.offset -= pQuery->rec.rows; pQuery->rec.rows = 0; @@ -3190,10 +3282,10 @@ void skipResults(SQueryRuntimeEnv *pRuntimeEnv) { int64_t numOfSkip = pQuery->limit.offset; pQuery->rec.rows -= numOfSkip; pQuery->limit.offset = 0; - + qDebug("QInfo:%p skip row:%"PRId64", new offset:%d, numOfRows remain:%" PRIu64, GET_QINFO_ADDR(pRuntimeEnv), numOfSkip, 0, pQuery->rec.rows); - + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t functionId = pQuery->pSelectExpr[i].base.functionId; int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; @@ -3230,7 +3322,7 @@ bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) { for (int32_t i = 0; i < pWindowResInfo->size; ++i) { SWindowResult *pResult = getWindowResult(pWindowResInfo, i); - if (!pResult->status.closed) { + if (!pResult->closed) { continue; } @@ -3268,10 +3360,10 @@ bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) { static SQueryStatusInfo getQueryStatusInfo(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { SQuery *pQuery = pRuntimeEnv->pQuery; STableQueryInfo* pTableQueryInfo = pQuery->current; - + assert((start <= pTableQueryInfo->lastKey && QUERY_IS_ASC_QUERY(pQuery)) || (start >= pTableQueryInfo->lastKey && !QUERY_IS_ASC_QUERY(pQuery))); - + SQueryStatusInfo info = { .status = pQuery->status, .windowIndex = pRuntimeEnv->windowResInfo.curIndex, @@ -3293,7 +3385,8 @@ static void setEnvBeforeReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatusI pStatus->cur = tsBufGetCursor(pRuntimeEnv->pTSBuf); // save the cursor if (pRuntimeEnv->pTSBuf) { SWITCH_ORDER(pRuntimeEnv->pTSBuf->cur.order); - tsBufNextPos(pRuntimeEnv->pTSBuf); + bool ret = tsBufNextPos(pRuntimeEnv->pTSBuf); + assert(ret); } // reverse order time range @@ -3351,7 +3444,7 @@ static void clearEnvAfterReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatus // update the pQuery->window.skey and pQuery->window.ekey to limit the scan scope of sliding query during reverse scan pTableQueryInfo->lastKey = pStatus->lastKey; pQuery->status = pStatus->status; - + pTableQueryInfo->win = pStatus->w; pQuery->window = pTableQueryInfo->win; } @@ -3367,7 +3460,7 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { SQInfo *pQInfo = (SQInfo *) GET_QINFO_ADDR(pRuntimeEnv); SQuery *pQuery = pRuntimeEnv->pQuery; STableQueryInfo *pTableQueryInfo = pQuery->current; - + setQueryStatus(pQuery, QUERY_NOT_COMPLETED); // store the start query position @@ -3420,7 +3513,7 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { pRuntimeEnv->windowResInfo.curIndex = qstatus.windowIndex; setQueryStatus(pQuery, QUERY_NOT_COMPLETED); pRuntimeEnv->scanFlag = REPEAT_SCAN; - + qDebug("QInfo:%p start to repeat scan data blocks due to query func required, qrange:%"PRId64"-%"PRId64, pQInfo, cond.twindow.skey, cond.twindow.ekey); @@ -3525,12 +3618,6 @@ void destroyTableQueryInfo(STableQueryInfo *pTableQueryInfo) { cleanupTimeWindowInfo(&pTableQueryInfo->windowResInfo); } -#define CHECK_QUERY_TIME_RANGE(_q, _tableInfo) \ - do { \ - assert((((_tableInfo)->lastKey >= (_tableInfo)->win.skey) && QUERY_IS_ASC_QUERY(_q)) || \ - (((_tableInfo)->lastKey <= (_tableInfo)->win.skey) && !QUERY_IS_ASC_QUERY(_q))); \ - } while (0) - /** * set output buffer for different group * @param pRuntimeEnv @@ -3644,9 +3731,8 @@ int32_t setAdditionalInfo(SQInfo *pQInfo, void* pTable, STableQueryInfo *pTableQ // both the master and supplement scan needs to set the correct ts comp start position if (pRuntimeEnv->pTSBuf != NULL) { if (pTableQueryInfo->cur.vgroupIndex == -1) { - pTableQueryInfo->tag = pRuntimeEnv->pCtx[0].tag.i64Key; - - tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, 0, pTableQueryInfo->tag); + tVariantAssign(&pTableQueryInfo->tag, &pRuntimeEnv->pCtx[0].tag); + tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, 0, &pTableQueryInfo->tag); // keep the cursor info of current meter pTableQueryInfo->cur = pRuntimeEnv->pTSBuf->cur; @@ -3671,7 +3757,7 @@ void setIntervalQueryRange(SQInfo *pQInfo, TSKEY key) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; STableQueryInfo *pTableQueryInfo = pQuery->current; - + if (pTableQueryInfo->queryRangeSet) { pTableQueryInfo->lastKey = key; } else { @@ -3702,7 +3788,7 @@ void setIntervalQueryRange(SQInfo *pQInfo, TSKEY key) { if (!QUERY_IS_ASC_QUERY(pQuery)) { assert(win.ekey == pQuery->window.ekey); } - + pWindowResInfo->prevSKey = w.skey; } @@ -3728,7 +3814,7 @@ bool needPrimaryTimestampCol(SQuery *pQuery, SDataBlockInfo *pDataBlockInfo) { */ STimeWindow *w = &pDataBlockInfo->window; STableQueryInfo* pTableQueryInfo = pQuery->current; - + bool loadPrimaryTS = (pTableQueryInfo->lastKey >= w->skey && pTableQueryInfo->lastKey <= w->ekey) || (pQuery->window.ekey >= w->skey && pQuery->window.ekey <= w->ekey) || requireTimestamp(pQuery); @@ -3755,27 +3841,27 @@ static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResInfo *pResultInfo, int32_ step = -1; } + SGroupResInfo* pGroupResInfo = &pQInfo->groupResInfo; + for (int32_t i = startIdx; (i < totalSet) && (i >= 0); i += step) { if (result[i].numOfRows == 0) { - pQInfo->offset = 0; pQInfo->groupIndex += 1; + pGroupResInfo->pos.rowId = 0; continue; } - assert(pQInfo->offset <= 1); - - int32_t numOfRowsToCopy = result[i].numOfRows - pQInfo->offset; - int32_t oldOffset = pQInfo->offset; + int32_t numOfRowsToCopy = result[i].numOfRows - pGroupResInfo->pos.rowId; + int32_t oldOffset = pGroupResInfo->pos.rowId; /* - * current output space is not enough to keep all the result data of this group, only copy partial results - * to SQuery object's result buffer + * current output space is not enough to accommodate all data of this page, only partial results + * will be copied to SQuery object's result buffer */ if (numOfRowsToCopy > pQuery->rec.capacity - numOfResult) { - numOfRowsToCopy = (int32_t)pQuery->rec.capacity - numOfResult; - pQInfo->offset += numOfRowsToCopy; + numOfRowsToCopy = (int32_t) pQuery->rec.capacity - numOfResult; + pGroupResInfo->pos.rowId += numOfRowsToCopy; } else { - pQInfo->offset = 0; + pGroupResInfo->pos.rowId = 0; pQInfo->groupIndex += 1; } @@ -3840,7 +3926,7 @@ static void updateWindowResNumOfRes(SQueryRuntimeEnv *pRuntimeEnv) { continue; } - pResult->numOfRows = MAX(pResult->numOfRows, pResult->resultInfo[j].numOfRes); + pResult->numOfRows = (uint16_t)(MAX(pResult->numOfRows, pResult->resultInfo[j].numOfRes)); } } } @@ -3849,7 +3935,7 @@ static void stableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBloc SArray *pDataBlock, __block_search_fn_t searchFn) { SQuery * pQuery = pRuntimeEnv->pQuery; STableQueryInfo* pTableQueryInfo = pQuery->current; - + SWindowResInfo * pWindowResInfo = &pTableQueryInfo->windowResInfo; pQuery->pos = QUERY_IS_ASC_QUERY(pQuery)? 0 : pDataBlockInfo->rows - 1; @@ -3941,10 +4027,10 @@ int32_t doFillGapsInResults(SQueryRuntimeEnv* pRuntimeEnv, tFilePage **pDst, int SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv); SQuery *pQuery = pRuntimeEnv->pQuery; SFillInfo* pFillInfo = pRuntimeEnv->pFillInfo; - + while (1) { int32_t ret = (int32_t)taosGenerateDataBlock(pFillInfo, (tFilePage**)pQuery->sdata, (int32_t)pQuery->rec.capacity); - + // todo apply limit output function /* reached the start position of according to offset value, return immediately */ if (pQuery->limit.offset == 0) { @@ -3955,7 +4041,7 @@ int32_t doFillGapsInResults(SQueryRuntimeEnv* pRuntimeEnv, tFilePage **pDst, int if (pQuery->limit.offset < ret) { qDebug("QInfo:%p initial numOfRows:%d, generate filled result:%d rows, offset:%" PRId64 ". Discard due to offset, remain:%" PRId64 ", new offset:%d", pQInfo, pFillInfo->numOfRows, ret, pQuery->limit.offset, ret - pQuery->limit.offset, 0); - + ret -= (int32_t)pQuery->limit.offset; // todo !!!!there exactly number of interpo is not valid. // todo refactor move to the beginning of buffer @@ -3963,14 +4049,14 @@ int32_t doFillGapsInResults(SQueryRuntimeEnv* pRuntimeEnv, tFilePage **pDst, int memmove(pDst[i]->data, pDst[i]->data + pQuery->pSelectExpr[i].bytes * pQuery->limit.offset, ret * pQuery->pSelectExpr[i].bytes); } - + pQuery->limit.offset = 0; return ret; } else { qDebug("QInfo:%p initial numOfRows:%d, generate filled result:%d rows, offset:%" PRId64 ". Discard due to offset, " "remain:%d, new offset:%" PRId64, pQInfo, pFillInfo->numOfRows, ret, pQuery->limit.offset, 0, pQuery->limit.offset - ret); - + pQuery->limit.offset -= ret; pQuery->rec.rows = 0; ret = 0; @@ -3986,18 +4072,22 @@ static void queryCostStatis(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQueryCostInfo *pSummary = &pRuntimeEnv->summary; - qDebug("QInfo:%p :cost summary: elapsed time:%"PRId64" us, total blocks:%d, load block statis:%d," - " load data block:%d, total rows:%"PRId64 ", check rows:%"PRId64, - pQInfo, pSummary->elapsedTime, pSummary->totalBlocks, pSummary->loadBlockStatis, + // add the merge time + pSummary->elapsedTime += pSummary->firstStageMergeTime; + + qDebug("QInfo:%p :cost summary: elapsed time:%"PRId64" us, first merge:%"PRId64" us, total blocks:%d, " + "load block statis:%d, load data block:%d, total rows:%"PRId64 ", check rows:%"PRId64, + pQInfo, pSummary->elapsedTime, pSummary->firstStageMergeTime, pSummary->totalBlocks, pSummary->loadBlockStatis, pSummary->loadBlocks, pSummary->totalRows, pSummary->totalCheckedRows); - qDebug("QInfo:%p :cost summary: internal size:%"PRId64, pQInfo, pSummary->internalSupSize); + qDebug("QInfo:%p :cost summary: internal size:%"PRId64"B, numOfWin:%"PRId64, pQInfo, pSummary->internalSupSize, + pSummary->numOfTimeWindows); } static void updateOffsetVal(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBlockInfo) { SQuery *pQuery = pRuntimeEnv->pQuery; STableQueryInfo* pTableQueryInfo = pQuery->current; - + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); if (pQuery->limit.offset == pBlockInfo->rows) { // current block will ignore completed @@ -4087,7 +4177,7 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) { assert(pRuntimeEnv->windowResInfo.prevSKey == TSKEY_INITIAL_VAL); STimeWindow w = TSWINDOW_INITIALIZER; - + SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; STableQueryInfo *pTableQueryInfo = pQuery->current; @@ -4136,21 +4226,21 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) { // set the abort info pQuery->pos = startPos; - + // reset the query start timestamp pTableQueryInfo->win.skey = ((TSKEY *)pColInfoData->pData)[startPos]; pQuery->window.skey = pTableQueryInfo->win.skey; *start = pTableQueryInfo->win.skey; - + pWindowResInfo->prevSKey = tw.skey; int32_t index = pRuntimeEnv->windowResInfo.curIndex; - + int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, NULL, binarySearchForKey, pDataBlock); pRuntimeEnv->windowResInfo.curIndex = index; // restore the window index - + qDebug("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, numOfRes:%d, lastKey:%"PRId64, GET_QINFO_ADDR(pRuntimeEnv), blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, numOfRes, pQuery->current->lastKey); - + return true; } else { // do nothing *start = tw.skey; @@ -4218,7 +4308,7 @@ static int32_t setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery) if (!isSTableQuery && (pQInfo->tableqinfoGroupInfo.numOfTables == 1) - && (cond.order == TSDB_ORDER_ASC) + && (cond.order == TSDB_ORDER_ASC) && (!QUERY_IS_INTERVAL_QUERY(pQuery)) && (!isGroupbyNormalCol(pQuery->pGroupbyExpr)) && (!isFixedOutputQuery(pRuntimeEnv)) @@ -4231,6 +4321,22 @@ static int32_t setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery) terrno = TSDB_CODE_SUCCESS; if (isFirstLastRowQuery(pQuery)) { pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo); + + // update the query time window + pQuery->window = cond.twindow; + + size_t numOfGroups = GET_NUM_OF_TABLEGROUP(pQInfo); + for(int32_t i = 0; i < numOfGroups; ++i) { + SArray *group = GET_TABLEGROUP(pQInfo, i); + + size_t t = taosArrayGetSize(group); + for (int32_t j = 0; j < t; ++j) { + STableQueryInfo *pCheckInfo = taosArrayGetP(group, j); + + pCheckInfo->win = pQuery->window; + pCheckInfo->lastKey = pCheckInfo->win.skey; + } + } } else if (isPointInterpoQuery(pQuery)) { pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo); } else { @@ -4243,21 +4349,25 @@ static int32_t setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery) static SFillColInfo* taosCreateFillColInfo(SQuery* pQuery) { int32_t numOfCols = pQuery->numOfOutput; int32_t offset = 0; - + SFillColInfo* pFillCol = calloc(numOfCols, sizeof(SFillColInfo)); + if (pFillCol == NULL) { + return NULL; + } + for(int32_t i = 0; i < numOfCols; ++i) { SExprInfo* pExprInfo = &pQuery->pSelectExpr[i]; - + pFillCol[i].col.bytes = pExprInfo->bytes; pFillCol[i].col.type = (int8_t)pExprInfo->type; pFillCol[i].col.offset = offset; pFillCol[i].flag = TSDB_COL_NORMAL; // always be ta normal column for table query pFillCol[i].functionId = pExprInfo->base.functionId; pFillCol[i].fillVal.i = pQuery->fillVal[i]; - + offset += pExprInfo->bytes; } - + return pFillCol; } @@ -4272,13 +4382,16 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo pRuntimeEnv->hasTagResults = hasTagValOutput(pQuery); setScanLimitationByResultBuffer(pQuery); + + // NOTE: pTableCheckInfo need to update the query time range and the lastKey info + // TODO fixme changeExecuteScanOrder(pQInfo, false); code = setupQueryHandle(tsdb, pQInfo, isSTableQuery); if (code != TSDB_CODE_SUCCESS) { return code; } - + pQInfo->tsdb = tsdb; pQInfo->vgId = vgId; @@ -4423,7 +4536,17 @@ static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) { } pQuery->current = *pTableQueryInfo; - CHECK_QUERY_TIME_RANGE(pQuery, *pTableQueryInfo); + if (QUERY_IS_ASC_QUERY(pQuery)) { + assert( + ((*pTableQueryInfo)->win.skey <= (*pTableQueryInfo)->win.ekey) && + ((*pTableQueryInfo)->lastKey >= (*pTableQueryInfo)->win.skey) && + ((*pTableQueryInfo)->win.skey >= pQuery->window.skey && (*pTableQueryInfo)->win.ekey <= pQuery->window.ekey)); + } else { + assert( + ((*pTableQueryInfo)->win.skey >= (*pTableQueryInfo)->win.ekey) && + ((*pTableQueryInfo)->lastKey <= (*pTableQueryInfo)->win.skey) && + ((*pTableQueryInfo)->win.skey <= pQuery->window.skey && (*pTableQueryInfo)->win.ekey >= pQuery->window.ekey)); + } if (!pRuntimeEnv->groupbyNormalCol) { setEnvForEachBlock(pQInfo, *pTableQueryInfo, &blockInfo); @@ -4439,8 +4562,10 @@ static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) { summary->totalRows += blockInfo.rows; stableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, pStatis, pDataBlock, binarySearchForKey); - qDebug("QInfo:%p check data block, uid:%"PRId64", tid:%d, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, lastKey:%" PRId64, - pQInfo, blockInfo.uid, blockInfo.tid, blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, pQuery->current->lastKey); + qDebug("QInfo:%p check data block completed, uid:%"PRId64", tid:%d, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, " + "lastKey:%" PRId64, + pQInfo, blockInfo.uid, blockInfo.tid, blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, + pQuery->current->lastKey); } if (terrno != TSDB_CODE_SUCCESS) { @@ -4501,8 +4626,7 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { if (pRuntimeEnv->pTSBuf != NULL) { if (pRuntimeEnv->cur.vgroupIndex == -1) { - int64_t tag = pRuntimeEnv->pCtx[0].tag.i64Key; - STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, 0, tag); + STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, 0, &pRuntimeEnv->pCtx[0].tag); // failed to find data with the specified tag value if (elem.vnode < 0) { @@ -4552,7 +4676,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { SArray *g1 = taosArrayInit(1, POINTER_BYTES); SArray *tx = taosArrayClone(group); taosArrayPush(g1, &tx); - + STableGroupInfo gp = {.numOfTables = taosArrayGetSize(tx), .pGroupList = g1}; // include only current table @@ -4560,9 +4684,9 @@ static void sequentialTableProcess(SQInfo *pQInfo) { tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle); pRuntimeEnv->pQueryHandle = NULL; } - + if (isFirstLastRowQuery(pQuery)) { - pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(pQInfo->tsdb, &cond, &gp, pQInfo); + assert(0); // last_row query switch to other routine to handle } else { pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(pQInfo->tsdb, &cond, &gp, pQInfo); } @@ -4574,10 +4698,10 @@ static void sequentialTableProcess(SQInfo *pQInfo) { } initCtxOutputBuf(pRuntimeEnv); - + SArray* s = tsdbGetQueriedTableList(pRuntimeEnv->pQueryHandle); assert(taosArrayGetSize(s) >= 1); - + setTagVal(pRuntimeEnv, taosArrayGetP(s, 0), pQInfo->tsdb); if (isFirstLastRowQuery(pQuery)) { assert(taosArrayGetSize(s) == 1); @@ -4590,13 +4714,13 @@ static void sequentialTableProcess(SQInfo *pQInfo) { pQuery->current = taosArrayGetP(first, 0); scanOneTableDataBlocks(pRuntimeEnv, pQuery->current->lastKey); - + int64_t numOfRes = getNumOfResult(pRuntimeEnv); if (numOfRes > 0) { pQuery->rec.rows += numOfRes; forwardCtxOutputBuf(pRuntimeEnv, numOfRes); } - + skipResults(pRuntimeEnv); pQInfo->groupIndex += 1; @@ -4655,18 +4779,17 @@ static void sequentialTableProcess(SQInfo *pQInfo) { SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; // no results generated for current group, continue to try the next group - taosArrayDestroy(s); + taosArrayDestroy(s); if (pWindowResInfo->size <= 0) { continue; } for (int32_t i = 0; i < pWindowResInfo->size; ++i) { - SWindowStatus *pStatus = &pWindowResInfo->pResult[i].status; - pStatus->closed = true; // enable return all results for group by normal columns + pWindowResInfo->pResult[i].closed = true; // enable return all results for group by normal columns SWindowResult *pResult = &pWindowResInfo->pResult[i]; for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { - pResult->numOfRows = MAX(pResult->numOfRows, pResult->resultInfo[j].numOfRes); + pResult->numOfRows = (uint16_t)(MAX(pResult->numOfRows, pResult->resultInfo[j].numOfRes)); } } @@ -4816,11 +4939,11 @@ static void doSaveContext(SQInfo *pQInfo) { SET_REVERSE_SCAN_FLAG(pRuntimeEnv); SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); SWITCH_ORDER(pQuery->order.order); - + if (pRuntimeEnv->pTSBuf != NULL) { pRuntimeEnv->pTSBuf->cur.order = pQuery->order.order; } - + STsdbQueryCond cond = { .order = pQuery->order.order, .colList = pQuery->colList, @@ -4924,7 +5047,6 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { el = scanMultiTableDataBlocks(pQInfo); qDebug("QInfo:%p reversed scan completed, elapsed time: %" PRId64 "ms", pQInfo, el); -// doCloseAllTimeWindowAfterScan(pQInfo); doRestoreContext(pQInfo); } else { qDebug("QInfo:%p no need to do reversed scan, query completed", pQInfo); @@ -4962,14 +5084,14 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { */ static void tableFixedOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - + SQuery *pQuery = pRuntimeEnv->pQuery; if (!pRuntimeEnv->topBotQuery && pQuery->limit.offset > 0) { // no need to execute, since the output will be ignore. return; } - + pQuery->current = pTableInfo; // set current query table info - + scanOneTableDataBlocks(pRuntimeEnv, pTableInfo->lastKey); finalizeQueryResult(pRuntimeEnv); @@ -4987,10 +5109,10 @@ static void tableFixedOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) static void tableMultiOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - + SQuery *pQuery = pRuntimeEnv->pQuery; pQuery->current = pTableInfo; - + // for ts_comp query, re-initialized is not allowed if (!isTSCompQuery(pQuery)) { resetCtxOutputBuf(pRuntimeEnv); @@ -5081,7 +5203,7 @@ static void tableIntervalProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { int32_t numOfFilled = 0; TSKEY newStartKey = TSKEY_INITIAL_VAL; - + // skip blocks without load the actual data block from file if no filter condition present skipTimeInterval(pRuntimeEnv, &newStartKey); if (pQuery->limit.offset > 0 && pQuery->numOfFilterCols == 0 && pRuntimeEnv->pFillInfo == NULL) { @@ -5108,7 +5230,7 @@ static void tableIntervalProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { taosFillSetStartInfo(pRuntimeEnv->pFillInfo, (int32_t)pQuery->rec.rows, pQuery->window.ekey); taosFillCopyInputDataFromFilePage(pRuntimeEnv->pFillInfo, (tFilePage**) pQuery->sdata); numOfFilled = 0; - + pQuery->rec.rows = doFillGapsInResults(pRuntimeEnv, (tFilePage **)pQuery->sdata, &numOfFilled); if (pQuery->rec.rows > 0 || Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { limitResults(pRuntimeEnv); @@ -5174,11 +5296,11 @@ static void tableQueryImpl(SQInfo *pQInfo) { // number of points returned during this query pQuery->rec.rows = 0; int64_t st = taosGetTimestampUs(); - + assert(pQInfo->tableqinfoGroupInfo.numOfTables == 1); SArray* g = GET_TABLEGROUP(pQInfo, 0); STableQueryInfo* item = taosArrayGetP(g, 0); - + // group by normal column, sliding window query, interval query are handled by interval query processor if (QUERY_IS_INTERVAL_QUERY(pQuery) || pRuntimeEnv->groupbyNormalCol) { // interval (down sampling operation) tableIntervalProcess(pQInfo, item); @@ -5202,15 +5324,13 @@ static void stableQueryImpl(SQInfo *pQInfo) { int64_t st = taosGetTimestampUs(); if (QUERY_IS_INTERVAL_QUERY(pQuery) || - (isFixedOutputQuery(pRuntimeEnv) && (!isPointInterpoQuery(pQuery)) && !pRuntimeEnv->groupbyNormalCol && - !isFirstLastRowQuery(pQuery))) { + (isFixedOutputQuery(pRuntimeEnv) && (!isPointInterpoQuery(pQuery)) && (!pRuntimeEnv->groupbyNormalCol))) { multiTableQueryProcess(pQInfo); } else { assert((pQuery->checkBuffer == 1 && pQuery->intervalTime == 0) || isPointInterpoQuery(pQuery) || isFirstLastRowQuery(pQuery) || pRuntimeEnv->groupbyNormalCol); sequentialTableProcess(pQInfo); - } // record the total elapsed time @@ -5222,7 +5342,7 @@ static int32_t getColumnIndexInSource(SQueryTableMsg *pQueryMsg, SSqlFuncMsg *pE if (TSDB_COL_IS_TAG(pExprMsg->colInfo.flag)) { if (pExprMsg->colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) { - return -1; + return TSDB_TBNAME_COLUMN_INDEX; } while(j < pQueryMsg->numOfTags) { @@ -5233,6 +5353,8 @@ static int32_t getColumnIndexInSource(SQueryTableMsg *pQueryMsg, SSqlFuncMsg *pE j += 1; } + } else if (pExprMsg->colInfo.flag == TSDB_COL_UDC) { // user specified column data + return TSDB_UD_COLUMN_INDEX; } else { while (j < pQueryMsg->numOfCols) { if (pExprMsg->colInfo.colId == pQueryMsg->colList[j].colId) { @@ -5371,11 +5493,15 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, int32_t numOfFilters = pColInfo->numOfFilters; if (numOfFilters > 0) { pColInfo->filters = calloc(numOfFilters, sizeof(SColumnFilterInfo)); + if (pColInfo->filters == NULL) { + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _cleanup; + } } for (int32_t f = 0; f < numOfFilters; ++f) { SColumnFilterInfo *pFilterMsg = (SColumnFilterInfo *)pMsg; - + SColumnFilterInfo *pColFilter = &pColInfo->filters[f]; pColFilter->filterstr = htons(pFilterMsg->filterstr); @@ -5385,6 +5511,11 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, pColFilter->len = htobe64(pFilterMsg->len); pColFilter->pz = (int64_t)calloc(1, (size_t)(pColFilter->len + 1 * TSDB_NCHAR_SIZE)); // note: null-terminator + if (pColFilter->pz == 0) { + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _cleanup; + } + memcpy((void *)pColFilter->pz, pMsg, (size_t)pColFilter->len); pMsg += (pColFilter->len + 1); } else { @@ -5398,6 +5529,11 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, } *pExpr = calloc(pQueryMsg->numOfOutput, POINTER_BYTES); + if (*pExpr == NULL) { + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _cleanup; + } + SSqlFuncMsg *pExprMsg = (SSqlFuncMsg *)pMsg; for (int32_t i = 0; i < pQueryMsg->numOfOutput; ++i) { @@ -5484,6 +5620,11 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, if (pQueryMsg->numOfTags > 0) { (*tagCols) = calloc(1, sizeof(SColumnInfo) * pQueryMsg->numOfTags); + if (*tagCols == NULL) { + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _cleanup; + } + for (int32_t i = 0; i < pQueryMsg->numOfTags; ++i) { SColumnInfo* pTagCol = (SColumnInfo*) pMsg; @@ -5500,6 +5641,12 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, // the tag query condition expression string is located at the end of query msg if (pQueryMsg->tagCondLen > 0) { *tagCond = calloc(1, pQueryMsg->tagCondLen); + + if (*tagCond == NULL) { + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _cleanup; + + } memcpy(*tagCond, pMsg, pQueryMsg->tagCondLen); pMsg += pQueryMsg->tagCondLen; } @@ -5591,9 +5738,19 @@ static int32_t createQFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SExprInfo * bytes = tDataTypeDesc[type].nSize; } else if (pExprs[i].base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX && pExprs[i].base.functionId == TSDB_FUNC_TAGPRJ) { // parse the normal column SSchema s = tGetTableNameColumnSchema(); - type = s.type; + type = s.type; bytes = s.bytes; - } else{ + } else if (pExprs[i].base.colInfo.colId <= TSDB_UD_COLUMN_INDEX) { + // it is a user-defined constant value column + assert(pExprs[i].base.functionId == TSDB_FUNC_PRJ); + + type = pExprs[i].base.arg[1].argType; + bytes = pExprs[i].base.arg[1].argBytes; + + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { + bytes += VARSTR_HEADER_SIZE; + } + } else { int32_t j = getColumnIndexInSource(pQueryMsg, &pExprs[i].base, pTagCols); assert(j < pQueryMsg->numOfCols || j < pQueryMsg->numOfTags); @@ -5629,18 +5786,19 @@ static int32_t createQFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SExprInfo * if (functId == TSDB_FUNC_TOP || functId == TSDB_FUNC_BOTTOM) { int32_t j = getColumnIndexInSource(pQueryMsg, &pExprs[i].base, pTagCols); - assert(j < pQueryMsg->numOfCols); - - SColumnInfo *pCol = &pQueryMsg->colList[j]; - - int32_t ret = - getResultDataInfo(pCol->type, pCol->bytes, functId, (int32_t)pExprs[i].base.arg[0].argValue.i64, - &pExprs[i].type, &pExprs[i].bytes, &pExprs[i].interBytes, tagLen, isSuperTable); - assert(ret == TSDB_CODE_SUCCESS); + if (j < 0 || j >= pQueryMsg->numOfCols) { + assert(0); + } else { + SColumnInfo *pCol = &pQueryMsg->colList[j]; + int32_t ret = + getResultDataInfo(pCol->type, pCol->bytes, functId, (int32_t)pExprs[i].base.arg[0].argValue.i64, + &pExprs[i].type, &pExprs[i].bytes, &pExprs[i].interBytes, tagLen, isSuperTable); + assert(ret == TSDB_CODE_SUCCESS); + } } } - *pExprInfo = pExprs; + *pExprInfo = pExprs; return TSDB_CODE_SUCCESS; } @@ -5680,6 +5838,9 @@ static int32_t createFilterInfo(void *pQInfo, SQuery *pQuery) { } pQuery->pFilterInfo = calloc(1, sizeof(SSingleColumnFilterInfo) * pQuery->numOfFilterCols); + if (pQuery->pFilterInfo == NULL) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } for (int32_t i = 0, j = 0; i < pQuery->numOfCols; ++i) { if (pQuery->colList[i].numOfFilters > 0) { @@ -5690,6 +5851,9 @@ static int32_t createFilterInfo(void *pQInfo, SQuery *pQuery) { pFilterInfo->numOfFilters = pQuery->colList[i].numOfFilters; pFilterInfo->pFilters = calloc(pFilterInfo->numOfFilters, sizeof(SColumnFilterElem)); + if (pFilterInfo->pFilters == NULL) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } for (int32_t f = 0; f < pFilterInfo->numOfFilters; ++f) { SColumnFilterElem *pSingleColFilter = &pFilterInfo->pFilters[f]; @@ -5766,7 +5930,7 @@ static void doUpdateExprColumnIndex(SQuery *pQuery) { // todo opt performance SColIndex *pColIndex = &pSqlExprMsg->colInfo; - if (!TSDB_COL_IS_TAG(pColIndex->flag)) { + if (TSDB_COL_IS_NORMAL_COL(pColIndex->flag)) { int32_t f = 0; for (f = 0; f < pQuery->numOfCols; ++f) { if (pColIndex->colId == pQuery->colList[f].colId) { @@ -5774,8 +5938,10 @@ static void doUpdateExprColumnIndex(SQuery *pQuery) { break; } } - - assert (f < pQuery->numOfCols); + + assert(f < pQuery->numOfCols); + } else if (pColIndex->colId <= TSDB_UD_COLUMN_INDEX) { + // do nothing for user-defined constant value result columns } else { int32_t f = 0; for (f = 0; f < pQuery->numOfTags; ++f) { @@ -5784,7 +5950,7 @@ static void doUpdateExprColumnIndex(SQuery *pQuery) { break; } } - + assert(f < pQuery->numOfTags || pColIndex->colId == TSDB_TBNAME_COLUMN_INDEX); } } @@ -5837,6 +6003,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList, if (pQuery == NULL) { goto _cleanup_query; } + pQInfo->runtimeEnv.pQuery = pQuery; pQuery->numOfCols = numOfCols; @@ -5922,6 +6089,10 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList, pQInfo->runtimeEnv.interBufSize = getOutputInterResultBufSize(pQuery); pQInfo->pBuf = calloc(pTableGroupInfo->numOfTables, sizeof(STableQueryInfo)); + if (pQInfo->pBuf == NULL) { + goto _cleanup; + } + int32_t index = 0; for(int32_t i = 0; i < numOfGroups; ++i) { @@ -5932,20 +6103,18 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList, if (p1 == NULL) { goto _cleanup; } + taosArrayPush(pQInfo->tableqinfoGroupInfo.pGroupList, &p1); for(int32_t j = 0; j < s; ++j) { STableKeyInfo* info = taosArrayGet(pa, j); + STableId* id = TSDB_TABLEID(info->pTable); - STableIdInfo* pTableId = taosArraySearch(pTableIdList, id, compareTableIdInfo); - if (pTableId != NULL ) { - window.skey = pTableId->key; - } else { - window.skey = pQueryMsg->window.skey; - } + window.skey = (pTableId != NULL)? pTableId->key:pQueryMsg->window.skey; void* buf = (char*)pQInfo->pBuf + index * sizeof(STableQueryInfo); + STableQueryInfo* item = createTableQueryInfo(&pQInfo->runtimeEnv, info->pTable, window, buf); if (item == NULL) { goto _cleanup; @@ -6040,7 +6209,7 @@ static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQ if ((code = doInitQInfo(pQInfo, pTSBuf, tsdb, vgId, isSTable)) != TSDB_CODE_SUCCESS) { goto _error; } - + return code; _error: @@ -6066,35 +6235,59 @@ static void freeQInfo(SQInfo *pQInfo) { return; } - SQuery *pQuery = pQInfo->runtimeEnv.pQuery; qDebug("QInfo:%p start to free QInfo", pQInfo); - for (int32_t col = 0; col < pQuery->numOfOutput; ++col) { - taosTFree(pQuery->sdata[col]); - } teardownQueryRuntimeEnv(&pQInfo->runtimeEnv); - for (int32_t i = 0; i < pQuery->numOfFilterCols; ++i) { - SSingleColumnFilterInfo *pColFilter = &pQuery->pFilterInfo[i]; - if (pColFilter->numOfFilters > 0) { - taosTFree(pColFilter->pFilters); + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + if (pQuery != NULL) { + if (pQuery->sdata != NULL) { + for (int32_t col = 0; col < pQuery->numOfOutput; ++col) { + taosTFree(pQuery->sdata[col]); + } + taosTFree(pQuery->sdata); } - } - if (pQuery->pSelectExpr != NULL) { - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - SExprInfo* pExprInfo = &pQuery->pSelectExpr[i]; + if (pQuery->fillVal != NULL) { + taosTFree(pQuery->fillVal); + } - if (pExprInfo->pExpr != NULL) { - tExprTreeDestroy(&pExprInfo->pExpr, NULL); + for (int32_t i = 0; i < pQuery->numOfFilterCols; ++i) { + SSingleColumnFilterInfo *pColFilter = &pQuery->pFilterInfo[i]; + if (pColFilter->numOfFilters > 0) { + taosTFree(pColFilter->pFilters); } } - taosTFree(pQuery->pSelectExpr); - } + if (pQuery->pSelectExpr != NULL) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + SExprInfo *pExprInfo = &pQuery->pSelectExpr[i]; - if (pQuery->fillVal != NULL) { - taosTFree(pQuery->fillVal); + if (pExprInfo->pExpr != NULL) { + tExprTreeDestroy(&pExprInfo->pExpr, NULL); + } + } + + taosTFree(pQuery->pSelectExpr); + } + + if (pQuery->pGroupbyExpr != NULL) { + taosArrayDestroy(pQuery->pGroupbyExpr->columnInfo); + taosTFree(pQuery->pGroupbyExpr); + } + + taosTFree(pQuery->tagColList); + taosTFree(pQuery->pFilterInfo); + + if (pQuery->colList != NULL) { + for (int32_t i = 0; i < pQuery->numOfCols; i++) { + SColumnInfo *column = pQuery->colList + i; + freeColumnFilterInfo(column->filters, column->numOfFilters); + } + taosTFree(pQuery->colList); + } + + taosTFree(pQuery); } // todo refactor, extract method to destroytableDataInfo @@ -6118,25 +6311,8 @@ static void freeQInfo(SQInfo *pQInfo) { taosHashCleanup(pQInfo->tableqinfoGroupInfo.map); tsdbDestroyTableGroup(&pQInfo->tableGroupInfo); taosArrayDestroy(pQInfo->arrTableIdInfo); - - if (pQuery->pGroupbyExpr != NULL) { - taosArrayDestroy(pQuery->pGroupbyExpr->columnInfo); - taosTFree(pQuery->pGroupbyExpr); - } - taosTFree(pQuery->tagColList); - taosTFree(pQuery->pFilterInfo); - if (pQuery->colList != NULL) { - for (int32_t i = 0; i < pQuery->numOfCols; i++) { - SColumnInfo* column = pQuery->colList + i; - freeColumnFilterInfo(column->filters, column->numOfFilters); - } - taosTFree(pQuery->colList); - } - - taosTFree(pQuery->sdata); - taosTFree(pQuery); pQInfo->signature = 0; qDebug("QInfo:%p QInfo is freed", pQInfo); @@ -6176,14 +6352,16 @@ static int32_t doDumpQueryResult(SQInfo *pQInfo, char *data) { // make sure file exist if (FD_VALID(fd)) { - int32_t s = lseek(fd, 0, SEEK_END); - UNUSED(s); - qDebug("QInfo:%p ts comp data return, file:%s, size:%d", pQInfo, pQuery->sdata[0]->data, s); + uint64_t s = lseek(fd, 0, SEEK_END); + + qDebug("QInfo:%p ts comp data return, file:%s, size:%"PRId64, pQInfo, pQuery->sdata[0]->data, s); if (lseek(fd, 0, SEEK_SET) >= 0) { - size_t sz = read(fd, data, s); - UNUSED(sz); + size_t sz = read(fd, data, (uint32_t)s); + if(sz < s) { // todo handle error + assert(0); + } } else { - // todo handle error + UNUSED(s); } close(fd); @@ -6193,7 +6371,7 @@ static int32_t doDumpQueryResult(SQInfo *pQInfo, char *data) { qError("QInfo:%p failed to open tmp file to send ts-comp data to client, path:%s, reason:%s", pQInfo, pQuery->sdata[0]->data, strerror(errno)); if (fd != -1) { - close(fd); + close(fd); } } @@ -6212,7 +6390,7 @@ static int32_t doDumpQueryResult(SQInfo *pQInfo, char *data) { qDebug("QInfo:%p results limitation reached, limitation:%"PRId64, pQInfo, pQuery->limit.limit); setQueryStatus(pQuery, QUERY_OVER); } - + return TSDB_CODE_SUCCESS; } @@ -6314,7 +6492,7 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qi pExprs = NULL; pGroupbyExpr = NULL; pTagColumnInfo = NULL; - + if ((*pQInfo) == NULL) { code = TSDB_CODE_QRY_OUT_OF_MEMORY; goto _over; @@ -6329,7 +6507,7 @@ _over: if (pGroupbyExpr != NULL) { taosArrayDestroy(pGroupbyExpr->columnInfo); free(pGroupbyExpr); - } + } free(pTagColumnInfo); free(pExprs); free(pExprMsg); @@ -6496,7 +6674,7 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co (*pRsp)->offset = 0; (*pRsp)->useconds = htobe64(pRuntimeEnv->summary.elapsedTime); } - + (*pRsp)->precision = htons(pQuery->precision); if (pQuery->rec.rows > 0 && pQInfo->code == TSDB_CODE_SUCCESS) { doDumpQueryResult(pQInfo, (*pRsp)->data); @@ -6573,7 +6751,7 @@ static void buildTagQueryResult(SQInfo* pQInfo) { if (numOfGroup == 0) { return; } - + SArray* pa = GET_TABLEGROUP(pQInfo, 0); size_t num = taosArrayGetSize(pa); @@ -6664,6 +6842,10 @@ static void buildTagQueryResult(SQInfo* pQInfo) { char *data = NULL, *dst = NULL; int16_t type = 0, bytes = 0; for(int32_t j = 0; j < pQuery->numOfOutput; ++j) { + // not assign value in case of user defined constant output column + if (pExprInfo[j].base.colInfo.flag == TSDB_COL_UDC) { + continue; + } if (pExprInfo[j].base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) { bytes = tbnameSchema.bytes; @@ -6674,7 +6856,7 @@ static void buildTagQueryResult(SQInfo* pQInfo) { } else { type = pExprInfo[j].type; bytes = pExprInfo[j].bytes; - + data = tsdbGetTableTagVal(item->pTable, pExprInfo[j].base.colInfo.colId, type, bytes); dst = pQuery->sdata[j]->data + count * pExprInfo[j].bytes; @@ -6710,12 +6892,16 @@ void freeqinfoFn(void *qhandle) { } void* qOpenQueryMgmt(int32_t vgId) { - const int32_t REFRESH_HANDLE_INTERVAL = 60; // every 30 seconds, refresh handle pool + const int32_t REFRESH_HANDLE_INTERVAL = 30; // every 30 seconds, refresh handle pool char cacheName[128] = {0}; sprintf(cacheName, "qhandle_%d", vgId); SQueryMgmt* pQueryMgmt = calloc(1, sizeof(SQueryMgmt)); + if (pQueryMgmt == NULL) { + terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + return NULL; + } pQueryMgmt->qinfoPool = taosCacheInit(TSDB_DATA_TYPE_BIGINT, REFRESH_HANDLE_INTERVAL, true, freeqinfoFn, cacheName); pQueryMgmt->closed = false; diff --git a/src/query/src/qParserImpl.c b/src/query/src/qParserImpl.c index d8f12da642..9629f24cc2 100644 --- a/src/query/src/qParserImpl.c +++ b/src/query/src/qParserImpl.c @@ -33,7 +33,7 @@ SSqlInfo qSQLParse(const char *pStr) { int32_t i = 0; while (1) { - SSQLToken t0 = {0}; + SStrToken t0 = {0}; if (pStr[i] == 0) { Parse(pParser, 0, t0, &sqlInfo); @@ -73,12 +73,12 @@ abort_parse: return sqlInfo; } -tSQLExprList *tSQLExprListAppend(tSQLExprList *pList, tSQLExpr *pNode, SSQLToken *pToken) { +tSQLExprList *tSQLExprListAppend(tSQLExprList *pList, tSQLExpr *pNode, SStrToken *pToken) { if (pList == NULL) { pList = calloc(1, sizeof(tSQLExprList)); } - if (pList->nAlloc <= pList->nExpr) { // + if (pList->nAlloc <= pList->nExpr) { pList->nAlloc = (pList->nAlloc << 1) + 4; pList->a = realloc(pList->a, pList->nAlloc * sizeof(pList->a[0])); if (pList->a == 0) { @@ -117,41 +117,45 @@ void tSQLExprListDestroy(tSQLExprList *pList) { free(pList); } -tSQLExpr *tSQLExprIdValueCreate(SSQLToken *pAliasToken, int32_t optrType) { - tSQLExpr *nodePtr = calloc(1, sizeof(tSQLExpr)); +tSQLExpr *tSQLExprIdValueCreate(SStrToken *pToken, int32_t optrType) { + tSQLExpr *pSQLExpr = calloc(1, sizeof(tSQLExpr)); + + if (pToken != NULL) { + pSQLExpr->token = *pToken; + } if (optrType == TK_INTEGER || optrType == TK_STRING || optrType == TK_FLOAT || optrType == TK_BOOL) { - toTSDBType(pAliasToken->type); + toTSDBType(pToken->type); - tVariantCreate(&nodePtr->val, pAliasToken); - nodePtr->nSQLOptr = optrType; + tVariantCreate(&pSQLExpr->val, pToken); + pSQLExpr->nSQLOptr = optrType; } else if (optrType == TK_NOW) { // default use microsecond - nodePtr->val.i64Key = taosGetTimestamp(TSDB_TIME_PRECISION_MICRO); - nodePtr->val.nType = TSDB_DATA_TYPE_BIGINT; - nodePtr->nSQLOptr = TK_TIMESTAMP; // TK_TIMESTAMP used to denote the time value is in microsecond + pSQLExpr->val.i64Key = taosGetTimestamp(TSDB_TIME_PRECISION_MICRO); + pSQLExpr->val.nType = TSDB_DATA_TYPE_BIGINT; + pSQLExpr->nSQLOptr = TK_TIMESTAMP; // TK_TIMESTAMP used to denote the time value is in microsecond } else if (optrType == TK_VARIABLE) { - int32_t ret = getTimestampInUsFromStr(pAliasToken->z, pAliasToken->n, &nodePtr->val.i64Key); + int32_t ret = getTimestampInUsFromStr(pToken->z, pToken->n, &pSQLExpr->val.i64Key); UNUSED(ret); - nodePtr->val.nType = TSDB_DATA_TYPE_BIGINT; - nodePtr->nSQLOptr = TK_TIMESTAMP; + pSQLExpr->val.nType = TSDB_DATA_TYPE_BIGINT; + pSQLExpr->nSQLOptr = TK_TIMESTAMP; } else { // it must be the column name (tk_id) if it is not the number assert(optrType == TK_ID || optrType == TK_ALL); - if (pAliasToken != NULL) { - nodePtr->colInfo = *pAliasToken; + if (pToken != NULL) { + pSQLExpr->colInfo = *pToken; } - nodePtr->nSQLOptr = optrType; + pSQLExpr->nSQLOptr = optrType; } - return nodePtr; + return pSQLExpr; } /* * pList is the parameters for function with id(optType) * function name is denoted by pFunctionToken */ -tSQLExpr *tSQLExprCreateFunction(tSQLExprList *pList, SSQLToken *pFuncToken, SSQLToken *endToken, int32_t optType) { +tSQLExpr *tSQLExprCreateFunction(tSQLExprList *pList, SStrToken *pFuncToken, SStrToken *endToken, int32_t optType) { if (pFuncToken == NULL) return NULL; tSQLExpr *pExpr = calloc(1, sizeof(tSQLExpr)); @@ -163,6 +167,8 @@ tSQLExpr *tSQLExprCreateFunction(tSQLExprList *pList, SSQLToken *pFuncToken, SSQ pExpr->operand.n = len; // raw field name pExpr->operand.type = pFuncToken->type; + + pExpr->token = pExpr->operand; return pExpr; } @@ -173,8 +179,14 @@ tSQLExpr *tSQLExprCreateFunction(tSQLExprList *pList, SSQLToken *pFuncToken, SSQ tSQLExpr *tSQLExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optrType) { tSQLExpr *pExpr = calloc(1, sizeof(tSQLExpr)); - if (optrType == TK_PLUS || optrType == TK_MINUS || optrType == TK_STAR || optrType == TK_DIVIDE || - optrType == TK_REM) { + if (pRight != NULL && pLeft != NULL) { + char* endPos = pRight->token.z + pRight->token.n; + pExpr->token.z = pLeft->token.z; + pExpr->token.n = (uint32_t)(endPos - pExpr->token.z); + pExpr->token.type = pLeft->token.type; + } + + if (optrType == TK_PLUS || optrType == TK_MINUS || optrType == TK_STAR || optrType == TK_DIVIDE || optrType == TK_REM) { /* * if a token is noted as the TK_TIMESTAMP, the time precision is microsecond * Otherwise, the time precision is adaptive, determined by the time precision from databases. @@ -373,7 +385,7 @@ void tVariantListDestroy(tVariantList *pList) { free(pList); } -tVariantList *tVariantListAppendToken(tVariantList *pList, SSQLToken *pAliasToken, uint8_t sortOrder) { +tVariantList *tVariantListAppendToken(tVariantList *pList, SStrToken *pToken, uint8_t sortOrder) { if (pList == NULL) { pList = calloc(1, sizeof(tVariantList)); } @@ -382,9 +394,9 @@ tVariantList *tVariantListAppendToken(tVariantList *pList, SSQLToken *pAliasToke return pList; } - if (pAliasToken) { + if (pToken) { tVariant t = {0}; - tVariantCreate(&t, pAliasToken); + tVariantCreate(&t, pToken); tVariantListItem *pItem = &pList->a[pList->nExpr++]; memcpy(pItem, &t, sizeof(tVariant)); @@ -420,7 +432,7 @@ void tFieldListDestroy(tFieldList *pList) { free(pList); } -void setDBName(SSQLToken *pCpxName, SSQLToken *pDB) { +void setDBName(SStrToken *pCpxName, SStrToken *pDB) { pCpxName->type = pDB->type; pCpxName->z = pDB->z; pCpxName->n = pDB->n; @@ -464,7 +476,7 @@ int32_t getTimestampInUsFromStrImpl(int64_t val, char unit, int64_t *result) { return 0; } -void tSQLSetColumnInfo(TAOS_FIELD *pField, SSQLToken *pName, TAOS_FIELD *pType) { +void tSQLSetColumnInfo(TAOS_FIELD *pField, SStrToken *pName, TAOS_FIELD *pType) { int32_t maxLen = sizeof(pField->name) / sizeof(pField->name[0]); // truncate the column name @@ -479,7 +491,7 @@ void tSQLSetColumnInfo(TAOS_FIELD *pField, SSQLToken *pName, TAOS_FIELD *pType) pField->bytes = pType->bytes; } -void tSQLSetColumnType(TAOS_FIELD *pField, SSQLToken *type) { +void tSQLSetColumnType(TAOS_FIELD *pField, SStrToken *type) { pField->type = -1; int32_t LENGTH_SIZE_OF_STR = 2; // in case of nchar and binary, there two bytes to keep the length of binary|nchar. @@ -517,9 +529,9 @@ void tSQLSetColumnType(TAOS_FIELD *pField, SSQLToken *type) { /* * extract the select info out of sql string */ -SQuerySQL *tSetQuerySQLElems(SSQLToken *pSelectToken, tSQLExprList *pSelection, tVariantList *pFrom, tSQLExpr *pWhere, - tVariantList *pGroupby, tVariantList *pSortOrder, SSQLToken *pInterval, - SSQLToken *pSliding, tVariantList *pFill, SLimitVal *pLimit, SLimitVal *pGLimit) { +SQuerySQL *tSetQuerySQLElems(SStrToken *pSelectToken, tSQLExprList *pSelection, tVariantList *pFrom, tSQLExpr *pWhere, + tVariantList *pGroupby, tVariantList *pSortOrder, SStrToken *pInterval, + SStrToken *pSliding, tVariantList *pFill, SLimitVal *pLimit, SLimitVal *pGLimit) { assert(pSelection != NULL); SQuerySQL *pQuery = calloc(1, sizeof(SQuerySQL)); @@ -611,7 +623,7 @@ void destroyAllSelectClause(SSubclauseInfo *pClause) { taosTFree(pClause->pClause); } -SCreateTableSQL *tSetCreateSQLElems(tFieldList *pCols, tFieldList *pTags, SSQLToken *pStableName, +SCreateTableSQL *tSetCreateSQLElems(tFieldList *pCols, tFieldList *pTags, SStrToken *pStableName, tVariantList *pTagVals, SQuerySQL *pSelect, int32_t type) { SCreateTableSQL *pCreate = calloc(1, sizeof(SCreateTableSQL)); @@ -644,7 +656,7 @@ SCreateTableSQL *tSetCreateSQLElems(tFieldList *pCols, tFieldList *pTags, SSQLTo return pCreate; } -SAlterTableSQL *tAlterTableSQLElems(SSQLToken *pMeterName, tFieldList *pCols, tVariantList *pVals, int32_t type) { +SAlterTableSQL *tAlterTableSQLElems(SStrToken *pMeterName, tFieldList *pCols, tVariantList *pVals, int32_t type) { SAlterTableSQL *pAlterTable = calloc(1, sizeof(SAlterTableSQL)); pAlterTable->name = *pMeterName; @@ -716,7 +728,7 @@ SSubclauseInfo* setSubclause(SSubclauseInfo* pSubclause, void *pSqlExprInfo) { return pSubclause; } -SSqlInfo* setSQLInfo(SSqlInfo *pInfo, void *pSqlExprInfo, SSQLToken *pMeterName, int32_t type) { +SSqlInfo* setSQLInfo(SSqlInfo *pInfo, void *pSqlExprInfo, SStrToken *pMeterName, int32_t type) { pInfo->type = type; if (type == TSDB_SQL_SELECT) { @@ -745,7 +757,7 @@ SSubclauseInfo* appendSelectClause(SSubclauseInfo *pQueryInfo, void *pSubclause) return pQueryInfo; } -void setCreatedTableName(SSqlInfo *pInfo, SSQLToken *pMeterName, SSQLToken *pIfNotExists) { +void setCreatedTableName(SSqlInfo *pInfo, SStrToken *pMeterName, SStrToken *pIfNotExists) { pInfo->pCreateTableInfo->name = *pMeterName; pInfo->pCreateTableInfo->existCheck = (pIfNotExists->n != 0); } @@ -760,7 +772,7 @@ void tTokenListBuyMoreSpace(tDCLSQL *pTokenList) { } } -tDCLSQL *tTokenListAppend(tDCLSQL *pTokenList, SSQLToken *pToken) { +tDCLSQL *tTokenListAppend(tDCLSQL *pTokenList, SStrToken *pToken) { if (pToken == NULL) return NULL; if (pTokenList == NULL) pTokenList = calloc(1, sizeof(tDCLSQL)); @@ -781,19 +793,19 @@ void setDCLSQLElems(SSqlInfo *pInfo, int32_t type, int32_t nParam, ...) { va_start(va, nParam); while (nParam-- > 0) { - SSQLToken *pToken = va_arg(va, SSQLToken *); + SStrToken *pToken = va_arg(va, SStrToken *); pInfo->pDCLInfo = tTokenListAppend(pInfo->pDCLInfo, pToken); } va_end(va); } -void setDropDBTableInfo(SSqlInfo *pInfo, int32_t type, SSQLToken* pToken, SSQLToken* existsCheck) { +void setDropDBTableInfo(SSqlInfo *pInfo, int32_t type, SStrToken* pToken, SStrToken* existsCheck) { pInfo->type = type; pInfo->pDCLInfo = tTokenListAppend(pInfo->pDCLInfo, pToken); pInfo->pDCLInfo->existsCheck = (existsCheck->n == 1); } -void setShowOptions(SSqlInfo *pInfo, int32_t type, SSQLToken* prefix, SSQLToken* pPatterns) { +void setShowOptions(SSqlInfo *pInfo, int32_t type, SStrToken* prefix, SStrToken* pPatterns) { if (pInfo->pDCLInfo == NULL) { pInfo->pDCLInfo = calloc(1, sizeof(tDCLSQL)); } @@ -816,7 +828,7 @@ void setShowOptions(SSqlInfo *pInfo, int32_t type, SSQLToken* prefix, SSQLToken* } } -void setCreateDBSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *pToken, SCreateDBInfo *pDB, SSQLToken *pIgExists) { +void setCreateDBSQL(SSqlInfo *pInfo, int32_t type, SStrToken *pToken, SCreateDBInfo *pDB, SStrToken *pIgExists) { pInfo->type = type; if (pInfo->pDCLInfo == NULL) { pInfo->pDCLInfo = calloc(1, sizeof(tDCLSQL)); @@ -827,7 +839,7 @@ void setCreateDBSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *pToken, SCreateDBI pInfo->pDCLInfo->dbOpt.ignoreExists = pIgExists->n; // sql.y has: ifnotexists(X) ::= IF NOT EXISTS. {X.n = 1;} } -void setCreateAcctSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *pName, SSQLToken *pPwd, SCreateAcctSQL *pAcctInfo) { +void setCreateAcctSQL(SSqlInfo *pInfo, int32_t type, SStrToken *pName, SStrToken *pPwd, SCreateAcctSQL *pAcctInfo) { pInfo->type = type; if (pInfo->pDCLInfo == NULL) { pInfo->pDCLInfo = calloc(1, sizeof(tDCLSQL)); @@ -843,7 +855,7 @@ void setCreateAcctSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *pName, SSQLToken } } -void setCreateUserSQL(SSqlInfo *pInfo, SSQLToken *pName, SSQLToken *pPasswd) { +void setCreateUserSQL(SSqlInfo *pInfo, SStrToken *pName, SStrToken *pPasswd) { pInfo->type = TSDB_SQL_CREATE_USER; if (pInfo->pDCLInfo == NULL) { pInfo->pDCLInfo = calloc(1, sizeof(tDCLSQL)); @@ -855,7 +867,7 @@ void setCreateUserSQL(SSqlInfo *pInfo, SSQLToken *pName, SSQLToken *pPasswd) { pInfo->pDCLInfo->user.passwd = *pPasswd; } -void setAlterUserSQL(SSqlInfo *pInfo, int16_t type, SSQLToken *pName, SSQLToken* pPwd, SSQLToken *pPrivilege) { +void setAlterUserSQL(SSqlInfo *pInfo, int16_t type, SStrToken *pName, SStrToken* pPwd, SStrToken *pPrivilege) { pInfo->type = TSDB_SQL_ALTER_USER; if (pInfo->pDCLInfo == NULL) { pInfo->pDCLInfo = calloc(1, sizeof(tDCLSQL)); @@ -880,7 +892,7 @@ void setAlterUserSQL(SSqlInfo *pInfo, int16_t type, SSQLToken *pName, SSQLToken* } } -void setKillSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *ip) { +void setKillSQL(SSqlInfo *pInfo, int32_t type, SStrToken *ip) { pInfo->type = type; if (pInfo->pDCLInfo == NULL) { pInfo->pDCLInfo = calloc(1, sizeof(tDCLSQL)); @@ -909,5 +921,5 @@ void setDefaultCreateDbOption(SCreateDBInfo *pDBInfo) { pDBInfo->quorum = -1; pDBInfo->keep = NULL; - memset(&pDBInfo->precision, 0, sizeof(SSQLToken)); + memset(&pDBInfo->precision, 0, sizeof(SStrToken)); } diff --git a/src/query/src/qResultbuf.c b/src/query/src/qResultbuf.c index 18d2520542..b3e97459d3 100644 --- a/src/query/src/qResultbuf.c +++ b/src/query/src/qResultbuf.c @@ -137,8 +137,15 @@ static char* doFlushPageToDisk(SDiskbasedResultBuf* pResultBuf, SPageInfo* pg) { } //3. write to disk. - fseek(pResultBuf->file, pg->info.offset, SEEK_SET); - fwrite(t, size, 1, pResultBuf->file); + int32_t ret = fseek(pResultBuf->file, pg->info.offset, SEEK_SET); + if (ret != 0) { // todo handle the error case + + } + + ret = (int32_t)fwrite(t, size, 1, pResultBuf->file); + if (ret != size) { // todo handle the error case + + } if (pResultBuf->fileSize < pg->info.offset + pg->info.length) { pResultBuf->fileSize = pg->info.offset + pg->info.length; diff --git a/src/query/src/qTokenizer.c b/src/query/src/qTokenizer.c index 227aded5f1..5fdabca166 100644 --- a/src/query/src/qTokenizer.c +++ b/src/query/src/qTokenizer.c @@ -85,7 +85,7 @@ static SKeyword keywordTable[] = { {"QUERIES", TK_QUERIES}, {"CONNECTIONS", TK_CONNECTIONS}, {"STREAMS", TK_STREAMS}, - {"CONFIGS", TK_CONFIGS}, + {"VARIABLES", TK_VARIABLES}, {"SCORES", TK_SCORES}, {"GRANTS", TK_GRANTS}, {"DOT", TK_DOT}, @@ -580,8 +580,8 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenType) { return 0; } -SSQLToken tStrGetToken(char* str, int32_t* i, bool isPrevOptr, uint32_t numOfIgnoreToken, uint32_t* ignoreTokenTypes) { - SSQLToken t0 = {0}; +SStrToken tStrGetToken(char* str, int32_t* i, bool isPrevOptr, uint32_t numOfIgnoreToken, uint32_t* ignoreTokenTypes) { + SStrToken t0 = {0}; // here we reach the end of sql string, null-terminated string if (str[*i] == 0) { diff --git a/src/query/src/qTsbuf.c b/src/query/src/qTsbuf.c index 25eb33ff7d..b264f6cdc9 100644 --- a/src/query/src/qTsbuf.c +++ b/src/query/src/qTsbuf.c @@ -8,7 +8,6 @@ static void TSBufUpdateVnodeInfo(STSBuf* pTSBuf, int32_t index, STSVnodeBlockInf static STSBuf* allocResForTSBuf(STSBuf* pTSBuf); static int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader); - /** * todo error handling * support auto closeable tmp file @@ -76,7 +75,7 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { return NULL; } - if ((int32_t)header.numOfVnode > pTSBuf->numOfAlloc) { + if (header.numOfVnode > pTSBuf->numOfAlloc) { pTSBuf->numOfAlloc = header.numOfVnode; STSVnodeBlockInfoEx* tmp = realloc(pTSBuf->pData, sizeof(STSVnodeBlockInfoEx) * pTSBuf->numOfAlloc); if (tmp == NULL) { @@ -225,14 +224,15 @@ static void writeDataToDisk(STSBuf* pTSBuf) { } STSBlock* pBlock = &pTSBuf->block; - - pBlock->numOfElem = pTSBuf->tsData.len / TSDB_KEYSIZE; + STSList* pTsData = &pTSBuf->tsData; + + pBlock->numOfElem = pTsData->len / TSDB_KEYSIZE; pBlock->compLen = - tsCompressTimestamp(pTSBuf->tsData.rawBuf, pTSBuf->tsData.len, pTSBuf->tsData.len / TSDB_KEYSIZE, pBlock->payload, - pTSBuf->tsData.allocSize, TWO_STAGE_COMP, pTSBuf->assistBuf, pTSBuf->bufSize); + tsCompressTimestamp(pTsData->rawBuf, pTsData->len, pTsData->len/TSDB_KEYSIZE, pBlock->payload, pTsData->allocSize, + TWO_STAGE_COMP, pTSBuf->assistBuf, pTSBuf->bufSize); int64_t r = fseek(pTSBuf->f, pTSBuf->fileSize, SEEK_SET); - UNUSED(r); + assert(r == 0); /* * format for output data: @@ -241,16 +241,22 @@ static void writeDataToDisk(STSBuf* pTSBuf) { * * both side has the compressed length is used to support load data forwards/backwords. */ - fwrite(&pBlock->tag, sizeof(pBlock->tag), 1, pTSBuf->f); + int32_t metaLen = 0; + metaLen += (int32_t)fwrite(&pBlock->tag.nType, 1, sizeof(pBlock->tag.nType), pTSBuf->f); + metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); + + if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR) { + metaLen += (int32_t)fwrite(pBlock->tag.pz, 1, (size_t)pBlock->tag.nLen, pTSBuf->f); + } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { + metaLen += (int32_t)fwrite(&pBlock->tag.i64Key, 1, sizeof(int64_t), pTSBuf->f); + } + fwrite(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f); - fwrite(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f); - fwrite(pBlock->payload, (size_t)pBlock->compLen, 1, pTSBuf->f); - fwrite(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f); - int32_t blockSize = sizeof(pBlock->tag) + sizeof(pBlock->numOfElem) + sizeof(pBlock->compLen) * 2 + pBlock->compLen; + int32_t blockSize = metaLen + sizeof(pBlock->numOfElem) + sizeof(pBlock->compLen) * 2 + pBlock->compLen; pTSBuf->fileSize += blockSize; pTSBuf->tsData.len = 0; @@ -298,9 +304,24 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { ret = fseek(pTSBuf->f, -offset, SEEK_CUR); UNUSED(ret); } - - size_t sz = fread(&pBlock->tag, sizeof(pBlock->tag), 1, pTSBuf->f); - UNUSED(sz); + + fread(&pBlock->tag.nType, sizeof(pBlock->tag.nType), 1, pTSBuf->f); + fread(&pBlock->tag.nLen, sizeof(pBlock->tag.nLen), 1, pTSBuf->f); + + // NOTE: mix types tags are not supported + size_t sz = 0; + if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR) { + char* tp = realloc(pBlock->tag.pz, pBlock->tag.nLen + 1); + assert(tp != NULL); + + memset(tp, 0, pBlock->tag.nLen + 1); + pBlock->tag.pz = tp; + + sz = fread(pBlock->tag.pz, (size_t)pBlock->tag.nLen, 1, pTSBuf->f); + } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { + sz = fread(&pBlock->tag.i64Key, sizeof(int64_t), 1, pTSBuf->f); + } + sz = fread(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f); UNUSED(sz); sz = fread(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f); @@ -361,7 +382,7 @@ static int32_t setCheckTSOrder(STSBuf* pTSBuf, const char* pData, int32_t len) { return TSDB_CODE_SUCCESS; } -void tsBufAppend(STSBuf* pTSBuf, int32_t vnodeId, int64_t tag, const char* pData, int32_t len) { +void tsBufAppend(STSBuf* pTSBuf, int32_t vnodeId, tVariant* tag, const char* pData, int32_t len) { STSVnodeBlockInfoEx* pBlockInfo = NULL; STSList* ptsData = &pTSBuf->tsData; @@ -375,15 +396,15 @@ void tsBufAppend(STSBuf* pTSBuf, int32_t vnodeId, int64_t tag, const char* pData } assert(pBlockInfo->info.vnode == vnodeId); - - if (pTSBuf->block.tag != tag && ptsData->len > 0) { + + if ((tVariantCompare(&pTSBuf->block.tag, tag) != 0) && ptsData->len > 0) { // new arrived data with different tags value, save current value into disk first writeDataToDisk(pTSBuf); } else { expandBuffer(ptsData, len); } - pTSBuf->block.tag = tag; + tVariantAssign(&pTSBuf->block.tag, tag); memcpy(ptsData->rawBuf + ptsData->len, pData, (size_t)len); // todo check return value @@ -465,7 +486,7 @@ static int32_t tsBufFindBlock(STSBuf* pTSBuf, STSVnodeBlockInfo* pBlockInfo, int return 0; } -static int32_t tsBufFindBlockByTag(STSBuf* pTSBuf, STSVnodeBlockInfo* pBlockInfo, int64_t tag) { +static int32_t tsBufFindBlockByTag(STSBuf* pTSBuf, STSVnodeBlockInfo* pBlockInfo, tVariant* tag) { bool decomp = false; int64_t offset = 0; @@ -484,7 +505,7 @@ static int32_t tsBufFindBlockByTag(STSBuf* pTSBuf, STSVnodeBlockInfo* pBlockInfo return -1; } - if (pTSBuf->block.tag == tag) { + if (tVariantCompare(&pTSBuf->block.tag, tag) == 0) { return i; } } @@ -669,8 +690,8 @@ STSElem tsBufGetElem(STSBuf* pTSBuf) { elem1.vnode = pTSBuf->pData[pCur->vgroupIndex].info.vnode; elem1.ts = *(TSKEY*)(pTSBuf->tsData.rawBuf + pCur->tsIndex * TSDB_KEYSIZE); - elem1.tag = pBlock->tag; - + tVariantAssign(&elem1.tag, &pBlock->tag); + return elem1; } @@ -800,7 +821,7 @@ STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_ return pTSBuf; } -STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t vnodeId, int64_t tag) { +STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t vnodeId, tVariant* tag) { STSElem elem = {.vnode = -1}; if (pTSBuf == NULL) { @@ -881,7 +902,9 @@ void tsBufDisplay(STSBuf* pTSBuf) { while (tsBufNextPos(pTSBuf)) { STSElem elem = tsBufGetElem(pTSBuf); - printf("%d-%" PRId64 "-%" PRId64 "\n", elem.vnode, elem.tag, elem.ts); + if (elem.tag.nType == TSDB_DATA_TYPE_BIGINT) { + printf("%d-%" PRId64 "-%" PRId64 "\n", elem.vnode, elem.tag.i64Key, elem.ts); + } } pTSBuf->cur.order = old; diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index a01eb33ec7..509362863c 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -45,8 +45,8 @@ int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, SQueryRuntimeEnv *pRun pWindowResInfo->curIndex = -1; pWindowResInfo->size = 0; pWindowResInfo->prevSKey = TSKEY_INITIAL_VAL; - - pRuntimeEnv->summary.internalSupSize += sizeof(SWindowResult) * threshold; + + SQueryCostInfo* pSummary = &pRuntimeEnv->summary; // use the pointer arraylist pWindowResInfo->pResult = calloc(threshold, sizeof(SWindowResult)); @@ -54,8 +54,11 @@ int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, SQueryRuntimeEnv *pRun return TSDB_CODE_QRY_OUT_OF_MEMORY; } - pRuntimeEnv->summary.internalSupSize += sizeof(SWindowResult) * threshold; - pRuntimeEnv->summary.internalSupSize += (pRuntimeEnv->pQuery->numOfOutput * sizeof(SResultInfo) + pRuntimeEnv->interBufSize) * pWindowResInfo->capacity; + pWindowResInfo->interval = pRuntimeEnv->pQuery->intervalTime; + + pSummary->internalSupSize += sizeof(SWindowResult) * threshold; + pSummary->internalSupSize += (pRuntimeEnv->pQuery->numOfOutput * sizeof(SResultInfo) + pRuntimeEnv->interBufSize) * pWindowResInfo->capacity; + pSummary->numOfTimeWindows = threshold; for (int32_t i = 0; i < pWindowResInfo->capacity; ++i) { int32_t code = createQueryResultInfo(pRuntimeEnv->pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, pRuntimeEnv->interBufSize); @@ -126,8 +129,8 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { for (int32_t i = 0; i < num; ++i) { SWindowResult *pResult = &pWindowResInfo->pResult[i]; - if (pResult->status.closed) { // remove the window slot from hash table - taosHashRemove(pWindowResInfo->hashList, (const char *)&pResult->window.skey, pWindowResInfo->type); + if (pResult->closed) { // remove the window slot from hash table + taosHashRemove(pWindowResInfo->hashList, (const char *)&pResult->skey, pWindowResInfo->type); } else { break; } @@ -149,12 +152,12 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { pWindowResInfo->size = remain; for (int32_t k = 0; k < pWindowResInfo->size; ++k) { SWindowResult *pResult = &pWindowResInfo->pResult[k]; - int32_t *p = (int32_t *)taosHashGet(pWindowResInfo->hashList, (const char *)&pResult->window.skey, + int32_t *p = (int32_t *)taosHashGet(pWindowResInfo->hashList, (const char *)&pResult->skey, tDataTypeDesc[pWindowResInfo->type].nSize); assert(p != NULL); int32_t v = (*p - num); assert(v >= 0 && v <= pWindowResInfo->size); - taosHashPut(pWindowResInfo->hashList, (char *)&pResult->window.skey, tDataTypeDesc[pWindowResInfo->type].nSize, + taosHashPut(pWindowResInfo->hashList, (char *)&pResult->skey, tDataTypeDesc[pWindowResInfo->type].nSize, (char *)&v, sizeof(int32_t)); } @@ -173,7 +176,7 @@ void clearClosedTimeWindow(SQueryRuntimeEnv *pRuntimeEnv) { int32_t numOfClosedTimeWindow(SWindowResInfo *pWindowResInfo) { int32_t i = 0; - while (i < pWindowResInfo->size && pWindowResInfo->pResult[i].status.closed) { + while (i < pWindowResInfo->size && pWindowResInfo->pResult[i].closed) { ++i; } @@ -184,11 +187,11 @@ void closeAllTimeWindow(SWindowResInfo *pWindowResInfo) { assert(pWindowResInfo->size >= 0 && pWindowResInfo->capacity >= pWindowResInfo->size); for (int32_t i = 0; i < pWindowResInfo->size; ++i) { - if (pWindowResInfo->pResult[i].status.closed) { + if (pWindowResInfo->pResult[i].closed) { continue; } - pWindowResInfo->pResult[i].status.closed = true; + pWindowResInfo->pResult[i].closed = true; } } @@ -204,7 +207,7 @@ void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_ } // get the result order - int32_t resultOrder = (pWindowResInfo->pResult[0].window.skey < pWindowResInfo->pResult[1].window.skey)? 1:-1; + int32_t resultOrder = (pWindowResInfo->pResult[0].skey < pWindowResInfo->pResult[1].skey)? 1:-1; if (order != resultOrder) { return; @@ -212,11 +215,12 @@ void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_ int32_t i = 0; if (order == QUERY_ASC_FORWARD_STEP) { - while (i < pWindowResInfo->size && (pWindowResInfo->pResult[i].window.ekey < lastKey)) { + TSKEY ekey = pWindowResInfo->pResult[i].skey + pWindowResInfo->interval; + while (i < pWindowResInfo->size && (ekey < lastKey)) { ++i; } } else if (order == QUERY_DESC_FORWARD_STEP) { - while (i < pWindowResInfo->size && (pWindowResInfo->pResult[i].window.skey > lastKey)) { + while (i < pWindowResInfo->size && (pWindowResInfo->pResult[i].skey > lastKey)) { ++i; } } @@ -227,11 +231,11 @@ void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_ } bool isWindowResClosed(SWindowResInfo *pWindowResInfo, int32_t slot) { - return (getWindowResult(pWindowResInfo, slot)->status.closed == true); + return (getWindowResult(pWindowResInfo, slot)->closed == true); } void closeTimeWindow(SWindowResInfo *pWindowResInfo, int32_t slot) { - getWindowResult(pWindowResInfo, slot)->status.closed = true; + getWindowResult(pWindowResInfo, slot)->closed = true; } void clearTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pWindowRes) { @@ -253,8 +257,8 @@ void clearTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pWindow pWindowRes->numOfRows = 0; pWindowRes->pos = (SPosInfo){-1, -1}; - pWindowRes->status.closed = false; - pWindowRes->window = TSWINDOW_INITIALIZER; + pWindowRes->closed = false; + pWindowRes->skey = TSKEY_INITIAL_VAL; } /** @@ -264,8 +268,8 @@ void clearTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pWindow */ void copyTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *dst, const SWindowResult *src) { dst->numOfRows = src->numOfRows; - dst->window = src->window; - dst->status = src->status; + dst->skey = src->skey; + dst->closed = src->closed; int32_t nOutputCols = pRuntimeEnv->pQuery->numOfOutput; diff --git a/src/query/src/sql.c b/src/query/src/sql.c index 75ef2f3218..f3f55791b0 100644 --- a/src/query/src/sql.c +++ b/src/query/src/sql.c @@ -99,7 +99,7 @@ #define YYCODETYPE unsigned short int #define YYNOCODE 274 #define YYACTIONTYPE unsigned short int -#define ParseTOKENTYPE SSQLToken +#define ParseTOKENTYPE SStrToken typedef union { int yyinit; ParseTOKENTYPE yy0; @@ -488,7 +488,7 @@ static const YYCODETYPE yyFallback[] = { 0, /* QUERIES => nothing */ 0, /* CONNECTIONS => nothing */ 0, /* STREAMS => nothing */ - 0, /* CONFIGS => nothing */ + 0, /* VARIABLES => nothing */ 0, /* SCORES => nothing */ 0, /* GRANTS => nothing */ 0, /* VNODES => nothing */ @@ -781,7 +781,7 @@ static const char *const yyTokenName[] = { /* 51 */ "QUERIES", /* 52 */ "CONNECTIONS", /* 53 */ "STREAMS", - /* 54 */ "CONFIGS", + /* 54 */ "VARIABLES", /* 55 */ "SCORES", /* 56 */ "GRANTS", /* 57 */ "VNODES", @@ -1017,7 +1017,7 @@ static const char *const yyRuleName[] = { /* 7 */ "cmd ::= SHOW QUERIES", /* 8 */ "cmd ::= SHOW CONNECTIONS", /* 9 */ "cmd ::= SHOW STREAMS", - /* 10 */ "cmd ::= SHOW CONFIGS", + /* 10 */ "cmd ::= SHOW VARIABLES", /* 11 */ "cmd ::= SHOW SCORES", /* 12 */ "cmd ::= SHOW GRANTS", /* 13 */ "cmd ::= SHOW VNODES", @@ -1702,7 +1702,7 @@ static const struct { { 209, -2 }, /* (7) cmd ::= SHOW QUERIES */ { 209, -2 }, /* (8) cmd ::= SHOW CONNECTIONS */ { 209, -2 }, /* (9) cmd ::= SHOW STREAMS */ - { 209, -2 }, /* (10) cmd ::= SHOW CONFIGS */ + { 209, -2 }, /* (10) cmd ::= SHOW VARIABLES */ { 209, -2 }, /* (11) cmd ::= SHOW SCORES */ { 209, -2 }, /* (12) cmd ::= SHOW GRANTS */ { 209, -2 }, /* (13) cmd ::= SHOW VNODES */ @@ -2029,8 +2029,8 @@ static void yy_reduce( case 9: /* cmd ::= SHOW STREAMS */ { setShowOptions(pInfo, TSDB_MGMT_TABLE_STREAMS, 0, 0); } break; - case 10: /* cmd ::= SHOW CONFIGS */ -{ setShowOptions(pInfo, TSDB_MGMT_TABLE_CONFIGS, 0, 0); } + case 10: /* cmd ::= SHOW VARIABLES */ +{ setShowOptions(pInfo, TSDB_MGMT_TABLE_VARIABLES, 0, 0); } break; case 11: /* cmd ::= SHOW SCORES */ { setShowOptions(pInfo, TSDB_MGMT_TABLE_SCORES, 0, 0); } @@ -2074,21 +2074,21 @@ static void yy_reduce( break; case 22: /* cmd ::= SHOW dbPrefix STABLES LIKE ids */ { - SSQLToken token; + SStrToken token; setDBName(&token, &yymsp[-3].minor.yy0); setShowOptions(pInfo, TSDB_MGMT_TABLE_METRIC, &token, &yymsp[0].minor.yy0); } break; case 23: /* cmd ::= SHOW dbPrefix VGROUPS */ { - SSQLToken token; + SStrToken token; setDBName(&token, &yymsp[-1].minor.yy0); setShowOptions(pInfo, TSDB_MGMT_TABLE_VGROUP, &token, 0); } break; case 24: /* cmd ::= SHOW dbPrefix VGROUPS ids */ { - SSQLToken token; + SStrToken token; setDBName(&token, &yymsp[-2].minor.yy0); setShowOptions(pInfo, TSDB_MGMT_TABLE_VGROUP, &token, &yymsp[0].minor.yy0); } @@ -2139,7 +2139,7 @@ static void yy_reduce( { setDCLSQLElems(pInfo, TSDB_SQL_CFG_LOCAL, 2, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } break; case 38: /* cmd ::= ALTER DATABASE ids alter_db_optr */ -{ SSQLToken t = {0}; setCreateDBSQL(pInfo, TSDB_SQL_ALTER_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy268, &t);} +{ SStrToken t = {0}; setCreateDBSQL(pInfo, TSDB_SQL_ALTER_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy268, &t);} break; case 39: /* cmd ::= ALTER ACCOUNT ids acct_optr */ { setCreateAcctSQL(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-1].minor.yy0, NULL, &yymsp[0].minor.yy149);} @@ -2277,6 +2277,7 @@ static void yy_reduce( yymsp[-1].minor.yy268 = yylhsminor.yy268; break; case 95: /* db_optr ::= db_optr fsync */ + case 107: /* alter_db_optr ::= alter_db_optr fsync */ yytestcase(yyruleno==107); { yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.fsyncPeriod = strtol(yymsp[0].minor.yy0.z, NULL, 10); } yymsp[-1].minor.yy268 = yylhsminor.yy268; break; @@ -2297,10 +2298,6 @@ static void yy_reduce( case 99: /* alter_db_optr ::= */ { setDefaultCreateDbOption(&yymsp[1].minor.yy268);} break; - case 107: /* alter_db_optr ::= alter_db_optr fsync */ -{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.fsyncPeriod = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy268 = yylhsminor.yy268; - break; case 108: /* typename ::= ids */ { yymsp[0].minor.yy0.type = 0; diff --git a/src/query/tests/tsBufTest.cpp b/src/query/tests/tsBufTest.cpp index f8738eec9c..b78c5314f2 100644 --- a/src/query/tests/tsBufTest.cpp +++ b/src/query/tests/tsBufTest.cpp @@ -32,14 +32,16 @@ void simpleTest() { // write 10 ts points int32_t num = 10; - int64_t tag = 1; + tVariant t = {0}; + t.nType = TSDB_DATA_TYPE_BIGINT; + t.i64Key = 1; int64_t* list = createTsList(10, 10000000, 30); - tsBufAppend(pTSBuf, 0, tag, (const char*)list, num * sizeof(int64_t)); + tsBufAppend(pTSBuf, 0, &t, (const char*)list, num * sizeof(int64_t)); EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC); EXPECT_EQ(pTSBuf->tsData.len, sizeof(int64_t) * num); - EXPECT_EQ(pTSBuf->block.tag, tag); + EXPECT_EQ(tVariantCompare(&pTSBuf->block.tag, &t), 0); EXPECT_EQ(pTSBuf->numOfVnodes, 1); tsBufFlush(pTSBuf); @@ -57,14 +59,16 @@ void largeTSTest() { // write 10 ts points int32_t num = 1000000; - int64_t tag = 1; + tVariant t = {0}; + t.nType = TSDB_DATA_TYPE_BIGINT; + t.i64Key = 1; int64_t* list = createTsList(num, 10000000, 30); - tsBufAppend(pTSBuf, 0, tag, (const char*)list, num * sizeof(int64_t)); + tsBufAppend(pTSBuf, 0, &t, (const char*)list, num * sizeof(int64_t)); // the data has been flush to disk, no data in cache EXPECT_EQ(pTSBuf->tsData.len, 0); - EXPECT_EQ(pTSBuf->block.tag, tag); + EXPECT_EQ(tVariantCompare(&pTSBuf->block.tag, &t), 0); EXPECT_EQ(pTSBuf->numOfVnodes, 1); EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC); @@ -80,14 +84,18 @@ void multiTagsTest() { STSBuf* pTSBuf = tsBufCreate(true, TSDB_ORDER_ASC); int32_t num = 10000; - int64_t tag = 1; + tVariant t = {0}; + t.nType = TSDB_DATA_TYPE_BIGINT; + int64_t start = 10000000; int32_t numOfTags = 50; int32_t step = 30; for (int32_t i = 0; i < numOfTags; ++i) { int64_t* list = createTsList(num, start, step); - tsBufAppend(pTSBuf, 0, i, (const char*)list, num * sizeof(int64_t)); + t.i64Key = i; + + tsBufAppend(pTSBuf, 0, &t, (const char*)list, num * sizeof(int64_t)); free(list); start += step * num; @@ -96,7 +104,7 @@ void multiTagsTest() { EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC); EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); - EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1); + EXPECT_EQ(pTSBuf->block.tag.i64Key, numOfTags - 1); EXPECT_EQ(pTSBuf->numOfVnodes, 1); tsBufFlush(pTSBuf); @@ -118,9 +126,14 @@ void multiVnodeTagsTest() { for (int32_t j = 0; j < 20; ++j) { // vnodeId:0 start = 10000000; + tVariant t = {0}; + t.nType = TSDB_DATA_TYPE_BIGINT; + for (int32_t i = 0; i < numOfTags; ++i) { int64_t* list = createTsList(num, start, step); - tsBufAppend(pTSBuf, j, i, (const char*)list, num * sizeof(int64_t)); + t.i64Key = i; + + tsBufAppend(pTSBuf, j, &t, (const char*)list, num * sizeof(int64_t)); free(list); start += step * num; @@ -131,11 +144,11 @@ void multiVnodeTagsTest() { EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC); EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); - EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1); + EXPECT_EQ(pTSBuf->block.tag.i64Key, numOfTags - 1); EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); - EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1); + EXPECT_EQ(pTSBuf->block.tag.i64Key, numOfTags - 1); tsBufFlush(pTSBuf); EXPECT_EQ(pTSBuf->tsData.len, 0); @@ -157,9 +170,14 @@ void loadDataTest() { for (int32_t j = 0; j < numOfVnode; ++j) { // vnodeId:0 int64_t start = 10000000; + tVariant t = {0}; + t.nType = TSDB_DATA_TYPE_BIGINT; + for (int32_t i = 0; i < numOfTags; ++i) { int64_t* list = createTsList(num, start, step); - tsBufAppend(pTSBuf, j, i, (const char*)list, num * sizeof(int64_t)); + t.i64Key = i; + + tsBufAppend(pTSBuf, j, &t, (const char*)list, num * sizeof(int64_t)); printf("%d - %" PRIu64 "\n", i, list[0]); free(list); @@ -172,11 +190,11 @@ void loadDataTest() { EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC); EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); - EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1); + EXPECT_EQ(pTSBuf->block.tag.i64Key, numOfTags - 1); EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); - EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1); + EXPECT_EQ(pTSBuf->block.tag.i64Key, numOfTags - 1); tsBufFlush(pTSBuf); EXPECT_EQ(pTSBuf->tsData.len, 0); @@ -230,16 +248,21 @@ void TSTraverse() { for (int32_t j = 0; j < numOfVnode; ++j) { // vnodeId:0 int64_t start = 10000000; + tVariant t = {0}; + t.nType = TSDB_DATA_TYPE_BIGINT; + for (int32_t i = 0; i < numOfTags; ++i) { int64_t* list = createTsList(num, start, step); - tsBufAppend(pTSBuf, j, i, (const char*)list, num * sizeof(int64_t)); + t.i64Key = i; + + tsBufAppend(pTSBuf, j, &t, (const char*)list, num * sizeof(int64_t)); printf("%d - %d - %" PRIu64 ", %" PRIu64 "\n", j, i, list[0], list[num - 1]); free(list); start += step * num; list = createTsList(num, start, step); - tsBufAppend(pTSBuf, j, i, (const char*)list, num * sizeof(int64_t)); + tsBufAppend(pTSBuf, j, &t, (const char*)list, num * sizeof(int64_t)); printf("%d - %d - %" PRIu64 ", %" PRIu64 "\n", j, i, list[0], list[num - 1]); free(list); @@ -272,12 +295,16 @@ void TSTraverse() { int32_t startVnode = 1; int32_t startTag = 2; - tsBufGetElemStartPos(pTSBuf, startVnode, startTag); + tVariant t = {0}; + t.nType = TSDB_DATA_TYPE_BIGINT; + t.i64Key = startTag; + + tsBufGetElemStartPos(pTSBuf, startVnode, &t); int32_t totalOutput = 10; while (1) { STSElem elem = tsBufGetElem(pTSBuf); - printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag, elem.ts); + printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag.i64Key, elem.ts); if (!tsBufNextPos(pTSBuf)) { break; @@ -286,7 +313,9 @@ void TSTraverse() { if (--totalOutput <= 0) { totalOutput = 10; - tsBufGetElemStartPos(pTSBuf, startVnode, --startTag); + startTag -= 1; + t.i64Key = startTag; + tsBufGetElemStartPos(pTSBuf, startVnode, &t); if (startTag == 0) { startVnode -= 1; @@ -316,13 +345,14 @@ void TSTraverse() { startVnode = 1; startTag = 2; + t.i64Key = startTag; - tsBufGetElemStartPos(pTSBuf, startVnode, startTag); + tsBufGetElemStartPos(pTSBuf, startVnode, &t); totalOutput = 10; while (1) { STSElem elem = tsBufGetElem(pTSBuf); - printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag, elem.ts); + printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag.i64Key, elem.ts); if (!tsBufNextPos(pTSBuf)) { break; @@ -331,7 +361,9 @@ void TSTraverse() { if (--totalOutput <= 0) { totalOutput = 10; - tsBufGetElemStartPos(pTSBuf, startVnode, --startTag); + startTag -= 1; + t.i64Key = startTag; + tsBufGetElemStartPos(pTSBuf, startVnode, &t); if (startTag < 0) { startVnode -= 1; @@ -375,12 +407,17 @@ void mergeDiffVnodeBufferTest() { int32_t num = 1000; int32_t numOfTags = 10; + tVariant t = {0}; + t.nType = TSDB_DATA_TYPE_BIGINT; + // vnodeId:0 int64_t start = 10000000; for (int32_t i = 0; i < numOfTags; ++i) { int64_t* list = createTsList(num, start, step); - tsBufAppend(pTSBuf1, 0, i, (const char*)list, num * sizeof(int64_t)); - tsBufAppend(pTSBuf2, 0, i, (const char*)list, num * sizeof(int64_t)); + t.i64Key = i; + + tsBufAppend(pTSBuf1, 0, &t, (const char*)list, num * sizeof(int64_t)); + tsBufAppend(pTSBuf2, 0, &t, (const char*)list, num * sizeof(int64_t)); free(list); @@ -403,6 +440,9 @@ void mergeIdenticalVnodeBufferTest() { STSBuf* pTSBuf1 = tsBufCreate(true, TSDB_ORDER_ASC); STSBuf* pTSBuf2 = tsBufCreate(true, TSDB_ORDER_ASC); + tVariant t = {0}; + t.nType = TSDB_DATA_TYPE_BIGINT; + int32_t step = 30; int32_t num = 1000; int32_t numOfTags = 10; @@ -411,17 +451,21 @@ void mergeIdenticalVnodeBufferTest() { int64_t start = 10000000; for (int32_t i = 0; i < numOfTags; ++i) { int64_t* list = createTsList(num, start, step); + t.i64Key = i; - tsBufAppend(pTSBuf1, 12, i, (const char*)list, num * sizeof(int64_t)); + tsBufAppend(pTSBuf1, 12, &t, (const char*)list, num * sizeof(int64_t)); free(list); start += step * num; } + + for (int32_t i = numOfTags; i < numOfTags * 2; ++i) { int64_t* list = createTsList(num, start, step); - tsBufAppend(pTSBuf2, 77, i, (const char*)list, num * sizeof(int64_t)); + t.i64Key = i; + tsBufAppend(pTSBuf2, 77, &t, (const char*)list, num * sizeof(int64_t)); free(list); start += step * num; @@ -438,7 +482,7 @@ void mergeIdenticalVnodeBufferTest() { STSElem elem = tsBufGetElem(pTSBuf1); EXPECT_EQ(elem.vnode, 12); - printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag, elem.ts); + printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag.i64Key, elem.ts); } tsBufDestroy(pTSBuf1); @@ -446,6 +490,8 @@ void mergeIdenticalVnodeBufferTest() { } } // namespace + +//TODO add binary tag value test case TEST(testCase, tsBufTest) { simpleTest(); largeTSTest(); @@ -453,7 +499,7 @@ TEST(testCase, tsBufTest) { multiVnodeTagsTest(); loadDataTest(); invalidFileTest(); - // randomIncTsTest(); +// randomIncTsTest(); TSTraverse(); mergeDiffVnodeBufferTest(); mergeIdenticalVnodeBufferTest(); diff --git a/src/query/tests/unitTest.cpp b/src/query/tests/unitTest.cpp index b59e0783a2..b6e1170d0e 100644 --- a/src/query/tests/unitTest.cpp +++ b/src/query/tests/unitTest.cpp @@ -16,7 +16,7 @@ namespace { int32_t testValidateName(char* name) { - SSQLToken token = {0}; + SStrToken token = {0}; token.z = name; token.n = strlen(name); token.type = 0; @@ -720,8 +720,8 @@ TEST(testCase, tGetToken_Test) { EXPECT_FALSE(type == TK_HEX); } -static SSQLToken createStrToken(char* s) { - SSQLToken t = {0};//.type = TK_STRING, .z = s, .n = strlen(s)}; +static SStrToken createStrToken(char* s) { + SStrToken t = {0};//.type = TK_STRING, .z = s, .n = strlen(s)}; t.type = TK_STRING; t.z = s; t.n = strlen(s); @@ -730,7 +730,7 @@ static SSQLToken createStrToken(char* s) { } TEST(testCase, isValidNumber_test) { - SSQLToken t1 = createStrToken("123abc"); + SStrToken t1 = createStrToken("123abc"); EXPECT_EQ(isValidNumber(&t1), TK_ILLEGAL); diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index 7913b1c5b0..3aecd127ef 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -563,7 +563,7 @@ static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerFqdn, uint16_t peerPort, uint32_t peerIp = taosGetIpFromFqdn(peerFqdn); if (peerIp == 0xFFFFFFFF) { tError("%s, failed to resolve FQDN:%s", pRpc->label, peerFqdn); - terrno = TSDB_CODE_RPC_APP_ERROR; + terrno = TSDB_CODE_RPC_FQDN_ERROR; return NULL; } diff --git a/src/sync/inc/syncInt.h b/src/sync/inc/syncInt.h index be1e01cb23..cd1252f4b4 100644 --- a/src/sync/inc/syncInt.h +++ b/src/sync/inc/syncInt.h @@ -74,7 +74,7 @@ typedef struct { uint32_t magic; uint32_t index; uint64_t fversion; - int32_t size; + int64_t size; } SFileInfo; typedef struct { diff --git a/src/sync/inc/taosTcpPool.h b/src/sync/inc/taosTcpPool.h index 1e410acc26..5f7ca9ede5 100644 --- a/src/sync/inc/taosTcpPool.h +++ b/src/sync/inc/taosTcpPool.h @@ -26,7 +26,7 @@ typedef void* tthread_h; typedef struct { int numOfThreads; uint32_t serverIp; - short port; + int16_t port; int bufferSize; void (*processBrokenLink)(void *ahandle); int (*processIncomingMsg)(void *ahandle, void *buffer); diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index eaa073348a..b92ef4cf26 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -108,8 +108,7 @@ static void syncModuleInitFunc() { tstrncpy(tsNodeFqdn, tsLocalFqdn, sizeof(tsNodeFqdn)); } -void *syncStart(const SSyncInfo *pInfo) -{ +void *syncStart(const SSyncInfo *pInfo) { const SSyncCfg *pCfg = &pInfo->syncCfg; SSyncNode *pNode = (SSyncNode *) calloc(sizeof(SSyncNode), 1); @@ -189,9 +188,8 @@ void *syncStart(const SSyncInfo *pInfo) return pNode; } -void syncStop(void *param) -{ - SSyncNode *pNode = param; +void syncStop(void *param) { + SSyncNode * pNode = param; SSyncPeer *pPeer; if (pNode == NULL) return; @@ -215,9 +213,8 @@ void syncStop(void *param) syncDecNodeRef(pNode); } -int32_t syncReconfig(void *param, const SSyncCfg *pNewCfg) -{ - SSyncNode *pNode = param; +int32_t syncReconfig(void *param, const SSyncCfg *pNewCfg) { + SSyncNode * pNode = param; int i, j; if (pNode == NULL) return TSDB_CODE_SYN_INVALID_CONFIG; @@ -283,10 +280,9 @@ int32_t syncReconfig(void *param, const SSyncCfg *pNewCfg) return 0; } -int32_t syncForwardToPeer(void *param, void *data, void *mhandle, int qtype) -{ - SSyncNode *pNode = param; - SSyncPeer *pPeer; +int32_t syncForwardToPeer(void *param, void *data, void *mhandle, int qtype) { + SSyncNode * pNode = param; + SSyncPeer * pPeer; SSyncHead *pSyncHead; SWalHead *pWalHead = data; int fwdLen; @@ -334,9 +330,8 @@ int32_t syncForwardToPeer(void *param, void *data, void *mhandle, int qtype) return code; } -void syncConfirmForward(void *param, uint64_t version, int32_t code) -{ - SSyncNode *pNode = param; +void syncConfirmForward(void *param, uint64_t version, int32_t code) { + SSyncNode *pNode = param; if (pNode == NULL) return; if (pNode->quorum <= 1) return; @@ -387,10 +382,9 @@ void syncRecover(void *param) { pthread_mutex_unlock(&(pNode->mutex)); } -int syncGetNodesRole(void *param, SNodesRole *pNodesRole) -{ +int syncGetNodesRole(void *param, SNodesRole *pNodesRole) { SSyncNode *pNode = param; - + pNodesRole->selfIndex = pNode->selfIndex; for (int i=0; ireplica; ++i) { pNodesRole->nodeId[i] = pNode->peerInfo[i]->nodeId; @@ -400,8 +394,7 @@ int syncGetNodesRole(void *param, SNodesRole *pNodesRole) return 0; } -static void syncAddArbitrator(SSyncNode *pNode) -{ +static void syncAddArbitrator(SSyncNode *pNode) { SSyncPeer *pPeer = pNode->peerInfo[TAOS_SYNC_MAX_REPLICA]; // if not configured, return right away @@ -413,9 +406,11 @@ static void syncAddArbitrator(SSyncNode *pNode) SNodeInfo nodeInfo; nodeInfo.nodeId = 0; - taosGetFqdnPortFromEp(tsArbitrator, nodeInfo.nodeFqdn, &nodeInfo.nodePort); - nodeInfo.nodePort += TSDB_PORT_SYNC; - + int ret = taosGetFqdnPortFromEp(tsArbitrator, nodeInfo.nodeFqdn, &nodeInfo.nodePort); + if (-1 == ret) { + nodeInfo.nodePort = tsArbitratorPort; + } + if (pPeer) { if ((strcmp(nodeInfo.nodeFqdn, pPeer->fqdn) == 0) && (nodeInfo.nodePort == pPeer->port)) { return; @@ -454,13 +449,11 @@ static void syncDecNodeRef(SSyncNode *pNode) } } -void syncAddPeerRef(SSyncPeer *pPeer) -{ +void syncAddPeerRef(SSyncPeer *pPeer) { atomic_add_fetch_8(&pPeer->refCount, 1); } -int syncDecPeerRef(SSyncPeer *pPeer) -{ +int syncDecPeerRef(SSyncPeer *pPeer) { if (atomic_sub_fetch_8(&pPeer->refCount, 1) == 0) { syncDecNodeRef(pPeer->pSyncNode); @@ -473,18 +466,16 @@ int syncDecPeerRef(SSyncPeer *pPeer) return 1; } -static void syncClosePeerConn(SSyncPeer *pPeer) -{ +static void syncClosePeerConn(SSyncPeer *pPeer) { taosTmrStopA(&pPeer->timer); taosClose(pPeer->syncFd); - if (pPeer->peerFd >=0) { + if (pPeer->peerFd >= 0) { pPeer->peerFd = -1; taosFreeTcpConn(pPeer->pConn); } } -static void syncRemovePeer(SSyncPeer *pPeer) -{ +static void syncRemovePeer(SSyncPeer *pPeer) { sInfo("%s, it is removed", pPeer->id); pPeer->ip = 0; @@ -492,8 +483,7 @@ static void syncRemovePeer(SSyncPeer *pPeer) syncDecPeerRef(pPeer); } -static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) -{ +static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) { uint32_t ip = taosGetIpFromFqdn(pInfo->nodeFqdn); if (ip == -1) return NULL; @@ -523,25 +513,24 @@ static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) return pPeer; } -void syncBroadcastStatus(SSyncNode *pNode) -{ +void syncBroadcastStatus(SSyncNode *pNode) { SSyncPeer *pPeer; for (int i = 0; i < pNode->replica; ++i) { - if ( i == pNode->selfIndex ) continue; + if (i == pNode->selfIndex) continue; pPeer = pNode->peerInfo[i]; syncSendPeersStatusMsgToPeer(pPeer, 1); } -} +} static void syncResetFlowCtrl(SSyncNode *pNode) { - for (int i = 0; i < pNode->replica; ++i) { pNode->peerInfo[i]->numOfRetrieves = 0; } - if (pNode->notifyFlowCtrl) - (*pNode->notifyFlowCtrl)(pNode->ahandle, 0); + if (pNode->notifyFlowCtrl) { + (*pNode->notifyFlowCtrl)(pNode->ahandle, 0); + } } static void syncChooseMaster(SSyncNode *pNode) { @@ -598,9 +587,9 @@ static void syncChooseMaster(SSyncNode *pNode) { } else { sDebug("vgId:%d, failed to choose master", pNode->vgId); } -} - -static SSyncPeer *syncCheckMaster(SSyncNode *pNode ) { +} + +static SSyncPeer *syncCheckMaster(SSyncNode *pNode) { int onlineNum = 0; int index = -1; int replica = pNode->replica; @@ -617,7 +606,7 @@ static SSyncPeer *syncCheckMaster(SSyncNode *pNode ) { replica = pNode->replica + 1; } - if (onlineNum <= replica*0.5) { + if (onlineNum <= replica * 0.5) { if (nodeRole != TAOS_SYNC_ROLE_UNSYNCED) { nodeRole = TAOS_SYNC_ROLE_UNSYNCED; pNode->peerInfo[pNode->selfIndex]->role = nodeRole; @@ -625,13 +614,13 @@ static SSyncPeer *syncCheckMaster(SSyncNode *pNode ) { sInfo("vgId:%d, change to unsynced state, online:%d replica:%d", pNode->vgId, onlineNum, replica); } } else { - for (int i=0; ireplica; ++i) { + for (int i = 0; i < pNode->replica; ++i) { SSyncPeer *pTemp = pNode->peerInfo[i]; - if ( pTemp->role != TAOS_SYNC_ROLE_MASTER ) continue; - if ( index < 0 ) { + if (pTemp->role != TAOS_SYNC_ROLE_MASTER) continue; + if (index < 0) { index = i; - } else { // multiple masters, it shall not happen - if ( i == pNode->selfIndex ) { + } else { // multiple masters, it shall not happen + if (i == pNode->selfIndex) { sError("%s, peer is master, work as slave instead", pTemp->id); nodeRole = TAOS_SYNC_ROLE_SLAVE; (*pNode->notifyRole)(pNode->ahandle, nodeRole); @@ -640,7 +629,7 @@ static SSyncPeer *syncCheckMaster(SSyncNode *pNode ) { } } - SSyncPeer *pMaster = (index>=0) ? pNode->peerInfo[index]:NULL; + SSyncPeer *pMaster = (index >= 0) ? pNode->peerInfo[index] : NULL; return pMaster; } @@ -649,7 +638,7 @@ static int syncValidateMaster(SSyncPeer *pPeer) { int code = 0; if (nodeRole == TAOS_SYNC_ROLE_MASTER && nodeVersion < pPeer->version) { - sDebug("%s, slave has higher version, restart all connections!!!", pPeer->id); + sDebug("%s, slave has higher version, restart all connections!!!", pPeer->id); nodeRole = TAOS_SYNC_ROLE_UNSYNCED; (*pNode->notifyRole)(pNode->ahandle, nodeRole); code = -1; @@ -658,13 +647,12 @@ static int syncValidateMaster(SSyncPeer *pPeer) { if ( i == pNode->selfIndex ) continue; syncRestartPeer(pNode->peerInfo[i]); } - } + } return code; } -static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus peersStatus[], int8_t newRole) -{ +static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus peersStatus[], int8_t newRole) { SSyncNode *pNode = pPeer->pSyncNode; int8_t peerOldRole = pPeer->role; int8_t selfOldRole = nodeRole; @@ -686,14 +674,14 @@ static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus peersStatus[], int8_t ne if (syncValidateMaster(pPeer) < 0) return; if (nodeRole == TAOS_SYNC_ROLE_UNSYNCED) { - if ( nodeVersion < pMaster->version) { + if (nodeVersion < pMaster->version) { syncRequired = 1; } else { sInfo("%s is master, work as slave, ver:%" PRIu64, pMaster->id, pMaster->version); nodeRole = TAOS_SYNC_ROLE_SLAVE; (*pNode->notifyRole)(pNode->ahandle, nodeRole); } - } else if ( nodeRole == TAOS_SYNC_ROLE_SLAVE && pMaster == pPeer) { + } else if (nodeRole == TAOS_SYNC_ROLE_SLAVE && pMaster == pPeer) { // nodeVersion = pMaster->version; } } else { @@ -734,20 +722,18 @@ static void syncRestartPeer(SSyncPeer *pPeer) { pPeer->sstatus = TAOS_SYNC_STATUS_INIT; int ret = strcmp(pPeer->fqdn, tsNodeFqdn); - if (ret > 0 || (ret == 0 && pPeer->port > tsSyncPort) ) - taosTmrReset(syncCheckPeerConnection, tsSyncTimer*1000, pPeer, syncTmrCtrl, &pPeer->timer); + if (ret > 0 || (ret == 0 && pPeer->port > tsSyncPort)) + taosTmrReset(syncCheckPeerConnection, tsSyncTimer * 1000, pPeer, syncTmrCtrl, &pPeer->timer); } -void syncRestartConnection(SSyncPeer *pPeer) -{ +void syncRestartConnection(SSyncPeer *pPeer) { if (pPeer->ip == 0) return; syncRestartPeer(pPeer); syncCheckRole(pPeer, NULL, TAOS_SYNC_ROLE_OFFLINE); } -static void syncProcessSyncRequest(char *msg, SSyncPeer *pPeer) -{ +static void syncProcessSyncRequest(char *msg, SSyncPeer *pPeer) { SSyncNode *pNode = pPeer->pSyncNode; sDebug("%s, sync-req is received", pPeer->id); @@ -782,8 +768,7 @@ static void syncProcessSyncRequest(char *msg, SSyncPeer *pPeer) } } -static void syncNotStarted(void *param, void *tmrId) -{ +static void syncNotStarted(void *param, void *tmrId) { SSyncPeer *pPeer = param; SSyncNode *pNode = pPeer->pSyncNode; @@ -803,14 +788,13 @@ static void syncTryRecoverFromMaster(void *param, void *tmrId) { pthread_mutex_unlock(&(pNode->mutex)); } -static void syncRecoverFromMaster(SSyncPeer *pPeer) -{ - SSyncNode *pNode = pPeer->pSyncNode; +static void syncRecoverFromMaster(SSyncPeer *pPeer) { + SSyncNode *pNode = pPeer->pSyncNode; - if ( nodeSStatus != TAOS_SYNC_STATUS_INIT) { + if (nodeSStatus != TAOS_SYNC_STATUS_INIT) { sDebug("%s, sync is already started, status:%d", pPeer->id, nodeSStatus); return; - } + } taosTmrStopA(&pPeer->timer); if (tsSyncNum >= tsMaxSyncNum) { @@ -840,9 +824,8 @@ static void syncRecoverFromMaster(SSyncPeer *pPeer) return; } -static void syncProcessFwdResponse(char *cont, SSyncPeer *pPeer) -{ - SSyncNode *pNode = pPeer->pSyncNode; +static void syncProcessFwdResponse(char *cont, SSyncPeer *pPeer) { + SSyncNode * pNode = pPeer->pSyncNode; SFwdRsp *pFwdRsp = (SFwdRsp *) cont; SSyncFwds *pSyncFwds = pNode->pSyncFwds; SFwdInfo *pFwdInfo; @@ -862,10 +845,8 @@ static void syncProcessFwdResponse(char *cont, SSyncPeer *pPeer) } } - -static void syncProcessForwardFromPeer(char *cont, SSyncPeer *pPeer) -{ - SSyncNode *pNode = pPeer->pSyncNode; +static void syncProcessForwardFromPeer(char *cont, SSyncPeer *pPeer) { + SSyncNode * pNode = pPeer->pSyncNode; SWalHead *pHead = (SWalHead *)cont; sDebug("%s, forward is received, ver:%" PRIu64, pPeer->id, pHead->version); @@ -884,9 +865,8 @@ static void syncProcessForwardFromPeer(char *cont, SSyncPeer *pPeer) return; } -static void syncProcessPeersStatusMsg(char *cont, SSyncPeer *pPeer) -{ - SSyncNode *pNode = pPeer->pSyncNode; +static void syncProcessPeersStatusMsg(char *cont, SSyncPeer *pPeer) { + SSyncNode * pNode = pPeer->pSyncNode; SPeersStatus *pPeersStatus = (SPeersStatus *)cont; sDebug("%s, status msg received, self:%s ver:%" PRIu64 " peer:%s ver:%" PRIu64 ", ack:%d", pPeer->id, @@ -909,10 +889,10 @@ static int syncReadPeerMsg(SSyncPeer *pPeer, SSyncHead *pHead, char *cont) { } // head.len = htonl(head.len); - if (pHead->len <0) { + if (pHead->len < 0) { sError("%s, invalid pkt length, len:%d", pPeer->id, pHead->len); return -1; - } + } int bytes = taosReadMsg(pPeer->peerFd, cont, pHead->len); if (bytes != pHead->len) { @@ -923,9 +903,8 @@ static int syncReadPeerMsg(SSyncPeer *pPeer, SSyncHead *pHead, char *cont) { return 0; } -static int syncProcessPeerMsg(void *param, void *buffer) -{ - SSyncPeer *pPeer = param; +static int syncProcessPeerMsg(void *param, void *buffer) { + SSyncPeer * pPeer = param; SSyncHead head; char *cont = (char *)buffer; @@ -953,8 +932,7 @@ static int syncProcessPeerMsg(void *param, void *buffer) #define statusMsgLen sizeof(SSyncHead)+sizeof(SPeersStatus)+sizeof(SPeerStatus)*TAOS_SYNC_MAX_REPLICA -static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack) -{ +static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack) { SSyncNode *pNode = pPeer->pSyncNode; char msg[statusMsgLen] = {0}; @@ -1011,7 +989,7 @@ static void syncSetupPeerConnection(SSyncPeer *pPeer) { firstPkt.port = tsSyncPort; firstPkt.sourceId = pNode->vgId; // tell arbitrator its vgId - if ( write(connFd, &firstPkt, sizeof(firstPkt)) == sizeof(firstPkt)) { + if (write(connFd, &firstPkt, sizeof(firstPkt)) == sizeof(firstPkt)) { sDebug("%s, connection to peer server is setup", pPeer->id); pPeer->peerFd = connFd; pPeer->role = TAOS_SYNC_ROLE_UNSYNCED; @@ -1024,8 +1002,7 @@ static void syncSetupPeerConnection(SSyncPeer *pPeer) { } } -static void syncCheckPeerConnection(void *param, void *tmrId) -{ +static void syncCheckPeerConnection(void *param, void *tmrId) { SSyncPeer *pPeer = param; SSyncNode *pNode = pPeer->pSyncNode; @@ -1037,8 +1014,7 @@ static void syncCheckPeerConnection(void *param, void *tmrId) pthread_mutex_unlock(&(pNode->mutex)); } -static void syncCreateRestoreDataThread(SSyncPeer *pPeer) -{ +static void syncCreateRestoreDataThread(SSyncPeer *pPeer) { taosTmrStopA(&pPeer->timer); pthread_attr_t thattr; @@ -1059,8 +1035,7 @@ static void syncCreateRestoreDataThread(SSyncPeer *pPeer) } } -static void syncProcessIncommingConnection(int connFd, uint32_t sourceIp) -{ +static void syncProcessIncommingConnection(int connFd, uint32_t sourceIp) { char ipstr[24]; int i; @@ -1137,8 +1112,7 @@ static void syncProcessBrokenLink(void *param) { syncDecNodeRef(pNode); } -static void syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle) -{ +static void syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle) { SSyncFwds *pSyncFwds = pNode->pSyncFwds; uint64_t time = taosGetTimestampMs(); @@ -1160,8 +1134,7 @@ static void syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle) sDebug("vgId:%d, fwd info is saved, ver:%" PRIu64 " fwds:%d ", pNode->vgId, version, pSyncFwds->fwds); } -static void syncRemoveConfirmedFwdInfo(SSyncNode *pNode) -{ +static void syncRemoveConfirmedFwdInfo(SSyncNode *pNode) { SSyncFwds *pSyncFwds = pNode->pSyncFwds; int fwds = pSyncFwds->fwds; @@ -1178,8 +1151,7 @@ static void syncRemoveConfirmedFwdInfo(SSyncNode *pNode) } } -static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code) -{ +static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code) { int confirm = 0; if (pFwdInfo->code == 0) pFwdInfo->code = code; @@ -1200,8 +1172,7 @@ static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code } } -static void syncMonitorFwdInfos(void *param, void *tmrId) -{ +static void syncMonitorFwdInfos(void *param, void *tmrId) { SSyncNode *pNode = param; SSyncFwds *pSyncFwds = pNode->pSyncFwds; uint64_t time = taosGetTimestampMs(); @@ -1220,6 +1191,3 @@ static void syncMonitorFwdInfos(void *param, void *tmrId) pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, 300, pNode, syncTmrCtrl); } - - - diff --git a/src/sync/src/syncRestore.c b/src/sync/src/syncRestore.c index 505882b209..2a0bee3726 100644 --- a/src/sync/src/syncRestore.c +++ b/src/sync/src/syncRestore.c @@ -28,7 +28,7 @@ static void syncRemoveExtraFile(SSyncPeer *pPeer, uint32_t sindex, uint32_t eind char fname[TSDB_FILENAME_LEN*3] = {0}; uint32_t magic; uint64_t fversion; - int32_t size; + int64_t size; uint32_t index = sindex; SSyncNode *pNode = pPeer->pSyncNode; @@ -48,8 +48,7 @@ static void syncRemoveExtraFile(SSyncPeer *pPeer, uint32_t sindex, uint32_t eind } } -static int syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) -{ +static int syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { SSyncNode *pNode = pPeer->pSyncNode; SFileInfo minfo; memset(&minfo, 0, sizeof(minfo)); /* = {0}; */ // master file info SFileInfo sinfo; memset(&sinfo, 0, sizeof(sinfo)); /* = {0}; */ // slave file info @@ -113,7 +112,7 @@ static int syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) close(dfd); if (ret<0) break; - sDebug("%s, %s is received, size:%d", pPeer->id, minfo.name, minfo.size); + sDebug("%s, %s is received, size:%" PRId64, pPeer->id, minfo.name, minfo.size); } @@ -130,8 +129,7 @@ static int syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) return code; } -static int syncRestoreWal(SSyncPeer *pPeer) -{ +static int syncRestoreWal(SSyncPeer *pPeer) { SSyncNode *pNode = pPeer->pSyncNode; int ret, code = -1; @@ -172,8 +170,7 @@ static char *syncProcessOneBufferedFwd(SSyncPeer *pPeer, char *offset) return offset; } -static int syncProcessBufferedFwd(SSyncPeer *pPeer) -{ +static int syncProcessBufferedFwd(SSyncPeer *pPeer) { SSyncNode *pNode = pPeer->pSyncNode; SRecvBuffer *pRecv = pNode->pRecv; int forwards = 0; @@ -201,8 +198,7 @@ static int syncProcessBufferedFwd(SSyncPeer *pPeer) return pRecv->code; } -int syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead) -{ +int syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead) { SSyncNode *pNode = pPeer->pSyncNode; SRecvBuffer *pRecv = pNode->pRecv; @@ -222,8 +218,7 @@ int syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead) return pRecv->code; } -static void syncCloseRecvBuffer(SSyncNode *pNode) -{ +static void syncCloseRecvBuffer(SSyncNode *pNode) { if (pNode->pRecv) { taosTFree(pNode->pRecv->buffer); } @@ -231,8 +226,7 @@ static void syncCloseRecvBuffer(SSyncNode *pNode) taosTFree(pNode->pRecv); } -static int syncOpenRecvBuffer(SSyncNode *pNode) -{ +static int syncOpenRecvBuffer(SSyncNode *pNode) { syncCloseRecvBuffer(pNode); SRecvBuffer *pRecv = calloc(sizeof(SRecvBuffer), 1); @@ -253,8 +247,7 @@ static int syncOpenRecvBuffer(SSyncNode *pNode) return 0; } -static int syncRestoreDataStepByStep(SSyncPeer *pPeer) -{ +static int syncRestoreDataStepByStep(SSyncPeer *pPeer) { SSyncNode *pNode = pPeer->pSyncNode; nodeSStatus = TAOS_SYNC_STATUS_FILE; uint64_t fversion = 0; @@ -292,10 +285,9 @@ static int syncRestoreDataStepByStep(SSyncPeer *pPeer) return 0; } -void *syncRestoreData(void *param) -{ - SSyncPeer *pPeer = (SSyncPeer *)param; - SSyncNode *pNode = pPeer->pSyncNode; +void *syncRestoreData(void *param) { + SSyncPeer *pPeer = (SSyncPeer *)param; + SSyncNode *pNode = pPeer->pSyncNode; taosBlockSIGPIPE(); __sync_fetch_and_add(&tsSyncNum, 1); @@ -326,4 +318,3 @@ void *syncRestoreData(void *param) return NULL; } - diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c index 1dd1cda343..8aa317b1ac 100644 --- a/src/sync/src/syncRetrieve.c +++ b/src/sync/src/syncRetrieve.c @@ -27,11 +27,10 @@ #include "tsync.h" #include "syncInt.h" -static int syncAddIntoWatchList(SSyncPeer *pPeer, char *name) -{ +static int syncAddIntoWatchList(SSyncPeer *pPeer, char *name) { sDebug("%s, start to monitor:%s", pPeer->id, name); - if (pPeer->notifyFd <=0) { + if (pPeer->notifyFd <= 0) { pPeer->watchNum = 0; pPeer->notifyFd = inotify_init1(IN_NONBLOCK); if (pPeer->notifyFd < 0) { @@ -70,9 +69,8 @@ static int syncAddIntoWatchList(SSyncPeer *pPeer, char *name) return 0; } -static int syncAreFilesModified(SSyncPeer *pPeer) -{ - if (pPeer->notifyFd <=0) return 0; +static int syncAreFilesModified(SSyncPeer *pPeer) { + if (pPeer->notifyFd <= 0) return 0; char buf[2048]; int len = read(pPeer->notifyFd, buf, sizeof(buf)); @@ -96,12 +94,11 @@ static int syncAreFilesModified(SSyncPeer *pPeer) } } - return code; + return code; } -static int syncRetrieveFile(SSyncPeer *pPeer) -{ - SSyncNode *pNode = pPeer->pSyncNode; +static int syncRetrieveFile(SSyncPeer *pPeer) { + SSyncNode * pNode = pPeer->pSyncNode; SFileInfo fileInfo; SFileAck fileAck; int code = -1; @@ -128,7 +125,7 @@ static int syncRetrieveFile(SSyncPeer *pPeer) // wait for the ack from peer ret = taosReadMsg(pPeer->syncFd, &(fileAck), sizeof(fileAck)); - if (ret <0) break; + if (ret < 0) break; // set the peer sync version pPeer->sversion = fileInfo.fversion; @@ -148,13 +145,13 @@ static int syncRetrieveFile(SSyncPeer *pPeer) // send the file to peer int sfd = open(name, O_RDONLY); - if ( sfd < 0 ) break; + if (sfd < 0) break; ret = taosTSendFile(pPeer->syncFd, sfd, NULL, fileInfo.size); - close(sfd); - if (ret <0) break; + close(sfd); + if (ret < 0) break; - sDebug("%s, %s is sent, size:%d", pPeer->id, name, fileInfo.size); + sDebug("%s, %s is sent, size:%" PRId64, pPeer->id, name, fileInfo.size); fileInfo.index++; // check if processed files are modified @@ -170,8 +167,7 @@ static int syncRetrieveFile(SSyncPeer *pPeer) /* if only a partial record is read out, set the IN_MODIFY flag in event, so upper layer will reload the file to get a complete record */ -static int syncReadOneWalRecord(int sfd, SWalHead *pHead, uint32_t *pEvent) -{ +static int syncReadOneWalRecord(int sfd, SWalHead *pHead, uint32_t *pEvent) { int ret; ret = read(sfd, pHead, sizeof(SWalHead)); @@ -185,7 +181,7 @@ static int syncReadOneWalRecord(int sfd, SWalHead *pHead, uint32_t *pEvent) } ret = read(sfd, pHead->cont, pHead->len); - if (ret <0) return -1; + if (ret < 0) return -1; if (ret != pHead->len) { // file is not at end yet, it shall be reloaded @@ -194,10 +190,9 @@ static int syncReadOneWalRecord(int sfd, SWalHead *pHead, uint32_t *pEvent) } return sizeof(SWalHead) + pHead->len; -} +} -static int syncMonitorLastWal(SSyncPeer *pPeer, char *name) -{ +static int syncMonitorLastWal(SSyncPeer *pPeer, char *name) { pPeer->watchNum = 0; taosClose(pPeer->notifyFd); pPeer->notifyFd = inotify_init1(IN_NONBLOCK); @@ -221,18 +216,17 @@ static int syncMonitorLastWal(SSyncPeer *pPeer, char *name) return -1; } - return 0; + return 0; } -static uint32_t syncCheckLastWalChanges(SSyncPeer *pPeer, uint32_t *pEvent) -{ +static uint32_t syncCheckLastWalChanges(SSyncPeer *pPeer, uint32_t *pEvent) { char buf[2048]; int len = read(pPeer->notifyFd, buf, sizeof(buf)); - if (len <0 && errno != EAGAIN) { - sError("%s, failed to read notify FD(%s)", pPeer->id, strerror(errno)); + if (len < 0 && errno != EAGAIN) { + sError("%s, failed to read notify FD(%s)", pPeer->id, strerror(errno)); return -1; } - + if (len == 0) return 0; struct inotify_event *event; @@ -248,8 +242,7 @@ static uint32_t syncCheckLastWalChanges(SSyncPeer *pPeer, uint32_t *pEvent) return 0; } -static int syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion, int64_t offset, uint32_t *pEvent) -{ +static int syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion, int64_t offset, uint32_t *pEvent) { SWalHead *pHead = (SWalHead *) malloc(640000); int code = -1; int32_t bytes = 0; @@ -261,9 +254,12 @@ static int syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion, sDebug("%s, retrieve last wal, offset:%" PRId64 " fversion:%" PRIu64, pPeer->id, offset, fversion); while (1) { - int wsize = syncReadOneWalRecord(sfd, pHead, pEvent); - if (wsize <0) break; - if (wsize == 0) { code = 0; break; } + int wsize = syncReadOneWalRecord(sfd, pHead, pEvent); + if (wsize < 0) break; + if (wsize == 0) { + code = 0; + break; + } sDebug("%s, last wal is forwarded, ver:%" PRIu64, pPeer->id, pHead->version); int ret = taosWriteMsg(pPeer->syncFd, pHead, wsize); @@ -286,8 +282,7 @@ static int syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion, return -1; } -static int syncProcessLastWal(SSyncPeer *pPeer, char *wname, uint32_t index) -{ +static int syncProcessLastWal(SSyncPeer *pPeer, char *wname, uint32_t index) { SSyncNode *pNode = pPeer->pSyncNode; int code = -1; char fname[TSDB_FILENAME_LEN * 2]; // full path to wal file @@ -350,12 +345,16 @@ static int syncProcessLastWal(SSyncPeer *pPeer, char *wname, uint32_t index) } if (code < 0) break; - if (pPeer->sversion >= fversion && fversion > 0) break; + if (pPeer->sversion >= fversion && fversion > 0) break; - index++; wname[0] = 0; + index++; + wname[0] = 0; code = (*pNode->getWalInfo)(pNode->ahandle, wname, &index); - if ( code < 0) break; - if ( wname[0] == 0 ) {code = 0; break;} + if (code < 0) break; + if (wname[0] == 0) { + code = 0; + break; + } // current last wal is closed, there is a new one sDebug("%s, last wal is closed, try new one", pPeer->id); @@ -366,9 +365,8 @@ static int syncProcessLastWal(SSyncPeer *pPeer, char *wname, uint32_t index) return code; } -static int syncRetrieveWal(SSyncPeer *pPeer) -{ - SSyncNode *pNode = pPeer->pSyncNode; +static int syncRetrieveWal(SSyncPeer *pPeer) { + SSyncNode * pNode = pPeer->pSyncNode; char fname[TSDB_FILENAME_LEN * 3]; char wname[TSDB_FILENAME_LEN * 2]; int32_t size; @@ -396,7 +394,7 @@ static int syncRetrieveWal(SSyncPeer *pPeer) // send wal file, // inotify is not required, old wal file won't be modified, even remove is ok - if ( stat(fname, &fstat) < 0 ) break; + if (stat(fname, &fstat) < 0) break; size = fstat.st_size; sDebug("%s, retrieve wal:%s size:%d", pPeer->id, fname, size); @@ -425,9 +423,8 @@ static int syncRetrieveWal(SSyncPeer *pPeer) return code; } -static int syncRetrieveDataStepByStep(SSyncPeer *pPeer) -{ - SSyncNode *pNode = pPeer->pSyncNode; +static int syncRetrieveDataStepByStep(SSyncPeer *pPeer) { + SSyncNode *pNode = pPeer->pSyncNode; SFirstPkt firstPkt; memset(&firstPkt, 0, sizeof(firstPkt)); @@ -462,9 +459,8 @@ static int syncRetrieveDataStepByStep(SSyncPeer *pPeer) return 0; } -void *syncRetrieveData(void *param) -{ - SSyncPeer *pPeer = (SSyncPeer *)param; +void *syncRetrieveData(void *param) { + SSyncPeer * pPeer = (SSyncPeer *)param; SSyncNode *pNode = pPeer->pSyncNode; taosBlockSIGPIPE(); diff --git a/src/sync/src/taosTcpPool.c b/src/sync/src/taosTcpPool.c index b523728bf9..2f064ceb36 100644 --- a/src/sync/src/taosTcpPool.c +++ b/src/sync/src/taosTcpPool.c @@ -48,8 +48,7 @@ static void *taosProcessTcpData(void *param); static SThreadObj *taosGetTcpThread(SPoolObj *pPool); static void taosStopPoolThread(SThreadObj* pThread); -void *taosOpenTcpThreadPool(SPoolInfo *pInfo) -{ +void *taosOpenTcpThreadPool(SPoolInfo *pInfo) { pthread_attr_t thattr; SPoolObj *pPool = calloc(sizeof(SPoolObj), 1); @@ -89,8 +88,7 @@ void *taosOpenTcpThreadPool(SPoolInfo *pInfo) return pPool; } -void taosCloseTcpThreadPool(void *param) -{ +void taosCloseTcpThreadPool(void *param) { SPoolObj *pPool = (SPoolObj *)param; SThreadObj *pThread; @@ -107,8 +105,7 @@ void taosCloseTcpThreadPool(void *param) uDebug("%p TCP pool is closed", pPool); } -void *taosAllocateTcpConn(void *param, void *pPeer, int connFd) -{ +void *taosAllocateTcpConn(void *param, void *pPeer, int connFd) { struct epoll_event event; SPoolObj *pPool = (SPoolObj *)param; @@ -145,9 +142,8 @@ void *taosAllocateTcpConn(void *param, void *pPeer, int connFd) return pConn; } -void taosFreeTcpConn(void *param) -{ - SConnObj *pConn = (SConnObj *)param; +void taosFreeTcpConn(void *param) { + SConnObj * pConn = (SConnObj *)param; SThreadObj *pThread = pConn->pThread; uDebug("%p TCP connection will be closed, fd:%d", pThread, pConn->fd); diff --git a/src/sync/src/tarbitrator.c b/src/sync/src/tarbitrator.c index 3538391a94..b704b1ecae 100644 --- a/src/sync/src/tarbitrator.c +++ b/src/sync/src/tarbitrator.c @@ -45,7 +45,7 @@ int main(int argc, char *argv[]) { for (int i=1; itsdbMeta; STsdbFileH *pFileH = pRepo->tsdbFileH; diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index ac3a6dac07..f8ff25ddab 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -28,6 +28,14 @@ #define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) #define QH_GET_NUM_OF_COLS(handle) ((size_t)(taosArrayGetSize((handle)->pColumns))) +#define GET_FILE_DATA_BLOCK_INFO(_checkInfo, _block) \ + ((SDataBlockInfo){.window = {.skey = (_block)->keyFirst, .ekey = (_block)->keyLast}, \ + .numOfCols = (_block)->numOfCols, \ + .rows = (_block)->numOfRows, \ + .tid = (_checkInfo)->tableId.tid, \ + .uid = (_checkInfo)->tableId.uid}) + + enum { TSDB_QUERY_TYPE_ALL = 1, TSDB_QUERY_TYPE_LAST = 2, @@ -119,7 +127,14 @@ typedef struct STsdbQueryHandle { SIOCostSummary cost; } STsdbQueryHandle; -static void changeQueryHandleForLastrowQuery(TsdbQueryHandleT pqHandle); +typedef struct STableGroupSupporter { + int32_t numOfCols; + SColIndex* pCols; + STSchema* pTagSchema; +} STableGroupSupporter; + +static STimeWindow changeTableGroupByLastrow(STableGroupInfo *groupList); + static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle); static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock); static int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order); @@ -242,10 +257,7 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab for (int32_t j = 0; j < gsize; ++j) { STableKeyInfo* pKeyInfo = (STableKeyInfo*) taosArrayGet(group, j); - STableCheckInfo info = { - .lastKey = pKeyInfo->lastKey, - .pTableObj = pKeyInfo->pTable, - }; + STableCheckInfo info = { .lastKey = pKeyInfo->lastKey, .pTableObj = pKeyInfo->pTable }; info.tableId = ((STable*)(pKeyInfo->pTable))->tableId; assert(info.pTableObj != NULL && (info.pTableObj->type == TSDB_NORMAL_TABLE || @@ -283,12 +295,9 @@ out_of_memory: } TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, void* qinfo) { + pCond->order = TSDB_ORDER_ASC; + pCond->twindow = changeTableGroupByLastrow(groupList); STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qinfo); - if (pQueryHandle != NULL) { - pQueryHandle->type = TSDB_QUERY_TYPE_LAST; - pQueryHandle->order = TSDB_ORDER_DESC; - changeQueryHandleForLastrowQuery(pQueryHandle); - } return pQueryHandle; } @@ -393,7 +402,7 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh SDataRow row = *(SDataRow *)SL_GET_NODE_DATA(node); TSKEY key = dataRowKey(row); // first timestamp in buffer - tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in mem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 + tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in imem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 "-%" PRId64 ", lastKey:%" PRId64 ", %p", pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, key, order, pIMem->keyFirst, pIMem->keyLast, pCheckInfo->lastKey, pHandle->qinfo); @@ -416,7 +425,7 @@ static void destroyTableMemIterator(STableCheckInfo* pCheckInfo) { tSkipListDestroyIter(pCheckInfo->iiter); } -SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order) { +static SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order) { SDataRow rmem = NULL, rimem = NULL; if (pCheckInfo->iter) { SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter); @@ -432,47 +441,46 @@ SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order) { } } - if (rmem != NULL && rimem != NULL) { - TSKEY r1 = dataRowKey(rmem); - TSKEY r2 = dataRowKey(rimem); - - if (r1 == r2) { // data ts are duplicated, ignore the data in mem - tSkipListIterNext(pCheckInfo->iter); - pCheckInfo->chosen = 1; - return rimem; - } else { - if (ASCENDING_TRAVERSE(order)) { - if (r1 < r2) { - pCheckInfo->chosen = 0; - return rmem; - } else { - pCheckInfo->chosen = 1; - return rimem; - } - } else { - if (r1 < r2) { - pCheckInfo->chosen = 1; - return rimem; - } else { - pCheckInfo->chosen = 0; - return rmem; - } - } - } + if (rmem == NULL && rimem == NULL) { + return NULL; } - // at least one (rmem or rimem) is absent here - if (rmem != NULL) { + if (rmem != NULL && rimem == NULL) { pCheckInfo->chosen = 0; return rmem; } - if (rimem != NULL) { + if (rmem == NULL && rimem != NULL) { pCheckInfo->chosen = 1; return rimem; } - return NULL; + TSKEY r1 = dataRowKey(rmem); + TSKEY r2 = dataRowKey(rimem); + + if (r1 == r2) { // data ts are duplicated, ignore the data in mem + tSkipListIterNext(pCheckInfo->iter); + pCheckInfo->chosen = 1; + return rimem; + } else { + if (ASCENDING_TRAVERSE(order)) { + if (r1 < r2) { + pCheckInfo->chosen = 0; + return rmem; + } else { + pCheckInfo->chosen = 1; + return rimem; + } + } else { + if (r1 < r2) { + pCheckInfo->chosen = 1; + return rimem; + } else { + pCheckInfo->chosen = 0; + return rmem; + } + } + } } static bool moveToNextRowInMem(STableCheckInfo* pCheckInfo) { @@ -672,14 +680,6 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo return code; } -#define GET_FILE_DATA_BLOCK_INFO(_checkInfo, _block) \ - ((SDataBlockInfo){.window = {.skey = (_block)->keyFirst, .ekey = (_block)->keyLast}, \ - .numOfCols = (_block)->numOfCols, \ - .rows = (_block)->numOfRows, \ - .tid = (_checkInfo)->tableId.tid, \ - .uid = (_checkInfo)->tableId.uid}) - - static int32_t doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo, int32_t slotIndex) { STsdbRepo *pRepo = pQueryHandle->pTsdb; int64_t st = taosGetTimestampUs(); @@ -736,6 +736,8 @@ static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBloc SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order); TSKEY key = (row != NULL)? dataRowKey(row):TSKEY_INITIAL_VAL; + tsdbDebug("%p key in mem:%"PRId64", %p", pQueryHandle, key, pQueryHandle->qinfo); + cur->pos = ASCENDING_TRAVERSE(pQueryHandle->order)? 0:(binfo.rows-1); if ((ASCENDING_TRAVERSE(pQueryHandle->order) && (key != TSKEY_INITIAL_VAL && key <= binfo.window.ekey)) || @@ -747,7 +749,8 @@ static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBloc // do not load file block into buffer int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order) ? 1 : -1; - cur->rows = tsdbReadRowsFromCache(pCheckInfo, binfo.window.skey - step, pQueryHandle->outputCapacity, &cur->win, pQueryHandle); + TSKEY maxKey = ASCENDING_TRAVERSE(pQueryHandle->order)? (binfo.window.skey - step):(binfo.window.ekey - step); + cur->rows = tsdbReadRowsFromCache(pCheckInfo, maxKey, pQueryHandle->outputCapacity, &cur->win, pQueryHandle); pQueryHandle->realNumOfRows = cur->rows; // update the last key value @@ -1072,8 +1075,8 @@ static void moveDataToFront(STsdbQueryHandle* pQueryHandle, int32_t numOfRows, i } } -static void getQualifiedRowsPos(STsdbQueryHandle* pQueryHandle, int32_t startPos, int32_t endPos, - int32_t numOfExisted, int32_t *start, int32_t *end) { +static void getQualifiedRowsPos(STsdbQueryHandle* pQueryHandle, int32_t startPos, int32_t endPos, int32_t numOfExisted, + int32_t* start, int32_t* end) { *start = -1; if (ASCENDING_TRAVERSE(pQueryHandle->order)) { @@ -1162,6 +1165,11 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* cur->mixBlock = true; } + tsdbDebug("%p uid:%" PRIu64",tid:%d start merge data block, file block range:%"PRIu64"-%"PRIu64" rows:%d, start:%d," + "end:%d, %p", + pQueryHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, blockInfo.window.skey, blockInfo.window.ekey, + blockInfo.rows, cur->pos, endPos, pQueryHandle->qinfo); + // compared with the data from in-memory buffer, to generate the correct timestamp array list int32_t numOfRows = 0; int32_t pos = cur->pos; @@ -1191,6 +1199,10 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* moveDataToFront(pQueryHandle, numOfRows, numOfCols); updateInfoAfterMerge(pQueryHandle, pCheckInfo, numOfRows, pos); doCheckGeneratedBlockRange(pQueryHandle); + + tsdbDebug("%p uid:%" PRIu64",tid:%d data block created, mixblock:%d, brange:%"PRIu64"-%"PRIu64" rows:%d, %p", + pQueryHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, cur->mixBlock, cur->win.skey, + cur->win.ekey, cur->rows, pQueryHandle->qinfo); return; } else if (pCheckInfo->iter != NULL || pCheckInfo->iiter != NULL) { SSkipListNode* node = NULL; @@ -1233,6 +1245,8 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* } int32_t end = doBinarySearchKey(pCols->cols[0].pData, pCols->numOfRows, key, order); + assert(end != -1); + if (tsArray[end] == key) { // the value of key in cache equals to the end timestamp value, ignore it moveToNextRowInMem(pCheckInfo); } @@ -1269,6 +1283,7 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* cur->win.ekey = ASCENDING_TRAVERSE(pQueryHandle->order)? tsArray[end]:tsArray[start]; cur->lastKey = cur->win.ekey + step; + cur->mixBlock = true; } } } @@ -1285,7 +1300,8 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* updateInfoAfterMerge(pQueryHandle, pCheckInfo, numOfRows, pos); doCheckGeneratedBlockRange(pQueryHandle); - tsdbDebug("%p uid:%" PRIu64",tid:%d data block created, brange:%"PRIu64"-%"PRIu64" rows:%d, %p", pQueryHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, cur->win.skey, + tsdbDebug("%p uid:%" PRIu64",tid:%d data block created, mixblock:%d, brange:%"PRIu64"-%"PRIu64" rows:%d, %p", + pQueryHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, cur->mixBlock, cur->win.skey, cur->win.ekey, cur->rows, pQueryHandle->qinfo); } @@ -1658,6 +1674,128 @@ static bool doHasDataInBuffer(STsdbQueryHandle* pQueryHandle) { return false; } +static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle) { + // filter the queried time stamp in the first place + STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle; + pQueryHandle->order = TSDB_ORDER_DESC; + + assert(pQueryHandle->window.skey == pQueryHandle->window.ekey); + + // starts from the buffer in case of descending timestamp order check data blocks + // todo consider the query time window, current last_row does not apply the query time window + size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + + int32_t i = 0; + while(i < numOfTables) { + STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); + if (pQueryHandle->window.skey <= pCheckInfo->pTableObj->lastKey && + pCheckInfo->pTableObj->lastKey != TSKEY_INITIAL_VAL) { + break; + } + + i++; + } + + // there are no data in all the tables + if (i == numOfTables) { + return; + } + + STableCheckInfo info = *(STableCheckInfo*) taosArrayGet(pQueryHandle->pTableCheckInfo, i); + taosArrayClear(pQueryHandle->pTableCheckInfo); + + info.lastKey = pQueryHandle->window.skey; + taosArrayPush(pQueryHandle->pTableCheckInfo, &info); + + // update the query time window according to the chosen last timestamp + pQueryHandle->window = (STimeWindow) {info.lastKey, TSKEY_INITIAL_VAL}; +} + +static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, STimeWindow* win, + STsdbQueryHandle* pQueryHandle) { + int numOfRows = 0; + int32_t numOfCols = (int32_t)taosArrayGetSize(pQueryHandle->pColumns); + win->skey = TSKEY_INITIAL_VAL; + + int64_t st = taosGetTimestampUs(); + STable* pTable = pCheckInfo->pTableObj; + + do { + SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order); + if (row == NULL) { + break; + } + + TSKEY key = dataRowKey(row); + if ((key > maxKey && ASCENDING_TRAVERSE(pQueryHandle->order)) || (key < maxKey && !ASCENDING_TRAVERSE(pQueryHandle->order))) { + tsdbDebug("%p key:%"PRIu64" beyond qrange:%"PRId64" - %"PRId64", no more data in buffer", pQueryHandle, key, pQueryHandle->window.skey, + pQueryHandle->window.ekey); + + break; + } + + if (win->skey == INT64_MIN) { + win->skey = key; + } + + win->ekey = key; + copyOneRowFromMem(pQueryHandle, maxRowsToRead, numOfRows, row, numOfCols, pTable); + + if (++numOfRows >= maxRowsToRead) { + moveToNextRowInMem(pCheckInfo); + break; + } + + } while(moveToNextRowInMem(pCheckInfo)); + + assert(numOfRows <= maxRowsToRead); + + // if the buffer is not full in case of descending order query, move the data in the front of the buffer + if (!ASCENDING_TRAVERSE(pQueryHandle->order) && numOfRows < maxRowsToRead) { + int32_t emptySize = maxRowsToRead - numOfRows; + + for(int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); + memmove((char*)pColInfo->pData, (char*)pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes); + } + } + + int64_t elapsedTime = taosGetTimestampUs() - st; + tsdbDebug("%p build data block from cache completed, elapsed time:%"PRId64" us, numOfRows:%d, numOfCols:%d, %p", pQueryHandle, + elapsedTime, numOfRows, numOfCols, pQueryHandle->qinfo); + + return numOfRows; +} + +static int32_t getAllTableList(STable* pSuperTable, SArray* list) { + SSkipListIterator* iter = tSkipListCreateIter(pSuperTable->pIndex); + while (tSkipListIterNext(iter)) { + SSkipListNode* pNode = tSkipListIterGet(iter); + + STable** pTable = (STable**) SL_GET_NODE_DATA((SSkipListNode*) pNode); + + STableKeyInfo info = {.pTable = *pTable, .lastKey = TSKEY_INITIAL_VAL}; + taosArrayPush(list, &info); + } + + tSkipListDestroyIter(iter); + return TSDB_CODE_SUCCESS; +} + +static void destroyHelper(void* param) { + if (param == NULL) { + return; + } + + + tQueryInfo* pInfo = (tQueryInfo*)param; + if (pInfo->optr != TSDB_RELATION_IN) { + taosTFree(pInfo->q); + } + + free(param); +} + // handle data in cache situation bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle; @@ -1698,6 +1836,11 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { return true; } else { STsdbQueryHandle* pSecQueryHandle = calloc(1, sizeof(STsdbQueryHandle)); + if (pSecQueryHandle == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return false; + } + pSecQueryHandle->order = TSDB_ORDER_ASC; pSecQueryHandle->window = (STimeWindow) {pQueryHandle->window.skey, INT64_MAX}; pSecQueryHandle->pTsdb = pQueryHandle->pTsdb; @@ -1709,6 +1852,7 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { pSecQueryHandle->outputCapacity = ((STsdbRepo*)pSecQueryHandle->pTsdb)->config.maxRowsPerFileBlock; if (tsdbInitReadHelper(&pSecQueryHandle->rhelper, (STsdbRepo*) pSecQueryHandle->pTsdb) != 0) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; free(pSecQueryHandle); return false; } @@ -1720,6 +1864,11 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { pSecQueryHandle->statis = calloc(numOfCols, sizeof(SDataStatis)); pSecQueryHandle->pColumns = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); + if (pSecQueryHandle->statis == NULL || pSecQueryHandle->pColumns == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + tsdbCleanupQueryHandle(pSecQueryHandle); + return false; + } for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData colInfo = {{0}, 0}; @@ -1727,6 +1876,12 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { colInfo.info = pCol->info; colInfo.pData = calloc(1, EXTRA_BYTES + pQueryHandle->outputCapacity * pCol->info.bytes); + if (colInfo.pData == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + tsdbCleanupQueryHandle(pSecQueryHandle); + return false; + } + taosArrayPush(pSecQueryHandle->pColumns, &colInfo); } @@ -1794,6 +1949,7 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { } if (pQueryHandle->checkFiles) { + // check if the query range overlaps with the file data block bool exists = true; int32_t code = getDataBlocksInFiles(pQueryHandle, &exists); @@ -1822,150 +1978,48 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { return ret; } -void changeQueryHandleForLastrowQuery(TsdbQueryHandleT pqHandle) { - STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pqHandle; - assert(!ASCENDING_TRAVERSE(pQueryHandle->order)); - - // starts from the buffer in case of descending timestamp order check data blocks +STimeWindow changeTableGroupByLastrow(STableGroupInfo *groupList) { + STimeWindow window = {INT64_MAX, INT64_MIN}; + // NOTE: starts from the buffer in case of descending timestamp order check data blocks // todo consider the query time window, current last_row does not apply the query time window - size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + size_t numOfGroups = taosArrayGetSize(groupList->pGroupList); + for(int32_t j = 0; j < numOfGroups; ++j) { + SArray* pGroup = taosArrayGetP(groupList->pGroupList, j); + TSKEY key = TSKEY_INITIAL_VAL; - TSKEY key = TSKEY_INITIAL_VAL; - int32_t index = -1; + STableKeyInfo keyInfo = {0}; - for(int32_t i = 0; i < numOfTables; ++i) { - STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); - if (pCheckInfo->pTableObj->lastKey > key) { - key = pCheckInfo->pTableObj->lastKey; - index = i; + size_t numOfTables = taosArrayGetSize(pGroup); + for(int32_t i = 0; i < numOfTables; ++i) { + STableKeyInfo* pKeyInfo = (STableKeyInfo*) taosArrayGet(pGroup, i); + TSKEY lastKey = ((STable*)(pKeyInfo->pTable))->lastKey; + + if (key < lastKey) { + key = lastKey; + + keyInfo.pTable = pKeyInfo->pTable; + keyInfo.lastKey = key; + pKeyInfo->lastKey = key; + + if (key < window.skey) { + window.skey = key; + } + + if (key > window.ekey) { + window.ekey = key; + } + } + } + + // more than one table in each group, only one table left for each group + if (numOfTables > 1) { + taosArrayClear(pGroup); + taosArrayPush(pGroup, &keyInfo); } } - if (index == -1) { - // todo add failure test cases - return; - } - - // erase all other elements in array list - size_t size = taosArrayGetSize(pQueryHandle->pTableCheckInfo); - for (int32_t i = 0; i < size; ++i) { - if (i == index) { - continue; - } - - STableCheckInfo* pTableCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); - tSkipListDestroyIter(pTableCheckInfo->iter); - - if (pTableCheckInfo->pDataCols != NULL) { - taosTFree(pTableCheckInfo->pDataCols->buf); - } - - taosTFree(pTableCheckInfo->pDataCols); - taosTFree(pTableCheckInfo->pCompInfo); - } - - STableCheckInfo info = *(STableCheckInfo*) taosArrayGet(pQueryHandle->pTableCheckInfo, index); - taosArrayClear(pQueryHandle->pTableCheckInfo); - - info.lastKey = key; - taosArrayPush(pQueryHandle->pTableCheckInfo, &info); - - // update the query time window according to the chosen last timestamp - pQueryHandle->window = (STimeWindow) {key, key}; -} - -static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle) { - // filter the queried time stamp in the first place - STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle; - pQueryHandle->order = TSDB_ORDER_DESC; - - assert(pQueryHandle->window.skey == pQueryHandle->window.ekey); - - // starts from the buffer in case of descending timestamp order check data blocks - // todo consider the query time window, current last_row does not apply the query time window - size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); - - int32_t i = 0; - while(i < numOfTables) { - STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); - if (pQueryHandle->window.skey <= pCheckInfo->pTableObj->lastKey && - pCheckInfo->pTableObj->lastKey != TSKEY_INITIAL_VAL) { - break; - } - - i++; - } - - // there are no data in all the tables - if (i == numOfTables) { - return; - } - - STableCheckInfo info = *(STableCheckInfo*) taosArrayGet(pQueryHandle->pTableCheckInfo, i); - taosArrayClear(pQueryHandle->pTableCheckInfo); - - info.lastKey = pQueryHandle->window.skey; - taosArrayPush(pQueryHandle->pTableCheckInfo, &info); - - // update the query time window according to the chosen last timestamp - pQueryHandle->window = (STimeWindow) {info.lastKey, TSKEY_INITIAL_VAL}; -} - -static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, STimeWindow* win, - STsdbQueryHandle* pQueryHandle) { - int numOfRows = 0; - int32_t numOfCols = (int32_t)taosArrayGetSize(pQueryHandle->pColumns); - win->skey = TSKEY_INITIAL_VAL; - - int64_t st = taosGetTimestampUs(); - STable* pTable = pCheckInfo->pTableObj; - - do { - SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order); - if (row == NULL) { - break; - } - - TSKEY key = dataRowKey(row); - if ((key > maxKey && ASCENDING_TRAVERSE(pQueryHandle->order)) || (key < maxKey && !ASCENDING_TRAVERSE(pQueryHandle->order))) { - tsdbDebug("%p key:%"PRIu64" beyond qrange:%"PRId64" - %"PRId64", no more data in buffer", pQueryHandle, key, pQueryHandle->window.skey, - pQueryHandle->window.ekey); - - break; - } - - if (win->skey == INT64_MIN) { - win->skey = key; - } - - win->ekey = key; - copyOneRowFromMem(pQueryHandle, maxRowsToRead, numOfRows, row, numOfCols, pTable); - - if (++numOfRows >= maxRowsToRead) { - moveToNextRowInMem(pCheckInfo); - break; - } - - } while(moveToNextRowInMem(pCheckInfo)); - - assert(numOfRows <= maxRowsToRead); - - // if the buffer is not full in case of descending order query, move the data in the front of the buffer - if (!ASCENDING_TRAVERSE(pQueryHandle->order) && numOfRows < maxRowsToRead) { - int32_t emptySize = maxRowsToRead - numOfRows; - - for(int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); - memmove((char*)pColInfo->pData, (char*)pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes); - } - } - - int64_t elapsedTime = taosGetTimestampUs() - st; - tsdbDebug("%p build data block from cache completed, elapsed time:%"PRId64" us, numOfRows:%d, numOfCols:%d, %p", pQueryHandle, - elapsedTime, numOfRows, numOfCols, pQueryHandle->qinfo); - - return numOfRows; + return window; } void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT* pQueryHandle, SDataBlockInfo* pDataBlockInfo) { @@ -2102,36 +2156,6 @@ SArray* tsdbRetrieveDataBlock(TsdbQueryHandleT* pQueryHandle, SArray* pIdList) { } } -static int32_t getAllTableList(STable* pSuperTable, SArray* list) { - SSkipListIterator* iter = tSkipListCreateIter(pSuperTable->pIndex); - while (tSkipListIterNext(iter)) { - SSkipListNode* pNode = tSkipListIterGet(iter); - - STable** pTable = (STable**) SL_GET_NODE_DATA((SSkipListNode*) pNode); - - STableKeyInfo info = {.pTable = *pTable, .lastKey = TSKEY_INITIAL_VAL}; - taosArrayPush(list, &info); - } - - tSkipListDestroyIter(iter); - return TSDB_CODE_SUCCESS; -} - -static void destroyHelper(void* param) { - if (param == NULL) { - return; - } - - - tQueryInfo* pInfo = (tQueryInfo*)param; - if (pInfo->optr != TSDB_RELATION_IN) { - taosTFree(pInfo->q); - } - -// tVariantDestroy(&(pInfo->q)); - free(param); -} - void filterPrepare(void* expr, void* param) { tExprNode* pExpr = (tExprNode*)expr; if (pExpr->_node.info != NULL) { @@ -2158,13 +2182,7 @@ void filterPrepare(void* expr, void* param) { } } -typedef struct STableGroupSupporter { - int32_t numOfCols; - SColIndex* pCols; - STSchema* pTagSchema; -} STableGroupSupporter; - -int32_t tableGroupComparFn(const void *p1, const void *p2, const void *param) { +static int32_t tableGroupComparFn(const void *p1, const void *p2, const void *param) { STableGroupSupporter* pTableGroupSupp = (STableGroupSupporter*) param; STable* pTable1 = ((STableKeyInfo*) p1)->pTable; STable* pTable2 = ((STableKeyInfo*) p2)->pTable; @@ -2217,8 +2235,19 @@ int32_t tableGroupComparFn(const void *p1, const void *p2, const void *param) { return 0; } -void createTableGroupImpl(SArray* pGroups, SArray* pTableList, size_t numOfTables, TSKEY skey, STableGroupSupporter* pSupp, - __ext_compar_fn_t compareFn) { +static int tsdbCheckInfoCompar(const void* key1, const void* key2) { + if (((STableCheckInfo*)key1)->tableId.tid < ((STableCheckInfo*)key2)->tableId.tid) { + return -1; + } else if (((STableCheckInfo*)key1)->tableId.tid > ((STableCheckInfo*)key2)->tableId.tid) { + return 1; + } else { + ASSERT(false); + return 0; + } +} + +void createTableGroupImpl(SArray* pGroups, SArray* pTableList, size_t numOfTables, TSKEY skey, + STableGroupSupporter* pSupp, __ext_compar_fn_t compareFn) { STable* pTable = taosArrayGetP(pTableList, 0); SArray* g = taosArrayInit(16, sizeof(STableKeyInfo)); @@ -2264,6 +2293,10 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC if (numOfOrderCols == 0 || size == 1) { // no group by tags clause or only one table SArray* sa = taosArrayInit(size, sizeof(STableKeyInfo)); + if (sa == NULL) { + taosArrayDestroy(pTableGroup); + return NULL; + } for(int32_t i = 0; i < size; ++i) { STableKeyInfo *pKeyInfo = taosArrayGet(pTableList, i); @@ -2278,20 +2311,19 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC taosArrayPush(pTableGroup, &sa); tsdbDebug("all %" PRIzu " tables belong to one group", size); } else { - STableGroupSupporter *pSupp = (STableGroupSupporter *) calloc(1, sizeof(STableGroupSupporter)); - pSupp->numOfCols = numOfOrderCols; - pSupp->pTagSchema = pTagSchema; - pSupp->pCols = pCols; + STableGroupSupporter sup = {0}; + sup.numOfCols = numOfOrderCols; + sup.pTagSchema = pTagSchema; + sup.pCols = pCols; - taosqsort(pTableList->pData, size, sizeof(STableKeyInfo), pSupp, tableGroupComparFn); - createTableGroupImpl(pTableGroup, pTableList, size, skey, pSupp, tableGroupComparFn); - taosTFree(pSupp); + taosqsort(pTableList->pData, size, sizeof(STableKeyInfo), &sup, tableGroupComparFn); + createTableGroupImpl(pTableGroup, pTableList, size, skey, &sup, tableGroupComparFn); } return pTableGroup; } -bool indexedNodeFilterFp(const void* pNode, void* param) { +static bool indexedNodeFilterFp(const void* pNode, void* param) { tQueryInfo* pInfo = (tQueryInfo*) param; STable* pTable = *(STable**)(SL_GET_NODE_DATA((SSkipListNode*)pNode)); @@ -2562,7 +2594,7 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) { tsdbDestroyHelper(&pQueryHandle->rhelper); SIOCostSummary* pCost = &pQueryHandle->cost; - tsdbDebug("%p :io-cost summary: statis-info:%"PRId64"us, datablock:%" PRId64"us, check data:%"PRId64"us, %p", + tsdbDebug("%p :io-cost summary: statis-info:%"PRId64" us, datablock:%" PRId64" us, check data:%"PRId64" us, %p", pQueryHandle, pCost->statisInfoLoadTime, pCost->blockLoadTime, pCost->checkForNextTime, pQueryHandle->qinfo); taosTFree(pQueryHandle); @@ -2589,14 +2621,3 @@ void tsdbDestroyTableGroup(STableGroupInfo *pGroupList) { taosArrayDestroy(pGroupList->pGroupList); } - -static int tsdbCheckInfoCompar(const void* key1, const void* key2) { - if (((STableCheckInfo*)key1)->tableId.tid < ((STableCheckInfo*)key2)->tableId.tid) { - return -1; - } else if (((STableCheckInfo*)key1)->tableId.tid > ((STableCheckInfo*)key2)->tableId.tid) { - return 1; - } else { - ASSERT(false); - return 0; - } -} diff --git a/src/util/inc/tcache.h b/src/util/inc/tcache.h index 11121fcf3b..3d6f9705ee 100644 --- a/src/util/inc/tcache.h +++ b/src/util/inc/tcache.h @@ -103,7 +103,7 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool ext * @param keepTime survival time in second * @return cached element */ -void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const void *pData, size_t dataSize, int keepTimeInSeconds); +void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const void *pData, size_t dataSize, int durationMS); /** * get data from cache @@ -113,16 +113,6 @@ void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const v */ void *taosCacheAcquireByKey(SCacheObj *pCacheObj, const void *key, size_t keyLen); -/** - * update the expire time of data in cache - * @param pCacheObj cache object - * @param key key - * @param keyLen keyLen - * @param expireTime new expire time of data - * @return - */ -//void* taosCacheUpdateExpireTimeByName(SCacheObj *pCacheObj, void *key, size_t keyLen, uint64_t expireTime); - /** * Add one reference count for the exist data, and assign this data for a new owner. * The new owner needs to invoke the taosCacheRelease when it does not need this data anymore. diff --git a/src/util/inc/tkvstore.h b/src/util/inc/tkvstore.h index 3b4e8e3757..b2b0ff05f5 100644 --- a/src/util/inc/tkvstore.h +++ b/src/util/inc/tkvstore.h @@ -58,7 +58,7 @@ int tdKVStoreStartCommit(SKVStore *pStore); int tdUpdateKVStoreRecord(SKVStore *pStore, uint64_t uid, void *cont, int contLen); int tdDropKVStoreRecord(SKVStore *pStore, uint64_t uid); int tdKVStoreEndCommit(SKVStore *pStore); -void tsdbGetStoreInfo(char *fname, uint32_t *magic, int32_t *size); +void tsdbGetStoreInfo(char *fname, uint32_t *magic, int64_t *size); #ifdef __cplusplus } diff --git a/src/util/inc/tlockfree.h b/src/util/inc/tlockfree.h index a81f597832..e960b601ca 100644 --- a/src/util/inc/tlockfree.h +++ b/src/util/inc/tlockfree.h @@ -36,6 +36,14 @@ typedef void (*_ref_fn_t)(const void* pObj); _ref_fn_t end; \ } _ref_func = {.begin = (s), .end = (e)}; +// set the initial reference count value +#define T_REF_INIT_VAL(x, _v) \ + do { \ + assert(_v >= 0); \ + atomic_store_32(&((x)->_ref.val), (_v)); \ + } while (0) + +// increase the reference count by 1 #define T_REF_INC(x) (atomic_add_fetch_32(&((x)->_ref.val), 1)) #define T_REF_INC_WITH_CB(x, p) \ diff --git a/src/util/inc/tstoken.h b/src/util/inc/tstoken.h index c1c6f2de7a..7aeb2da6b6 100644 --- a/src/util/inc/tstoken.h +++ b/src/util/inc/tstoken.h @@ -28,11 +28,11 @@ extern "C" { #define TSQL_TBNAME_L "tbname" // used to denote the minimum unite in sql parsing -typedef struct SSQLToken { +typedef struct SStrToken { uint32_t n; uint32_t type; char * z; -} SSQLToken; +} SStrToken; /** * tokenizer for sql string @@ -52,7 +52,7 @@ uint32_t tSQLGetToken(char *z, uint32_t *tokenType); * @param ignoreTokenTypes * @return */ -SSQLToken tStrGetToken(char *str, int32_t *i, bool isPrevOptr, uint32_t numOfIgnoreToken, uint32_t *ignoreTokenTypes); +SStrToken tStrGetToken(char *str, int32_t *i, bool isPrevOptr, uint32_t numOfIgnoreToken, uint32_t *ignoreTokenTypes); /** * check if it is a keyword or not @@ -76,7 +76,7 @@ bool isKeyWord(const char *z, int32_t len); * @param pToken * @return token type, if it is not a number, TK_ILLEGAL will return */ -static FORCE_INLINE int32_t isValidNumber(const SSQLToken* pToken) { +static FORCE_INLINE int32_t isValidNumber(const SStrToken* pToken) { const char* z = pToken->z; int32_t type = TK_ILLEGAL; diff --git a/src/util/src/hash.c b/src/util/src/hash.c index 714f36f1cb..9634175db7 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -377,6 +377,12 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe } } + if (pe->num == 0) { + assert(pe->next == NULL); + } else { + assert(pe->next != NULL); + } + if (pHashObj->type == HASH_ENTRY_LOCK) { taosWUnLockLatch(&pe->latch); } @@ -390,22 +396,8 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe if (pRes != NULL) { atomic_sub_fetch_64(&pHashObj->size, 1); FREE_HASH_NODE(pHashObj, pRes); - - if (pe->num == 0) { - assert(pe->next == NULL); - } else { - assert(pe->next != NULL); - } - return 0; } else { - - if (pe->num == 0) { - assert(pe->next == NULL); - } else { - assert(pe->next != NULL); - } - return -1; } } @@ -657,7 +649,7 @@ void taosHashTableResize(SHashObj *pHashObj) { } int64_t st = taosGetTimestampUs(); - void *pNewEntryList = realloc(pHashObj->hashList, sizeof(void*) * newSize); + void *pNewEntryList = realloc(pHashObj->hashList, sizeof(void *) * newSize); if (pNewEntryList == NULL) { // todo handle error // uDebug("cache resize failed due to out of memory, capacity remain:%d", pHashObj->capacity); return; diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index bd903c8c23..dfa982b848 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -189,12 +189,12 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool ext return pCacheObj; } -void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const void *pData, size_t dataSize, int duration) { +void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const void *pData, size_t dataSize, int durationMS) { if (pCacheObj == NULL || pCacheObj->pHashTable == NULL || pCacheObj->deleting == 1) { return NULL; } - SCacheDataNode *pNode1 = taosCreateCacheNode(key, keyLen, pData, dataSize, duration); + SCacheDataNode *pNode1 = taosCreateCacheNode(key, keyLen, pData, dataSize, durationMS); if (pNode1 == NULL) { uError("cache:%s, key:%p, failed to added into cache, out of memory", pCacheObj->name, key); return NULL; diff --git a/src/util/src/tcompare.c b/src/util/src/tcompare.c index 9564588254..ba711ced8f 100644 --- a/src/util/src/tcompare.c +++ b/src/util/src/tcompare.c @@ -220,8 +220,14 @@ static int32_t compareStrPatternComp(const void* pLeft, const void* pRight) { char pattern[128] = {0}; memcpy(pattern, varDataVal(pRight), varDataLen(pRight)); assert(varDataLen(pRight) < 128); - - int32_t ret = patternMatch(pattern, varDataVal(pLeft), varDataLen(pLeft), &pInfo); + + size_t sz = varDataLen(pLeft); + char *buf = malloc(sz + 1); + memcpy(buf, varDataVal(pLeft), sz); + buf[sz] = 0; + + int32_t ret = patternMatch(pattern, buf, sz, &pInfo); + free(buf); return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; } diff --git a/src/util/src/tkvstore.c b/src/util/src/tkvstore.c index 696819e5d0..6ba1d87d92 100644 --- a/src/util/src/tkvstore.c +++ b/src/util/src/tkvstore.c @@ -332,7 +332,7 @@ int tdKVStoreEndCommit(SKVStore *pStore) { return 0; } -void tsdbGetStoreInfo(char *fname, uint32_t *magic, int32_t *size) { +void tsdbGetStoreInfo(char *fname, uint32_t *magic, int64_t *size) { char buf[TD_KVSTORE_HEADER_SIZE] = "\0"; SStoreInfo info = {0}; @@ -349,7 +349,7 @@ void tsdbGetStoreInfo(char *fname, uint32_t *magic, int32_t *size) { close(fd); *magic = info.magic; - *size = (int32_t)offset; + *size = offset; return; diff --git a/src/util/src/tsched.c b/src/util/src/tsched.c index cf7f5c10d4..f014dd0fab 100644 --- a/src/util/src/tsched.c +++ b/src/util/src/tsched.c @@ -123,11 +123,6 @@ void *taosProcessSchedQueue(void *param) { while (1) { if (tsem_wait(&pSched->fullSem) != 0) { - if (errno == EINTR) { - /* sem_wait is interrupted by interrupt, ignore and continue */ - uDebug("wait %s fullSem was interrupted", pSched->label); - continue; - } uError("wait %s fullSem failed(%s)", pSched->label, strerror(errno)); } if (pSched->stop) { @@ -163,12 +158,8 @@ int taosScheduleTask(void *qhandle, SSchedMsg *pMsg) { return 0; } - while (tsem_wait(&pSched->emptySem) != 0) { - if (errno != EINTR) { - uError("wait %s emptySem failed(%s)", pSched->label, strerror(errno)); - break; - } - uDebug("wait %s emptySem was interrupted", pSched->label); + if (tsem_wait(&pSched->emptySem) != 0) { + uError("wait %s emptySem failed(%s)", pSched->label, strerror(errno)); } if (pthread_mutex_lock(&pSched->queueMutex) != 0) diff --git a/src/util/src/version.c b/src/util/src/version.c deleted file mode 100644 index 6f7b29ebb9..0000000000 --- a/src/util/src/version.c +++ /dev/null @@ -1,7 +0,0 @@ -char version[12] = "2.0.2.0"; -char compatible_version[12] = "2.0.0.0"; -char gitinfo[48] = "d711657139620f6c50f362597020705b8ad26bd2"; -char gitinfoOfInternal[48] = "1d74ae24c541ffbb280e8630883c0236cd45f8c7"; -char buildinfo[64] = "Built by root at 2020-08-24 16:31"; - -void libtaos_2_0_2_0_Linux_x64_beta() {}; diff --git a/src/util/src/version.c.in b/src/util/src/version.c.in new file mode 100644 index 0000000000..21c78a0eb4 --- /dev/null +++ b/src/util/src/version.c.in @@ -0,0 +1,7 @@ +char version[12] = "${TD_VER_NUMBER}"; +char compatible_version[12] = "${TD_VER_COMPATIBLE}"; +char gitinfo[48] = "${TD_VER_GIT}"; +char gitinfoOfInternal[48] = "${TD_VER_GIT_INTERNAL}"; +char buildinfo[64] = "Built at ${TD_VER_DATE}"; + +void libtaos_${TD_LIB_VER_NUMBER}_${TD_VER_OSTYPE}_${TD_VER_CPUTYPE}_${TD_VER_VERTYPE}() {}; diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 3c9da7fd23..c294e86839 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -41,7 +41,7 @@ static int32_t vnodeReadCfg(SVnodeObj *pVnode); static int32_t vnodeSaveVersion(SVnodeObj *pVnode); static int32_t vnodeReadVersion(SVnodeObj *pVnode); static int vnodeProcessTsdbStatus(void *arg, int status); -static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int32_t *size, uint64_t *fversion); +static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion); static int vnodeGetWalInfo(void *ahandle, char *name, uint32_t *index); static void vnodeNotifyRole(void *ahandle, int8_t role); static void vnodeCtrlFlow(void *handle, int32_t mseconds); @@ -290,11 +290,18 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { pVnode->sync = syncStart(&syncInfo); if (pVnode->sync == NULL) { + vError("vgId:%d, failed to open sync module, replica:%d reason:%s", pVnode->vgId, pVnode->syncCfg.replica, + tstrerror(terrno)); vnodeCleanUp(pVnode); return terrno; } pVnode->qMgmt = qOpenQueryMgmt(pVnode->vgId); + if (pVnode->qMgmt == NULL) { + vnodeCleanUp(pVnode); + return terrno; + } + pVnode->events = NULL; pVnode->status = TAOS_VN_STATUS_READY; vDebug("vgId:%d, vnode is opened in %s, pVnode:%p", pVnode->vgId, rootDir, pVnode); @@ -536,7 +543,7 @@ static int vnodeProcessTsdbStatus(void *arg, int status) { return 0; } -static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int32_t *size, uint64_t *fversion) { +static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion) { SVnodeObj *pVnode = ahandle; *fversion = pVnode->fversion; return tsdbGetFileInfo(pVnode->tsdb, name, index, eindex, size); @@ -549,7 +556,7 @@ static int vnodeGetWalInfo(void *ahandle, char *name, uint32_t *index) { static void vnodeNotifyRole(void *ahandle, int8_t role) { SVnodeObj *pVnode = ahandle; - vInfo("vgId:%d, sync role changed from %d to %d", pVnode->vgId, pVnode->role, role); + vInfo("vgId:%d, sync role changed from %s to %s", pVnode->vgId, syncRole[pVnode->role], syncRole[role]); pVnode->role = role; dnodeSendStatusMsgToMnode(); diff --git a/tests/examples/C#/TDengineDriver.cs b/tests/examples/C#/TDengineDriver.cs index 2797c362c1..b6f143e181 100644 --- a/tests/examples/C#/TDengineDriver.cs +++ b/tests/examples/C#/TDengineDriver.cs @@ -20,16 +20,17 @@ using System.Runtime.InteropServices; namespace TDengineDriver { enum TDengineDataType { - TSDB_DATA_TYPE_BOOL = 1, - TSDB_DATA_TYPE_TINYINT = 2, - TSDB_DATA_TYPE_SMALLINT = 3, - TSDB_DATA_TYPE_INT = 4, - TSDB_DATA_TYPE_BIGINT = 5, - TSDB_DATA_TYPE_FLOAT = 6, - TSDB_DATA_TYPE_DOUBLE = 7, - TSDB_DATA_TYPE_BINARY = 8, - TSDB_DATA_TYPE_TIMESTAMP = 9, - TSDB_DATA_TYPE_NCHAR = 10 + TSDB_DATA_TYPE_NULL = 0, // 1 bytes + TSDB_DATA_TYPE_BOOL = 1, // 1 bytes + TSDB_DATA_TYPE_TINYINT = 2, // 1 bytes + TSDB_DATA_TYPE_SMALLINT = 3, // 2 bytes + TSDB_DATA_TYPE_INT = 4, // 4 bytes + TSDB_DATA_TYPE_BIGINT = 5, // 8 bytes + TSDB_DATA_TYPE_FLOAT = 6, // 4 bytes + TSDB_DATA_TYPE_DOUBLE = 7, // 8 bytes + TSDB_DATA_TYPE_BINARY = 8, // string + TSDB_DATA_TYPE_TIMESTAMP = 9,// 8 bytes + TSDB_DATA_TYPE_NCHAR = 10 // unicode string } enum TDengineInitOption @@ -79,54 +80,53 @@ namespace TDengineDriver class TDengine { public const int TSDB_CODE_SUCCESS = 0; - - [DllImport("taos.dll", EntryPoint = "taos_init", CallingConvention = CallingConvention.StdCall)] + + [DllImport("taos.dll", EntryPoint = "taos_init", CallingConvention = CallingConvention.Cdecl)] static extern public void Init(); - [DllImport("taos.dll", EntryPoint = "taos_options", CallingConvention = CallingConvention.StdCall)] + [DllImport("taos.dll", EntryPoint = "taos_cleanup", CallingConvention = CallingConvention.Cdecl)] + static extern public void Cleanup(); + + [DllImport("taos.dll", EntryPoint = "taos_options", CallingConvention = CallingConvention.Cdecl)] static extern public void Options(int option, string value); - [DllImport("taos.dll", EntryPoint = "taos_connect", CallingConvention = CallingConvention.StdCall)] - static extern public long Connect(string ip, string user, string password, string db, int port); + [DllImport("taos.dll", EntryPoint = "taos_connect", CallingConvention = CallingConvention.Cdecl)] + static extern public IntPtr Connect(string ip, string user, string password, string db, short port); - [DllImport("taos.dll", EntryPoint = "taos_errstr", CallingConvention = CallingConvention.StdCall)] - static extern private IntPtr taos_errstr(long taos); - static public string Error(long conn) + [DllImport("taos.dll", EntryPoint = "taos_errstr", CallingConvention = CallingConvention.Cdecl)] + static extern private IntPtr taos_errstr(IntPtr res); + static public string Error(IntPtr res) { - IntPtr errPtr = taos_errstr(conn); + IntPtr errPtr = taos_errstr(res); return Marshal.PtrToStringAnsi(errPtr); } - [DllImport("taos.dll", EntryPoint = "taos_errno", CallingConvention = CallingConvention.StdCall)] - static extern public int ErrorNo(long taos); + [DllImport("taos.dll", EntryPoint = "taos_errno", CallingConvention = CallingConvention.Cdecl)] + static extern public int ErrorNo(IntPtr res); - [DllImport("taos.dll", EntryPoint = "taos_query", CallingConvention = CallingConvention.StdCall)] - static extern public int Query(long taos, string sqlstr); + [DllImport("taos.dll", EntryPoint = "taos_query", CallingConvention = CallingConvention.Cdecl)] + static extern public IntPtr Query(IntPtr conn, string sqlstr); - [DllImport("taos.dll", EntryPoint = "taos_affected_rows", CallingConvention = CallingConvention.StdCall)] - static extern public int AffectRows(long taos); + [DllImport("taos.dll", EntryPoint = "taos_affected_rows", CallingConvention = CallingConvention.Cdecl)] + static extern public int AffectRows(IntPtr res); - [DllImport("taos.dll", EntryPoint = "taos_use_result", CallingConvention = CallingConvention.StdCall)] - static extern public long UseResult(long taos); + [DllImport("taos.dll", EntryPoint = "taos_field_count", CallingConvention = CallingConvention.Cdecl)] + static extern public int FieldCount(IntPtr res); - [DllImport("taos.dll", EntryPoint = "taos_field_count", CallingConvention = CallingConvention.StdCall)] - static extern public int FieldCount(long taos); - - [DllImport("taos.dll", EntryPoint = "taos_fetch_fields", CallingConvention = CallingConvention.StdCall)] - static extern private IntPtr taos_fetch_fields(long res); - static public List FetchFields(long taos) + [DllImport("taos.dll", EntryPoint = "taos_fetch_fields", CallingConvention = CallingConvention.Cdecl)] + static extern private IntPtr taos_fetch_fields(IntPtr res); + static public List FetchFields(IntPtr res) { const int fieldSize = 68; List metas = new List(); - long result = TDengine.UseResult(taos); - if (result == 0) + if (res == IntPtr.Zero) { return metas; } - int fieldCount = FieldCount(taos); - IntPtr fieldsPtr = taos_fetch_fields(result); + int fieldCount = FieldCount(res); + IntPtr fieldsPtr = taos_fetch_fields(res); for (int i = 0; i < fieldCount; ++i) { @@ -134,21 +134,21 @@ namespace TDengineDriver TDengineMeta meta = new TDengineMeta(); meta.name = Marshal.PtrToStringAnsi(fieldsPtr + offset); - meta.size = Marshal.ReadInt16(fieldsPtr + offset + 64); - meta.type = Marshal.ReadByte(fieldsPtr + offset + 66); + meta.type = Marshal.ReadByte(fieldsPtr + offset + 65); + meta.size = Marshal.ReadInt16(fieldsPtr + offset + 66); metas.Add(meta); } return metas; } - [DllImport("taos.dll", EntryPoint = "taos_fetch_row", CallingConvention = CallingConvention.StdCall)] - static extern public IntPtr FetchRows(long res); + [DllImport("taos.dll", EntryPoint = "taos_fetch_row", CallingConvention = CallingConvention.Cdecl)] + static extern public IntPtr FetchRows(IntPtr res); - [DllImport("taos.dll", EntryPoint = "taos_free_result", CallingConvention = CallingConvention.StdCall)] - static extern public IntPtr FreeResult(long res); + [DllImport("taos.dll", EntryPoint = "taos_free_result", CallingConvention = CallingConvention.Cdecl)] + static extern public IntPtr FreeResult(IntPtr res); - [DllImport("taos.dll", EntryPoint = "taos_close", CallingConvention = CallingConvention.StdCall)] - static extern public int Close(long taos); + [DllImport("taos.dll", EntryPoint = "taos_close", CallingConvention = CallingConvention.Cdecl)] + static extern public int Close(IntPtr taos); } } \ No newline at end of file diff --git a/tests/examples/C#/TDengineTest.cs b/tests/examples/C#/TDengineTest.cs index 235775f68f..6b3f1160ad 100644 --- a/tests/examples/C#/TDengineTest.cs +++ b/tests/examples/C#/TDengineTest.cs @@ -28,7 +28,7 @@ namespace TDengineDriver private string configDir; private string user; private string password; - private int port = 0; + private short port = 0; //sql parameters private string dbName; @@ -43,7 +43,7 @@ namespace TDengineDriver private long batchRows; private long beginTimestamp = 1551369600000L; - private long conn = 0; + private IntPtr conn = IntPtr.Zero; private long rowsInserted = 0; static void Main(string[] args) @@ -191,7 +191,7 @@ namespace TDengineDriver { string db = ""; this.conn = TDengine.Connect(this.host, this.user, this.password, db, this.port); - if (this.conn == 0) + if (this.conn == IntPtr.Zero) { Console.WriteLine("Connect to TDengine failed"); ExitProgram(); @@ -211,58 +211,62 @@ namespace TDengineDriver StringBuilder sql = new StringBuilder(); sql.Append("create database if not exists ").Append(this.dbName); - int code = TDengine.Query(this.conn, sql.ToString()); - if (code == TDengine.TSDB_CODE_SUCCESS) + IntPtr res = TDengine.Query(this.conn, sql.ToString()); + if (res != IntPtr.Zero) { Console.WriteLine(sql.ToString() + " success"); } else { - Console.WriteLine(sql.ToString() + " failure, reason: " + TDengine.Error(conn)); + Console.WriteLine(sql.ToString() + " failure, reason: " + TDengine.Error(res)); ExitProgram(); } + TDengine.FreeResult(res); sql.Clear(); sql.Append("use ").Append(this.dbName); - code = TDengine.Query(this.conn, sql.ToString()); - if (code == TDengine.TSDB_CODE_SUCCESS) + res = TDengine.Query(this.conn, sql.ToString()); + if (res != IntPtr.Zero) { Console.WriteLine(sql.ToString() + " success"); } else { - Console.WriteLine(sql.ToString() + " failure, reason: " + TDengine.Error(this.conn)); + Console.WriteLine(sql.ToString() + " failure, reason: " + TDengine.Error(res)); ExitProgram(); } + TDengine.FreeResult(res); sql.Clear(); - sql.Append("create table if not exists ").Append(this.stableName).Append("(ts timestamp, v1 int) tags(t1 int)"); - code = TDengine.Query(this.conn, sql.ToString()); - if (code == TDengine.TSDB_CODE_SUCCESS) + sql.Append("create table if not exists ").Append(this.stableName).Append("(ts timestamp, v1 bool, v2 tinyint, v3 smallint, v4 int, v5 bigint, v6 float, v7 double, v8 binary(10), v9 nchar(10)) tags(t1 int)"); + res = TDengine.Query(this.conn, sql.ToString()); + if (res != IntPtr.Zero) { Console.WriteLine(sql.ToString() + " success"); } else { - Console.WriteLine(sql.ToString() + " failure, reason: " + TDengine.Error(this.conn)); + Console.WriteLine(sql.ToString() + " failure, reason: " + TDengine.Error(res)); ExitProgram(); } + TDengine.FreeResult(res); for (int i = 0; i < this.tableCount; i++) { sql.Clear(); sql = sql.Append("create table if not exists ").Append(this.tablePrefix).Append(i) .Append(" using ").Append(this.stableName).Append(" tags(").Append(i).Append(")"); - code = TDengine.Query(this.conn, sql.ToString()); - if (code == TDengine.TSDB_CODE_SUCCESS) + res = TDengine.Query(this.conn, sql.ToString()); + if (res != IntPtr.Zero) { Console.WriteLine(sql.ToString() + " success"); } else { - Console.WriteLine(sql.ToString() + " failure, reason: " + TDengine.Error(this.conn)); + Console.WriteLine(sql.ToString() + " failure, reason: " + TDengine.Error(res)); ExitProgram(); } + TDengine.FreeResult(res); } Console.WriteLine("create db and table success"); @@ -287,16 +291,22 @@ namespace TDengineDriver for (int batch = 0; batch < this.batchRows; ++batch) { long rows = loop * this.batchRows + batch; - sql.Append("(").Append(this.beginTimestamp + rows).Append(",").Append(rows).Append(")"); + sql.Append("(") + .Append(this.beginTimestamp + rows) + .Append(", 1, 2, 3,") + .Append(rows) + .Append(", 5, 6, 7, 'abc', 'def')"); } - int code = TDengine.Query(conn, sql.ToString()); - if (code != TDengine.TSDB_CODE_SUCCESS) + IntPtr res = TDengine.Query(this.conn, sql.ToString()); + if (res == IntPtr.Zero) { - Console.WriteLine(sql.ToString() + " failure, reason: " + TDengine.Error(conn)); + Console.WriteLine(sql.ToString() + " failure, reason: " + TDengine.Error(res)); } - int affectRows = TDengine.AffectRows(conn); + int affectRows = TDengine.AffectRows(res); this.rowsInserted += affectRows; + + TDengine.FreeResult(res); } } @@ -316,51 +326,45 @@ namespace TDengineDriver System.DateTime start = new System.DateTime(); long queryRows = 0; - - for (int i = 0; i < this.tableCount; ++i) + + for (int i = 0; i < 1/*this.tableCount*/; ++i) { String sql = "select * from " + this.dbName + "." + tablePrefix + i; Console.WriteLine(sql); - int code = TDengine.Query(conn, sql); - if (code != TDengine.TSDB_CODE_SUCCESS) + IntPtr res = TDengine.Query(conn, sql); + if (res == IntPtr.Zero) { - Console.WriteLine(sql + " failure, reason: " + TDengine.Error(conn)); + Console.WriteLine(sql + " failure, reason: " + TDengine.Error(res)); ExitProgram(); } - int fieldCount = TDengine.FieldCount(conn); - //Console.WriteLine("field count: " + fieldCount); + int fieldCount = TDengine.FieldCount(res); + Console.WriteLine("field count: " + fieldCount); - List metas = TDengine.FetchFields(conn); + List metas = TDengine.FetchFields(res); for (int j = 0; j < metas.Count; j++) { TDengineMeta meta = (TDengineMeta)metas[j]; - //Console.WriteLine("index:" + j + ", type:" + meta.type + ", typename:" + meta.TypeName() + ", name:" + meta.name + ", size:" + meta.size); - } - - long result = TDengine.UseResult(conn); - if (result == 0) - { - Console.WriteLine(sql + " result set is null"); - return; + Console.WriteLine("index:" + j + ", type:" + meta.type + ", typename:" + meta.TypeName() + ", name:" + meta.name + ", size:" + meta.size); } IntPtr rowdata; - while ((rowdata = TDengine.FetchRows(result)) != IntPtr.Zero) + StringBuilder builder = new StringBuilder(); + while ((rowdata = TDengine.FetchRows(res)) != IntPtr.Zero) { queryRows++; for (int fields = 0; fields < fieldCount; ++fields) { TDengineMeta meta = metas[fields]; - int offset = 8 * fields; + int offset = IntPtr.Size * fields; IntPtr data = Marshal.ReadIntPtr(rowdata, offset); - //Console.Write("---"); + builder.Append("---"); if (data == IntPtr.Zero) { - //Console.Write("NULL"); + builder.Append("NULL"); continue; } @@ -368,55 +372,61 @@ namespace TDengineDriver { case TDengineDataType.TSDB_DATA_TYPE_BOOL: bool v1 = Marshal.ReadByte(data) == 0 ? false : true; - //Console.Write(v1); + builder.Append(v1); break; case TDengineDataType.TSDB_DATA_TYPE_TINYINT: byte v2 = Marshal.ReadByte(data); - //Console.Write(v2); + builder.Append(v2); break; case TDengineDataType.TSDB_DATA_TYPE_SMALLINT: short v3 = Marshal.ReadInt16(data); - //Console.Write(v3); + builder.Append(v3); break; case TDengineDataType.TSDB_DATA_TYPE_INT: int v4 = Marshal.ReadInt32(data); - //Console.Write(v4); + builder.Append(v4); break; case TDengineDataType.TSDB_DATA_TYPE_BIGINT: long v5 = Marshal.ReadInt64(data); - //Console.Write(v5); + builder.Append(v5); break; case TDengineDataType.TSDB_DATA_TYPE_FLOAT: float v6 = (float)Marshal.PtrToStructure(data, typeof(float)); - //Console.Write(v6); + builder.Append(v6); break; case TDengineDataType.TSDB_DATA_TYPE_DOUBLE: double v7 = (double)Marshal.PtrToStructure(data, typeof(double)); - //Console.Write(v7); + builder.Append(v7); break; case TDengineDataType.TSDB_DATA_TYPE_BINARY: string v8 = Marshal.PtrToStringAnsi(data); - //Console.Write(v8); + builder.Append(v8); break; case TDengineDataType.TSDB_DATA_TYPE_TIMESTAMP: long v9 = Marshal.ReadInt64(data); - //Console.Write(v9); + builder.Append(v9); break; case TDengineDataType.TSDB_DATA_TYPE_NCHAR: string v10 = Marshal.PtrToStringAnsi(data); - //Console.Write(v10); + builder.Append(v10); break; } } - //Console.WriteLine("---"); + builder.Append("---"); + + if (queryRows <= 10) + { + Console.WriteLine(builder.ToString()); + } + builder.Clear(); } - if (TDengine.ErrorNo(conn) != 0) + if (TDengine.ErrorNo(res) != 0) { - Console.Write("Query is not complete， Error {0:G}", TDengine.ErrorNo(conn), TDengine.Error(conn)); + Console.Write("Query is not complete, Error {0:G}", TDengine.ErrorNo(res), TDengine.Error(res)); } - TDengine.FreeResult(result); + TDengine.FreeResult(res); } System.DateTime end = new System.DateTime(); @@ -428,14 +438,15 @@ namespace TDengineDriver public void CloseConnection() { - if (conn != 0) + if (this.conn != IntPtr.Zero) { - TDengine.Close(conn); + TDengine.Close(this.conn); } } static void ExitProgram() { + TDengine.Cleanup(); System.Environment.Exit(0); } } diff --git a/tests/perftest-scripts/tdengineTestWriteLoop.sh b/tests/perftest-scripts/tdengineTestWriteLoop.sh index e1ee4418dc..cb9d87d0c1 100755 --- a/tests/perftest-scripts/tdengineTestWriteLoop.sh +++ b/tests/perftest-scripts/tdengineTestWriteLoop.sh @@ -24,22 +24,46 @@ function runTest { for r in ${!rowsPerRequest[@]}; do for c in `seq 1 $clients`; do totalRPR=0 - OUTPUT_FILE=tdengineTestWrite-RPR${rowsPerRequest[$r]}-clients$c.out + if $v16 ; then + OUTPUT_FILE=tdengineTestWrite-v16-RPR${rowsPerRequest[$r]}-clients$c.out + else + OUTPUT_FILE=tdengineTestWrite-v20-RPR${rowsPerRequest[$r]}-clients$c.out + fi for i in `seq 1 $NUM_LOOP`; do - restartTaosd - $TAOSD_DIR/taos -s "drop database db" > /dev/null 2>&1 - printTo "loop i:$i, $TDTEST_DIR/tdengineTest \ + if ! $printresultonly ; then + restartTaosd + $TAOSD_DIR/taos -s "drop database db" > /dev/null 2>&1 + + if $v16 ; then + printTo "loop i:$i, $TDTEST_DIR/tdengineTest \ -dataDir $DATA_DIR \ -numOfFiles $NUM_OF_FILES \ - -w -clients $c \ - -rowsPerRequest ${rowsPerRequest[$r]}" - $TDTEST_DIR/tdengineTest \ - -dataDir $DATA_DIR \ - -numOfFiles $NUM_OF_FILES \ - -w -clients $c \ - -rowsPerRequest ${rowsPerRequest[$r]} \ - | tee $OUTPUT_FILE + -writeClients $c \ + -rowsPerRequest ${rowsPerRequest[$r]} \ + | tee $OUTPUT_FILE" + $TDTEST_DIR/tdengineTest \ + -dataDir $DATA_DIR \ + -numOfFiles $NUM_OF_FILES \ + -writeClients $c \ + -rowsPerRequest ${rowsPerRequest[$r]} \ + | tee $OUTPUT_FILE + else + printTo "loop i:$i, $TDTEST_DIR/tdengineTest \ + -dataDir $DATA_DIR \ + -numOfFiles $NUM_OF_FILES \ + -w -clients $c \ + -rowsPerRequest ${rowsPerRequest[$r]} \ + | tee $OUTPUT_FILE" + $TDTEST_DIR/tdengineTest \ + -dataDir $DATA_DIR \ + -numOfFiles $NUM_OF_FILES \ + -w -clients $c \ + -rowsPerRequest ${rowsPerRequest[$r]} \ + | tee $OUTPUT_FILE + fi + fi + RPR=`cat $OUTPUT_FILE | grep speed | awk '{print $(NF-1)}'` totalRPR=`echo "scale=4; $totalRPR + $RPR" | bc` printTo "rows:${rowsPerRequest[$r]}, clients:$c, i:$i RPR:$RPR" @@ -86,25 +110,30 @@ function restartTaosd { ################ Main ################ -master=false -develop=true +v16=false +v20=true verbose=false clients=1 +printresultonly=false while : ; do case $1 in + printresultonly) + printresultonly=true + shift ;; + -v) verbose=true shift ;; - master) - master=true - develop=false + v16) + v16=true + v20=false shift ;; - develop) - master=false - develop=true + v20) + v16=false + v20=true shift ;; -c) @@ -120,19 +149,24 @@ while : ; do esac done -if $master ; then - echo "Test master branch.." - cp /mnt/root/cfg/master/taos.cfg /etc/taos/taos.cfg - WORK_DIR=/mnt/root/TDengine.master +if $v16 ; then + echo "Test v16 branch.." + WORK_DIR=/mnt/root/TDengine.v16 + cp /mnt/root/cfg/v16/taos.cfg /etc/taos/taos.cfg else - echo "Test develop branch.." - cp /mnt/root/cfg/develop/taos.cfg /etc/taos/taos.cfg + echo "Test v20 branch.." + cp /mnt/root/cfg/v20/taos.cfg /etc/taos/taos.cfg WORK_DIR=/mnt/root/TDengine fi TAOSD_DIR=$WORK_DIR/debug/build/bin TDTEST_DIR=$WORK_DIR/tests/comparisonTest/tdengine +if [ ! -f $TDTEST_DIR/tdengineTest ]; then + echo "Please build tdengineTest first!" + exit 1 +fi + runTest echo "Test done!" diff --git a/tests/pytest/crash_gen.py b/tests/pytest/crash_gen.py index 7d3eb959c0..7588e03e17 100755 --- a/tests/pytest/crash_gen.py +++ b/tests/pytest/crash_gen.py @@ -161,6 +161,21 @@ class WorkerThread: logger.debug("[TRD] Thread Coordinator not running any more, worker thread now stopping...") break + # Before we fetch the task and run it, let's ensure we properly "use" the database + try: + if (gConfig.per_thread_db_connection): # most likely TRUE + if not self._dbConn.isOpen: # might have been closed during server auto-restart + self._dbConn.open() + self.useDb() # might encounter exceptions. TODO: catch + except taos.error.ProgrammingError as err: + errno = Helper.convertErrno(err.errno) + if errno in [0x383, 0x386, 0x00B, 0x014] : # invalid database, dropping, Unable to establish connection, Database not ready + # ignore + dummy = 0 + else: + print("\nCaught programming error. errno=0x{:X}, msg={} ".format(errno, err.msg)) + raise + # Fetch a task from the Thread Coordinator logger.debug( "[TRD] Worker thread [{}] about to fetch task".format(self._tid)) task = tc.fetchTask() @@ -324,10 +339,12 @@ class ThreadCoordinator: logger.debug("[STT] transition ended") # Due to limitation (or maybe not) of the Python library, # we cannot share connections across threads - if sm.hasDatabase(): - for t in self._pool.threadList: - logger.debug("[DB] use db for all worker threads") - t.useDb() + # Here we are in main thread, we cannot operate the connections created in workers + # Moving below to task loop + # if sm.hasDatabase(): + # for t in self._pool.threadList: + # logger.debug("[DB] use db for all worker threads") + # t.useDb() # t.execSql("use db") # main thread executing "use # db" on behalf of every worker thread except taos.error.ProgrammingError as err: @@ -387,7 +404,7 @@ class ThreadCoordinator: transitionFailed = self._doTransition() # To start, we end step -1 first except taos.error.ProgrammingError as err: transitionFailed = True - errno2 = err.errno if (err.errno > 0) else 0x80000000 + err.errno # correct error scheme + errno2 = Helper.convertErrno(err.errno) # correct error scheme errMsg = "Transition failed: errno=0x{:X}, msg: {}".format(errno2, err) logger.info(errMsg) self._execStats.registerFailure(errMsg) @@ -468,6 +485,10 @@ class ThreadCoordinator: # We define a class to run a number of threads in locking steps. +class Helper: + @classmethod + def convertErrno(cls, errno): + return errno if (errno > 0) else 0x80000000 + errno class ThreadPool: def __init__(self, numThreads, maxSteps): @@ -613,8 +634,7 @@ class DbConn: def resetDb(self): # reset the whole database, etc. if (not self.isOpen): - raise RuntimeError( - "Cannot reset database until connection is open") + raise RuntimeError("Cannot reset database until connection is open") # self._tdSql.prepare() # Recreate database, etc. self.execute('drop database if exists db') @@ -681,8 +701,7 @@ class DbConnRest(DbConn): def close(self): if (not self.isOpen): - raise RuntimeError( - "Cannot clean up database until connection is open") + raise RuntimeError("Cannot clean up database until connection is open") # Do nothing for REST logger.debug("[DB] REST Database connection closed") self.isOpen = False @@ -747,27 +766,32 @@ class DbConnRest(DbConn): class MyTDSql: - def __init__(self): + def __init__(self, hostAddr, cfgPath): + # Make the DB connection + self._conn = taos.connect(host=hostAddr, config=cfgPath) + self._cursor = self._conn.cursor() + self.queryRows = 0 self.queryCols = 0 self.affectedRows = 0 - def init(self, cursor, log=True): - self.cursor = cursor + # def init(self, cursor, log=True): + # self.cursor = cursor # if (log): # caller = inspect.getframeinfo(inspect.stack()[1][0]) # self.cursor.log(caller.filename + ".sql") def close(self): - self.cursor.close() + self._conn.close() # TODO: very important, cursor close does NOT close DB connection! + self._cursor.close() def query(self, sql): self.sql = sql try: - self.cursor.execute(sql) - self.queryResult = self.cursor.fetchall() + self._cursor.execute(sql) + self.queryResult = self._cursor.fetchall() self.queryRows = len(self.queryResult) - self.queryCols = len(self.cursor.description) + self.queryCols = len(self._cursor.description) except Exception as e: # caller = inspect.getframeinfo(inspect.stack()[1][0]) # args = (caller.filename, caller.lineno, sql, repr(e)) @@ -778,7 +802,7 @@ class MyTDSql: def execute(self, sql): self.sql = sql try: - self.affectedRows = self.cursor.execute(sql) + self.affectedRows = self._cursor.execute(sql) except Exception as e: # caller = inspect.getframeinfo(inspect.stack()[1][0]) # args = (caller.filename, caller.lineno, sql, repr(e)) @@ -791,13 +815,13 @@ class DbConnNative(DbConn): # Class variables _lock = threading.Lock() _connInfoDisplayed = False + totalConnections = 0 # Not private def __init__(self): super().__init__() self._type = self.TYPE_NATIVE self._conn = None - self._cursor = None - + # self._cursor = None def getBuildPath(self): selfPath = os.path.dirname(os.path.realpath(__file__)) @@ -814,7 +838,8 @@ class DbConnNative(DbConn): buildPath = root[:len(root) - len("/build/bin")] break if buildPath == None: - raise RuntimeError("Failed to determine buildPath, selfPath={}".format(selfPath)) + raise RuntimeError("Failed to determine buildPath, selfPath={}, projPath={}" + .format(selfPath, projPath)) return buildPath @@ -822,33 +847,40 @@ class DbConnNative(DbConn): cfgPath = self.getBuildPath() + "/test/cfg" hostAddr = "127.0.0.1" - with self._lock: # force single threading for opening DB connections - if not self._connInfoDisplayed: - self.__class__._connInfoDisplayed = True # updating CLASS variable - logger.info("Initiating TAOS native connection to {}, using config at {}".format(hostAddr, cfgPath)) - - self._conn = taos.connect(host=hostAddr, config=cfgPath) # TODO: make configurable - self._cursor = self._conn.cursor() + cls = self.__class__ # Get the class, to access class variables + with cls._lock: # force single threading for opening DB connections. # TODO: whaaat??!!! + if not cls._connInfoDisplayed: + cls._connInfoDisplayed = True # updating CLASS variable + logger.info("Initiating TAOS native connection to {}, using config at {}".format(hostAddr, cfgPath)) + # Make the connection + # self._conn = taos.connect(host=hostAddr, config=cfgPath) # TODO: make configurable + # self._cursor = self._conn.cursor() + # Record the count in the class + self._tdSql = MyTDSql(hostAddr, cfgPath) # making DB connection + cls.totalConnections += 1 - self._cursor.execute('reset query cache') + self._tdSql.execute('reset query cache') # self._cursor.execute('use db') # do this at the beginning of every # Open connection - self._tdSql = MyTDSql() - self._tdSql.init(self._cursor) - + # self._tdSql = MyTDSql() + # self._tdSql.init(self._cursor) + def close(self): if (not self.isOpen): - raise RuntimeError( - "Cannot clean up database until connection is open") + raise RuntimeError("Cannot clean up database until connection is open") self._tdSql.close() + # Decrement the class wide counter + cls = self.__class__ # Get the class, to access class variables + with cls._lock: + cls.totalConnections -= 1 + logger.debug("[DB] Database connection closed") self.isOpen = False def execute(self, sql): if (not self.isOpen): - raise RuntimeError( - "Cannot execute database commands until connection is open") + raise RuntimeError("Cannot execute database commands until connection is open") logger.debug("[SQL] Executing SQL: {}".format(sql)) self._lastSql = sql nRows = self._tdSql.execute(sql) @@ -1528,7 +1560,7 @@ class Task(): try: self._executeInternal(te, wt) # TODO: no return value? except taos.error.ProgrammingError as err: - errno2 = err.errno if (err.errno > 0) else 0x80000000 + err.errno # correct error scheme + errno2 = Helper.convertErrno(err.errno) if (gConfig.continue_on_exception): # user choose to continue self.logDebug("[=] Continue after TAOS exception: errno=0x{:X}, msg: {}, SQL: {}".format( errno2, err, wt.getDbConn().getLastSql())) @@ -1678,9 +1710,8 @@ class ExecutionStats: logger.info( "| Total Elapsed Time (from wall clock): {:.3f} seconds".format( self._elapsedTime)) - logger.info( - "| Top numbers written: {}".format( - TaskExecutor.getBoundedList())) + logger.info("| Top numbers written: {}".format(TaskExecutor.getBoundedList())) + logger.info("| Total Number of Active DB Native Connections: {}".format(DbConnNative.totalConnections)) logger.info( "----------------------------------------------------------------------") @@ -1789,7 +1820,7 @@ class TdSuperTable: try: dbc.query("select TBNAME from db.{}".format(self._stName)) # TODO: analyze result set later except taos.error.ProgrammingError as err: - errno2 = err.errno if (err.errno > 0) else 0x80000000 + err.errno + errno2 = Helper.convertErrno(err.errno) logger.debug("[=] Failed to get tables from super table: errno=0x{:X}, msg: {}".format(errno2, err)) raise @@ -1891,7 +1922,7 @@ class TaskReadData(StateTransitionTask): if aggExpr not in ['stddev(speed)']: #TODO: STDDEV not valid for super tables?! dbc.execute("select {} from db.{}".format(aggExpr, sTable.getName())) except taos.error.ProgrammingError as err: - errno2 = err.errno if (err.errno > 0) else 0x80000000 + err.errno + errno2 = Helper.convertErrno(err.errno) logger.debug("[=] Read Failure: errno=0x{:X}, msg: {}, SQL: {}".format(errno2, err, dbc.getLastSql())) raise @@ -1920,9 +1951,8 @@ class TaskDropSuperTable(StateTransitionTask): self.execWtSql(wt, "drop table {}".format( regTableName)) # nRows always 0, like MySQL except taos.error.ProgrammingError as err: - # correcting for strange error number scheme - errno2 = err.errno if ( - err.errno > 0) else 0x80000000 + err.errno + # correcting for strange error number scheme + errno2 = Helper.convertErrno(err.errno) if (errno2 in [0x362]): # mnode invalid table name isSuccess = False logger.debug( @@ -2429,7 +2459,11 @@ class ServiceManagerThread: for line in iter(out.readline, b''): # print("Finished reading a line: {}".format(line)) # print("Adding item to queue...") - line = line.decode("utf-8").rstrip() + try: + line = line.decode("utf-8").rstrip() + except UnicodeError: + print("\nNon-UTF8 server output: {}\n".format(line)) + # This might block, and then causing "out" buffer to block queue.put(line) self._printProgress("_i") @@ -2455,7 +2489,7 @@ class ServiceManagerThread: def svcErrorReader(self, err: IO, queue): for line in iter(err.readline, b''): - print("\nTD Svc STDERR: {}".format(line)) + print("\nTDengine Service (taosd) ERROR (from stderr): {}".format(line)) class TdeSubProcess: diff --git a/tests/pytest/insert/nchar.py b/tests/pytest/insert/nchar.py index b8e365f143..3319aa3c56 100644 --- a/tests/pytest/insert/nchar.py +++ b/tests/pytest/insert/nchar.py @@ -35,6 +35,8 @@ class TDTestCase: tdSql.checkRows(2) tdSql.checkData(1, 1, '涛思数据') + tdSql.error("insert into tb values (now, 'taosdata001')") + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/tag_lite/bigint.py b/tests/pytest/tag_lite/bigint.py index ecc6fad44d..f83961aaad 100644 --- a/tests/pytest/tag_lite/bigint.py +++ b/tests/pytest/tag_lite/bigint.py @@ -575,6 +575,20 @@ class TDTestCase: # TSIM: system sh/exec.sh -n dnode1 -s stop -x SIGINT # convert end + tdSql.execute("create database db") + tdSql.execute("use db") + tdSql.execute( + "create table if not exists st (ts timestamp, tagtype int) tags(dev bigint)") + + tdSql.execute( + 'CREATE TABLE if not exists dev_001 using st tags(%d)' % (pow(2, 63) - 1)) + tdSql.execute( + 'CREATE TABLE if not exists dev_002 using st tags(%d)' % (-1 * pow(2, 63) + 1)) + + print("==============step2") + tdSql.query("show tables") + tdSql.checkRows(2) + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/tag_lite/binary.py b/tests/pytest/tag_lite/binary.py index a5757dc5cd..4cbae63bec 100644 --- a/tests/pytest/tag_lite/binary.py +++ b/tests/pytest/tag_lite/binary.py @@ -579,6 +579,20 @@ class TDTestCase: # TSIM: system sh/exec.sh -n dnode1 -s stop -x SIGINT # convert end + tdSql.execute("create database db") + tdSql.execute("use db") + tdSql.execute( + "create table if not exists st (ts timestamp, tagtype int) tags(dev binary(5))") + tdSql.error( + 'CREATE TABLE if not exists dev_001 using st tags("dev_001")') + + tdSql.execute( + 'CREATE TABLE if not exists dev_002 using st tags("dev")') + + print("==============step2") + tdSql.query("show tables") + tdSql.checkRows(1) + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/tag_lite/float.py b/tests/pytest/tag_lite/float.py index 7b93cb0ac9..9c4d20e82b 100644 --- a/tests/pytest/tag_lite/float.py +++ b/tests/pytest/tag_lite/float.py @@ -575,6 +575,23 @@ class TDTestCase: # TSIM: system sh/exec.sh -n dnode1 -s stop -x SIGINT # convert end + tdSql.execute("create database db") + tdSql.execute("use db") + tdSql.execute( + "create table if not exists st (ts timestamp, tagtype int) tags(dev float)") + tdSql.error( + 'CREATE TABLE if not exists dev_001 using st tags(%f)' % -3.4E38 - 1) + tdSql.error( + 'CREATE TABLE if not exists dev_001 using st tags(%f)' % 3.4E38 + 1) + + tdSql.execute( + 'CREATE TABLE if not exists dev_001 using st tags(%f)' % 3.4E38) + tdSql.execute( + 'CREATE TABLE if not exists dev_002 using st tags(%f)' % -3.4E38) + + tdSql.query("show tables") + tdSql.checkRows(2) + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/tag_lite/int.py b/tests/pytest/tag_lite/int.py index 99d4a69624..d5a6917389 100644 --- a/tests/pytest/tag_lite/int.py +++ b/tests/pytest/tag_lite/int.py @@ -574,6 +574,24 @@ class TDTestCase: # TSIM: system sh/exec.sh -n dnode1 -s stop -x SIGINT # convert end + tdSql.execute("create database db") + tdSql.execute("use db") + tdSql.execute( + "create table if not exists st (ts timestamp, tagtype int) tags(dev int)") + tdSql.error( + 'CREATE TABLE if not exists dev_001 using st tags(%d)' % pow(2, 31)) + tdSql.error( + 'CREATE TABLE if not exists dev_001 using st tags(%d)' % (-1 * pow(2, 31))) + + tdSql.execute( + 'CREATE TABLE if not exists dev_001 using st tags(%d)' % (pow(2, 31) - 1)) + tdSql.execute( + 'CREATE TABLE if not exists dev_002 using st tags(%d)' % (-1 * pow(2, 31) + 1)) + + print("==============step2") + tdSql.query("show tables") + tdSql.checkRows(2) + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/tag_lite/nchar.py b/tests/pytest/tag_lite/nchar.py new file mode 100644 index 0000000000..851cc32b56 --- /dev/null +++ b/tests/pytest/tag_lite/nchar.py @@ -0,0 +1,48 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + + print("==============step1") + tdSql.execute( + "create table if not exists st (ts timestamp, tagtype int) tags(dev nchar(5))") + tdSql.error( + 'CREATE TABLE if not exists dev_001 using st tags("dev_001")') + tdSql.execute( + 'CREATE TABLE if not exists dev_002 using st tags("dev")') + + + print("==============step2") + tdSql.query("show tables") + tdSql.checkRows(1) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/tag_lite/smallint.py b/tests/pytest/tag_lite/smallint.py index 089af55a34..c060e3f82b 100644 --- a/tests/pytest/tag_lite/smallint.py +++ b/tests/pytest/tag_lite/smallint.py @@ -574,6 +574,23 @@ class TDTestCase: # TSIM: # TSIM: system sh/exec.sh -n dnode1 -s stop -x SIGINT # convert end + tdSql.execute("create database db") + tdSql.execute("use db") + tdSql.execute( + "create table if not exists st (ts timestamp, tagtype int) tags(dev smallint)") + tdSql.error( + 'CREATE TABLE if not exists dev_001 using st tags(%d)' % pow(2, 15)) + tdSql.error( + 'CREATE TABLE if not exists dev_001 using st tags(%d)' % (-1 * pow(2, 15))) + + tdSql.execute( + 'CREATE TABLE if not exists dev_001 using st tags(%d)' % (pow(2, 15) - 1)) + tdSql.execute( + 'CREATE TABLE if not exists dev_002 using st tags(%d)' % (-1 * pow(2, 15) + 1)) + + print("==============step2") + tdSql.query("show tables") + tdSql.checkRows(2) def stop(self): tdSql.close() diff --git a/tests/pytest/tag_lite/tinyint.py b/tests/pytest/tag_lite/tinyint.py index 55e33d013b..089dd46569 100644 --- a/tests/pytest/tag_lite/tinyint.py +++ b/tests/pytest/tag_lite/tinyint.py @@ -575,6 +575,24 @@ class TDTestCase: # TSIM: system sh/exec.sh -n dnode1 -s stop -x SIGINT # convert end + tdSql.execute("create database db") + tdSql.execute("use db") + tdSql.execute( + "create table if not exists st (ts timestamp, tagtype int) tags(dev tinyint)") + tdSql.error( + 'CREATE TABLE if not exists dev_001 using st tags(%d)' % pow(2, 7)) + tdSql.error( + 'CREATE TABLE if not exists dev_001 using st tags(%d)' % (-1 * pow(2, 7))) + + tdSql.execute( + 'CREATE TABLE if not exists dev_001 using st tags(%d)' % (pow(2, 7) - 1)) + tdSql.execute( + 'CREATE TABLE if not exists dev_002 using st tags(%d)' % (-1 * pow(2, 7) + 1)) + + print("==============step2") + tdSql.query("show tables") + tdSql.checkRows(2) + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/script/general/parser/bug.sim b/tests/script/general/parser/bug.sim deleted file mode 100644 index 2a46ad1fd6..0000000000 --- a/tests/script/general/parser/bug.sim +++ /dev/null @@ -1,43 +0,0 @@ -system sh/stop_dnodes.sh - -system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 - -system sh/cfg.sh -n dnode1 -c dDebugFlag -v 135 -system sh/cfg.sh -n dnode1 -c mDebugFlag -v 135 -system sh/cfg.sh -n dnode1 -c sdbDebugFlag -v 135 -system sh/cfg.sh -n dnode1 -c rpcDebugFlag -v 135 -system sh/cfg.sh -n dnode1 -c cDebugFlag -v 135 -system sh/cfg.sh -n dnode1 -c monitorDebugflag -v 135 -system sh/cfg.sh -n dnode1 -c httpDebugFlag -v 135 -system sh/cfg.sh -n dnode1 -c uDebugFlag -v 135 - -system sh/cfg.sh -n dnode1 -c httpCacheSessions -v 10 -system sh/cfg.sh -n dnode1 -c httpMaxThreads -v 10 -system sh/cfg.sh -n dnode1 -c httpEnableCompress -v 0 - -system sh/cfg.sh -n dnode1 -c maxVnodeConnections -v 30000 -system sh/cfg.sh -n dnode1 -c maxMgmtConnections -v 30000 -system sh/cfg.sh -n dnode1 -c maxMeterConnections -v 30000 -system sh/cfg.sh -n dnode1 -c maxShellConns -v 30000 - -system sh/exec.sh -n dnode1 -s start -sql connect - -sql create database db; -sql use db; -sql create table t (ts timestamp, i int); -sql create table st1 (ts timestamp, f1 int) tags(t1 int); -sql create table st2 (ts timestamp, f2 int) tags(t2 int); - -sql create table t1 using st1 tags(1); -sql create table t2 using st2 tags(1); - -sql insert into t1 values(1575880055000, 1); -sql insert into t2 values(1575880055000, 2); - -sql select st1.ts, st1.f1, st2.f2 from db.st1, db.st2 where st1.t1=st2.t2 and st1.ts=st2.ts - -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'select st1.ts, st1.f1, st2.f2 from db.st1, db.st2 where st1.t1=st2.t2 and st1.ts=st2.ts' 127.0.0.1:7111/restful/sql - -system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/general/parser/constCol.sim b/tests/script/general/parser/constCol.sim new file mode 100644 index 0000000000..a196ba2b50 --- /dev/null +++ b/tests/script/general/parser/constCol.sim @@ -0,0 +1,354 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c walLevel -v 0 + +system sh/cfg.sh -n dnode1 -c dDebugFlag -v 135 +system sh/cfg.sh -n dnode1 -c mDebugFlag -v 135 +system sh/cfg.sh -n dnode1 -c sdbDebugFlag -v 135 +system sh/cfg.sh -n dnode1 -c rpcDebugFlag -v 135 +system sh/cfg.sh -n dnode1 -c cDebugFlag -v 135 +system sh/cfg.sh -n dnode1 -c monitorDebugflag -v 135 +system sh/cfg.sh -n dnode1 -c httpDebugFlag -v 135 +system sh/cfg.sh -n dnode1 -c uDebugFlag -v 135 + +system sh/cfg.sh -n dnode1 -c httpCacheSessions -v 10 +system sh/cfg.sh -n dnode1 -c httpMaxThreads -v 10 +system sh/cfg.sh -n dnode1 -c httpEnableCompress -v 0 + +system sh/cfg.sh -n dnode1 -c maxVnodeConnections -v 30000 +system sh/cfg.sh -n dnode1 -c maxMgmtConnections -v 30000 +system sh/cfg.sh -n dnode1 -c maxMeterConnections -v 30000 +system sh/cfg.sh -n dnode1 -c maxShellConns -v 30000 + +system sh/exec.sh -n dnode1 -s start +sql connect + +sql create database db; +sql use db; +sql create table t (ts timestamp, i int); +sql create table st1 (ts timestamp, f1 int) tags(t1 int); +sql create table st2 (ts timestamp, f2 int) tags(t2 int); + +sql create table t1 using st1 tags(1); +sql create table t2 using st2 tags(1); + +sql insert into t1 values(1575880055000, 1); +sql insert into t1 values(1575880059000, 1); +sql insert into t1 values(1575880069000, 1); + +sql insert into t2 values(1575880055000, 2); + +sql select st1.ts, st1.f1, st2.f2 from db.st1, db.st2 where st1.t1=st2.t2 and st1.ts=st2.ts + +system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'select st1.ts, st1.f1, st2.f2 from db.st1, db.st2 where st1.t1=st2.t2 and st1.ts=st2.ts' 127.0.0.1:7111/restful/sql + +print ==============select with user-defined columns +sql select 'abc' as f, ts,f1 from t1 +if $rows != 3 then + return -1 +endi + +if $data00 != @abc@ then + return -1 +endi + +if $data01 != @19-12-09 16:27:35.000@ then + return -1 +endi + +if $data02 != 1 then + return -1 +endi + +sql select 'abc', ts, f1 from t1 +if $rows != 3 then + return -1 +endi + +if $data01 != @19-12-09 16:27:35.000@ then + return -1 +endi + +if $data02 != 1 then + return -1 +endi + +if $data10 != @abc@ then + return -1 +endi + +sql select 'abc' from t1 +if $rows != 3 then + return -1 +endi + +if $data00 != @abc@ then + return -1 +endi + +sql select 'abc' as f1 from t1 +if $rows != 3 then + return -1 +endi + +if $data00 != @abc@ then + return -1 +endi + +sql select 1 from t1 +if $rows != 3 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +if $data10 != 1 then + return -1 +endi + +if $data20 != 1 then + return -1 +endi + +sql select 1 as f1 from t1 +if $rows != 3 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +sql select 1 as f, f1 from t1 +if $rows != 3 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +if $data01 != 1 then + return -1 +endi + +sql select 1.123 as f, f1 from t1 +if $rows != 3 then + return -1 +endi + +if $data00 != 1.123000000 then + print expect 1.123000000 , actual:$data00 + return -1 +endi + +if $data10 != 1.123000000 then + print expect 1.123000000 , actual:$data10 + return -1 +endi + +sql select 1, f1 from t1 +if $rows != 3 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +if $data01 != 1 then + return -1 +endi + +if $data10 != 1 then + return -1 +endi + +sql select 1.2391, f1 from t1 +if $rows != 3 then + return -1 +endi + +if $data00 != 1.239100000 then + print expect 1.239100000 actual: $data00 + return -1 +endi + +if $data10 != 1.239100000 then + print expect 1.239100000 actual: $data00 + return -1 +endi + +if $data01 != 1 then + return -1 +endi + +print ===================== user-defined columns with agg functions +sql select 1 as t, count(*) from t1 +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +if $data01 != 3 then + return -1 +endi + +sql select 1, sum(f1) from t1 +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +if $data01 != 3 then + return -1 +endi + +sql select 1,2,3, sum(f1)*99, 4,5,6,7,8,9,10 from t1 +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +if $data01 != 2 then + return -1 +endi + +if $data02 != 3 then + return -1 +endi + +if $data03 != 297.000000000 then + print expect 297.000000000, actual:$data03 + return -1 +endi + +sql select sum(f1)*avg(f1)+12, 1,2,3,4,5,6,7,8,9,10 from t1 +if $rows != 1 then + return -1 +endi + +if $data00 != 15.000000000 then + print expect 15.000000000 actual:$data00 + return -1 +endi + +if $data01 != 1 then + return -1 +endi + +if $data02 != 2 then + return -1 +endi + +sql select 1.2987, f1, 'k' from t1 where f1=1 +if $rows != 3 then + return -1 +endi + +if $data00 != 1.298700000 then + print expect 1.298700000 actual:$data00 + return -1 +endi + +if $data01 != 1 then + return -1 +endi + +if $data02 != @k@ then + return -1 +endi + +print ====================user-defined columns with union +sql select f1, 'f1' from t1 union all select f1, 'f1' from t1; +if $rows != 6 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +if $data01 != @f1@ then + return -1 +endi + +if $data10 != 1 then + return -1 +endi + +if $data11 != @f1@ then + return -1 +endi + +print =====================udc with join +sql select st1.ts, st1.f1, st2.f2, 'abc', 1.9827 from db.st1, db.st2 where st1.t1=st2.t2 and st1.ts=st2.ts +if $rows != 1 then + return -1 +endi + +if $data00 != @19-12-09 16:27:35.000@ then + print expect @19-12-09 16:27:35.000@ actual:$data00 + return -1 +endi + +if $data01 != 1 then + return -1 +endi + +if $data02 != 2 then + return -1 +endi + +if $data03 != @abc@ then + return -1 +endi + +if $data04 != 1.982700000 then + print expect 1.982700000 actual:$data04 + return -1 +endi + +print ======================udc with interval +sql select count(*), 'uuu' from t1 interval(1s) order by ts desc; +if $rows != 3 then + return -1 +endi + +print ======================udc with tags +sql select t1,'abc',tbname from st1 +if $rows != 1 then + return -1 +endi + +if $data01 != @abc@ then + return -1 +endi + +if $data02 != @t1@ then + return -1 +endi + +print ======================udc with arithmetic +sql select 1+1 from t1 +if $rows != 3 then + return -1 +endi + +sql_error select from t1 +sql_error select abc from t1 +sql_error select abc as tu from t1 + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/general/parser/fill_stb.sim b/tests/script/general/parser/fill_stb.sim index 402b79d6b5..f0cd058352 100644 --- a/tests/script/general/parser/fill_stb.sim +++ b/tests/script/general/parser/fill_stb.sim @@ -133,12 +133,18 @@ endi if $data74 != -4.000000000 then return -1 endi + ## fill(value) + group by sql select max(c1), max(c2), max(c3), max(c4), max(c5) from $stb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, -1, -2, -3, -4, -5, -6, -7, -8) group by t1 $val = $rowNum * 2 +print $rowNum, $val + $val = $val - 1 $val = $val * $tbNum -if $rows != $val then +print ==================== $val + +if $rows != 190 then + print expect 190, actual:$rows return -1 endi if $data06 != 0 then diff --git a/tests/script/general/parser/join.sim b/tests/script/general/parser/join.sim index 882f561ae1..1bb0ff5448 100644 --- a/tests/script/general/parser/join.sim +++ b/tests/script/general/parser/join.sim @@ -330,7 +330,10 @@ sql_error select join_tb1.* from $tb1 , $tb2 where join_tb1.ts != join_tb0.ts an sql_error select join_tb1.* from $tb1 , $tb1 where join_tb1.ts = join_tb1.ts and join_tb1.ts >= 100000; sql_error select join_tb1.* from $tb1 , $tb1 where join_tb1.ts = join_tb1.ts order by ts; sql_error select join_tb1.* from $tb1 , $tb1 where join_tb1.ts = join_tb1.ts order by join_tb1.c7; - +sql_error select * from join_tb0, join_tb1 +sql_error select last_row(*) from join_tb0, join_tb1 +sql_error select last_row(*) from $tb1, $tb2 where join_tb1.ts < now +sql_error select last_row(*) from $tb1, $tb2 where join_tb1.ts = join_tb2.ts print ==================================super table join ============================== # select duplicate columns diff --git a/tests/script/general/parser/lastrow_query.sim b/tests/script/general/parser/lastrow_query.sim index 98eb5a8d6d..7954a8d228 100644 --- a/tests/script/general/parser/lastrow_query.sim +++ b/tests/script/general/parser/lastrow_query.sim @@ -128,3 +128,27 @@ if $rows != 86399 then return -1 endi +sql select t1,t1,count(*),t1,t1 from lr_stb0 where ts>'2018-09-24 00:00:00.000' and ts<'2018-09-25 00:00:00.000' interval(1h) fill(NULL) group by t1 order by ts DESC limit 30 +if $rows != 48 then + return -1 +endi + +sql select t1,t1,count(*),t1,t1 from lr_stb0 where ts>'2018-09-24 00:00:00.000' and ts<'2018-09-25 00:00:00.000' interval(1h) fill(NULL) group by t1 order by ts DESC limit 2 +if $rows != 4 then + return -1 +endi + +sql select t1,t1,count(*),tbname,t1,t1,tbname from lr_stb0 where ts>'2018-09-24 00:00:00.000' and ts<'2018-09-25 00:00:00.000' interval(1s) fill(NULL) group by tbname, t1 order by ts desc slimit 1 soffset 1 limit 250000 offset 1 +if $rows != 86399 then + return -1 +endi + +sql select t1,t1,count(*),t1,t1 from lr_stb0 where ts>'2018-09-24 00:00:00.000' and ts<'2018-09-25 00:00:00.000' interval(1h) fill(NULL) group by t1 order by ts desc limit 1 +if $rows != 2 then + return -1 +endi + +sql select t1,t1,count(*),t1,t1 from lr_stb0 where ts>'2018-09-24 00:00:00.000' and ts<'2018-09-25 00:00:00.000' interval(1h) fill(NULL) group by t1 order by ts desc limit 25 offset 1 +if $rows != 46 then + return -1 +endi diff --git a/tests/script/general/parser/nchar.sim b/tests/script/general/parser/nchar.sim index 8d4e468f0e..bdac5ace55 100644 --- a/tests/script/general/parser/nchar.sim +++ b/tests/script/general/parser/nchar.sim @@ -223,6 +223,7 @@ if $rows != 2 then endi print data00 = $data00 if $data00 != 5 then + print expect 5, actual: $data00 return -1 endi if $data10 != 5 then diff --git a/tests/script/general/parser/testSuite.sim b/tests/script/general/parser/testSuite.sim index aafba2d328..4e26d14cfd 100644 --- a/tests/script/general/parser/testSuite.sim +++ b/tests/script/general/parser/testSuite.sim @@ -97,7 +97,7 @@ run general/parser/topbot.sim sleep 2000 run general/parser/union.sim sleep 2000 -run general/parser/bug.sim +run general/parser/constCol.sim sleep 2000 run general/parser/sliding.sim diff --git a/tests/script/general/user/monitor.sim b/tests/script/general/user/monitor.sim index 208b85f16c..eb543612f5 100644 --- a/tests/script/general/user/monitor.sim +++ b/tests/script/general/user/monitor.sim @@ -11,9 +11,9 @@ sleep 3000 sql connect print ========== step2 -sql drop database log -x step21 - return -1 -step21: +#sql drop database log -x step21 +# return -1 +#step21: sql drop table log.dn -x step22 return -1 step22: diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index a48584b0ed..40598332d9 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -147,7 +147,6 @@ cd ../../../debug; make ./test.sh -f general/parser/join.sim ./test.sh -f general/parser/join_multivnode.sim ./test.sh -f general/parser/binary_escapeCharacter.sim -./test.sh -f general/parser/bug.sim ./test.sh -f general/parser/repeatAlter.sim ./test.sh -f general/parser/union.sim ./test.sh -f general/parser/topbot.sim @@ -294,6 +293,7 @@ cd ../../../debug; make ./test.sh -f unique/stable/replica3_dnode6.sim ./test.sh -f unique/stable/replica3_vnode3.sim +./test.sh -f unique/mnode/mgmt21.sim ./test.sh -f unique/mnode/mgmt22.sim ./test.sh -f unique/mnode/mgmt23.sim ./test.sh -f unique/mnode/mgmt24.sim diff --git a/tests/script/jenkins/unique.txt b/tests/script/jenkins/unique.txt index 06edb8890a..b271f5b726 100644 --- a/tests/script/jenkins/unique.txt +++ b/tests/script/jenkins/unique.txt @@ -1,22 +1,11 @@ cd ../../../debug; cmake .. cd ../../../debug; make -./test.sh -f unique/account/account_create.sim -./test.sh -f unique/account/account_delete.sim -./test.sh -f unique/account/account_len.sim -./test.sh -f unique/account/authority.sim -./test.sh -f unique/account/basic.sim -./test.sh -f unique/account/paras.sim -./test.sh -f unique/account/pass_alter.sim -./test.sh -f unique/account/pass_len.sim -./test.sh -f unique/account/usage.sim -./test.sh -f unique/account/user_create.sim -./test.sh -f unique/account/user_len.sim - ./test.sh -f unique/big/balance.sim ./test.sh -f unique/big/maxvnodes.sim ./test.sh -f unique/big/tcp.sim +./test.sh -f unique/cluster/alter.sim ./test.sh -f unique/cluster/balance1.sim ./test.sh -f unique/cluster/balance2.sim ./test.sh -f unique/cluster/balance3.sim @@ -61,6 +50,7 @@ cd ../../../debug; make ./test.sh -f unique/stable/replica3_dnode6.sim ./test.sh -f unique/stable/replica3_vnode3.sim +./test.sh -f unique/mnode/mgmt21.sim ./test.sh -f unique/mnode/mgmt22.sim ./test.sh -f unique/mnode/mgmt23.sim ./test.sh -f unique/mnode/mgmt24.sim @@ -77,31 +67,16 @@ cd ../../../debug; make ./test.sh -f unique/vnode/replica3_repeat.sim ./test.sh -f unique/vnode/replica3_vgroup.sim -./test.sh -f general/stream/metrics_1.sim -./test.sh -f general/stream/metrics_del.sim -./test.sh -f general/stream/metrics_n.sim -./test.sh -f general/stream/metrics_replica1_vnoden.sim -#./test.sh -f general/stream/new_stream.sim -./test.sh -f general/stream/restart_stream.sim -./test.sh -f general/stream/stream_1.sim -./test.sh -f general/stream/stream_2.sim -./test.sh -f general/stream/stream_3.sim -./test.sh -f general/stream/stream_restart.sim -./test.sh -f general/stream/table_1.sim -./test.sh -f general/stream/table_del.sim -./test.sh -f general/stream/table_n.sim -./test.sh -f general/stream/table_replica1_vnoden.sim - ./test.sh -f unique/arbitrator/check_cluster_cfg_para.sim -./test.sh -f unique/arbitrator/dn2_mn1_cache_file_sync.sim +#./test.sh -f unique/arbitrator/dn2_mn1_cache_file_sync.sim ./test.sh -f unique/arbitrator/dn3_mn1_full_createTableFail.sim -./test.sh -f unique/arbitrator/dn3_mn1_full_dropDnodeFail.sim ./test.sh -f unique/arbitrator/dn3_mn1_multiCreateDropTable.sim -./test.sh -f unique/arbitrator/dn3_mn1_nw_disable_timeout_autoDropDnode.sim -./test.sh -f unique/arbitrator/dn3_mn1_replica2_wal1_AddDelDnode.sim +#./test.sh -f unique/arbitrator/dn3_mn1_nw_disable_timeout_autoDropDnode.sim +#./test.sh -f unique/arbitrator/dn3_mn1_replica2_wal1_AddDelDnode.sim ./test.sh -f unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim ./test.sh -f unique/arbitrator/dn3_mn1_replica_change.sim -./test.sh -f unique/arbitrator/dn3_mn1_stopDnode_timeout.sim +#./test.sh -f unique/arbitrator/dn3_mn1_stopDnode_timeout.sim +# lower the priority while file corruption #./test.sh -f unique/arbitrator/dn3_mn1_vnode_change.sim #./test.sh -f unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim #./test.sh -f unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim @@ -126,6 +101,7 @@ cd ../../../debug; make ./test.sh -f unique/arbitrator/replica_changeWithArbitrator.sim ./test.sh -f unique/arbitrator/sync_replica2_alterTable_add.sim ./test.sh -f unique/arbitrator/sync_replica2_alterTable_drop.sim + ./test.sh -f unique/arbitrator/sync_replica2_dropDb.sim ./test.sh -f unique/arbitrator/sync_replica2_dropTable.sim ./test.sh -f unique/arbitrator/sync_replica3_alterTable_add.sim diff --git a/tests/script/tmp/182.sim b/tests/script/tmp/182.sim new file mode 100644 index 0000000000..27e064dc9b --- /dev/null +++ b/tests/script/tmp/182.sim @@ -0,0 +1,40 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 + +system sh/cfg.sh -n dnode1 -c walLevel -v 2 +system sh/cfg.sh -n dnode2 -c walLevel -v 2 +system sh/cfg.sh -n dnode3 -c walLevel -v 2 +system sh/cfg.sh -n dnode4 -c walLevel -v 2 + +system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1 +system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1 +system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1 +system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1 + +system sh/cfg.sh -n dnode1 -c activeCode -v eglxDLzRpslJWl7OxrPZ2K3sQ5631AP9SVpezsaz2dhJWl7OxrPZ2ElaXs7Gs9nYSVpezsaz2djGIj5StnQ3ZvLHcsE8cwcN + +system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4 + +system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 100000 +system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 100000 +system sh/cfg.sh -n dnode3 -c maxTablesPerVnode -v 100000 +system sh/cfg.sh -n dnode4 -c maxTablesPerVnode -v 100000 + +system sh/cfg.sh -n dnode1 -c http -v 1 +system sh/cfg.sh -n dnode2 -c http -v 1 +system sh/cfg.sh -n dnode3 -c http -v 1 +system sh/cfg.sh -n dnode4 -c http -v 1 + + +system sh/cfg.sh -n dnode1 -c firstEp -v 127.0.0.1:6030 +system sh/cfg.sh -n dnode1 -c secondEp -v 127.0.0.1:6030 +system sh/cfg.sh -n dnode1 -c serverPort -v 6030 +system sh/cfg.sh -n dnode1 -c fqdn -v 127.0.0.1 +#system sh/exec.sh -n dnode1 -s start diff --git a/tests/script/tmp/prepare.sim b/tests/script/tmp/prepare.sim index f59eebede0..8b8f206233 100644 --- a/tests/script/tmp/prepare.sim +++ b/tests/script/tmp/prepare.sim @@ -32,10 +32,17 @@ system sh/cfg.sh -n dnode2 -c http -v 1 system sh/cfg.sh -n dnode3 -c http -v 1 system sh/cfg.sh -n dnode4 -c http -v 1 - +return +# for crash_gen +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 2 +system sh/cfg.sh -n dnode1 -c rpcMaxTime -v 101 +system sh/cfg.sh -n dnode1 -c cache -v 2 +system sh/cfg.sh -n dnode1 -c keep -v 36500 system sh/cfg.sh -n dnode1 -c walLevel -v 2 +# for windows + system sh/cfg.sh -n dnode1 -c firstEp -v 152.136.17.116:6030 system sh/cfg.sh -n dnode1 -c secondEp -v 152.136.17.116:6030 system sh/cfg.sh -n dnode1 -c serverPort -v 6030 diff --git a/tests/script/unique/account/usage.sim b/tests/script/unique/account/usage.sim index 3f12c46137..7fde365201 100644 --- a/tests/script/unique/account/usage.sim +++ b/tests/script/unique/account/usage.sim @@ -112,21 +112,41 @@ print =============== step4 sql insert into d1.t1 values(now + 1s, 1) sql insert into d1.t1 values(now + 2s, 2) -# no write auth -sleep 3000 +sleep 10000 +print no write auth sql_error insert into d1.t1 values(now + 3s, 2) sql_error insert into d1.t1 values(now + 4s, 2) sql alter account root pass "taosdata" tseries 10 storage 36 streams 10 dbs 5 users 5 -sleep 3000 +sleep 10000 +print has write auth sql insert into d1.t1 values(now + 5s, 1) sql insert into d1.t1 values(now + 6s, 2) # no write auth -sleep 3000 +sleep 10000 +print no write auth sql_error insert into d1.t1 values(now + 7s, 2) sql_error insert into d1.t1 values(now + 8s, 2) +print =============== step5 +sql alter account root pass "taosdata" tseries 10 storage 3600 streams 10 dbs 5 users 5 state all +sleep 10000 + +sql insert into d1.t1 values(now + 11s, 1) +sql insert into d1.t1 values(now + 12s, 2) + +sql alter account root pass "taosdata" tseries 10 storage 3600 streams 10 dbs 5 users 5 state no +sleep 10000 +print no write auth +sql_error insert into d1.t1 values(now + 13s, 2) +sql_error insert into d1.t1 values(now + 14s, 2) + +sql alter account root pass "taosdata" tseries 10 storage 3600 streams 10 dbs 5 users 5 state all +sleep 10000 +print has write auth +sql insert into d1.t1 values(now + 15s, 1) +sql insert into d1.t1 values(now + 16s, 2) print =============== check grant sql_error create database d6 diff --git a/tests/script/unique/mnode/mgmt21.sim b/tests/script/unique/mnode/mgmt21.sim new file mode 100644 index 0000000000..53ad0eebe7 --- /dev/null +++ b/tests/script/unique/mnode/mgmt21.sim @@ -0,0 +1,44 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 + +system sh/cfg.sh -n dnode1 -c numOfMnodes -v 2 +system sh/cfg.sh -n dnode2 -c numOfMnodes -v 2 + +print ============== step1 +system sh/exec.sh -n dnode2 -s start +sleep 10000 + +system sh/exec.sh -n dnode1 -s start +sql connect + +sql show mnodes +print dnode1 ==> $data2_1 +print dnode2 ==> $data2_2 +if $data2_1 != master then + return -1 +endi + +print ============== step2 +sql create dnode $hostname2 + +$x = 0 +show2: + $x = $x + 1 + sleep 2000 + if $x == 10 then + return -1 + endi + +sql show mnodes +print dnode1 ==> $data2_1 +print dnode2 ==> $data2_2 +if $data2_1 != master then + goto show2 +endi +if $data2_2 != slave then + goto show2 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/unique/mnode/testSuite.sim b/tests/script/unique/mnode/testSuite.sim index 33df24b860..b9adbe06a2 100644 --- a/tests/script/unique/mnode/testSuite.sim +++ b/tests/script/unique/mnode/testSuite.sim @@ -1,3 +1,4 @@ +run unique/mnode/mgmt21.sim run unique/mnode/mgmt22.sim run unique/mnode/mgmt23.sim run unique/mnode/mgmt24.sim diff --git a/tests/script/wtest.bat b/tests/script/wtest.bat index 1574b5013e..6cdd63b42d 100644 --- a/tests/script/wtest.bat +++ b/tests/script/wtest.bat @@ -6,8 +6,8 @@ echo Start TDengine Testing Case ... set "SCRIPT_DIR=%~dp0" echo SCRIPT_DIR: %SCRIPT_DIR% -set "BUILD_DIR=%~dp0..\..\debug\build\bin" -set "TSIM=%~dp0..\..\debug\build\bin\tsim" +set "BUILD_DIR=%~dp0..\..\debug\32\build\bin" +set "TSIM=%~dp0..\..\debug\32\build\bin\tsim" echo BUILD_DIR: %BUILD_DIR% set "SIM_DIR=%~dp0..\..\sim" @@ -47,7 +47,7 @@ echo qdebugFlag 143 >> %TAOS_CFG% echo udebugFlag 143 >> %TAOS_CFG% set "FILE_NAME=windows\testSuite.sim" -set "FIRSTEP=localhost" +set "FIRSTEP=192.168.1.182" if "%1" == "-f" set "FILE_NAME=%2" if "%1" == "-h" set "FIRSTEP=%2" if "%3" == "-f" set "FILE_NAME=%4" diff --git a/tests/test/c/CMakeLists.txt b/tests/test/c/CMakeLists.txt index cc8bc78e33..536727100d 100644 --- a/tests/test/c/CMakeLists.txt +++ b/tests/test/c/CMakeLists.txt @@ -10,8 +10,8 @@ IF (TD_LINUX) #add_executable(insertPerTable insertPerTable.c) #target_link_libraries(insertPerTable taos_static pthread) - add_executable(insertPerRow insertPerRow.c) - target_link_libraries(insertPerRow taos_static pthread) + #add_executable(insertPerRow insertPerRow.c) + #target_link_libraries(insertPerRow taos_static pthread) #add_executable(importOneRow importOneRow.c) #target_link_libraries(importOneRow taos_static pthread) @@ -22,6 +22,12 @@ IF (TD_LINUX) #add_executable(hashPerformance hashPerformance.c) #target_link_libraries(hashPerformance taos_static tutil common pthread) - add_executable(createTablePerformance createTablePerformance.c) - target_link_libraries(createTablePerformance taos_static tutil common pthread) + #add_executable(createTablePerformance createTablePerformance.c) + #target_link_libraries(createTablePerformance taos_static tutil common pthread) + + #add_executable(createNormalTable createNormalTable.c) + #target_link_libraries(createNormalTable taos_static tutil common pthread) + + #add_executable(queryPerformance queryPerformance.c) + #target_link_libraries(queryPerformance taos_static tutil common pthread) ENDIF() diff --git a/tests/test/c/createNormalTable.c b/tests/test/c/createNormalTable.c new file mode 100644 index 0000000000..18a648b9e1 --- /dev/null +++ b/tests/test/c/createNormalTable.c @@ -0,0 +1,218 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "taoserror.h" +#include "taos.h" +#include "tulog.h" +#include "tutil.h" +#include "tglobal.h" +#include "hash.h" + +#define MAX_RANDOM_POINTS 20000 +#define GREEN "\033[1;32m" +#define NC "\033[0m" + +char dbName[32] = "db"; +char stableName[64] = "st"; +int32_t numOfThreads = 30; +int32_t numOfTables = 100000; +int32_t replica = 1; +int32_t numOfColumns = 2; + +typedef struct { + int32_t tableBeginIndex; + int32_t tableEndIndex; + int32_t threadIndex; + char dbName[32]; + char stableName[64]; + float createTableSpeed; + pthread_t thread; +} SThreadInfo; + +void shellParseArgument(int argc, char *argv[]); +void *threadFunc(void *param); +void createDbAndSTable(); + +int main(int argc, char *argv[]) { + shellParseArgument(argc, argv); + taos_init(); + createDbAndSTable(); + + pPrint("%d threads are spawned to create table", numOfThreads); + + pthread_attr_t thattr; + pthread_attr_init(&thattr); + pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + SThreadInfo *pInfo = (SThreadInfo *)calloc(numOfThreads, sizeof(SThreadInfo)); + + int32_t numOfTablesPerThread = numOfTables / numOfThreads; + numOfTables = numOfTablesPerThread * numOfThreads; + for (int i = 0; i < numOfThreads; ++i) { + pInfo[i].tableBeginIndex = i * numOfTablesPerThread; + pInfo[i].tableEndIndex = (i + 1) * numOfTablesPerThread; + pInfo[i].threadIndex = i; + strcpy(pInfo[i].dbName, dbName); + strcpy(pInfo[i].stableName, stableName); + pthread_create(&(pInfo[i].thread), &thattr, threadFunc, (void *)(pInfo + i)); + } + + taosMsleep(300); + for (int i = 0; i < numOfThreads; i++) { + pthread_join(pInfo[i].thread, NULL); + } + + float createTableSpeed = 0; + for (int i = 0; i < numOfThreads; ++i) { + createTableSpeed += pInfo[i].createTableSpeed; + } + + pPrint("%s total speed:%.1f tables/second, threads:%d %s", GREEN, createTableSpeed, numOfThreads, NC); + + pthread_attr_destroy(&thattr); + free(pInfo); +} + +void createDbAndSTable() { + pPrint("start to create db and stable"); + char qstr[64000]; + + TAOS *con = taos_connect(NULL, "root", "taosdata", NULL, 0); + if (con == NULL) { + pError("failed to connect to DB, reason:%s", taos_errstr(con)); + exit(1); + } + + sprintf(qstr, "create database if not exists %s replica %d", dbName, replica); + TAOS_RES *pSql = taos_query(con, qstr); + int32_t code = taos_errno(pSql); + if (code != 0) { + pError("failed to create database:%s, sql:%s, code:%d reason:%s", dbName, qstr, taos_errno(con), taos_errstr(con)); + exit(0); + } + taos_free_result(pSql); + + sprintf(qstr, "use %s", dbName); + pSql = taos_query(con, qstr); + code = taos_errno(pSql); + if (code != 0) { + pError("failed to use db, code:%d reason:%s", taos_errno(con), taos_errstr(con)); + exit(0); + } + taos_free_result(pSql); + + taos_close(con); +} + +void *threadFunc(void *param) { + SThreadInfo *pInfo = (SThreadInfo *)param; + char qstr[65000]; + int code; + + TAOS *con = taos_connect(NULL, "root", "taosdata", NULL, 0); + if (con == NULL) { + pError("index:%d, failed to connect to DB, reason:%s", pInfo->threadIndex, taos_errstr(con)); + exit(1); + } + + sprintf(qstr, "use %s", pInfo->dbName); + TAOS_RES *pSql = taos_query(con, qstr); + taos_free_result(pSql); + + int64_t startMs = taosGetTimestampMs(); + + for (int32_t t = pInfo->tableBeginIndex; t < pInfo->tableEndIndex; ++t) { + sprintf(qstr, "create table %s%d (ts timestamp, i int)", stableName, t); + TAOS_RES *pSql = taos_query(con, qstr); + code = taos_errno(pSql); + if (code != 0) { + pError("failed to create table %s%d, reason:%s", stableName, t, tstrerror(code)); + } + taos_free_result(pSql); + } + + float createTableSpeed = 0; + for (int i = 0; i < numOfThreads; ++i) { + createTableSpeed += pInfo[i].createTableSpeed; + } + + int64_t endMs = taosGetTimestampMs(); + int32_t totalTables = pInfo->tableEndIndex - pInfo->tableBeginIndex; + float seconds = (endMs - startMs) / 1000.0; + float speed = totalTables / seconds; + pInfo->createTableSpeed = speed; + + pPrint("thread:%d, time:%.2f sec, speed:%.1f tables/second, ", pInfo->threadIndex, seconds, speed); + taos_close(con); + + return 0; +} + +void printHelp() { + char indent[10] = " "; + printf("Used to test the performance while create table\n"); + + printf("%s%s\n", indent, "-c"); + printf("%s%s%s%s\n", indent, indent, "Configuration directory, default is ", configDir); + printf("%s%s\n", indent, "-d"); + printf("%s%s%s%s\n", indent, indent, "The name of the database to be created, default is ", dbName); + printf("%s%s\n", indent, "-s"); + printf("%s%s%s%s\n", indent, indent, "The name of the super table to be created, default is ", stableName); + printf("%s%s\n", indent, "-t"); + printf("%s%s%s%d\n", indent, indent, "numOfThreads, default is ", numOfThreads); + printf("%s%s\n", indent, "-n"); + printf("%s%s%s%d\n", indent, indent, "numOfTables, default is ", numOfTables); + printf("%s%s\n", indent, "-r"); + printf("%s%s%s%d\n", indent, indent, "replica, default is ", replica); + printf("%s%s\n", indent, "-columns"); + printf("%s%s%s%d\n", indent, indent, "numOfColumns, default is ", numOfColumns); + + exit(EXIT_SUCCESS); +} + +void shellParseArgument(int argc, char *argv[]) { + for (int i = 1; i < argc; i++) { + if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) { + printHelp(); + exit(0); + } else if (strcmp(argv[i], "-d") == 0) { + strcpy(dbName, argv[++i]); + } else if (strcmp(argv[i], "-c") == 0) { + strcpy(configDir, argv[++i]); + } else if (strcmp(argv[i], "-s") == 0) { + strcpy(stableName, argv[++i]); + } else if (strcmp(argv[i], "-t") == 0) { + numOfThreads = atoi(argv[++i]); + } else if (strcmp(argv[i], "-n") == 0) { + numOfTables = atoi(argv[++i]); + } else if (strcmp(argv[i], "-r") == 0) { + replica = atoi(argv[++i]); + } else if (strcmp(argv[i], "-columns") == 0) { + numOfColumns = atoi(argv[++i]); + } else { + } + } + + pPrint("%s dbName:%s %s", GREEN, dbName, NC); + pPrint("%s stableName:%s %s", GREEN, stableName, NC); + pPrint("%s configDir:%s %s", GREEN, configDir, NC); + pPrint("%s numOfTables:%d %s", GREEN, numOfTables, NC); + pPrint("%s numOfThreads:%d %s", GREEN, numOfThreads, NC); + pPrint("%s numOfColumns:%d %s", GREEN, numOfColumns, NC); + pPrint("%s replica:%d %s", GREEN, replica, NC); + + pPrint("%s start create table performace test %s", GREEN, NC); +} diff --git a/tests/test/c/queryPerformance.c b/tests/test/c/queryPerformance.c new file mode 100644 index 0000000000..5e7a4333de --- /dev/null +++ b/tests/test/c/queryPerformance.c @@ -0,0 +1,167 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "taos.h" +#include "tulog.h" +#include "ttimer.h" +#include "tutil.h" +#include "tglobal.h" + +#define MAX_RANDOM_POINTS 20000 +#define GREEN "\033[1;32m" +#define NC "\033[0m" + +typedef struct { + int64_t startTimeMs; + int64_t endTimeMs; + int threadIndex; + pthread_t thread; +} SInfo; + +void *syncTest(void *param); +void shellParseArgument(int argc, char *argv[]); +void insertData(); + +int64_t numOfThreads = 100; +char sql[10240] = "show dnodes"; +int32_t loopTimes = 1000; + +int main(int argc, char *argv[]) { + shellParseArgument(argc, argv); + taos_init(); + insertData(); +} + +void insertData() { + struct timeval systemTime; + int64_t st, et; + + gettimeofday(&systemTime, NULL); + st = systemTime.tv_sec * 1000000 + systemTime.tv_usec; + + pPrint("%" PRId64 " threads are spawned to query", numOfThreads); + + pthread_attr_t thattr; + pthread_attr_init(&thattr); + pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + SInfo *pInfo = (SInfo *)malloc(sizeof(SInfo) * numOfThreads); + + // Start threads to write + for (int i = 0; i < numOfThreads; ++i) { + pInfo[i].threadIndex = i; + pthread_create(&(pInfo[i].thread), &thattr, syncTest, (void *)(pInfo + i)); + } + + taosMsleep(300); + for (int i = 0; i < numOfThreads; i++) { + pthread_join(pInfo[i].thread, NULL); + } + + gettimeofday(&systemTime, NULL); + et = systemTime.tv_sec * 1000000 + systemTime.tv_usec; + double mseconds = (et - st) / 1000.0; + + int64_t request = loopTimes * numOfThreads; + float avg = mseconds / request;; + float qps = 1000 / avg * numOfThreads; + + pPrint( + "%sall threads:%ld finished, use %.1lf ms, qps:%f, avg:%f %s", + GREEN, numOfThreads, mseconds, qps, avg, NC); + + pPrint("threads exit"); + + pthread_attr_destroy(&thattr); + free(pInfo); +} + +void *syncTest(void *param) { + TAOS * con; + SInfo * pInfo = (SInfo *)param; + struct timeval systemTime; + + pPrint("thread:%d, start to run", pInfo->threadIndex); + char fqdn[TSDB_FQDN_LEN]; + uint16_t port; + + taosGetFqdnPortFromEp(tsFirst, fqdn, &port); + + con = taos_connect(fqdn, "root", "taosdata", NULL, port); + if (con == NULL) { + pError("index:%d, failed to connect to DB, reason:%s", pInfo->threadIndex, taos_errstr(con)); + exit(1); + } + + for (int i = 0; i < loopTimes; ++i) { + void *tres = taos_query(con, sql); + + TAOS_ROW row = taos_fetch_row(tres); + if (row == NULL) { + taos_free_result(tres); + exit(0); + } + + do { + row = taos_fetch_row(tres); + } while( row != NULL); + + taos_free_result(tres); + } + + gettimeofday(&systemTime, NULL); + + return NULL; +} + +void printHelp() { + char indent[10] = " "; + printf("Used to test the query performance of TDengine\n"); + + printf("%s%s\n", indent, "-c"); + printf("%s%s%s%s\n", indent, indent, "Configuration directory, default is ", configDir); + printf("%s%s\n", indent, "-s"); + printf("%s%s%s%s\n", indent, indent, "The sql to be executed, default is %s", sql); + printf("%s%s\n", indent, "-l"); + printf("%s%s%s%d\n", indent, indent, "Loop Times per thread, default is ", loopTimes); + printf("%s%s\n", indent, "-t"); + printf("%s%s%s%" PRId64 "\n", indent, indent, "Number of threads to be used, default is ", numOfThreads); + + exit(EXIT_SUCCESS); +} + +void shellParseArgument(int argc, char *argv[]) { + for (int i = 1; i < argc; i++) { + if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) { + printHelp(); + exit(0); + } else if (strcmp(argv[i], "-c") == 0) { + strcpy(configDir, argv[++i]); + } else if (strcmp(argv[i], "-s") == 0) { + strcpy(sql, argv[++i]); + } else if (strcmp(argv[i], "-l") == 0) { + loopTimes = atoi(argv[++i]); + } else if (strcmp(argv[i], "-t") == 0) { + numOfThreads = atoi(argv[++i]); + } else { + } + } + + pPrint("%ssql:%s%s", GREEN, sql, NC); + pPrint("%sloopTImes:%d%s", GREEN, loopTimes, NC); + pPrint("%snumOfThreads:%" PRId64 "%s", GREEN, numOfThreads, NC); + pPrint("%sstart to run%s", GREEN, NC); +} diff --git a/tests/tsim/inc/sim.h b/tests/tsim/inc/sim.h index 18af21a506..6f3bc7099d 100644 --- a/tests/tsim/inc/sim.h +++ b/tests/tsim/inc/sim.h @@ -96,20 +96,20 @@ enum { struct _script_t; typedef struct _cmd_t { - short cmdno; - short nlen; - char name[MAX_SIM_CMD_NAME_LEN]; - bool (*parseCmd)(char *, struct _cmd_t *, int); - bool (*executeCmd)(struct _script_t *script, char *option); + int16_t cmdno; + int16_t nlen; + char name[MAX_SIM_CMD_NAME_LEN]; + bool (*parseCmd)(char *, struct _cmd_t *, int); + bool (*executeCmd)(struct _script_t *script, char *option); struct _cmd_t *next; } SCommand; typedef struct { - short cmdno; - short jump; // jump position - short errorJump; // sql jump flag, while '-x' exist in sql cmd, this flag - // will be SQL_JUMP_TRUE, otherwise is SQL_JUMP_FALSE */ - short lineNum; // correspodning line number in original file + int16_t cmdno; + int16_t jump; // jump position + int16_t errorJump; // sql jump flag, while '-x' exist in sql cmd, this flag + // will be SQL_JUMP_TRUE, otherwise is SQL_JUMP_FALSE */ + int16_t lineNum; // correspodning line number in original file int optionOffset; // relative option offset } SCmdLine; @@ -120,7 +120,7 @@ typedef struct _var_t { } SVariable; typedef struct _script_t { - int type; + int type; bool killed; void *taos; @@ -130,10 +130,10 @@ typedef struct _script_t { char system_exit_code[12]; char system_ret_content[MAX_SYSTEM_RESULT_LEN]; - int varLen; - int linePos; // current cmd position - int numOfLines; // number of lines in the script - int bgScriptLen; + int varLen; + int linePos; // current cmd position + int numOfLines; // number of lines in the script + int bgScriptLen; char fileName[MAX_FILE_NAME_LEN]; // script file name char error[MAX_ERROR_LEN]; char *optionBuffer; diff --git a/tests/tsim/inc/simParse.h b/tests/tsim/inc/simParse.h index cff66e6c4f..d3f92add71 100644 --- a/tests/tsim/inc/simParse.h +++ b/tests/tsim/inc/simParse.h @@ -33,21 +33,21 @@ enum { /* label stack */ typedef struct { - char top; /* number of labels */ - short pos[MAX_NUM_LABLES]; /* the position of the label */ - char label[MAX_NUM_LABLES][MAX_LABEL_LEN]; /* name of the label */ + char top; /* number of labels */ + int16_t pos[MAX_NUM_LABLES]; /* the position of the label */ + char label[MAX_NUM_LABLES][MAX_LABEL_LEN]; /* name of the label */ } SLabel; /* block definition */ typedef struct { - char top; /* the number of blocks stacked */ - char type[MAX_NUM_BLOCK]; /* the block type */ - short *pos[MAX_NUM_BLOCK]; /* position of the jump for if/elif/case */ - short back[MAX_NUM_BLOCK]; /* go back, endw and continue */ - char numJump[MAX_NUM_BLOCK]; - short *jump[MAX_NUM_BLOCK][MAX_NUM_JUMP]; /* break or elif */ - char sexp[MAX_NUM_BLOCK][40]; /*switch expression */ - char sexpLen[MAX_NUM_BLOCK]; /*switch expression length */ + char top; /* the number of blocks stacked */ + char type[MAX_NUM_BLOCK]; /* the block type */ + int16_t *pos[MAX_NUM_BLOCK]; /* position of the jump for if/elif/case */ + int16_t back[MAX_NUM_BLOCK]; /* go back, endw and continue */ + char numJump[MAX_NUM_BLOCK]; + int16_t *jump[MAX_NUM_BLOCK][MAX_NUM_JUMP]; /* break or elif */ + char sexp[MAX_NUM_BLOCK][40]; /*switch expression */ + char sexpLen[MAX_NUM_BLOCK]; /*switch expression length */ } SBlock; bool simParseExpression(char *token, int lineNum);