xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch 'develop' into compress_float

This commit is contained in:
tickduan 2021-07-19 09:47:02 +08:00
parent 04aa94dd0e 614b5a3352
commit a215ba73e2
146 changed files with 11672 additions and 4367 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
cmake/define.inc
View File

@ -83,6 +83,8 @@ IF (TD_ARM_64)
ADD_DEFINITIONS(-DUSE_LIBICONV)
MESSAGE(STATUS "arm64 is defined")
SET(COMMON_FLAGS "-Wall -Werror -fPIC -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE")
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/lua/src)
ENDIF ()
IF (TD_ARM_32)
@ -91,6 +93,8 @@ IF (TD_ARM_32)
ADD_DEFINITIONS(-DUSE_LIBICONV)
MESSAGE(STATUS "arm32 is defined")
SET(COMMON_FLAGS "-Wall -Werror -fPIC -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE -Wno-pointer-to-int-cast -Wno-int-to-pointer-cast -Wno-incompatible-pointer-types ")
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/lua/src)
ENDIF ()
IF (TD_MIPS_64)
@ -143,6 +147,7 @@ IF (TD_LINUX)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/cJson/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/lz4/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/lua/src)
ENDIF ()
IF (TD_DARWIN_64)
@ -164,6 +169,7 @@ IF (TD_DARWIN_64)
SET(RELEASE_FLAGS "-Og")
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/cJson/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/lz4/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/lua/src)
ENDIF ()
IF (TD_WINDOWS)
@ -194,6 +200,7 @@ IF (TD_WINDOWS)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/regex)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/wepoll/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/MsvcLibX/include)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/lua/src)
ENDIF ()
IF (TD_WINDOWS_64)

2
cmake/install.inc
View File

@ -32,7 +32,7 @@ ELSEIF (TD_WINDOWS)
#INSTALL(TARGETS taos RUNTIME DESTINATION driver)
#INSTALL(TARGETS shell RUNTIME DESTINATION .)
IF (TD_MVN_INSTALLED)
INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.32-dist.jar DESTINATION connector/jdbc)
INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-*-dist.jar DESTINATION connector/jdbc)
ENDIF ()
ELSEIF (TD_DARWIN)
SET(TD_MAKE_INSTALL_SH "${TD_COMMUNITY_DIR}/packaging/tools/make_install.sh")

98
documentation20/cn/02.getting-started/docs.md
View File

@ -2,25 +2,25 @@
## <a class="anchor" id="install"></a>快捷安装
TDengine软件分为服务器、客户端和报警模块三部分，目前2.0版服务器仅能在Linux系统上安装和运行，后续会支持Windows、mac OS等系统。客户端可以在Windows或Linux上安装和运行。任何OS的应用也可以选择RESTful接口连接服务器taosd。CPU支持X64/ARM64/MIPS64/Alpha64，后续会支持ARM32、RISC-V等CPU架构。用户可根据需求选择通过[源码](https://www.taosdata.com/cn/getting-started/#通过源码安装)或者[安装包](https://www.taosdata.com/cn/getting-started/#通过安装包安装)来安装。
TDengine 软件分为服务器、客户端和报警模块三部分，目前 2.0 版服务器仅能在 Linux 系统上安装和运行，后续会支持 Windows、Mac OS 等系统。客户端可以在 Windows 或 Linux 上安装和运行。任何 OS 的应用也可以选择 RESTful 接口连接服务器 taosd。CPU 支持 X64/ARM64/MIPS64/Alpha64，后续会支持 ARM32、RISC-V 等 CPU 架构。用户可根据需求选择通过 [源码](https://www.taosdata.com/cn/getting-started/#通过源码安装) 或者 [安装包](https://www.taosdata.com/cn/getting-started/#通过安装包安装) 来安装。
### <a class="anchor" id="source-install"></a>通过源码安装
请参考我们的[TDengine github主页](https://github.com/taosdata/TDengine)下载源码并安装.
请参考我们的 [TDengine github 主页](https://github.com/taosdata/TDengine) 下载源码并安装.
### 通过Docker容器运行
### 通过 Docker 容器运行
暂时不建议生产环境采用 Docker 来部署 TDengine 的客户端或服务端，但在开发环境下或初次尝试时，使用 Docker 方式部署是十分方便的。特别是，利用 Docker，可以方便地在 Mac OSX 和 Windows 环境下尝试 TDengine。
暂时不建议生产环境采用 Docker 来部署 TDengine 的客户端或服务端，但在开发环境下或初次尝试时，使用 Docker 方式部署是十分方便的。特别是，利用 Docker，可以方便地在 Mac OS X 和 Windows 环境下尝试 TDengine。
详细操作方法请参照 [通过Docker快速体验TDengine](https://www.taosdata.com/cn/documentation/getting-started/docker)。
详细操作方法请参照 [通过 Docker 快速体验 TDengine](https://www.taosdata.com/cn/documentation/getting-started/docker)。
### <a class="anchor" id="package-install"></a>通过安装包安装
TDengine的安装非常简单，从下载到安装成功仅仅只要几秒钟。服务端安装包包含客户端和连接器，我们提供三种安装包，您可以根据需要选择：
TDengine 的安装非常简单，从下载到安装成功仅仅只要几秒钟。服务端安装包包含客户端和连接器，我们提供三种安装包，您可以根据需要选择：
安装包下载在[这里](https://www.taosdata.com/cn/getting-started/#通过安装包安装)。
安装包下载在 [这里](https://www.taosdata.com/cn/getting-started/#通过安装包安装)。
具体的安装过程，请参见[TDengine多种安装包的安装和卸载](https://www.taosdata.com/blog/2019/08/09/566.html)以及[视频教程](https://www.taosdata.com/blog/2020/11/11/1941.html)。
具体的安装过程，请参见 [TDengine 多种安装包的安装和卸载](https://www.taosdata.com/blog/2019/08/09/566.html) 以及 [视频教程](https://www.taosdata.com/blog/2020/11/11/1941.html)。
## <a class="anchor" id="start"></a>轻松启动
@ -53,21 +53,21 @@ $ systemctl status taosd
如果系统中不支持 systemd，也可以用手动运行 /usr/local/taos/bin/taosd 方式启动 TDengine 服务。
## <a class="anchor" id="console"></a>TDengine命令行程序
## <a class="anchor" id="console"></a>TDengine 命令行程序
执行TDengine命令行程序，您只要在Linux终端执行`taos`即可。
执行 TDengine 命令行程序，您只要在 Linux 终端执行 `taos` 即可。
```bash
$ taos
```
如果TDengine终端连接服务成功，将会打印出欢迎消息和版本信息。如果失败，则会打印错误消息出来（请参考[FAQ](https://www.taosdata.com/cn/documentation/faq/)来解决终端连接服务端失败的问题）。TDengine终端的提示符号如下：
如果 TDengine 终端连接服务成功，将会打印出欢迎消息和版本信息。如果失败，则会打印错误消息出来（请参考 [FAQ](https://www.taosdata.com/cn/documentation/faq/) 来解决终端连接服务端失败的问题）。TDengine 终端的提示符号如下：
```cmd
taos>
```
在TDengine终端中，用户可以通过SQL命令来创建/删除数据库、表等，并进行插入查询操作。在终端中运行的SQL语句需要以分号结束来运行。示例：
在 TDengine 终端中，用户可以通过 SQL 命令来创建/删除数据库、表等，并进行插入查询操作。在终端中运行的 SQL 语句需要以分号结束来运行。示例：
```mysql
create database demo;
@ -76,24 +76,24 @@ create table t (ts timestamp, speed int);
insert into t values ('2019-07-15 00:00:00', 10);
insert into t values ('2019-07-15 01:00:00', 20);
select * from t;
ts | speed |
===================================
19-07-15 00:00:00.000| 10|
19-07-15 01:00:00.000| 20|
Query OK, 2 row(s) in set (0.001700s)
ts | speed |
========================================
2019-07-15 00:00:00.000 | 10 |
2019-07-15 01:00:00.000 | 20 |
Query OK, 2 row(s) in set (0.003128s)
```
除执行SQL语句外，系统管理员还可以从TDengine终端检查系统运行状态，添加删除用户账号等。
除执行 SQL 语句外，系统管理员还可以从 TDengine 终端检查系统运行状态，添加删除用户账号等。
### 命令行参数
您可通过配置命令行参数来改变TDengine终端的行为。以下为常用的几个命令行参数：
您可通过配置命令行参数来改变 TDengine 终端的行为。以下为常用的几个命令行参数：
- -c, --config-dir: 指定配置文件目录，默认为_/etc/taos_
- -h, --host: 指定服务的IP地址，默认为本地服务
- -s, --commands: 在不进入终端的情况下运行TDengine命令
- -u, -- user: 连接TDengine服务器的用户名，缺省为root
- -p, --password: 连接TDengine服务器的密码，缺省为taosdata
- -c, --config-dir: 指定配置文件目录，默认为 _/etc/taos_
- -h, --host: 指定服务的 FQDN 地址（也可以使用 IP），默认为连接本地服务
- -s, --commands: 在不进入终端的情况下运行 TDengine 命令
- -u, --user: 连接 TDengine 服务器的用户名，缺省为 root
- -p, --password: 连接TDengine服务器的密码，缺省为 taosdata
- -?, --help: 打印出所有命令行参数
示例：
@ -102,7 +102,7 @@ Query OK, 2 row(s) in set (0.001700s)
$ taos -h 192.168.0.1 -s "use db; show tables;"
```
### 运行SQL命令脚本
### 运行 SQL 命令脚本
TDengine 终端可以通过 `source` 命令来运行 SQL 命令脚本.
@ -110,27 +110,27 @@ TDengine 终端可以通过 `source` 命令来运行 SQL 命令脚本.
taos> source <filename>;
```
### Shell小技巧
### Shell 小技巧
- 可以使用上下光标键查看历史输入的指令
- 修改用户密码。在 shell 中使用 alter user 指令
- 修改用户密码，在 shell 中使用 alter user 指令
- ctrl+c 中止正在进行中的查询
- 执行 `RESET QUERY CACHE` 清空本地缓存的表 schema
## <a class="anchor" id="demo"></a>TDengine 极速体验
启动TDengine的服务，在Linux终端执行taosdemo
启动 TDengine 的服务，在 Linux 终端执行 taosdemo
```bash
$ taosdemo
```
该命令将在数据库test下面自动创建一张超级表meters，该超级表下有1万张表，表名为"t0" 到"t9999"，每张表有10万条记录，每条记录有 （f1, f2， f3）三个字段，时间戳从"2017-07-14 10:40:00 000" 到"2017-07-14 10:41:39 999"，每张表带有标签areaid和loc, areaid被设置为1到10, loc被设置为"beijing"或者“shanghai"。
该命令将在数据库 test 下面自动创建一张超级表 meters，该超级表下有 1 万张表，表名为 "t0" 到 "t9999"，每张表有 1 万条记录，每条记录有 (ts, current, voltage, phase) 四个字段，时间戳从 "2017-07-14 10:40:00 000" 到 "2017-07-14 10:40:09 999"，每张表带有标签 location 和 groupdId，groupdId 被设置为 1 到 10， location 被设置为 "beijing" 或者 "shanghai"。
执行这条命令大概需要10分钟，最后共插入10亿条记录。
执行这条命令大概需要几分钟，最后共插入 1 亿条记录。
在TDengine客户端输入查询命令，体验查询速度。
在 TDengine 客户端输入查询命令，体验查询速度。
- 查询超级表下记录总条数：
@ -138,49 +138,43 @@ $ taosdemo
taos> select count(*) from test.meters;
```
- 查询10亿条记录的平均值、最大值、最小值等：
- 查询 1 亿条记录的平均值、最大值、最小值等：
```mysql
taos> select avg(f1), max(f2), min(f3) from test.meters;
taos> select avg(current), max(voltage), min(phase) from test.meters;
```
- 查询loc="beijing"的记录总条数：
- 查询 location="beijing" 的记录总条数：
```mysql
taos> select count(*) from test.meters where loc="beijing";
taos> select count(*) from test.meters where location="beijing";
```
- 查询areaid=10的所有记录的平均值、最大值、最小值等：
- 查询 groupdId=10 的所有记录的平均值、最大值、最小值等：
```mysql
taos> select avg(f1), max(f2), min(f3) from test.meters where areaid=10;
taos> select avg(current), max(voltage), min(phase) from test.meters where groupdId=10;
```
- 对表t10按10s进行平均值、最大值和最小值聚合统计：
- 对表 t10 按 10s 进行平均值、最大值和最小值聚合统计：
```mysql
taos> select avg(f1), max(f2), min(f3) from test.t10 interval(10s);
taos> select avg(current), max(voltage), min(phase) from test.t10 interval(10s);
```
**Note:** taosdemo命令本身带有很多选项，配置表的数目、记录条数等等，请执行 `taosdemo --help`详细列出。您可以设置不同参数进行体验。
**Note:** taosdemo 命令本身带有很多选项，配置表的数目、记录条数等等，请执行 `taosdemo --help` 详细列出。您可以设置不同参数进行体验。
## 客户端和报警模块
如果客户端和服务端运行在不同的电脑上，可以单独安装客户端。Linux和Windows安装包如下：
如果客户端和服务端运行在不同的电脑上，可以单独安装客户端。Linux 和 Windows 安装包可以在 [这里](https://www.taosdata.com/cn/getting-started/#客户端) 下载。
- TDengine-client-2.0.10.0-Linux-x64.tar.gz(3.0M)
- TDengine-client-2.0.10.0-Windows-x64.exe(2.8M)
- TDengine-client-2.0.10.0-Windows-x86.exe(2.8M)
报警模块的Linux安装包如下（请参考[报警模块的使用方法](https://github.com/taosdata/TDengine/blob/master/alert/README_cn.md)）：
- TDengine-alert-2.0.10.0-Linux-x64.tar.gz (8.1M)
报警模块的 Linux 和 Windows 安装包请在 [所有下载链接](https://www.taosdata.com/cn/all-downloads/) 页面搜索“TDengine Alert Linux”章节或“TDengine Alert Windows”章节进行下载。使用方法请参考 [报警模块的使用方法](https://github.com/taosdata/TDengine/blob/master/alert/README_cn.md)。
## <a class="anchor" id="platforms"></a>支持平台列表
### TDengine服务器支持的平台列表
### TDengine 服务器支持的平台列表
| | **CentOS 6/7/8** | **Ubuntu 16/18/20** | **Other Linux** | **统信 UOS** | **银河/中标麒麟** | **凝思 V60/V80** | **华为 EulerOS** |
| -------------- | --------------------- | ------------------------ | --------------- | --------------- | ------------------------- | --------------------- | --------------------- |
@ -201,9 +195,9 @@ taos> select avg(f1), max(f2), min(f3) from test.t10 interval(10s);
### TDengine客户端和连接器支持的平台列表
### TDengine 客户端和连接器支持的平台列表
目前TDengine的连接器可支持的平台广泛，目前包括：X64/X86/ARM64/ARM32/MIPS/Alpha等硬件平台，以及Linux/Win64/Win32等开发环境。
目前 TDengine 的连接器可支持的平台广泛，目前包括：X64/X86/ARM64/ARM32/MIPS/Alpha 等硬件平台，以及 Linux/Win64/Win32 等开发环境。
对照矩阵如下：
@ -220,5 +214,5 @@ taos> select avg(f1), max(f2), min(f3) from test.t10 interval(10s);
注： ● 表示经过官方测试验证， ○ 表示非官方测试验证。
请跳转到 [连接器](https://www.taosdata.com/cn/documentation/connector)查看更详细的信息。
请跳转到 [连接器](https://www.taosdata.com/cn/documentation/connector) 查看更详细的信息。

8
packaging/tools/install_client.sh
View File

@ -182,7 +182,13 @@ function install_jemalloc() {
${csudo} /usr/bin/install -c -d /usr/local/share/man/man3
${csudo} /usr/bin/install -c -m 644 ${jemalloc_dir}/share/man/man3/jemalloc.3 /usr/local/share/man/man3
fi
${csudo} ldconfig
if [ -d /etc/ld.so.conf.d ]; then
${csudo} echo "/usr/local/lib" > /etc/ld.so.conf.d/jemalloc.conf
${csudo} ldconfig
else
echo "/etc/ld.so.conf.d not found!"
fi
fi
}

17
packaging/tools/makeclient.sh
View File

@ -41,10 +41,10 @@ fi
if [ "$osType" != "Darwin" ]; then
if [ "$pagMode" == "lite" ]; then
#strip ${build_dir}/bin/taosd
#strip ${build_dir}/bin/taosd
strip ${build_dir}/bin/taos
bin_files="${build_dir}/bin/taos ${script_dir}/remove_client.sh"
else
else
bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo \
${script_dir}/remove_client.sh ${script_dir}/set_core.sh ${script_dir}/get_client.sh ${script_dir}/taosd-dump-cfg.gdb"
fi
@ -139,7 +139,7 @@ if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then
cp -r ${examples_dir}/C# ${install_dir}/examples
fi
# Copy driver
mkdir -p ${install_dir}/driver
mkdir -p ${install_dir}/driver
cp ${lib_files} ${install_dir}/driver
# Copy connector
@ -168,7 +168,7 @@ fi
# exit 1
cd ${release_dir}
cd ${release_dir}
# install_dir has been distinguishes cluster from edege, so comments this code
pkg_name=${install_dir}-${osType}-${cpuType}
@ -195,6 +195,15 @@ if [ "$pagMode" == "lite" ]; then
pkg_name=${pkg_name}-Lite
fi
if [ "$verType" == "beta" ]; then
pkg_name=${pkg_name}-${verType}
elif [ "$verType" == "stable" ]; then
pkg_name=${pkg_name}
else
echo "unknow verType, nor stable or beta"
exit 1
fi
if [ "$osType" != "Darwin" ]; then
tar -zcv -f "$(basename ${pkg_name}).tar.gz" $(basename ${install_dir}) --remove-files || :
else

61
packaging/tools/makeclient_power.sh
View File

@ -41,10 +41,10 @@ fi
if [ "$osType" != "Darwin" ]; then
# if [ "$pagMode" == "lite" ]; then
# strip ${build_dir}/bin/powerd
# strip ${build_dir}/bin/powerd
# strip ${build_dir}/bin/power
# bin_files="${build_dir}/bin/power ${script_dir}/remove_client_power.sh"
# else
# else
# bin_files="${build_dir}/bin/power ${build_dir}/bin/powerdemo ${script_dir}/remove_client_power.sh ${script_dir}/set_core.sh"
# fi
lib_files="${build_dir}/lib/libtaos.so.${version}"
@ -67,6 +67,39 @@ mkdir -p ${install_dir}
mkdir -p ${install_dir}/inc && cp ${header_files} ${install_dir}/inc
mkdir -p ${install_dir}/cfg && cp ${cfg_dir}/taos.cfg ${install_dir}/cfg/taos.cfg
if [ -f ${build_dir}/bin/jemalloc-config ]; then
mkdir -p ${install_dir}/jemalloc/{bin,lib,lib/pkgconfig,include/jemalloc,share/doc/jemalloc,share/man/man3}
cp ${build_dir}/bin/jemalloc-config ${install_dir}/jemalloc/bin
if [ -f ${build_dir}/bin/jemalloc.sh ]; then
cp ${build_dir}/bin/jemalloc.sh ${install_dir}/jemalloc/bin
fi
if [ -f ${build_dir}/bin/jeprof ]; then
cp ${build_dir}/bin/jeprof ${install_dir}/jemalloc/bin
fi
if [ -f ${build_dir}/include/jemalloc/jemalloc.h ]; then
cp ${build_dir}/include/jemalloc/jemalloc.h ${install_dir}/jemalloc/include/jemalloc
fi
if [ -f ${build_dir}/lib/libjemalloc.so.2 ]; then
cp ${build_dir}/lib/libjemalloc.so.2 ${install_dir}/jemalloc/lib
ln -sf libjemalloc.so.2 ${install_dir}/jemalloc/lib/libjemalloc.so
fi
if [ -f ${build_dir}/lib/libjemalloc.a ]; then
cp ${build_dir}/lib/libjemalloc.a ${install_dir}/jemalloc/lib
fi
if [ -f ${build_dir}/lib/libjemalloc_pic.a ]; then
cp ${build_dir}/lib/libjemalloc_pic.a ${install_dir}/jemalloc/lib
fi
if [ -f ${build_dir}/lib/pkgconfig/jemalloc.pc ]; then
cp ${build_dir}/lib/pkgconfig/jemalloc.pc ${install_dir}/jemalloc/lib/pkgconfig
fi
if [ -f ${build_dir}/share/doc/jemalloc/jemalloc.html ]; then
cp ${build_dir}/share/doc/jemalloc/jemalloc.html ${install_dir}/jemalloc/share/doc/jemalloc
fi
if [ -f ${build_dir}/share/man/man3/jemalloc.3 ]; then
cp ${build_dir}/share/man/man3/jemalloc.3 ${install_dir}/jemalloc/share/man/man3
fi
fi
sed -i '/dataDir/ {s/taos/power/g}' ${install_dir}/cfg/taos.cfg
sed -i '/logDir/ {s/taos/power/g}' ${install_dir}/cfg/taos.cfg
sed -i "s/TDengine/PowerDB/g" ${install_dir}/cfg/taos.cfg
@ -77,11 +110,11 @@ if [ "$osType" != "Darwin" ]; then
strip ${build_dir}/bin/taos
cp ${build_dir}/bin/taos ${install_dir}/bin/power
cp ${script_dir}/remove_power.sh ${install_dir}/bin
else
else
cp ${build_dir}/bin/taos ${install_dir}/bin/power
cp ${script_dir}/remove_power.sh ${install_dir}/bin
cp ${build_dir}/bin/taosdemo ${install_dir}/bin/powerdemo
cp ${build_dir}/bin/taosdump ${install_dir}/bin/powerdump
cp ${build_dir}/bin/taosdemo ${install_dir}/bin/powerdemo
cp ${build_dir}/bin/taosdump ${install_dir}/bin/powerdump
cp ${script_dir}/set_core.sh ${install_dir}/bin
cp ${script_dir}/get_client.sh ${install_dir}/bin
cp ${script_dir}/taosd-dump-cfg.gdb ${install_dir}/bin
@ -158,15 +191,15 @@ if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then
cp -r ${examples_dir}/JDBC ${install_dir}/examples
cp -r ${examples_dir}/matlab ${install_dir}/examples
sed -i '/password/ {s/taosdata/powerdb/g}' ${install_dir}/examples/matlab/TDengineDemo.m
cp -r ${examples_dir}/python ${install_dir}/examples
cp -r ${examples_dir}/python ${install_dir}/examples
sed -i '/password/ {s/taosdata/powerdb/g}' ${install_dir}/examples/python/read_example.py
cp -r ${examples_dir}/R ${install_dir}/examples
sed -i '/password/ {s/taosdata/powerdb/g}' ${install_dir}/examples/R/command.txt
cp -r ${examples_dir}/go ${install_dir}/examples
cp -r ${examples_dir}/go ${install_dir}/examples
sed -i '/root/ {s/taosdata/powerdb/g}' ${install_dir}/examples/go/taosdemo.go
fi
# Copy driver
mkdir -p ${install_dir}/driver
mkdir -p ${install_dir}/driver
cp ${lib_files} ${install_dir}/driver
# Copy connector
@ -188,11 +221,11 @@ if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then
echo "WARNING: go connector not found, please check if want to use it!"
fi
cp -r ${connector_dir}/python ${install_dir}/connector
sed -i '/password/ {s/taosdata/powerdb/g}' ${install_dir}/connector/python/taos/cinterface.py
sed -i '/password/ {s/taosdata/powerdb/g}' ${install_dir}/connector/python/taos/subscription.py
sed -i '/self._password/ {s/taosdata/powerdb/g}' ${install_dir}/connector/python/taos/connection.py
fi
# Copy release note
@ -200,7 +233,7 @@ fi
# exit 1
cd ${release_dir}
cd ${release_dir}
if [ "$verMode" == "cluster" ]; then
pkg_name=${install_dir}-${osType}-${cpuType}
@ -217,8 +250,8 @@ fi
if [ "$verType" == "beta" ]; then
pkg_name=${pkg_name}-${verType}
elif [ "$verType" == "stable" ]; then
pkg_name=${pkg_name}
elif [ "$verType" == "stable" ]; then
pkg_name=${pkg_name}
else
echo "unknow verType, nor stable or beta"
exit 1

12
src/client/CMakeLists.txt
View File

@ -21,7 +21,7 @@ IF (TD_LINUX)
IF (TD_LINUX_64)
TARGET_LINK_LIBRARIES(taos lua)
ENDIF ()
SET_TARGET_PROPERTIES(taos PROPERTIES CLEAN_DIRECT_OUTPUT 1)
#set version of .so
@ -37,13 +37,13 @@ ELSEIF (TD_DARWIN)
# set the static lib name
ADD_LIBRARY(taos_static STATIC ${SRC})
TARGET_LINK_LIBRARIES(taos_static common query trpc tutil pthread m)
TARGET_LINK_LIBRARIES(taos_static common query trpc tutil pthread m lua)
SET_TARGET_PROPERTIES(taos_static PROPERTIES OUTPUT_NAME "taos_static")
SET_TARGET_PROPERTIES(taos_static PROPERTIES CLEAN_DIRECT_OUTPUT 1)
# generate dynamic library (*.dylib)
ADD_LIBRARY(taos SHARED ${SRC})
TARGET_LINK_LIBRARIES(taos common query trpc tutil pthread m)
TARGET_LINK_LIBRARIES(taos common query trpc tutil pthread m lua)
SET_TARGET_PROPERTIES(taos PROPERTIES CLEAN_DIRECT_OUTPUT 1)
#set version of .dylib
@ -68,19 +68,19 @@ ELSEIF (TD_WINDOWS)
IF (NOT TD_GODLL)
SET_TARGET_PROPERTIES(taos PROPERTIES LINK_FLAGS /DEF:${TD_COMMUNITY_DIR}/src/client/src/taos.def)
ENDIF ()
TARGET_LINK_LIBRARIES(taos trpc tutil query)
TARGET_LINK_LIBRARIES(taos trpc tutil query lua)
ELSEIF (TD_DARWIN)
SET(CMAKE_MACOSX_RPATH 1)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/jni/linux)
ADD_LIBRARY(taos_static STATIC ${SRC})
TARGET_LINK_LIBRARIES(taos_static query trpc tutil pthread m)
TARGET_LINK_LIBRARIES(taos_static query trpc tutil pthread m lua)
SET_TARGET_PROPERTIES(taos_static PROPERTIES OUTPUT_NAME "taos_static")
# generate dynamic library (*.dylib)
ADD_LIBRARY(taos SHARED ${SRC})
TARGET_LINK_LIBRARIES(taos query trpc tutil pthread m)
TARGET_LINK_LIBRARIES(taos query trpc tutil pthread m lua)
SET_TARGET_PROPERTIES(taos PROPERTIES CLEAN_DIRECT_OUTPUT 1)

5
src/client/inc/tscUtil.h
View File

@ -107,6 +107,7 @@ SParamInfo* tscAddParamToDataBlock(STableDataBlocks* pDataBlock, char type, uint
uint32_t offset);
void* tscDestroyBlockArrayList(SArray* pDataBlockList);
void* tscDestroyUdfArrayList(SArray* pUdfList);
void* tscDestroyBlockHashTable(SHashObj* pBlockHashTable, bool removeMeta);
int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock);
@ -261,6 +262,7 @@ void tscVgroupTableCopy(SVgroupTableInfo* info, SVgroupTableInfo* pInfo);
int tscGetSTableVgroupInfo(SSqlObj* pSql, SQueryInfo* pQueryInfo);
int tscGetTableMeta(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo);
int tscGetTableMetaEx(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, bool createIfNotExists);
int32_t tscGetUdfFromNode(SSqlObj *pSql, SQueryInfo* pQueryInfo);
void tscResetForNextRetrieve(SSqlRes* pRes);
void executeQuery(SSqlObj* pSql, SQueryInfo* pQueryInfo);
@ -307,10 +309,9 @@ bool hasMoreClauseToTry(SSqlObj* pSql);
void tscFreeQueryInfo(SSqlCmd* pCmd, bool removeMeta);
void tscTryQueryNextVnode(SSqlObj *pSql, __async_cb_func_t fp);
void tscAsyncQuerySingleRowForNextVnode(void *param, TAOS_RES *tres, int numOfRows);
void tscTryQueryNextClause(SSqlObj* pSql, __async_cb_func_t fp);
int tscSetMgmtEpSetFromCfg(const char *first, const char *second, SRpcCorEpSet *corEpSet);
int32_t getMultiTableMetaFromMnode(SSqlObj *pSql, SArray* pNameList, SArray* pVgroupNameList, __async_cb_func_t fp);
int32_t getMultiTableMetaFromMnode(SSqlObj *pSql, SArray* pNameList, SArray* pVgroupNameList, SArray* pUdfList, __async_cb_func_t fp, bool metaClone);
int tscTransferTableNameList(SSqlObj *pSql, const char *pNameList, int32_t length, SArray* pNameArray);

1
src/client/inc/tsclient.h
View File

@ -146,6 +146,7 @@ typedef struct {
SInsertStatementParam insertParam;
char reserve1[3]; // fix bus error on arm32
int32_t count; // todo remove it
bool subCmd;
char reserve2[3]; // fix bus error on arm32
int16_t numOfCols;

59
src/client/src/tscGlobalmerge.c
View File

@ -602,6 +602,15 @@ static void doExecuteFinalMerge(SOperatorInfo* pOperator, int32_t numOfExpr, SSD
if (functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TS_DUMMY) {
continue;
}
if (functionId < 0) {
SUdfInfo* pUdfInfo = taosArrayGet(pInfo->udfInfo, -1 * functionId - 1);
doInvokeUdf(pUdfInfo, &pCtx[j], 0, TSDB_UDF_FUNC_MERGE);
continue;
}
aAggs[functionId].mergeFunc(&pCtx[j]);
}
} else {
@ -610,6 +619,15 @@ static void doExecuteFinalMerge(SOperatorInfo* pOperator, int32_t numOfExpr, SSD
if (functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TS_DUMMY) {
continue;
}
if (functionId < 0) {
SUdfInfo* pUdfInfo = taosArrayGet(pInfo->udfInfo, -1 * functionId - 1);
doInvokeUdf(pUdfInfo, &pCtx[j], 0, TSDB_UDF_FUNC_FINALIZE);
continue;
}
aAggs[functionId].xFinalize(&pCtx[j]);
}
@ -626,7 +644,11 @@ static void doExecuteFinalMerge(SOperatorInfo* pOperator, int32_t numOfExpr, SSD
}
for(int32_t j = 0; j < numOfExpr; ++j) {
aAggs[pCtx[j].functionId].init(&pCtx[j]);
if (pCtx[j].functionId < 0) {
continue;
}
aAggs[pCtx[j].functionId].init(&pCtx[j], pCtx[j].resultInfo);
}
for (int32_t j = 0; j < numOfExpr; ++j) {
@ -638,6 +660,15 @@ static void doExecuteFinalMerge(SOperatorInfo* pOperator, int32_t numOfExpr, SSD
if (functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TS_DUMMY) {
continue;
}
if (functionId < 0) {
SUdfInfo* pUdfInfo = taosArrayGet(pInfo->udfInfo, -1 * functionId - 1);
doInvokeUdf(pUdfInfo, &pCtx[j], 0, TSDB_UDF_FUNC_MERGE);
continue;
}
aAggs[functionId].mergeFunc(&pCtx[j]);
}
}
@ -651,6 +682,15 @@ static void doExecuteFinalMerge(SOperatorInfo* pOperator, int32_t numOfExpr, SSD
if (functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TS_DUMMY) {
continue;
}
if (functionId < 0) {
SUdfInfo* pUdfInfo = taosArrayGet(pInfo->udfInfo, -1 * functionId - 1);
doInvokeUdf(pUdfInfo, &pCtx[j], 0, TSDB_UDF_FUNC_MERGE);
continue;
}
aAggs[functionId].mergeFunc(&pCtx[j]);
}
}
@ -871,7 +911,6 @@ SSDataBlock* doGlobalAggregate(void* param, bool* newgroup) {
{
if (pAggInfo->hasDataBlockForNewGroup) {
pAggInfo->binfo.pRes->info.rows = 0;
pAggInfo->hasPrev = false; // now we start from a new group data set.
// not belongs to the same group, return the result of current group;
@ -880,7 +919,13 @@ SSDataBlock* doGlobalAggregate(void* param, bool* newgroup) {
{ // reset output buffer
for(int32_t j = 0; j < pOperator->numOfOutput; ++j) {
aAggs[pAggInfo->binfo.pCtx[j].functionId].init(&pAggInfo->binfo.pCtx[j]);
SQLFunctionCtx* pCtx = &pAggInfo->binfo.pCtx[j];
if (pCtx->functionId < 0) {
clearOutputBuf(&pAggInfo->binfo, &pAggInfo->bufCapacity);
continue;
}
aAggs[pCtx->functionId].init(pCtx, pCtx->resultInfo);
}
}
@ -933,6 +978,14 @@ SSDataBlock* doGlobalAggregate(void* param, bool* newgroup) {
continue;
}
if (functionId < 0) {
SUdfInfo* pUdfInfo = taosArrayGet(pAggInfo->udfInfo, -1 * functionId - 1);
doInvokeUdf(pUdfInfo, &pAggInfo->binfo.pCtx[j], 0, TSDB_UDF_FUNC_FINALIZE);
continue;
}
aAggs[functionId].xFinalize(&pAggInfo->binfo.pCtx[j]);
}

686
src/client/src/tscSQLParser.c
View File

File diff suppressed because it is too large Load Diff

330
src/client/src/tscServer.c
View File

@ -13,7 +13,10 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <tscompression.h>
#include "os.h"
#include "qPlan.h"
#include "qTableMeta.h"
#include "tcmdtype.h"
#include "tlockfree.h"
#include "trpc.h"
@ -21,10 +24,8 @@
#include "tscLog.h"
#include "tscProfile.h"
#include "tscUtil.h"
#include "qTableMeta.h"
#include "tsclient.h"
#include "ttimer.h"
#include "qPlan.h"
int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo) = {0};
@ -502,6 +503,7 @@ int doBuildAndSendMsg(SSqlObj *pSql) {
pCmd->command == TSDB_SQL_INSERT ||
pCmd->command == TSDB_SQL_CONNECT ||
pCmd->command == TSDB_SQL_HB ||
pCmd->command == TSDB_SQL_RETRIEVE_FUNC ||
pCmd->command == TSDB_SQL_STABLEVGROUP) {
pRes->code = tscBuildMsg[pCmd->command](pSql, NULL);
}
@ -544,7 +546,7 @@ int tscBuildAndSendRequest(SSqlObj *pSql, SQueryInfo* pQueryInfo) {
type = pQueryInfo->type;
// while numOfTables equals to 0, it must be Heartbeat
assert((pQueryInfo->numOfTables == 0 && pQueryInfo->command == TSDB_SQL_HB) || pQueryInfo->numOfTables > 0);
assert((pQueryInfo->numOfTables == 0 && (pQueryInfo->command == TSDB_SQL_HB || pSql->cmd.command == TSDB_SQL_RETRIEVE_FUNC)) || pQueryInfo->numOfTables > 0);
}
tscDebug("0x%"PRIx64" SQL cmd:%s will be processed, name:%s, type:%d", pSql->self, sqlCmd[pCmd->command], name, type);
@ -1031,6 +1033,34 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
pMsg += sizeof(int32_t);
}
// support only one udf
if (pQueryInfo->pUdfInfo != NULL && taosArrayGetSize(pQueryInfo->pUdfInfo) > 0) {
pQueryMsg->udfContentOffset = htonl((int32_t) (pMsg - pCmd->payload));
for(int32_t i = 0; i < taosArrayGetSize(pQueryInfo->pUdfInfo); ++i) {
SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, i);
*(int8_t*) pMsg = pUdfInfo->resType;
pMsg += sizeof(pUdfInfo->resType);
*(int16_t*) pMsg = htons(pUdfInfo->resBytes);
pMsg += sizeof(pUdfInfo->resBytes);
STR_TO_VARSTR(pMsg, pUdfInfo->name);
pMsg += varDataTLen(pMsg);
*(int32_t*) pMsg = htonl(pUdfInfo->funcType);
pMsg += sizeof(pUdfInfo->funcType);
*(int32_t*) pMsg = htonl(pUdfInfo->bufSize);
pMsg += sizeof(pUdfInfo->bufSize);
pQueryMsg->udfContentLen = htonl(pUdfInfo->contLen);
memcpy(pMsg, pUdfInfo->content, pUdfInfo->contLen);
pMsg += pUdfInfo->contLen;
}
}
memcpy(pMsg, pSql->sqlstr, sqlLen);
pMsg += sqlLen;
@ -1066,6 +1096,18 @@ int32_t tscBuildCreateDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
return TSDB_CODE_SUCCESS;
}
int32_t tscBuildCreateFuncMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SSqlCmd *pCmd = &pSql->cmd;
SCreateFuncMsg *pCreateFuncMsg = (SCreateFuncMsg *)pCmd->payload;
pCmd->msgType = TSDB_MSG_TYPE_CM_CREATE_FUNCTION;
pCmd->payloadLen = sizeof(SCreateFuncMsg) + htonl(pCreateFuncMsg->codeLen);
return TSDB_CODE_SUCCESS;
}
int32_t tscBuildCreateDnodeMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SSqlCmd *pCmd = &pSql->cmd;
pCmd->payloadLen = sizeof(SCreateDnodeMsg);
@ -1190,6 +1232,17 @@ int32_t tscBuildDropDbMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
return TSDB_CODE_SUCCESS;
}
int32_t tscBuildDropFuncMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SSqlCmd *pCmd = &pSql->cmd;
pCmd->msgType = TSDB_MSG_TYPE_CM_DROP_FUNCTION;
pCmd->payloadLen = sizeof(SDropFuncMsg);
return TSDB_CODE_SUCCESS;
}
int32_t tscBuildDropTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SSqlCmd *pCmd = &pSql->cmd;
pCmd->payloadLen = sizeof(SCMDropTableMsg);
@ -1293,11 +1346,19 @@ int32_t tscBuildShowMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
SShowMsg *pShowMsg = (SShowMsg *)pCmd->payload;
SShowInfo *pShowInfo = &pInfo->pMiscInfo->showOpt;
SShowMsg *pShowMsg = (SShowMsg *)pCmd->payload;
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, 0);
if (pShowInfo->showType == TSDB_MGMT_TABLE_FUNCTION) {
pShowMsg->type = pShowInfo->showType;
pShowMsg->payloadLen = 0;
pCmd->payloadLen = sizeof(SShowMsg);
if (tNameIsEmpty(&pTableMetaInfo->name)) {
return TSDB_CODE_SUCCESS;
}
if (tNameIsEmpty(&pTableMetaInfo->name)) {
pthread_mutex_lock(&pObj->mutex);
tstrncpy(pShowMsg->db, pObj->db, sizeof(pShowMsg->db));
pthread_mutex_unlock(&pObj->mutex);
@ -1305,7 +1366,6 @@ int32_t tscBuildShowMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
tNameGetFullDbName(&pTableMetaInfo->name, pShowMsg->db);
}
SShowInfo *pShowInfo = &pInfo->pMiscInfo->showOpt;
pShowMsg->type = pShowInfo->showType;
if (pShowInfo->showType != TSDB_MGMT_TABLE_VNODES) {
@ -1818,6 +1878,29 @@ int tscBuildSTableVgroupMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
return TSDB_CODE_SUCCESS;
}
int tscBuildRetrieveFuncMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SSqlCmd *pCmd = &pSql->cmd;
char *pMsg = pCmd->payload;
SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd);
int32_t numOfFuncs = (int32_t)taosArrayGetSize(pQueryInfo->pUdfInfo);
SRetrieveFuncMsg *pRetrieveFuncMsg = (SRetrieveFuncMsg *)pMsg;
pRetrieveFuncMsg->num = htonl(numOfFuncs);
pMsg += sizeof(SRetrieveFuncMsg);
for(int32_t i = 0; i < numOfFuncs; ++i) {
SUdfInfo* pUdf = taosArrayGet(pQueryInfo->pUdfInfo, i);
STR_TO_NET_VARSTR(pMsg, pUdf->name);
pMsg += varDataNetTLen(pMsg);
}
pCmd->msgType = TSDB_MSG_TYPE_CM_RETRIEVE_FUNC;
pCmd->payloadLen = (int32_t)(pMsg - pCmd->payload);
return TSDB_CODE_SUCCESS;
}
int tscBuildHeartBeatMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
SSqlCmd *pCmd = &pSql->cmd;
STscObj *pObj = pSql->pTscObj;
@ -1872,7 +1955,6 @@ static int32_t tableMetaMsgConvert(STableMetaMsg* pMetaMsg) {
pMetaMsg->vgroup.vgId = htonl(pMetaMsg->vgroup.vgId);
pMetaMsg->uid = htobe64(pMetaMsg->uid);
// pMetaMsg->contLen = htonl(pMetaMsg->contLen);
pMetaMsg->numOfColumns = htons(pMetaMsg->numOfColumns);
if ((pMetaMsg->tableType != TSDB_SUPER_TABLE) &&
@ -1937,13 +2019,14 @@ static void doAddTableMetaToLocalBuf(STableMeta* pTableMeta, STableMetaMsg* pMet
int32_t len = (int32_t) strnlen(pTableMeta->sTableName, TSDB_TABLE_FNAME_LEN);
// The super tableMeta already exists, create it according to tableMeta and add it to hash map
STableMeta* pSupTableMeta = createSuperTableMeta(pMetaMsg);
if (updateSTable) {
STableMeta* pSupTableMeta = createSuperTableMeta(pMetaMsg);
uint32_t size = tscGetTableMetaSize(pSupTableMeta);
int32_t code = taosHashPut(tscTableMetaInfo, pTableMeta->sTableName, len, pSupTableMeta, size);
assert(code == TSDB_CODE_SUCCESS);
uint32_t size = tscGetTableMetaSize(pSupTableMeta);
int32_t code = taosHashPut(tscTableMetaInfo, pTableMeta->sTableName, len, pSupTableMeta, size);
assert(code == TSDB_CODE_SUCCESS);
tfree(pSupTableMeta);
tfree(pSupTableMeta);
}
CChildTableMeta* cMeta = tscCreateChildMeta(pTableMeta);
taosHashPut(tscTableMetaInfo, pMetaMsg->tableFname, strlen(pMetaMsg->tableFname), cMeta, sizeof(CChildTableMeta));
@ -2033,12 +2116,64 @@ static SVgroupsInfo* createVgroupInfoFromMsg(char* pMsg, int32_t* size, uint64_t
return pVgroupInfo;
}
int tscProcessRetrieveFuncRsp(SSqlObj* pSql) {
SSqlCmd* pCmd = &pSql->cmd;
SUdfFuncMsg* pFuncMsg = (SUdfFuncMsg *)pSql->res.pRsp;
SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd);
pFuncMsg->num = htonl(pFuncMsg->num);
assert(pFuncMsg->num == taosArrayGetSize(pQueryInfo->pUdfInfo));
char* pMsg = pFuncMsg->content;
for(int32_t i = 0; i < pFuncMsg->num; ++i) {
SFunctionInfoMsg* pFunc = (SFunctionInfoMsg*) pMsg;
for(int32_t j = 0; j < pFuncMsg->num; ++j) {
SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, j);
if (strcmp(pUdfInfo->name, pFunc->name) != 0) {
continue;
}
if (pUdfInfo->content) {
continue;
}
pUdfInfo->resBytes = htons(pFunc->resBytes);
pUdfInfo->resType = pFunc->resType;
pUdfInfo->funcType = htonl(pFunc->funcType);
pUdfInfo->contLen = htonl(pFunc->len);
pUdfInfo->bufSize = htonl(pFunc->bufSize);
pUdfInfo->content = malloc(pUdfInfo->contLen);
memcpy(pUdfInfo->content, pFunc->content, pUdfInfo->contLen);
pMsg += sizeof(SFunctionInfoMsg) + pUdfInfo->contLen;
}
}
// master sqlObj locates in param
SSqlObj* parent = (SSqlObj*)taosAcquireRef(tscObjRef, (int64_t)pSql->param);
if(parent == NULL) {
return pSql->res.code;
}
SQueryInfo* parQueryInfo = tscGetQueryInfo(&parent->cmd);
assert(parent->signature == parent && (int64_t)pSql->param == parent->self);
taosArrayDestroy(parQueryInfo->pUdfInfo);
parQueryInfo->pUdfInfo = pQueryInfo->pUdfInfo; // assigned to parent sql obj.
pQueryInfo->pUdfInfo = NULL;
return TSDB_CODE_SUCCESS;
}
int tscProcessMultiTableMetaRsp(SSqlObj *pSql) {
char *rsp = pSql->res.pRsp;
SMultiTableMeta *pMultiMeta = (SMultiTableMeta *)rsp;
pMultiMeta->numOfTables = htonl(pMultiMeta->numOfTables);
pMultiMeta->numOfVgroup = htonl(pMultiMeta->numOfVgroup);
pMultiMeta->numOfUdf = htonl(pMultiMeta->numOfUdf);
rsp += sizeof(SMultiTableMeta);
@ -2048,46 +2183,66 @@ int tscProcessMultiTableMetaRsp(SSqlObj *pSql) {
}
SSqlCmd *pParentCmd = &pParentSql->cmd;
SHashObj *pSet = taosHashInit(pMultiMeta->numOfVgroup, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
SHashObj *pSet = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
char* buf = NULL;
char* pMsg = pMultiMeta->meta;
if (pMultiMeta->compressed) {
buf = malloc(pMultiMeta->rawLen - sizeof(SMultiTableMeta));
int32_t len = tsDecompressString(pMultiMeta->meta, pMultiMeta->contLen - sizeof(SMultiTableMeta), 1,
buf, pMultiMeta->rawLen - sizeof(SMultiTableMeta), ONE_STAGE_COMP, NULL, 0);
assert(len == pMultiMeta->rawLen - sizeof(SMultiTableMeta));
pMsg = buf;
}
for (int32_t i = 0; i < pMultiMeta->numOfTables; i++) {
STableMetaMsg *pMetaMsg = (STableMetaMsg *)pMsg;
int32_t code = tableMetaMsgConvert(pMetaMsg);
if (code != TSDB_CODE_SUCCESS) {
taosHashCleanup(pSet);
taosReleaseRef(tscObjRef, pParentSql->self);
tfree(buf);
return code;
}
bool freeMeta = false;
STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg);
if (!tIsValidSchema(pTableMeta->schema, pTableMeta->tableInfo.numOfColumns, pTableMeta->tableInfo.numOfTags)) {
tscError("0x%"PRIx64" invalid table meta from mnode, name:%s", pSql->self, pMetaMsg->tableFname);
tfree(pTableMeta);
taosHashCleanup(pSet);
taosReleaseRef(tscObjRef, pParentSql->self);
tfree(buf);
return TSDB_CODE_TSC_INVALID_VALUE;
}
SName sn = {0};
tNameFromString(&sn, pMetaMsg->tableFname, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
const char* tableName = tNameGetTableName(&sn);
size_t keyLen = strlen(tableName);
if (pMultiMeta->metaClone == 1 || pTableMeta->tableType == TSDB_SUPER_TABLE) {
STableMetaVgroupInfo p = {.pTableMeta = pTableMeta,};
STableMetaVgroupInfo p = {.pTableMeta = pTableMeta,};
taosHashPut(pParentCmd->pTableMetaMap, tableName, keyLen, &p, sizeof(STableMetaVgroupInfo));
const char* tableName = tNameGetTableName(&sn);
size_t keyLen = strlen(tableName);
taosHashPut(pParentCmd->pTableMetaMap, tableName, keyLen, &p, sizeof(STableMetaVgroupInfo));
} else {
freeMeta = true;
}
bool addToBuf = false;
if (taosHashGet(pSet, &pMetaMsg->uid, sizeof(pMetaMsg->uid)) == NULL) {
addToBuf = true;
taosHashPut(pSet, &pMetaMsg->uid, sizeof(pMetaMsg->uid), "", 0);
// for each super table, only update meta information once
bool updateStableMeta = false;
if (pTableMeta->tableType == TSDB_CHILD_TABLE && taosHashGet(pSet, &pMetaMsg->suid, sizeof(pMetaMsg->suid)) == NULL) {
updateStableMeta = true;
taosHashPut(pSet, &pTableMeta->suid, sizeof(pMetaMsg->suid), "", 0);
}
// create the tableMeta and add it into the TableMeta map
doAddTableMetaToLocalBuf(pTableMeta, pMetaMsg, addToBuf);
doAddTableMetaToLocalBuf(pTableMeta, pMetaMsg, updateStableMeta);
// if the vgroup is not updated in current process, update it.
// for each vgroup, only update the information once.
int64_t vgId = pMetaMsg->vgroup.vgId;
if (pTableMeta->tableType != TSDB_SUPER_TABLE && taosHashGet(pSet, &vgId, sizeof(vgId)) == NULL) {
doUpdateVgroupInfo(pTableMeta, &pMetaMsg->vgroup);
@ -2095,6 +2250,9 @@ int tscProcessMultiTableMetaRsp(SSqlObj *pSql) {
}
pMsg += pMetaMsg->contLen;
if (freeMeta) {
tfree(pTableMeta);
}
}
for(int32_t i = 0; i < pMultiMeta->numOfVgroup; ++i) {
@ -2105,16 +2263,53 @@ int tscProcessMultiTableMetaRsp(SSqlObj *pSql) {
assert(p != NULL);
int32_t size = 0;
if (p->pVgroupInfo!= NULL) {
tscVgroupInfoClear(p->pVgroupInfo);
//tfree(p->pTableMeta);
}
p->pVgroupInfo = createVgroupInfoFromMsg(pMsg, &size, pSql->self);
pMsg += size;
}
SQueryInfo* pQueryInfo = tscGetQueryInfo(pParentCmd);
if (pMultiMeta->numOfUdf > 0) {
assert(pQueryInfo->pUdfInfo != NULL);
}
for(int32_t i = 0; i < pMultiMeta->numOfUdf; ++i) {
SFunctionInfoMsg* pFunc = (SFunctionInfoMsg*) pMsg;
for(int32_t j = 0; j < pMultiMeta->numOfUdf; ++j) {
SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, j);
if (strcmp(pUdfInfo->name, pFunc->name) != 0) {
continue;
}
if (pUdfInfo->content) {
continue;
}
pUdfInfo->resBytes = htons(pFunc->resBytes);
pUdfInfo->resType = pFunc->resType;
pUdfInfo->funcType = htonl(pFunc->funcType);
pUdfInfo->contLen = htonl(pFunc->len);
pUdfInfo->bufSize = htonl(pFunc->bufSize);
pUdfInfo->content = malloc(pUdfInfo->contLen);
memcpy(pUdfInfo->content, pFunc->content, pUdfInfo->contLen);
pMsg += sizeof(SFunctionInfoMsg) + pUdfInfo->contLen;
}
}
pSql->res.code = TSDB_CODE_SUCCESS;
pSql->res.numOfTotal = pMultiMeta->numOfTables;
tscDebug("0x%"PRIx64" load multi-tableMeta from mnode, numOfTables:%d", pSql->self, pMultiMeta->numOfTables);
taosHashCleanup(pSet);
taosReleaseRef(tscObjRef, pParentSql->self);
tfree(buf);
return TSDB_CODE_SUCCESS;
}
@ -2497,7 +2692,7 @@ static int32_t getTableMetaFromMnode(SSqlObj *pSql, STableMetaInfo *pTableMetaIn
return code;
}
int32_t getMultiTableMetaFromMnode(SSqlObj *pSql, SArray* pNameList, SArray* pVgroupNameList, __async_cb_func_t fp) {
int32_t getMultiTableMetaFromMnode(SSqlObj *pSql, SArray* pNameList, SArray* pVgroupNameList, SArray* pUdfList, __async_cb_func_t fp, bool metaClone) {
SSqlObj *pNew = calloc(1, sizeof(SSqlObj));
if (NULL == pNew) {
tscError("0x%"PRIx64" failed to allocate sqlobj to get multiple table meta", pSql->self);
@ -2510,8 +2705,9 @@ int32_t getMultiTableMetaFromMnode(SSqlObj *pSql, SArray* pNameList, SArray* pVg
int32_t numOfTable = (int32_t) taosArrayGetSize(pNameList);
int32_t numOfVgroupList = (int32_t) taosArrayGetSize(pVgroupNameList);
int32_t numOfUdf = pUdfList ? (int32_t)taosArrayGetSize(pUdfList) : 0;
int32_t size = (numOfTable + numOfVgroupList) * TSDB_TABLE_FNAME_LEN + sizeof(SMultiTableInfoMsg);
int32_t size = (numOfTable + numOfVgroupList) * TSDB_TABLE_FNAME_LEN + TSDB_FUNC_NAME_LEN * numOfUdf + sizeof(SMultiTableInfoMsg);
if (TSDB_CODE_SUCCESS != tscAllocPayload(&pNew->cmd, size)) {
tscError("0x%"PRIx64" malloc failed for payload to get table meta", pSql->self);
tscFreeSqlObj(pNew);
@ -2519,14 +2715,16 @@ int32_t getMultiTableMetaFromMnode(SSqlObj *pSql, SArray* pNameList, SArray* pVg
}
SMultiTableInfoMsg* pInfo = (SMultiTableInfoMsg*) pNew->cmd.payload;
pInfo->metaClone = metaClone? 1:0;
pInfo->numOfTables = htonl((uint32_t) taosArrayGetSize(pNameList));
pInfo->numOfVgroups = htonl((uint32_t) taosArrayGetSize(pVgroupNameList));
pInfo->numOfUdfs = htonl(numOfUdf);
char* start = pInfo->tableNames;
int32_t len = 0;
for(int32_t i = 0; i < numOfTable; ++i) {
char* name = taosArrayGetP(pNameList, i);
if (i < numOfTable - 1 || numOfVgroupList > 0) {
if (i < numOfTable - 1 || numOfVgroupList > 0 || numOfUdf > 0) {
len = sprintf(start, "%s,", name);
} else {
len = sprintf(start, "%s", name);
@ -2537,7 +2735,7 @@ int32_t getMultiTableMetaFromMnode(SSqlObj *pSql, SArray* pNameList, SArray* pVg
for(int32_t i = 0; i < numOfVgroupList; ++i) {
char* name = taosArrayGetP(pVgroupNameList, i);
if (i < numOfVgroupList - 1) {
if (i < numOfVgroupList - 1 || numOfUdf > 0) {
len = sprintf(start, "%s,", name);
} else {
len = sprintf(start, "%s", name);
@ -2546,12 +2744,23 @@ int32_t getMultiTableMetaFromMnode(SSqlObj *pSql, SArray* pNameList, SArray* pVg
start += len;
}
for(int32_t i = 0; i < numOfUdf; ++i) {
SUdfInfo * u = taosArrayGet(pUdfList, i);
if (i < numOfUdf - 1) {
len = sprintf(start, "%s,", u->name);
} else {
len = sprintf(start, "%s", u->name);
}
start += len;
}
pNew->cmd.payloadLen = (int32_t) ((start - pInfo->tableNames) + sizeof(SMultiTableInfoMsg));
pNew->cmd.msgType = TSDB_MSG_TYPE_CM_TABLES_META;
registerSqlObj(pNew);
tscDebug("0x%"PRIx64" new pSqlObj:0x%"PRIx64" to get %d tableMeta, vgroupInfo:%d, msg size:%d", pSql->self,
pNew->self, numOfTable, numOfVgroupList, pNew->cmd.payloadLen);
tscDebug("0x%"PRIx64" new pSqlObj:0x%"PRIx64" to get %d tableMeta, vgroupInfo:%d, udf:%d, msg size:%d", pSql->self,
pNew->self, numOfTable, numOfVgroupList, numOfUdf, pNew->cmd.payloadLen);
pNew->fp = fp;
pNew->param = (void *)pSql->self;
@ -2622,6 +2831,60 @@ int tscGetTableMetaEx(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo, bool create
return tscGetTableMetaImpl(pSql, pTableMetaInfo, createIfNotExists);
}
int32_t tscGetUdfFromNode(SSqlObj *pSql, SQueryInfo* pQueryInfo) {
SSqlObj *pNew = calloc(1, sizeof(SSqlObj));
if (NULL == pNew) {
tscError("%p malloc failed for new sqlobj to get user-defined functions", pSql);
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
pNew->pTscObj = pSql->pTscObj;
pNew->signature = pNew;
pNew->cmd.command = TSDB_SQL_RETRIEVE_FUNC;
if (tscAddQueryInfo(&pNew->cmd) != TSDB_CODE_SUCCESS) {
tscError("%p malloc failed for new queryinfo", pSql);
tscFreeSqlObj(pNew);
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
SQueryInfo *pNewQueryInfo = tscGetQueryInfo(&pNew->cmd);
pNewQueryInfo->pUdfInfo = taosArrayInit(4, sizeof(SUdfInfo));
for(int32_t i = 0; i < taosArrayGetSize(pQueryInfo->pUdfInfo); ++i) {
SUdfInfo info = {0};
SUdfInfo* p1 = taosArrayGet(pQueryInfo->pUdfInfo, i);
info = *p1;
info.name = strdup(p1->name);
taosArrayPush(pNewQueryInfo->pUdfInfo, &info);
}
pNew->cmd.active = pNewQueryInfo;
if (TSDB_CODE_SUCCESS != tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE + pSql->cmd.payloadLen)) {
tscError("%p malloc failed for payload to get table meta", pSql);
tscFreeSqlObj(pNew);
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
tscDebug("%p new pSqlObj:%p to retrieve udf", pSql, pNew);
registerSqlObj(pNew);
pNew->fp = tscTableMetaCallBack;
pNew->param = (void *)pSql->self;
tscDebug("%p metaRid from %" PRId64 " to %" PRId64 , pSql, pSql->metaRid, pNew->self);
pSql->metaRid = pNew->self;
int32_t code = tscBuildAndSendRequest(pNew, NULL);
if (code == TSDB_CODE_SUCCESS) {
code = TSDB_CODE_TSC_ACTION_IN_PROGRESS; // notify application that current process needs to be terminated
}
return code;
}
/**
* retrieve table meta from mnode, and then update the local table meta hashmap.
* @param pSql sql object
@ -2723,6 +2986,7 @@ void tscInitMsgsFp() {
tscBuildMsg[TSDB_SQL_CREATE_DB] = tscBuildCreateDbMsg;
tscBuildMsg[TSDB_SQL_CREATE_USER] = tscBuildUserMsg;
tscBuildMsg[TSDB_SQL_CREATE_FUNCTION] = tscBuildCreateFuncMsg;
tscBuildMsg[TSDB_SQL_CREATE_ACCT] = tscBuildAcctMsg;
tscBuildMsg[TSDB_SQL_ALTER_ACCT] = tscBuildAcctMsg;
@ -2731,6 +2995,7 @@ void tscInitMsgsFp() {
tscBuildMsg[TSDB_SQL_DROP_USER] = tscBuildDropUserAcctMsg;
tscBuildMsg[TSDB_SQL_DROP_ACCT] = tscBuildDropUserAcctMsg;
tscBuildMsg[TSDB_SQL_DROP_DB] = tscBuildDropDbMsg;
tscBuildMsg[TSDB_SQL_DROP_FUNCTION] = tscBuildDropFuncMsg;
tscBuildMsg[TSDB_SQL_SYNC_DB_REPLICA] = tscBuildSyncDbReplicaMsg;
tscBuildMsg[TSDB_SQL_DROP_TABLE] = tscBuildDropTableMsg;
tscBuildMsg[TSDB_SQL_ALTER_USER] = tscBuildUserMsg;
@ -2746,6 +3011,7 @@ void tscInitMsgsFp() {
tscBuildMsg[TSDB_SQL_USE_DB] = tscBuildUseDbMsg;
// tscBuildMsg[TSDB_SQL_META] = tscBuildTableMetaMsg;
tscBuildMsg[TSDB_SQL_STABLEVGROUP] = tscBuildSTableVgroupMsg;
tscBuildMsg[TSDB_SQL_RETRIEVE_FUNC] = tscBuildRetrieveFuncMsg;
tscBuildMsg[TSDB_SQL_HB] = tscBuildHeartBeatMsg;
tscBuildMsg[TSDB_SQL_SHOW] = tscBuildShowMsg;
@ -2764,6 +3030,7 @@ void tscInitMsgsFp() {
tscProcessMsgRsp[TSDB_SQL_META] = tscProcessTableMetaRsp;
tscProcessMsgRsp[TSDB_SQL_STABLEVGROUP] = tscProcessSTableVgroupRsp;
tscProcessMsgRsp[TSDB_SQL_MULTI_META] = tscProcessMultiTableMetaRsp;
tscProcessMsgRsp[TSDB_SQL_RETRIEVE_FUNC] = tscProcessRetrieveFuncRsp;
tscProcessMsgRsp[TSDB_SQL_SHOW] = tscProcessShowRsp;
tscProcessMsgRsp[TSDB_SQL_RETRIEVE] = tscProcessRetrieveRspFromNode; // rsp handled by same function.
@ -2793,3 +3060,4 @@ void tscInitMsgsFp() {
tscKeepConn[TSDB_SQL_FETCH] = 1;
tscKeepConn[TSDB_SQL_HB] = 1;
}

36
src/client/src/tscSql.c
View File

@ -566,7 +566,7 @@ static bool tscKillQueryInDnode(SSqlObj* pSql) {
SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd);
if ((pQueryInfo == NULL) || tscIsTwoStageSTableQuery(pQueryInfo, 0)) {
if ((pQueryInfo == NULL) || pQueryInfo->globalMerge) {
return true;
}
@ -679,7 +679,7 @@ static void tscKillSTableQuery(SSqlObj *pSql) {
SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd);
if (!tscIsTwoStageSTableQuery(pQueryInfo, 0)) {
if (!pQueryInfo->globalMerge) {
return;
}
@ -730,7 +730,7 @@ void taos_stop_query(TAOS_RES *res) {
SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd);
if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) {
if (pQueryInfo->globalMerge) {
assert(pSql->rpcRid <= 0);
tscKillSTableQuery(pSql);
} else {
@ -945,30 +945,35 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) {
return TSDB_CODE_TSC_DISCONNECTED;
}
SSqlObj* pSql = calloc(1, sizeof(SSqlObj));
pSql->pTscObj = taos;
pSql->signature = pSql;
pSql->fp = NULL; // todo set the correct callback function pointer
pSql->cmd.pTableMetaMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
int32_t length = (int32_t)strlen(tableNameList);
if (length == 0) {
return TSDB_CODE_SUCCESS;
}
if (length > MAX_TABLE_NAME_LENGTH) {
tscError("0x%"PRIx64" tableNameList too long, length:%d, maximum allowed:%d", pSql->self, length, MAX_TABLE_NAME_LENGTH);
tscFreeSqlObj(pSql);
tscError("tableNameList too long, length:%d, maximum allowed:%d", length, MAX_TABLE_NAME_LENGTH);
return TSDB_CODE_TSC_INVALID_OPERATION;
}
char *str = calloc(1, length + 1);
if (str == NULL) {
tscError("0x%"PRIx64" failed to allocate sql string buffer", pSql->self);
tscFreeSqlObj(pSql);
tscError("failed to allocate sql string buffer, size:%d", length);
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
strtolower(str, tableNameList);
SArray* plist = taosArrayInit(4, POINTER_BYTES);
SArray* vgroupList = taosArrayInit(4, POINTER_BYTES);
if (plist == NULL || vgroupList == NULL) {
tfree(str);
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
SSqlObj* pSql = calloc(1, sizeof(SSqlObj));
tscAllocPayload(&pSql->cmd, 1024);
pSql->pTscObj = taos;
pSql->signature = pSql;
int32_t code = (uint8_t) tscTransferTableNameList(pSql, str, length, plist);
free(str);
@ -978,10 +983,11 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) {
return code;
}
pSql->cmd.pTableMetaMap = taosHashInit(taosArrayGetSize(plist), taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
registerSqlObj(pSql);
tscDebug("0x%"PRIx64" load multiple table meta, tableNameList: %s pObj:%p", pSql->self, tableNameList, pObj);
code = getMultiTableMetaFromMnode(pSql, plist, vgroupList, loadMultiTableMetaCallback);
code = getMultiTableMetaFromMnode(pSql, plist, vgroupList, NULL, loadMultiTableMetaCallback, false);
if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
code = TSDB_CODE_SUCCESS;
}

111
src/client/src/tscSubquery.c
View File

@ -23,6 +23,7 @@
#include "tscSubquery.h"
#include "qTableMeta.h"
#include "tsclient.h"
#include "qUdf.h"
#include "qUtil.h"
#include "qPlan.h"
@ -32,7 +33,7 @@ typedef struct SInsertSupporter {
} SInsertSupporter;
static void freeJoinSubqueryObj(SSqlObj* pSql);
static bool tscHasRemainDataInSubqueryResultSet(SSqlObj *pSql);
//static bool tscHasRemainDataInSubqueryResultSet(SSqlObj *pSql);
static int32_t tsCompare(int32_t order, int64_t left, int64_t right) {
if (left == right) {
@ -107,6 +108,9 @@ bool subAndCheckDone(SSqlObj *pSql, SSqlObj *pParentSql, int idx) {
subState->states[idx] = 1;
bool done = allSubqueryDone(pParentSql);
if (!done) {
tscDebug("0x%"PRIx64" sub:%p,%d completed, total:%d", pParentSql->self, pSql, idx, pParentSql->subState.numOfSub);
}
pthread_mutex_unlock(&subState->mutex);
return done;
}
@ -416,7 +420,9 @@ static void tscDestroyJoinSupporter(SJoinSupporter* pSupporter) {
}
// tscFieldInfoClear(&pSupporter->fieldsInfo);
if (pSupporter->fieldsInfo.internalField != NULL) {
taosArrayDestroy(pSupporter->fieldsInfo.internalField);
}
if (pSupporter->pTSBuf != NULL) {
tsBufDestroy(pSupporter->pTSBuf);
pSupporter->pTSBuf = NULL;
@ -430,7 +436,8 @@ static void tscDestroyJoinSupporter(SJoinSupporter* pSupporter) {
}
if (pSupporter->pVgroupTables != NULL) {
taosArrayDestroy(pSupporter->pVgroupTables);
//taosArrayDestroy(pSupporter->pVgroupTables);
tscFreeVgroupTableInfo(pSupporter->pVgroupTables);
pSupporter->pVgroupTables = NULL;
}
@ -705,6 +712,15 @@ static void updateQueryTimeRange(SQueryInfo* pQueryInfo, STimeWindow* win) {
pQueryInfo->window = *win;
}
int32_t tagValCompar(const void* p1, const void* p2) {
const STidTags* t1 = (const STidTags*) varDataVal(p1);
const STidTags* t2 = (const STidTags*) varDataVal(p2);
__compar_fn_t func = getComparFunc(t1->padding, 0);
return func(t1->tag, t2->tag);
}
int32_t tidTagsCompar(const void* p1, const void* p2) {
const STidTags* t1 = (const STidTags*) (p1);
const STidTags* t2 = (const STidTags*) (p2);
@ -713,28 +729,7 @@ int32_t tidTagsCompar(const void* p1, const void* p2) {
return (t1->vgId > t2->vgId) ? 1 : -1;
}
tstr* tag1 = (tstr*) t1->tag;
tstr* tag2 = (tstr*) t2->tag;
if (tag1->len != tag2->len) {
return (tag1->len > tag2->len)? 1: -1;
}
return strncmp(tag1->data, tag2->data, tag1->len);
}
int32_t tagValCompar(const void* p1, const void* p2) {
const STidTags* t1 = (const STidTags*) varDataVal(p1);
const STidTags* t2 = (const STidTags*) varDataVal(p2);
tstr* tag1 = (tstr*) t1->tag;
tstr* tag2 = (tstr*) t2->tag;
if (tag1->len != tag2->len) {
return (tag1->len > tag2->len)? 1: -1;
}
return memcmp(tag1->data, tag2->data, tag1->len);
return tagValCompar(p1, p2);
}
void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArray* tables) {
@ -864,6 +859,12 @@ static bool checkForDuplicateTagVal(SSchema* pColSchema, SJoinSupporter* p1, SSq
return true;
}
static void setTidTagType(SJoinSupporter* p, uint8_t type) {
for (int32_t i = 0; i < p->num; ++i) {
STidTags * tag = (STidTags*) varDataVal(p->pIdTagList + i * p->tagSize);
tag->padding = type;
}
}
static int32_t getIntersectionOfTableTuple(SQueryInfo* pQueryInfo, SSqlObj* pParentSql, SArray* resList) {
int16_t joinNum = pParentSql->subState.numOfSub;
@ -883,13 +884,17 @@ static int32_t getIntersectionOfTableTuple(SQueryInfo* pQueryInfo, SSqlObj* pPar
for (int32_t i = 0; i < joinNum; i++) {
SJoinSupporter* p = pParentSql->pSubs[i]->param;
setTidTagType(p, pColSchema->type);
ctxlist[i].p = p;
ctxlist[i].res = taosArrayInit(p->num, size);
tscDebug("Join %d - num:%d", i, p->num);
// sort according to the tag valu
qsort(p->pIdTagList, p->num, p->tagSize, tagValCompar);
if (p->pIdTagList != NULL) {
qsort(p->pIdTagList, p->num, p->tagSize, tagValCompar);
}
if (!checkForDuplicateTagVal(pColSchema, p, pParentSql)) {
for (int32_t j = 0; j <= i; j++) {
@ -1173,7 +1178,7 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow
// no data exists in next vnode, mark the <tid, tags> query completed
// only when there is no subquery exits any more, proceeds to get the intersect of the <tid, tags> tuple sets.
if (!subAndCheckDone(pSql, pParentSql, pSupporter->subqueryIndex)) {
tscDebug("0x%"PRIx64" tagRetrieve:%p,%d completed, total:%d", pParentSql->self, tres, pSupporter->subqueryIndex, pParentSql->subState.numOfSub);
//tscDebug("0x%"PRIx64" tagRetrieve:%p,%d completed, total:%d", pParentSql->self, tres, pSupporter->subqueryIndex, pParentSql->subState.numOfSub);
return;
}
@ -1441,7 +1446,7 @@ static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfR
}
if (!subAndCheckDone(pSql, pParentSql, pSupporter->subqueryIndex)) {
tscDebug("0x%"PRIx64" sub:0x%"PRIx64",%d completed, total:%d", pParentSql->self, pSql->self, pSupporter->subqueryIndex, pState->numOfSub);
//tscDebug("0x%"PRIx64" sub:0x%"PRIx64",%d completed, total:%d", pParentSql->self, pSql->self, pSupporter->subqueryIndex, pState->numOfSub);
return;
}
@ -1888,7 +1893,7 @@ int32_t tscCreateJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter
int16_t type = 0;
int32_t inter = 0;
getResultDataInfo(s->type, s->bytes, TSDB_FUNC_TID_TAG, 0, &type, &bytes, &inter, 0, 0);
getResultDataInfo(s->type, s->bytes, TSDB_FUNC_TID_TAG, 0, &type, &bytes, &inter, 0, 0, NULL);
SSchema s1 = {.colId = s->colId, .type = (uint8_t)type, .bytes = bytes};
pSupporter->tagSize = s1.bytes;
@ -2287,6 +2292,7 @@ int32_t tscHandleFirstRoundStableQuery(SSqlObj *pSql) {
SArray* pColList = pNewQueryInfo->colList;
pNewQueryInfo->colList = NULL;
pNewQueryInfo->fillType = TSDB_FILL_NONE;
tscClearSubqueryInfo(pCmd);
tscFreeSqlResult(pSql);
@ -2542,7 +2548,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) {
SSqlObj* pSub = pSql->pSubs[j];
SRetrieveSupport* pSupport = pSub->param;
tscDebug("0x%"PRIx64" sub:%p launch subquery, orderOfSub:%d.", pSql->self, pSub, pSupport->subqueryIndex);
tscDebug("0x%"PRIx64" sub:0x%"PRIx64" launch subquery, orderOfSub:%d.", pSql->self, pSub->self, pSupport->subqueryIndex);
tscBuildAndSendRequest(pSub, NULL);
}
@ -2793,6 +2799,28 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p
tscFreeRetrieveSup(pSql);
// set the command flag must be after the semaphore been correctly set.
if (pParentSql->cmd.command != TSDB_SQL_RETRIEVE_EMPTY_RESULT) {
pParentSql->cmd.command = TSDB_SQL_RETRIEVE_GLOBALMERGE;
SQueryInfo *pQueryInfo2 = tscGetQueryInfo(&pParentSql->cmd);
size_t size = tscNumOfExprs(pQueryInfo);
for (int32_t j = 0; j < size; ++j) {
SExprInfo* pExprInfo = tscExprGet(pQueryInfo2, j);
int32_t functionId = pExprInfo->base.functionId;
if (functionId < 0) {
SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo2->pUdfInfo, -1 * functionId - 1);
code = initUdfInfo(pUdfInfo);
if (code != TSDB_CODE_SUCCESS) {
pParentSql->res.code = code;
tscAsyncResultOnError(pParentSql);
}
}
}
}
if (pParentSql->res.code == TSDB_CODE_SUCCESS) {
(*pParentSql->fp)(pParentSql->param, pParentSql, 0);
} else {
@ -2861,8 +2889,8 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR
assert(pRes->numOfRows == numOfRows);
int64_t num = atomic_add_fetch_64(&pState->numOfRetrievedRows, numOfRows);
tscDebug("0x%"PRIx64" sub:%p retrieve numOfRows:%d totalNumOfRows:%" PRIu64 " from ep:%s, orderOfSub:%d",
pParentSql->self, pSql, pRes->numOfRows, pState->numOfRetrievedRows, pSql->epSet.fqdn[pSql->epSet.inUse], idx);
tscDebug("0x%"PRIx64" sub:0x%"PRIx64" retrieve numOfRows:%d totalNumOfRows:%" PRIu64 " from ep:%s, orderOfSub:%d",
pParentSql->self, pSql->self, pRes->numOfRows, pState->numOfRetrievedRows, pSql->epSet.fqdn[pSql->epSet.inUse], idx);
if (num > tsMaxNumOfOrderedResults && tscIsProjectionQueryOnSTable(pQueryInfo, 0) && !(tscGetQueryInfo(&pParentSql->cmd)->distinctTag)) {
tscError("0x%"PRIx64" sub:0x%"PRIx64" num of OrderedRes is too many, max allowed:%" PRId32 " , current:%" PRId64,
@ -3047,9 +3075,10 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows)
pParentObj->cmd.insertParam.schemaAttached = 1;
}
}
if (!subAndCheckDone(tres, pParentObj, pSupporter->index)) {
tscDebug("0x%"PRIx64" insert:%p,%d completed, total:%d", pParentObj->self, tres, pSupporter->index, pParentObj->subState.numOfSub);
// concurrency problem, other thread already release pParentObj
//tscDebug("0x%"PRIx64" insert:%p,%d completed, total:%d", pParentObj->self, tres, suppIdx, pParentObj->subState.numOfSub);
return;
}
@ -3460,20 +3489,20 @@ static UNUSED_FUNC bool tscHasRemainDataInSubqueryResultSet(SSqlObj *pSql) {
SQueryInfo *pQueryInfo = tscGetQueryInfo(pCmd);
if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) {
bool allSubqueryExhausted = true;
for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) {
if (pSql->pSubs[i] == NULL) {
continue;
}
SSqlRes *pRes1 = &pSql->pSubs[i]->res;
SSqlCmd *pCmd1 = &pSql->pSubs[i]->cmd;
SQueryInfo *pQueryInfo1 = tscGetQueryInfo(pCmd1);
assert(pQueryInfo1->numOfTables == 1);
STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo1, 0);
/*
* if the global limitation is not reached, and current result has not exhausted, or next more vnodes are
* available, goes on
@ -3484,14 +3513,14 @@ static UNUSED_FUNC bool tscHasRemainDataInSubqueryResultSet(SSqlObj *pSql) {
break;
}
}
hasData = !allSubqueryExhausted;
} else { // otherwise, in case inner join, if any subquery exhausted, query completed.
for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) {
if (pSql->pSubs[i] == 0) {
continue;
}
SSqlRes * pRes1 = &pSql->pSubs[i]->res;
SQueryInfo *pQueryInfo1 = tscGetQueryInfo(&pSql->pSubs[i]->cmd);

7
src/client/src/tscSystem.c
View File

@ -28,6 +28,7 @@
#include "tconfig.h"
#include "ttimezone.h"
#include "tlocale.h"
#include "qScript.h"
// global, not configurable
#define TSC_VAR_NOT_RELEASE 1
@ -148,6 +149,8 @@ void taos_init_imp(void) {
taosInitNotes();
rpcInit();
scriptEnvPoolInit();
tscDebug("starting to initialize TAOS client ...");
tscDebug("Local End Point is:%s", tsLocalEp);
}
@ -202,7 +205,9 @@ void taos_cleanup(void) {
if (atomic_val_compare_exchange_32(&sentinel, TSC_VAR_NOT_RELEASE, TSC_VAR_RELEASED) != TSC_VAR_NOT_RELEASE) {
return;
}
if (tscEmbedded == 0) {
scriptEnvPoolCleanup();
}
taosHashCleanup(tscTableMetaInfo);
tscTableMetaInfo = NULL;

207
src/client/src/tscUtil.c
View File

@ -34,54 +34,54 @@ static void freeQueryInfoImpl(SQueryInfo* pQueryInfo);
int32_t converToStr(char *str, int type, void *buf, int32_t bufSize, int32_t *len) {
int32_t n = 0;
switch (type) {
case TSDB_DATA_TYPE_NULL:
n = sprintf(str, "null");
break;
case TSDB_DATA_TYPE_BOOL:
n = sprintf(str, (*(int8_t*)buf) ? "true" : "false");
break;
case TSDB_DATA_TYPE_TINYINT:
n = sprintf(str, "%d", *(int8_t*)buf);
break;
case TSDB_DATA_TYPE_SMALLINT:
n = sprintf(str, "%d", *(int16_t*)buf);
break;
case TSDB_DATA_TYPE_INT:
n = sprintf(str, "%d", *(int32_t*)buf);
break;
case TSDB_DATA_TYPE_BIGINT:
case TSDB_DATA_TYPE_TIMESTAMP:
n = sprintf(str, "%" PRId64, *(int64_t*)buf);
break;
case TSDB_DATA_TYPE_FLOAT:
n = sprintf(str, "%f", GET_FLOAT_VAL(buf));
break;
case TSDB_DATA_TYPE_DOUBLE:
n = sprintf(str, "%f", GET_DOUBLE_VAL(buf));
break;
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
if (bufSize < 0) {
tscError("invalid buf size");
return TSDB_CODE_TSC_INVALID_VALUE;
}
*str = '"';
memcpy(str + 1, buf, bufSize);
*(str + bufSize + 1) = '"';
n = bufSize + 2;
break;
default:
tscError("unsupported type:%d", type);
return TSDB_CODE_TSC_INVALID_VALUE;
@ -216,6 +216,15 @@ bool tscIsProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableIndex) {
for (int32_t i = 0; i < numOfExprs; ++i) {
int32_t functionId = tscExprGet(pQueryInfo, i)->base.functionId;
if (functionId < 0) {
SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, -1 * functionId - 1);
if (pUdfInfo->funcType == TSDB_UDF_TYPE_AGGREGATE) {
return false;
}
continue;
}
if (functionId != TSDB_FUNC_PRJ &&
functionId != TSDB_FUNC_TAGPRJ &&
functionId != TSDB_FUNC_TAG &&
@ -266,6 +275,16 @@ bool tscIsProjectionQuery(SQueryInfo* pQueryInfo) {
f != TSDB_FUNC_DERIVATIVE) {
return false;
}
if (f < 0) {
SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, -1 * f - 1);
if (pUdfInfo->funcType == TSDB_UDF_TYPE_AGGREGATE) {
return false;
}
continue;
}
}
return true;
@ -297,7 +316,7 @@ bool tscHasColumnFilter(SQueryInfo* pQueryInfo) {
size_t size = taosArrayGetSize(pQueryInfo->colList);
for (int32_t i = 0; i < size; ++i) {
SColumn* pCol = taosArrayGet(pQueryInfo->colList, i);
SColumn* pCol = taosArrayGetP(pQueryInfo->colList, i);
if (pCol->info.flist.numOfFilters > 0) {
return true;
}
@ -338,6 +357,10 @@ bool tsIsArithmeticQueryOnAggResult(SQueryInfo* pQueryInfo) {
}
}
if (tscIsProjectionQuery(pQueryInfo)) {
return false;
}
return false;
}
@ -526,6 +549,15 @@ bool isSimpleAggregateRv(SQueryInfo* pQueryInfo) {
}
int32_t functionId = pExpr->base.functionId;
if (functionId < 0) {
SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, -1 * functionId - 1);
if (pUdfInfo->funcType == TSDB_UDF_TYPE_AGGREGATE) {
return true;
}
continue;
}
if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TS_DUMMY) {
continue;
}
@ -1186,7 +1218,7 @@ void handleDownstreamOperator(SSqlObj** pSqlObjList, int32_t numOfUpstream, SQue
SOperatorInfo* pSourceOperator = createDummyInputOperator(pSqlObjList[0], pSchema, numOfCol1, pFilterInfo, numOfFilterCols);
pOutput->precision = pSqlObjList[0]->res.precision;
SSchema* schema = NULL;
if (px->numOfTables > 1) {
SOperatorInfo** p = calloc(px->numOfTables, POINTER_BYTES);
@ -1302,6 +1334,12 @@ void tscFreeQueryInfo(SSqlCmd* pCmd, bool removeMeta) {
tfree(pUpQueryInfo);
}
if (pQueryInfo->udfCopy) {
pQueryInfo->pUdfInfo = taosArrayDestroy(pQueryInfo->pUdfInfo);
} else {
pQueryInfo->pUdfInfo = tscDestroyUdfArrayList(pQueryInfo->pUdfInfo);
}
freeQueryInfoImpl(pQueryInfo);
clearAllTableMetaInfo(pQueryInfo, removeMeta);
@ -1535,6 +1573,47 @@ void* tscDestroyBlockArrayList(SArray* pDataBlockList) {
return NULL;
}
void freeUdfInfo(SUdfInfo* pUdfInfo) {
if (pUdfInfo == NULL) {
return;
}
if (pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY]) {
(*(udfDestroyFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY])(&pUdfInfo->init);
}
tfree(pUdfInfo->name);
if (pUdfInfo->path) {
unlink(pUdfInfo->path);
}
tfree(pUdfInfo->path);
tfree(pUdfInfo->content);
taosCloseDll(pUdfInfo->handle);
}
void* tscDestroyUdfArrayList(SArray* pUdfList) {
if (pUdfList == NULL) {
return NULL;
}
size_t size = taosArrayGetSize(pUdfList);
for (int32_t i = 0; i < size; i++) {
SUdfInfo* udf = taosArrayGet(pUdfList, i);
freeUdfInfo(udf);
}
taosArrayDestroy(pUdfList);
return NULL;
}
void* tscDestroyBlockHashTable(SHashObj* pBlockHashTable, bool removeMeta) {
if (pBlockHashTable == NULL) {
return NULL;
@ -1782,7 +1861,7 @@ static void extractTableNameList(SInsertStatementParam *pInsertParam, bool freeB
}
int32_t tscMergeTableDataBlocks(SInsertStatementParam *pInsertParam, bool freeBlockMap) {
const int INSERT_HEAD_SIZE = sizeof(SMsgDesc) + sizeof(SSubmitMsg);
const int INSERT_HEAD_SIZE = sizeof(SMsgDesc) + sizeof(SSubmitMsg);
void* pVnodeDataBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false);
SArray* pVnodeDataBlockList = taosArrayInit(8, POINTER_BYTES);
@ -2126,7 +2205,7 @@ void tscFieldInfoCopy(SFieldInfo* pFieldInfo, const SFieldInfo* pSrc, const SArr
assert(pfield->pExpr->pExpr != NULL);
p.pExpr = calloc(1, sizeof(SExprInfo));
tscExprAssign(p.pExpr, pfield->pExpr);
}
}
taosArrayPush(pFieldInfo->internalField, &p);
}
@ -2985,8 +3064,9 @@ int32_t tscQueryInfoCopy(SQueryInfo* pQueryInfo, const SQueryInfo* pSrc) {
pQueryInfo->tsBuf = NULL;
pQueryInfo->fillType = pSrc->fillType;
pQueryInfo->fillVal = NULL;
pQueryInfo->numOfFillVal = 0;;
pQueryInfo->clauseLimit = pSrc->clauseLimit;
pQueryInfo->prjOffset = pSrc->prjOffset;
pQueryInfo->prjOffset = pSrc->prjOffset;
pQueryInfo->numOfTables = 0;
pQueryInfo->window = pSrc->window;
pQueryInfo->sessionWindow = pSrc->sessionWindow;
@ -3020,11 +3100,12 @@ int32_t tscQueryInfoCopy(SQueryInfo* pQueryInfo, const SQueryInfo* pSrc) {
}
if (pSrc->fillType != TSDB_FILL_NONE) {
pQueryInfo->fillVal = malloc(pSrc->fieldsInfo.numOfOutput * sizeof(int64_t));
pQueryInfo->fillVal = calloc(1, pSrc->fieldsInfo.numOfOutput * sizeof(int64_t));
if (pQueryInfo->fillVal == NULL) {
code = TSDB_CODE_TSC_OUT_OF_MEMORY;
goto _error;
}
pQueryInfo->numOfFillVal = pSrc->fieldsInfo.numOfOutput;
memcpy(pQueryInfo->fillVal, pSrc->fillVal, pSrc->fieldsInfo.numOfOutput * sizeof(int64_t));
}
@ -3056,6 +3137,14 @@ int32_t tscQueryInfoCopy(SQueryInfo* pQueryInfo, const SQueryInfo* pSrc) {
tscAddTableMetaInfo(pQueryInfo, &p1->name, pMeta, p1->vgroupList, p1->tagColList, p1->pVgroupTables);
}
SArray *pUdfInfo = NULL;
if (pSrc->pUdfInfo) {
pUdfInfo = taosArrayDup(pSrc->pUdfInfo);
}
pQueryInfo->pUdfInfo = pUdfInfo;
pQueryInfo->udfCopy = true;
_error:
return code;
}
@ -3102,7 +3191,9 @@ void tscVgroupTableCopy(SVgroupTableInfo* info, SVgroupTableInfo* pInfo) {
info->vgInfo.epAddr[j].fqdn = strdup(pInfo->vgInfo.epAddr[j].fqdn);
}
info->itemList = taosArrayDup(pInfo->itemList);
if (pInfo->itemList) {
info->itemList = taosArrayDup(pInfo->itemList);
}
}
SArray* tscVgroupTableInfoDup(SArray* pVgroupTables) {
@ -3353,6 +3444,11 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, __async_cb_func_t
SQueryInfo* pNewQueryInfo = tscGetQueryInfo(pnCmd);
if (pQueryInfo->pUdfInfo) {
pNewQueryInfo->pUdfInfo = taosArrayDup(pQueryInfo->pUdfInfo);
pNewQueryInfo->udfCopy = true;
}
pNewQueryInfo->command = pQueryInfo->command;
pnCmd->active = pNewQueryInfo;
@ -3366,11 +3462,13 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, __async_cb_func_t
pNewQueryInfo->tsBuf = NULL;
pNewQueryInfo->fillType = pQueryInfo->fillType;
pNewQueryInfo->fillVal = NULL;
pNewQueryInfo->numOfFillVal = 0;
pNewQueryInfo->clauseLimit = pQueryInfo->clauseLimit;
pNewQueryInfo->prjOffset = pQueryInfo->prjOffset;
pNewQueryInfo->numOfTables = 0;
pNewQueryInfo->pTableMetaInfo = NULL;
pNewQueryInfo->bufLen = pQueryInfo->bufLen;
pNewQueryInfo->buf = malloc(pQueryInfo->bufLen);
if (pNewQueryInfo->buf == NULL) {
terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
@ -3396,11 +3494,14 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, __async_cb_func_t
}
if (pQueryInfo->fillType != TSDB_FILL_NONE) {
pNewQueryInfo->fillVal = malloc(pQueryInfo->fieldsInfo.numOfOutput * sizeof(int64_t));
//just make memory memory sanitizer happy
//refator later
pNewQueryInfo->fillVal = calloc(1, pQueryInfo->fieldsInfo.numOfOutput * sizeof(int64_t));
if (pNewQueryInfo->fillVal == NULL) {
terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
goto _error;
}
pNewQueryInfo->numOfFillVal = pQueryInfo->fieldsInfo.numOfOutput;
memcpy(pNewQueryInfo->fillVal, pQueryInfo->fillVal, pQueryInfo->fieldsInfo.numOfOutput * sizeof(int64_t));
}
@ -4210,7 +4311,7 @@ int32_t createProjectionExpr(SQueryInfo* pQueryInfo, STableMetaInfo* pTableMetaI
int32_t inter = 0;
getResultDataInfo(pSource->base.colType, pSource->base.colBytes, functionId, 0, &pse->resType,
&pse->resBytes, &inter, 0, false);
&pse->resBytes, &inter, 0, false, NULL);
pse->colType = pse->resType;
pse->colBytes = pse->resBytes;
@ -4277,8 +4378,14 @@ static int32_t createGlobalAggregateExpr(SQueryAttr* pQueryAttr, SQueryInfo* pQu
functionId = TSDB_FUNC_STDDEV;
}
SUdfInfo* pUdfInfo = NULL;
if (functionId < 0) {
pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, -1 * functionId - 1);
}
getResultDataInfo(pExpr->base.colType, pExpr->base.colBytes, functionId, 0, &pse->resType, &pse->resBytes, &inter,
0, false);
0, false, pUdfInfo);
}
}
@ -4354,7 +4461,8 @@ int32_t tscCreateQueryFromQueryInfo(SQueryInfo* pQueryInfo, SQueryAttr* pQueryAt
pQueryAttr->order = pQueryInfo->order;
pQueryAttr->fillType = pQueryInfo->fillType;
pQueryAttr->havingNum = pQueryInfo->havingFieldNum;
pQueryAttr->pUdfInfo = pQueryInfo->pUdfInfo;
if (pQueryInfo->order.order == TSDB_ORDER_ASC) { // TODO refactor
pQueryAttr->window = pQueryInfo->window;
} else {
@ -4420,7 +4528,7 @@ int32_t tscCreateQueryFromQueryInfo(SQueryInfo* pQueryInfo, SQueryAttr* pQueryAt
if (pQueryAttr->fillType != TSDB_FILL_NONE) {
pQueryAttr->fillVal = calloc(pQueryAttr->numOfOutput, sizeof(int64_t));
memcpy(pQueryAttr->fillVal, pQueryInfo->fillVal, pQueryAttr->numOfOutput * sizeof(int64_t));
memcpy(pQueryAttr->fillVal, pQueryInfo->fillVal, pQueryInfo->numOfFillVal * sizeof(int64_t));
}
pQueryAttr->srcRowSize = 0;
@ -4466,8 +4574,13 @@ static int32_t doAddTableName(char* nextStr, char** str, SArray* pNameArray, SSq
strncpy(tablename, *str, TSDB_TABLE_FNAME_LEN);
len = (int32_t) strlen(tablename);
} else {
memcpy(tablename, *str, nextStr - (*str));
len = (int32_t)(nextStr - (*str));
if (len >= TSDB_TABLE_NAME_LEN) {
sprintf(pCmd->payload, "table name too long");
return TSDB_CODE_TSC_INVALID_OPERATION;
}
memcpy(tablename, *str, nextStr - (*str));
tablename[len] = '\0';
}
@ -4479,9 +4592,8 @@ static int32_t doAddTableName(char* nextStr, char** str, SArray* pNameArray, SSq
// Check if the table name available or not
if (tscValidateName(&sToken) != TSDB_CODE_SUCCESS) {
code = TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH;
sprintf(pCmd->payload, "table name is invalid");
return code;
return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH;
}
SName name = {0};
@ -4497,6 +4609,15 @@ static int32_t doAddTableName(char* nextStr, char** str, SArray* pNameArray, SSq
return TSDB_CODE_SUCCESS;
}
int32_t nameComparFn(const void* n1, const void* n2) {
int32_t ret = strcmp(*(char**)n1, *(char**)n2);
if (ret == 0) {
return 0;
} else {
return ret > 0? 1:-1;
}
}
int tscTransferTableNameList(SSqlObj *pSql, const char *pNameList, int32_t length, SArray* pNameArray) {
SSqlCmd *pCmd = &pSql->cmd;
@ -4532,12 +4653,44 @@ int tscTransferTableNameList(SSqlObj *pSql, const char *pNameList, int32_t lengt
}
}
if (taosArrayGetSize(pNameArray) > TSDB_MULTI_TABLEMETA_MAX_NUM) {
size_t len = taosArrayGetSize(pNameArray);
if (len == 1) {
return TSDB_CODE_SUCCESS;
}
if (len > TSDB_MULTI_TABLEMETA_MAX_NUM) {
code = TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH;
sprintf(pCmd->payload, "tables over the max number");
return code;
}
taosArraySort(pNameArray, nameComparFn);
int32_t pos = 0;
for(int32_t i = 1; i < len; ++i) {
char** p1 = taosArrayGet(pNameArray, pos);
char** p2 = taosArrayGet(pNameArray, i);
if (strcmp(*p1, *p2) == 0) {
// do nothing
} else {
if (pos + 1 != i) {
char* p = taosArrayGetP(pNameArray, pos + 1);
tfree(p);
taosArraySet(pNameArray, pos + 1, p2);
pos += 1;
} else {
pos += 1;
}
}
}
for(int32_t i = pos + 1; i < pNameArray->size; ++i) {
char* p = taosArrayGetP(pNameArray, i);
tfree(p);
}
pNameArray->size = pos + 1;
return TSDB_CODE_SUCCESS;
}

3
src/common/inc/tcmdtype.h
View File

@ -41,8 +41,10 @@ enum {
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_MGMT, "mgmt" )
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_DB, "create-db" )
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_TABLE, "create-table" )
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_FUNCTION, "create-function" )
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_DROP_DB, "drop-db" )
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_DROP_TABLE, "drop-table" )
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_DROP_FUNCTION, "drop-function" )
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_ACCT, "create-acct" )
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_CREATE_USER, "create-user" )
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_DROP_ACCT, "drop-acct" )
@ -74,6 +76,7 @@ enum {
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_STABLEVGROUP, "stable-vgroup" )
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_MULTI_META, "multi-meta" )
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_HB, "heart-beat" )
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_RETRIEVE_FUNC, "retrieve-function" )
// SQL below for client local
TSDB_DEFINE_SQL_TYPE( TSDB_SQL_LOCAL, "local" )

8
src/common/inc/tdataformat.h
View File

@ -31,6 +31,14 @@ extern "C" {
memcpy(varDataVal(x), (str), __len); \
} while (0);
#define STR_TO_NET_VARSTR(x, str) \
do { \
VarDataLenT __len = (VarDataLenT)strlen(str); \
*(VarDataLenT *)(x) = htons(__len); \
memcpy(varDataVal(x), (str), __len); \
} while (0);
#define STR_WITH_MAXSIZE_TO_VARSTR(x, str, _maxs) \
do { \
char *_e = stpncpy(varDataVal(x), (str), (_maxs)-VARSTR_HEADER_SIZE); \

71
src/common/src/tarithoperator.c
View File

@ -18,7 +18,58 @@
#include "ttype.h"
#include "tutil.h"
#include "tarithoperator.h"
#include "tcompare.h"
//GET_TYPED_DATA(v, double, _right_type, (char *)&((right)[i]));
#define ARRAY_LIST_OP_DIV(left, right, _left_type, _right_type, len1, len2, out, op, _res_type, _ord) \
{ \
int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : MAX(len1, len2) - 1; \
int32_t step = ((_ord) == TSDB_ORDER_ASC) ? 1 : -1; \
\
if ((len1) == (len2)) { \
for (; i < (len2) && i >= 0; i += step, (out) += 1) { \
if (isNull((char *)&((left)[i]), _left_type) || isNull((char *)&((right)[i]), _right_type)) { \
SET_DOUBLE_NULL(out); \
continue; \
} \
double v, z = 0.0; \
GET_TYPED_DATA(v, double, _right_type, (char *)&((right)[i])); \
if (getComparFunc(TSDB_DATA_TYPE_DOUBLE, 0)(&v, &z) == 0) { \
SET_DOUBLE_NULL(out); \
continue; \
} \
*(out) = (double)(left)[i] op(right)[i]; \
} \
} else if ((len1) == 1) { \
for (; i >= 0 && i < (len2); i += step, (out) += 1) { \
if (isNull((char *)(left), _left_type) || isNull((char *)&(right)[i], _right_type)) { \
SET_DOUBLE_NULL(out); \
continue; \
} \
double v, z = 0.0; \
GET_TYPED_DATA(v, double, _right_type, (char *)&((right)[i])); \
if (getComparFunc(TSDB_DATA_TYPE_DOUBLE, 0)(&v, &z) == 0) { \
SET_DOUBLE_NULL(out); \
continue; \
} \
*(out) = (double)(left)[0] op(right)[i]; \
} \
} else if ((len2) == 1) { \
for (; i >= 0 && i < (len1); i += step, (out) += 1) { \
if (isNull((char *)&(left)[i], _left_type) || isNull((char *)(right), _right_type)) { \
SET_DOUBLE_NULL(out); \
continue; \
} \
double v, z = 0.0; \
GET_TYPED_DATA(v, double, _right_type, (char *)&((right)[0])); \
if (getComparFunc(TSDB_DATA_TYPE_DOUBLE, 0)(&v, &z) == 0) { \
SET_DOUBLE_NULL(out); \
continue; \
} \
*(out) = (double)(left)[i] op(right)[0]; \
} \
} \
}
#define ARRAY_LIST_OP(left, right, _left_type, _right_type, len1, len2, out, op, _res_type, _ord) \
{ \
int32_t i = ((_ord) == TSDB_ORDER_ASC) ? 0 : MAX(len1, len2) - 1; \
@ -62,6 +113,12 @@
SET_DOUBLE_NULL(out); \
continue; \
} \
double v, z = 0.0; \
GET_TYPED_DATA(v, double, _right_type, (char *)&((right)[i])); \
if (getComparFunc(TSDB_DATA_TYPE_DOUBLE, 0)(&v, &z) == 0) { \
SET_DOUBLE_NULL(out); \
continue; \
} \
*(out) = (double)(left)[i] - ((int64_t)(((double)(left)[i]) / (right)[i])) * (right)[i]; \
} \
} else if (len1 == 1) { \
@ -70,6 +127,12 @@
SET_DOUBLE_NULL(out); \
continue; \
} \
double v, z = 0.0; \
GET_TYPED_DATA(v, double, _right_type, (char *)&((right)[i])); \
if (getComparFunc(TSDB_DATA_TYPE_DOUBLE, 0)(&v, &z) == 0) { \
SET_DOUBLE_NULL(out); \
continue; \
} \
*(out) = (double)(left)[0] - ((int64_t)(((double)(left)[0]) / (right)[i])) * (right)[i]; \
} \
} else if ((len2) == 1) { \
@ -78,6 +141,12 @@
SET_DOUBLE_NULL(out); \
continue; \
} \
double v, z = 0.0; \
GET_TYPED_DATA(v, double, _right_type, (char *)&((right)[0])); \
if (getComparFunc(TSDB_DATA_TYPE_DOUBLE, 0)(&v, &z) == 0) { \
SET_DOUBLE_NULL(out); \
continue; \
} \
*(out) = (double)(left)[i] - ((int64_t)(((double)(left)[i]) / (right)[0])) * (right)[0]; \
} \
} \
@ -90,7 +159,7 @@
#define ARRAY_LIST_MULTI(left, right, _left_type, _right_type, len1, len2, out, _ord) \
ARRAY_LIST_OP(left, right, _left_type, _right_type, len1, len2, out, *, TSDB_DATA_TYPE_DOUBLE, _ord)
#define ARRAY_LIST_DIV(left, right, _left_type, _right_type, len1, len2, out, _ord) \
ARRAY_LIST_OP(left, right, _left_type, _right_type, len1, len2, out, /, TSDB_DATA_TYPE_DOUBLE, _ord)
ARRAY_LIST_OP_DIV(left, right, _left_type, _right_type, len1, len2, out, /, TSDB_DATA_TYPE_DOUBLE, _ord)
#define ARRAY_LIST_REM(left, right, _left_type, _right_type, len1, len2, out, _ord) \
ARRAY_LIST_OP_REM(left, right, _left_type, _right_type, len1, len2, out, %, TSDB_DATA_TYPE_DOUBLE, _ord)

4
src/connector/jdbc/CMakeLists.txt
View File

@ -8,8 +8,8 @@ IF (TD_MVN_INSTALLED)
ADD_CUSTOM_COMMAND(OUTPUT ${JDBC_CMD_NAME}
POST_BUILD
COMMAND mvn -Dmaven.test.skip=true install -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml
COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.32-dist.jar ${LIBRARY_OUTPUT_PATH}
COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-*-dist.jar ${LIBRARY_OUTPUT_PATH}
COMMAND mvn -Dmaven.test.skip=true clean -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml
COMMENT "build jdbc driver")
ADD_CUSTOM_TARGET(${JDBC_TARGET_NAME} ALL WORKING_DIRECTORY ${EXECUTABLE_OUTPUT_PATH} DEPENDS ${JDBC_CMD_NAME})
ENDIF ()
ENDIF ()

6
src/connector/jdbc/pom.xml
View File

@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>com.taosdata.jdbc</groupId>
<artifactId>taos-jdbcdriver</artifactId>
<version>2.0.32</version>
<version>2.0.33</version>
<packaging>jar</packaging>
<name>JDBCDriver</name>
<url>https://github.com/taosdata/TDengine/tree/master/src/connector/jdbc</url>
@ -40,7 +40,7 @@
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.13</version>
<version>4.13.1</version>
<scope>test</scope>
</dependency>
<!-- for restful -->
@ -57,7 +57,7 @@
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>29.0-jre</version>
<version>30.0-jre</version>
</dependency>
</dependencies>

492
src/connector/jdbc/readme.md
View File

@ -1,54 +1,62 @@
# Java Connector
## TAOS-JDBCDriver 概述
TDengine 提供了遵循 JDBC 标准（3.0）API 规范的 `taos-jdbcdriver` 实现，可在 maven 的中央仓库 [Sonatype Repository][1] 搜索下载。
TDengine 为了方便 Java 应用使用，提供了遵循 JDBC 标准(3.0)API 规范的 `taos-jdbcdriver` 实现。目前可以通过 [Sonatype Repository][1] 搜索并下载。
`taos-jdbcdriver` 的实现包括 2 种形式： JDBC-JNI 和 JDBC-RESTful（taos-jdbcdriver-2.0.18 开始支持 JDBC-RESTful）。 JDBC-JNI 通过调用客户端 libtaos.so（或 taos.dll ）的本地方法实现， JDBC-RESTful 则在内部封装了 RESTful 接口实现。
由于 TDengine 是使用 c 语言开发的，使用 taos-jdbcdriver 驱动包时需要依赖系统对应的本地函数库。
![tdengine-connector](https://www.taosdata.com/cn/documentation/user/pages/images/tdengine-jdbc-connector.png)
* libtaos.so
在 linux 系统中成功安装 TDengine 后，依赖的本地函数库 libtaos.so 文件会被自动拷贝至 /usr/lib/libtaos.so，该目录包含在 Linux 自动扫描路径上，无需单独指定。
* taos.dll
在 windows 系统中安装完客户端之后，驱动包依赖的 taos.dll 文件会自动拷贝到系统默认搜索路径 C:/Windows/System32 下，同样无需要单独指定。
> 注意：在 windows 环境开发时需要安装 TDengine 对应的 windows 版本客户端，由于目前没有提供 Linux 环境单独的客户端，需要安装 TDengine 才能使用。
上图显示了 3 种 Java 应用使用连接器访问 TDengine 的方式：
TDengine 的 JDBC 驱动实现尽可能的与关系型数据库驱动保持一致，但时序空间数据库与关系对象型数据库服务的对象和技术特征的差异导致 taos-jdbcdriver 并未完全实现 JDBC 标准规范。在使用时需要注意以下几点：
* JDBC-JNI：Java 应用在物理节点1（pnode1）上使用 JDBC-JNI 的 API ，直接调用客户端 API（libtaos.so 或 taos.dll）将写入和查询请求发送到位于物理节点2（pnode2）上的 taosd 实例。
* RESTful：应用将 SQL 发送给位于物理节点2（pnode2）上的 RESTful 连接器，再调用客户端 API（libtaos.so）。
* JDBC-RESTful：Java 应用通过 JDBC-RESTful 的 API ，将 SQL 封装成一个 RESTful 请求，发送给物理节点2的 RESTful 连接器。
* TDengine 不提供针对单条数据记录的删除和修改的操作，驱动中也没有支持相关方法。
* 由于不支持删除和修改，所以也不支持事务操作。
* 目前不支持表间的 union 操作。
* 目前不支持嵌套查询(nested query)，`对每个 Connection 的实例，至多只能有一个打开的 ResultSet 实例；如果在 ResultSet还没关闭的情况下执行了新的查询，TSDBJDBCDriver 则会自动关闭上一个 ResultSet`。
TDengine 的 JDBC 驱动实现尽可能与关系型数据库驱动保持一致，但时序空间数据库与关系对象型数据库服务的对象和技术特征存在差异，导致 `taos-jdbcdriver` 与传统的 JDBC driver 也存在一定差异。在使用时需要注意以下几点：
* TDengine 目前不支持针对单条数据记录的删除操作。
* 目前不支持事务操作。
* 目前不支持嵌套查询（nested query）。
* 对每个 Connection 的实例，至多只能有一个打开的 ResultSet 实例；如果在 ResultSet 还没关闭的情况下执行了新的查询，taos-jdbcdriver 会自动关闭上一个 ResultSet。
## TAOS-JDBCDriver 版本以及支持的 TDengine 版本和 JDK 版本
## JDBC-JNI和JDBC-RESTful的对比
| taos-jdbcdriver 版本 | TDengine 版本 | JDK 版本 |
| --- | --- | --- |
| 1.0.3 | 1.6.1.x 及以上 | 1.8.x |
| 1.0.2 | 1.6.1.x 及以上 | 1.8.x |
| 1.0.1 | 1.6.1.x 及以上 | 1.8.x |
<table >
<tr align="center"><th>对比项</th><th>JDBC-JNI</th><th>JDBC-RESTful</th></tr>
<tr align="center">
<td>支持的操作系统</td>
<td>linux、windows</td>
<td>全平台</td>
</tr>
<tr align="center">
<td>是否需要安装 client</td>
<td>需要</td>
<td>不需要</td>
</tr>
<tr align="center">
<td>server 升级后是否需要升级 client</td>
<td>需要</td>
<td>不需要</td>
</tr>
<tr align="center">
<td>写入性能</td>
<td colspan="2">JDBC-RESTful 是 JDBC-JNI 的 50%～90% </td>
</tr>
<tr align="center">
<td>查询性能</td>
<td colspan="2">JDBC-RESTful 与 JDBC-JNI 没有差别</td>
</tr>
</table>
## TDengine DataType 和 Java DataType
注意：与 JNI 方式不同，RESTful 接口是无状态的，因此 `USE db_name` 指令没有效果，RESTful 下所有对表名、超级表名的引用都需要指定数据库名前缀。
TDengine 目前支持时间戳、数字、字符、布尔类型，与 Java 对应类型转换如下：
| TDengine DataType | Java DataType |
| --- | --- |
| TIMESTAMP | java.sql.Timestamp |
| INT | java.lang.Integer |
| BIGINT | java.lang.Long |
| FLOAT | java.lang.Float |
| DOUBLE | java.lang.Double |
| SMALLINT, TINYINT |java.lang.Short |
| BOOL | java.lang.Boolean |
| BINARY, NCHAR | java.lang.String |
## 如何获取 TAOS-JDBCDriver
## 如何获取 taos-jdbcdriver
### maven 仓库
目前 taos-jdbcdriver 已经发布到 [Sonatype Repository][1] 仓库，且各大仓库都已同步。
* [sonatype][8]
* [mvnrepository][9]
* [maven.aliyun][10]
@ -56,56 +64,86 @@ TDengine 目前支持时间戳、数字、字符、布尔类型，与 Java 对
maven 项目中使用如下 pom.xml 配置即可：
```xml
<dependencies>
<dependency>
<groupId>com.taosdata.jdbc</groupId>
<artifactId>taos-jdbcdriver</artifactId>
<version>1.0.3</version>
</dependency>
</dependencies>
<dependency>
<groupId>com.taosdata.jdbc</groupId>
<artifactId>taos-jdbcdriver</artifactId>
<version>2.0.18</version>
</dependency>
```
### 源码编译打包
下载 [TDengine][3] 源码之后，进入 taos-jdbcdriver 源码目录 `src/connector/jdbc` 执行 `mvn clean package` 即可生成相应 jar 包。
下载 [TDengine][3] 源码之后，进入 taos-jdbcdriver 源码目录 `src/connector/jdbc` 执行 `mvn clean package -Dmaven.test.skip=true` 即可生成相应 jar 包。
## 使用说明
## JDBC的使用说明
### 获取连接
如下所示配置即可获取 TDengine Connection：
#### 指定URL获取连接
通过指定URL获取连接，如下所示：
```java
Class.forName("com.taosdata.jdbc.TSDBDriver");
String jdbcUrl = "jdbc:TAOS://127.0.0.1:6030/log?user=root&password=taosdata";
Class.forName("com.taosdata.jdbc.rs.RestfulDriver");
String jdbcUrl = "jdbc:TAOS-RS://taosdemo.com:6041/test?user=root&password=taosdata";
Connection conn = DriverManager.getConnection(jdbcUrl);
```
> 端口 6030 为默认连接端口，JDBC URL 中的 log 为系统本身的监控数据库。
以上示例，使用 **JDBC-RESTful** 的 driver，建立了到 hostname 为 taosdemo.com，端口为 6041，数据库名为 test 的连接。这个 URL 中指定用户名（user）为 root，密码（password）为 taosdata。
使用 JDBC-RESTful 接口，不需要依赖本地函数库。与 JDBC-JNI 相比，仅需要：
1. driverClass 指定为“com.taosdata.jdbc.rs.RestfulDriver”；
2. jdbcUrl 以“jdbc:TAOS-RS://”开头；
3. 使用 6041 作为连接端口。
如果希望获得更好的写入和查询性能，Java 应用可以使用 **JDBC-JNI** 的driver，如下所示：
```java
Class.forName("com.taosdata.jdbc.TSDBDriver");
String jdbcUrl = "jdbc:TAOS://taosdemo.com:6030/test?user=root&password=taosdata";
Connection conn = DriverManager.getConnection(jdbcUrl);
```
以上示例，使用了 JDBC-JNI 的 driver，建立了到 hostname 为 taosdemo.com，端口为 6030（TDengine 的默认端口），数据库名为 test 的连接。这个 URL 中指定用户名（user）为 root，密码（password）为 taosdata。
**注意**：使用 JDBC-JNI 的 driver，taos-jdbcdriver 驱动包时需要依赖系统对应的本地函数库。
* libtaos.so
在 linux 系统中成功安装 TDengine 后，依赖的本地函数库 libtaos.so 文件会被自动拷贝至 /usr/lib/libtaos.so，该目录包含在 Linux 自动扫描路径上，无需单独指定。
* taos.dll
在 windows 系统中安装完客户端之后，驱动包依赖的 taos.dll 文件会自动拷贝到系统默认搜索路径 C:/Windows/System32 下，同样无需要单独指定。
> 在 windows 环境开发时需要安装 TDengine 对应的 [windows 客户端][14]，Linux 服务器安装完 TDengine 之后默认已安装 client，也可以单独安装 [Linux 客户端][15] 连接远程 TDengine Server。
JDBC-JNI 的使用请参见[视频教程](https://www.taosdata.com/blog/2020/11/11/1955.html)。
TDengine 的 JDBC URL 规范格式为：
`jdbc:TSDB://{host_ip}:{port}/[database_name]?[user={user}|&password={password}|&charset={charset}|&cfgdir={config_dir}|&locale={locale}|&timezone={timezone}]`
其中，`{}` 中的内容必须，`[]` 中为可选。配置参数说明如下：
`jdbc:[TAOS|TAOS-RS]://[host_name]:[port]/[database_name]?[user={user}|&password={password}|&charset={charset}|&cfgdir={config_dir}|&locale={locale}|&timezone={timezone}]`
url中的配置参数如下：
* user：登录 TDengine 用户名，默认值 root。
* password：用户登录密码，默认值 taosdata。
* charset：客户端使用的字符集，默认值为系统字符集。
* cfgdir：客户端配置文件目录路径，Linux OS 上默认值 /etc/taos ，Windows OS 上默认值 C:/TDengine/cfg。
* charset：客户端使用的字符集，默认值为系统字符集。
* locale：客户端语言环境，默认值系统当前 locale。
* timezone：客户端使用的时区，默认值为系统当前时区。
以上参数可以在 3 处配置，`优先级由高到低`分别如下：
1. JDBC URL 参数
如上所述，可以在 JDBC URL 的参数中指定。
2. java.sql.DriverManager.getConnection(String jdbcUrl, Properties connProps)
#### 指定URL和Properties获取连接
除了通过指定的 URL 获取连接，还可以使用 Properties 指定建立连接时的参数，如下所示：
```java
public Connection getConn() throws Exception{
Class.forName("com.taosdata.jdbc.TSDBDriver");
String jdbcUrl = "jdbc:TAOS://127.0.0.1:0/log?user=root&password=taosdata";
// Class.forName("com.taosdata.jdbc.rs.RestfulDriver");
String jdbcUrl = "jdbc:TAOS://taosdemo.com:6030/test?user=root&password=taosdata";
// String jdbcUrl = "jdbc:TAOS-RS://taosdemo.com:6041/test?user=root&password=taosdata";
Properties connProps = new Properties();
connProps.setProperty(TSDBDriver.PROPERTY_KEY_USER, "root");
connProps.setProperty(TSDBDriver.PROPERTY_KEY_PASSWORD, "taosdata");
connProps.setProperty(TSDBDriver.PROPERTY_KEY_CONFIG_DIR, "/etc/taos");
connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");
@ -114,22 +152,68 @@ public Connection getConn() throws Exception{
}
```
3. 客户端配置文件 taos.cfg
以上示例，建立一个到 hostname 为 taosdemo.com，端口为 6030，数据库名为 test 的连接。注释为使用 JDBC-RESTful 时的方法。这个连接在 url 中指定了用户名(user)为 root，密码（password）为 taosdata，并在 connProps 中指定了使用的字符集、语言环境、时区等信息。
linux 系统默认配置文件为 /var/lib/taos/taos.cfg，windows 系统默认配置文件路径为 C:\TDengine\cfg\taos.cfg。
```properties
# client default username
# defaultUser root
properties 中的配置参数如下：
* TSDBDriver.PROPERTY_KEY_USER：登录 TDengine 用户名，默认值 root。
* TSDBDriver.PROPERTY_KEY_PASSWORD：用户登录密码，默认值 taosdata。
* TSDBDriver.PROPERTY_KEY_CONFIG_DIR：客户端配置文件目录路径，Linux OS 上默认值 /etc/taos ，Windows OS 上默认值 C:/TDengine/cfg。
* TSDBDriver.PROPERTY_KEY_CHARSET：客户端使用的字符集，默认值为系统字符集。
* TSDBDriver.PROPERTY_KEY_LOCALE：客户端语言环境，默认值系统当前 locale。
* TSDBDriver.PROPERTY_KEY_TIME_ZONE：客户端使用的时区，默认值为系统当前时区。
# client default password
# defaultPass taosdata
#### 使用客户端配置文件建立连接
当使用 JDBC-JNI 连接 TDengine 集群时，可以使用客户端配置文件，在客户端配置文件中指定集群的 firstEp、secondEp参数。
如下所示：
1. 在 Java 应用中不指定 hostname 和 port
```java
public Connection getConn() throws Exception{
Class.forName("com.taosdata.jdbc.TSDBDriver");
String jdbcUrl = "jdbc:TAOS://:/test?user=root&password=taosdata";
Properties connProps = new Properties();
connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8");
connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8");
connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8");
Connection conn = DriverManager.getConnection(jdbcUrl, connProps);
return conn;
}
```
2. 在配置文件中指定 firstEp 和 secondEp
```
# first fully qualified domain name (FQDN) for TDengine system
firstEp cluster_node1:6030
# second fully qualified domain name (FQDN) for TDengine system, for cluster only
secondEp cluster_node2:6030
# default system charset
# charset UTF-8
# charset UTF-8
# system locale
# locale en_US.UTF-8
```
以上示例，jdbc 会使用客户端的配置文件，建立到 hostname 为 cluster_node1、端口为 6030、数据库名为 test 的连接。当集群中 firstEp 节点失效时，JDBC 会尝试使用 secondEp 连接集群。
TDengine 中，只要保证 firstEp 和 secondEp 中一个节点有效，就可以正常建立到集群的连接。
> 注意：这里的配置文件指的是调用 JDBC Connector 的应用程序所在机器上的配置文件，Linux OS 上默认值 /etc/taos/taos.cfg ，Windows OS 上默认值 C://TDengine/cfg/taos.cfg。
#### 配置参数的优先级
通过以上 3 种方式获取连接，如果配置参数在 url、Properties、客户端配置文件中有重复，则参数的`优先级由高到低`分别如下：
1. JDBC URL 参数，如上所述，可以在 JDBC URL 的参数中指定。
2. Properties connProps
3. 客户端配置文件 taos.cfg
例如：在 url 中指定了 password 为 taosdata，在 Properties 中指定了 password 为 taosdemo，那么，JDBC 会使用 url 中的 password 建立连接。
> 更多详细配置请参考[客户端配置][13]
### 创建数据库和表
@ -146,6 +230,7 @@ stmt.executeUpdate("use db");
// create table
stmt.executeUpdate("create table if not exists tb (ts timestamp, temperature int, humidity float)");
```
> 注意：如果不使用 `use db` 指定数据库，则后续对表的操作都需要增加数据库名称作为前缀，如 db.tb。
### 插入数据
@ -156,6 +241,7 @@ int affectedRows = stmt.executeUpdate("insert into tb values(now, 23, 10.3) (now
System.out.println("insert " + affectedRows + " rows.");
```
> now 为系统内部函数，默认为服务器当前时间。
> `now + 1s` 代表服务器当前时间往后加 1 秒，数字后面代表时间单位：a(毫秒), s(秒), m(分), h(小时), d(天)，w(周), n(月), y(年)。
@ -177,8 +263,150 @@ while(resultSet.next()){
System.out.printf("%s, %d, %s\n", ts, temperature, humidity);
}
```
> 查询和操作关系型数据库一致，使用下标获取返回字段内容时从 1 开始，建议使用字段名称获取。
### 处理异常
在报错后，通过SQLException可以获取到错误的信息和错误码：
```java
try (Statement statement = connection.createStatement()) {
// executeQuery
ResultSet resultSet = statement.executeQuery(sql);
// print result
printResult(resultSet);
} catch (SQLException e) {
System.out.println("ERROR Message: " + e.getMessage());
System.out.println("ERROR Code: " + e.getErrorCode());
e.printStackTrace();
}
```
JDBC连接器可能报错的错误码包括3种：JDBC driver本身的报错（错误码在0x2301到0x2350之间），JNI方法的报错（错误码在0x2351到0x2400之间），TDengine其他功能模块的报错。
具体的错误码请参考：
* https://github.com/taosdata/TDengine/blob/develop/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBErrorNumbers.java
* https://github.com/taosdata/TDengine/blob/develop/src/inc/taoserror.h
### <a class="anchor" id="stmt-java"></a>通过参数绑定写入数据
从 2.1.2.0 版本开始，TDengine 的 **JDBC-JNI** 实现大幅改进了参数绑定方式对数据写入（INSERT）场景的支持。采用这种方式写入数据时，能避免 SQL 语法解析的资源消耗，从而在很多情况下显著提升写入性能。（注意：**JDBC-RESTful** 实现并不提供参数绑定这种使用方式。）
```java
Statement stmt = conn.createStatement();
Random r = new Random();
// INSERT 语句中，VALUES 部分允许指定具体的数据列；如果采取自动建表，则 TAGS 部分需要设定全部 TAGS 列的参数值：
TSDBPreparedStatement s = (TSDBPreparedStatement) conn.prepareStatement("insert into ? using weather_test tags (?, ?) (ts, c1, c2) values(?, ?, ?)");
// 设定数据表名：
s.setTableName("w1");
// 设定 TAGS 取值：
s.setTagInt(0, r.nextInt(10));
s.setTagString(1, "Beijing");
int numOfRows = 10;
// VALUES 部分以逐列的方式进行设置：
ArrayList<Long> ts = new ArrayList<>();
for (int i = 0; i < numOfRows; i++){
ts.add(System.currentTimeMillis() + i);
}
s.setTimestamp(0, ts);
ArrayList<Integer> s1 = new ArrayList<>();
for (int i = 0; i < numOfRows; i++){
s1.add(r.nextInt(100));
}
s.setInt(1, s1);
ArrayList<String> s2 = new ArrayList<>();
for (int i = 0; i < numOfRows; i++){
s2.add("test" + r.nextInt(100));
}
s.setString(2, s2, 10);
// AddBatch 之后，缓存并未清空。为避免混乱，并不推荐在 ExecuteBatch 之前再次绑定新一批的数据：
s.columnDataAddBatch();
// 执行绑定数据后的语句：
s.columnDataExecuteBatch();
// 执行语句后清空缓存。在清空之后，可以复用当前的对象，绑定新的一批数据（可以是新表名、新 TAGS 值、新 VALUES 值）：
s.columnDataClearBatch();
// 执行完毕，释放资源：
s.columnDataCloseBatch();
```
用于设定 TAGS 取值的方法总共有：
```java
public void setTagNull(int index, int type)
public void setTagBoolean(int index, boolean value)
public void setTagInt(int index, int value)
public void setTagByte(int index, byte value)
public void setTagShort(int index, short value)
public void setTagLong(int index, long value)
public void setTagTimestamp(int index, long value)
public void setTagFloat(int index, float value)
public void setTagDouble(int index, double value)
public void setTagString(int index, String value)
public void setTagNString(int index, String value)
```
用于设定 VALUES 数据列的取值的方法总共有：
```java
public void setInt(int columnIndex, ArrayList<Integer> list) throws SQLException
public void setFloat(int columnIndex, ArrayList<Float> list) throws SQLException
public void setTimestamp(int columnIndex, ArrayList<Long> list) throws SQLException
public void setLong(int columnIndex, ArrayList<Long> list) throws SQLException
public void setDouble(int columnIndex, ArrayList<Double> list) throws SQLException
public void setBoolean(int columnIndex, ArrayList<Boolean> list) throws SQLException
public void setByte(int columnIndex, ArrayList<Byte> list) throws SQLException
public void setShort(int columnIndex, ArrayList<Short> list) throws SQLException
public void setString(int columnIndex, ArrayList<String> list, int size) throws SQLException
public void setNString(int columnIndex, ArrayList<String> list, int size) throws SQLException
```
其中 setString 和 setNString 都要求用户在 size 参数里声明表定义中对应列的列宽。
### <a class="anchor" id="subscribe"></a>订阅
#### 创建
```java
TSDBSubscribe sub = ((TSDBConnection)conn).subscribe("topic", "select * from meters", false);
```
`subscribe` 方法的三个参数含义如下：
* topic：订阅的主题（即名称），此参数是订阅的唯一标识
* sql：订阅的查询语句，此语句只能是 `select` 语句，只应查询原始数据，只能按时间正序查询数据
* restart：如果订阅已经存在，是重新开始，还是继续之前的订阅
如上面的例子将使用 SQL 语句 `select * from meters` 创建一个名为 `topic` 的订阅，如果这个订阅已经存在，将继续之前的查询进度，而不是从头开始消费所有的数据。
#### 消费数据
```java
int total = 0;
while(true) {
TSDBResultSet rs = sub.consume();
int count = 0;
while(rs.next()) {
count++;
}
total += count;
System.out.printf("%d rows consumed, total %d\n", count, total);
Thread.sleep(1000);
}
```
`consume` 方法返回一个结果集，其中包含从上次 `consume` 到目前为止的所有新数据。请务必按需选择合理的调用 `consume` 的频率（如例子中的 `Thread.sleep(1000)`），否则会给服务端造成不必要的压力。
#### 关闭订阅
```java
sub.close(true);
```
`close` 方法关闭一个订阅。如果其参数为 `true` 表示保留订阅进度信息，后续可以创建同名订阅继续消费数据；如为 `false` 则不保留订阅进度。
### 关闭资源
@ -187,12 +415,17 @@ resultSet.close();
stmt.close();
conn.close();
```
> `注意务必要将 connection 进行关闭`，否则会出现连接泄露。
## 与连接池使用
**HikariCP**
* 引入相应 HikariCP maven 依赖：
```xml
<dependency>
<groupId>com.zaxxer</groupId>
@ -202,31 +435,34 @@ conn.close();
```
* 使用示例如下：
```java
public static void main(String[] args) throws SQLException {
HikariConfig config = new HikariConfig();
// jdbc properties
config.setJdbcUrl("jdbc:TAOS://127.0.0.1:6030/log");
config.setUsername("root");
config.setPassword("taosdata");
config.setMinimumIdle(3); //minimum number of idle connection
// connection pool configurations
config.setMinimumIdle(10); //minimum number of idle connection
config.setMaximumPoolSize(10); //maximum number of connection in the pool
config.setConnectionTimeout(10000); //maximum wait milliseconds for get connection from pool
config.setIdleTimeout(60000); // max idle time for recycle idle connection
config.setConnectionTestQuery("describe log.dn"); //validation query
config.setValidationTimeout(3000); //validation query timeout
config.setConnectionTimeout(30000); //maximum wait milliseconds for get connection from pool
config.setMaxLifetime(0); // maximum life time for each connection
config.setIdleTimeout(0); // max idle time for recycle idle connection
config.setConnectionTestQuery("select server_status()"); //validation query
HikariDataSource ds = new HikariDataSource(config); //create datasource
Connection connection = ds.getConnection(); // get connection
Statement statement = connection.createStatement(); // get statement
//query or insert
//query or insert
// ...
connection.close(); // put back to conneciton pool
}
```
> 通过 HikariDataSource.getConnection() 获取连接后，使用完成后需要调用 close() 方法，实际上它并不会关闭连接，只是放回连接池中。
> 更多 HikariCP 使用问题请查看[官方说明][5]
@ -243,40 +479,32 @@ conn.close();
```
* 使用示例如下：
```java
public static void main(String[] args) throws Exception {
Properties properties = new Properties();
properties.put("driverClassName","com.taosdata.jdbc.TSDBDriver");
properties.put("url","jdbc:TAOS://127.0.0.1:6030/log");
properties.put("username","root");
properties.put("password","taosdata");
properties.put("maxActive","10"); //maximum number of connection in the pool
properties.put("initialSize","3");//initial number of connection
properties.put("maxWait","10000");//maximum wait milliseconds for get connection from pool
properties.put("minIdle","3");//minimum number of connection in the pool
properties.put("timeBetweenEvictionRunsMillis","3000");// the interval milliseconds to test connection
properties.put("minEvictableIdleTimeMillis","60000");//the minimum milliseconds to keep idle
properties.put("maxEvictableIdleTimeMillis","90000");//the maximum milliseconds to keep idle
properties.put("validationQuery","describe log.dn"); //validation query
properties.put("testWhileIdle","true"); // test connection while idle
properties.put("testOnBorrow","false"); // don't need while testWhileIdle is true
properties.put("testOnReturn","false"); // don't need while testWhileIdle is true
//create druid datasource
DataSource ds = DruidDataSourceFactory.createDataSource(properties);
Connection connection = ds.getConnection(); // get connection
DruidDataSource dataSource = new DruidDataSource();
// jdbc properties
dataSource.setDriverClassName("com.taosdata.jdbc.TSDBDriver");
dataSource.setUrl(url);
dataSource.setUsername("root");
dataSource.setPassword("taosdata");
// pool configurations
dataSource.setInitialSize(10);
dataSource.setMinIdle(10);
dataSource.setMaxActive(10);
dataSource.setMaxWait(30000);
dataSource.setValidationQuery("select server_status()");
Connection connection = dataSource.getConnection(); // get connection
Statement statement = connection.createStatement(); // get statement
//query or insert
// ...
connection.close(); // put back to conneciton pool
}
```
> 更多 druid 使用问题请查看[官方说明][6]
**注意事项**
@ -291,29 +519,64 @@ server_status()|
Query OK, 1 row(s) in set (0.000141s)
```
## 与框架使用
* Spring JdbcTemplate 中使用 taos-jdbcdriver，可参考 [SpringJdbcTemplate][11]
* Springboot + Mybatis 中使用，可参考 [springbootdemo][12]
## <a class="anchor" id="version"></a>TAOS-JDBCDriver 版本以及支持的 TDengine 版本和 JDK 版本
| taos-jdbcdriver 版本 | TDengine 版本 | JDK 版本 |
| -------------------- | ----------------- | -------- |
| 2.0.31 | 2.1.3.0 及以上 | 1.8.x |
| 2.0.22 - 2.0.30 | 2.0.18.0 - 2.1.2.x | 1.8.x |
| 2.0.12 - 2.0.21 | 2.0.8.0 - 2.0.17.x | 1.8.x |
| 2.0.4 - 2.0.11 | 2.0.0.0 - 2.0.7.x | 1.8.x |
| 1.0.3 | 1.6.1.x 及以上 | 1.8.x |
| 1.0.2 | 1.6.1.x 及以上 | 1.8.x |
| 1.0.1 | 1.6.1.x 及以上 | 1.8.x |
## TDengine DataType 和 Java DataType
TDengine 目前支持时间戳、数字、字符、布尔类型，与 Java 对应类型转换如下：
| TDengine DataType | Java DataType |
| ----------------- | ------------------ |
| TIMESTAMP | java.sql.Timestamp |
| INT | java.lang.Integer |
| BIGINT | java.lang.Long |
| FLOAT | java.lang.Float |
| DOUBLE | java.lang.Double |
| SMALLINT | java.lang.Short |
| TINYINT | java.lang.Byte |
| BOOL | java.lang.Boolean |
| BINARY | byte array |
| NCHAR | java.lang.String |
## 常见问题
* java.lang.UnsatisfiedLinkError: no taos in java.library.path
**原因**：程序没有找到依赖的本地函数库 taos。
**解决方法**：windows 下可以将 C:\TDengine\driver\taos.dll 拷贝到 C:\Windows\System32\ 目录下，linux 下将建立如下软链 ` ln -s /usr/local/taos/driver/libtaos.so.x.x.x.x /usr/lib/libtaos.so` 即可。
**解决方法**：windows 下可以将 C:\TDengine\driver\taos.dll 拷贝到 C:\Windows\System32\ 目录下，linux 下将建立如下软链 `ln -s /usr/local/taos/driver/libtaos.so.x.x.x.x /usr/lib/libtaos.so` 即可。
* java.lang.UnsatisfiedLinkError: taos.dll Can't load AMD 64 bit on a IA 32-bit platform
**原因**：目前 TDengine 只支持 64 位 JDK。
**解决方法**：重新安装 64 位 JDK。
* 其它问题请参考 [Issues][7]
[1]: https://search.maven.org/artifact/com.taosdata.jdbc/taos-jdbcdriver
[2]: https://mvnrepository.com/artifact/com.taosdata.jdbc/taos-jdbcdriver
[3]: https://github.com/taosdata/TDengine
@ -324,6 +587,9 @@ Query OK, 1 row(s) in set (0.000141s)
[8]: https://search.maven.org/artifact/com.taosdata.jdbc/taos-jdbcdriver
[9]: https://mvnrepository.com/artifact/com.taosdata.jdbc/taos-jdbcdriver
[10]: https://maven.aliyun.com/mvn/search
[11]: https://github.com/taosdata/TDengine/tree/develop/tests/examples/JDBC/SpringJdbcTemplate
[11]: https://github.com/taosdata/TDengine/tree/develop/tests/examples/JDBC/SpringJdbcTemplate
[12]: https://github.com/taosdata/TDengine/tree/develop/tests/examples/JDBC/springbootdemo
[13]: https://www.taosdata.com/cn/documentation/administrator/#%E5%AE%A2%E6%88%B7%E7%AB%AF%E9%85%8D%E7%BD%AE
[13]: https://www.taosdata.com/cn/documentation/administrator/#client
[14]: https://www.taosdata.com/cn/all-downloads/#TDengine-Windows-Client
[15]: https://www.taosdata.com/cn/getting-started/#%E5%AE%A2%E6%88%B7%E7%AB%AF

42
src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractStatement.java
View File

@ -2,6 +2,7 @@ package com.taosdata.jdbc;
import java.sql.*;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
public abstract class AbstractStatement extends WrapperImpl implements Statement {
@ -196,13 +197,44 @@ public abstract class AbstractStatement extends WrapperImpl implements Statement
if (batchedArgs == null || batchedArgs.isEmpty())
throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_BATCH_IS_EMPTY);
String clientInfo = getConnection().getClientInfo(TSDBDriver.PROPERTY_KEY_BATCH_ERROR_IGNORE);
boolean batchErrorIgnore = clientInfo == null ? TSDBConstants.DEFAULT_BATCH_ERROR_IGNORE : Boolean.parseBoolean(clientInfo);
if (batchErrorIgnore) {
return executeBatchIgnoreException();
}
return executeBatchThrowException();
}
private int[] executeBatchIgnoreException() {
return batchedArgs.stream().mapToInt(sql -> {
try {
boolean isSelect = execute(sql);
if (isSelect) {
return SUCCESS_NO_INFO;
} else {
return getUpdateCount();
}
} catch (SQLException e) {
return EXECUTE_FAILED;
}
}).toArray();
}
private int[] executeBatchThrowException() throws BatchUpdateException {
int[] res = new int[batchedArgs.size()];
for (int i = 0; i < batchedArgs.size(); i++) {
boolean isSelect = execute(batchedArgs.get(i));
if (isSelect) {
res[i] = SUCCESS_NO_INFO;
} else {
res[i] = getUpdateCount();
try {
boolean isSelect = execute(batchedArgs.get(i));
if (isSelect) {
res[i] = SUCCESS_NO_INFO;
} else {
res[i] = getUpdateCount();
}
} catch (SQLException e) {
String reason = e.getMessage();
int[] updateCounts = Arrays.copyOfRange(res, 0, i);
throw new BatchUpdateException(reason, updateCounts, e);
}
}
return res;

2
src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConstants.java
View File

@ -74,6 +74,8 @@ public abstract class TSDBConstants {
public static final String DEFAULT_PRECISION = "ms";
public static final boolean DEFAULT_BATCH_ERROR_IGNORE = false;
public static int typeName2JdbcType(String type) {
switch (type.toUpperCase()) {
case "TIMESTAMP":

5
src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java
View File

@ -100,6 +100,11 @@ public class TSDBDriver extends AbstractDriver {
*/
public static final String PROPERTY_KEY_TIMESTAMP_FORMAT = "timestampFormat";
/**
* continue process commands in executeBatch
*/
public static final String PROPERTY_KEY_BATCH_ERROR_IGNORE = "batchErrorIgnore";
private TSDBDatabaseMetaData dbMetaData = null;
static {

20
src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSetRowData.java
View File

@ -19,6 +19,7 @@ import com.taosdata.jdbc.utils.NullType;
import java.math.BigDecimal;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collections;
@ -463,6 +464,25 @@ public class TSDBResultSetRowData {
data.set(col, tsObj);
}
/**
* this implementation is used for TDengine old version
*/
public void setTimestamp(int col, long ts) {
//TODO: this implementation contains logical error
// when precision is us the (long ts) is 16 digital number
// when precision is ms, the (long ts) is 13 digital number
// we need a JNI function like this:
// public void setTimestamp(int col, long epochSecond, long nanoAdjustment)
if (ts < 1_0000_0000_0000_0L) {
data.set(col, new Timestamp(ts));
} else {
long epochSec = ts / 1000_000L;
long nanoAdjustment = ts % 1000_000L * 1000L;
Timestamp timestamp = Timestamp.from(Instant.ofEpochSecond(epochSec, nanoAdjustment));
data.set(col, timestamp);
}
}
public Timestamp getTimestamp(int col, int nativeType) {
Object obj = data.get(col - 1);
if (obj == null)

17
src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/Utils.java
View File

@ -5,7 +5,6 @@ import com.google.common.collect.RangeSet;
import com.google.common.collect.TreeRangeSet;
import com.taosdata.jdbc.enums.TimestampPrecision;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.sql.Date;
import java.sql.Time;
@ -110,15 +109,27 @@ public class Utils {
return rawSql;
// toLowerCase
String preparedSql = rawSql.trim().toLowerCase();
String[] clause = new String[]{"values\\s*\\(.*?\\)", "tags\\s*\\(.*?\\)", "where\\s*.*"};
String[] clause = new String[]{"tags\\s*\\([\\s\\S]*?\\)", "where[\\s\\S]*"};
Map<Integer, Integer> placeholderPositions = new HashMap<>();
RangeSet<Integer> clauseRangeSet = TreeRangeSet.create();
findPlaceholderPosition(preparedSql, placeholderPositions);
// find tags and where clause's position
findClauseRangeSet(preparedSql, clause, clauseRangeSet);
// find values clause's position
findValuesClauseRangeSet(preparedSql, clauseRangeSet);
return transformSql(rawSql, parameters, placeholderPositions, clauseRangeSet);
}
private static void findValuesClauseRangeSet(String preparedSql, RangeSet<Integer> clauseRangeSet) {
Matcher matcher = Pattern.compile("(values|,)\\s*(\\([^)]*\\))").matcher(preparedSql);
while (matcher.find()) {
int start = matcher.start(2);
int end = matcher.end(2);
clauseRangeSet.add(Range.closedOpen(start, end));
}
}
private static void findClauseRangeSet(String preparedSql, String[] regexArr, RangeSet<Integer> clauseRangeSet) {
clauseRangeSet.clear();
for (String regex : regexArr) {
@ -126,7 +137,7 @@ public class Utils {
while (matcher.find()) {
int start = matcher.start();
int end = matcher.end();
clauseRangeSet.add(Range.closed(start, end));
clauseRangeSet.add(Range.closedOpen(start, end));
}
}
}

4
src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java
View File

@ -841,13 +841,13 @@ public class TSDBPreparedStatementTest {
}
@Test
public void setBytes() throws SQLException, IOException {
public void setBytes() throws SQLException {
// given
long ts = System.currentTimeMillis();
byte[] f8 = "{\"name\": \"john\", \"age\": 10, \"address\": \"192.168.1.100\"}".getBytes();
// when
pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis()));
pstmt_insert.setTimestamp(1, new Timestamp(ts));
pstmt_insert.setBytes(9, f8);
int result = pstmt_insert.executeUpdate();

144
src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BatchErrorIgnoreTest.java Normal file
View File

@ -0,0 +1,144 @@
package com.taosdata.jdbc.cases;
import org.junit.*;
import java.sql.*;
import java.util.stream.IntStream;
public class BatchErrorIgnoreTest {
private static final String host = "127.0.0.1";
@Test
public void batchErrorThrowException() throws SQLException {
// given
Connection conn = DriverManager.getConnection("jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata");
// when
try (Statement stmt = conn.createStatement()) {
IntStream.range(1, 6).mapToObj(i -> "insert into test.t" + i + " values(now, " + i + ")").forEach(sql -> {
try {
stmt.addBatch(sql);
} catch (SQLException e) {
e.printStackTrace();
}
});
stmt.addBatch("insert into t11 values(now, 11)");
IntStream.range(6, 11).mapToObj(i -> "insert into test.t" + i + " values(now, " + i + "),(now + 1s, " + (10 * i) + ")").forEach(sql -> {
try {
stmt.addBatch(sql);
} catch (SQLException e) {
e.printStackTrace();
}
});
stmt.addBatch("select count(*) from test.weather");
stmt.executeBatch();
} catch (BatchUpdateException e) {
int[] updateCounts = e.getUpdateCounts();
Assert.assertEquals(5, updateCounts.length);
Assert.assertEquals(1, updateCounts[0]);
Assert.assertEquals(1, updateCounts[1]);
Assert.assertEquals(1, updateCounts[2]);
Assert.assertEquals(1, updateCounts[3]);
Assert.assertEquals(1, updateCounts[4]);
}
}
@Test
public void batchErrorIgnore() throws SQLException {
// given
Connection conn = DriverManager.getConnection("jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata&batchErrorIgnore=true");
// when
int[] results = null;
try (Statement stmt = conn.createStatement()) {
IntStream.range(1, 6).mapToObj(i -> "insert into test.t" + i + " values(now, " + i + ")").forEach(sql -> {
try {
stmt.addBatch(sql);
} catch (SQLException e) {
e.printStackTrace();
}
});
stmt.addBatch("insert into t11 values(now, 11)");
IntStream.range(6, 11).mapToObj(i -> "insert into test.t" + i + " values(now, " + i + "),(now + 1s, " + (10 * i) + ")").forEach(sql -> {
try {
stmt.addBatch(sql);
} catch (SQLException e) {
e.printStackTrace();
}
});
stmt.addBatch("select count(*) from test.weather");
results = stmt.executeBatch();
} catch (SQLException e) {
e.printStackTrace();
}
// then
assert results != null;
Assert.assertEquals(12, results.length);
Assert.assertEquals(1, results[0]);
Assert.assertEquals(1, results[1]);
Assert.assertEquals(1, results[2]);
Assert.assertEquals(1, results[3]);
Assert.assertEquals(1, results[4]);
Assert.assertEquals(Statement.EXECUTE_FAILED, results[5]);
Assert.assertEquals(2, results[6]);
Assert.assertEquals(2, results[7]);
Assert.assertEquals(2, results[8]);
Assert.assertEquals(2, results[9]);
Assert.assertEquals(2, results[10]);
Assert.assertEquals(Statement.SUCCESS_NO_INFO, results[11]);
}
@Before
public void before() {
try {
Connection conn = DriverManager.getConnection("jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata");
Statement stmt = conn.createStatement();
stmt.execute("use test");
stmt.execute("drop table if exists weather");
stmt.execute("create table weather (ts timestamp, f1 float) tags(t1 int)");
IntStream.range(1, 11).mapToObj(i -> "create table t" + i + " using weather tags(" + i + ")").forEach(sql -> {
try {
stmt.execute(sql);
} catch (SQLException e) {
e.printStackTrace();
}
});
stmt.close();
conn.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
@BeforeClass
public static void beforeClass() {
try {
Connection conn = DriverManager.getConnection("jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata");
Statement stmt = conn.createStatement();
stmt.execute("drop database if exists test");
stmt.execute("create database if not exists test");
stmt.close();
conn.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
@AfterClass
public static void afterClass() {
try {
Connection conn = DriverManager.getConnection("jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata");
Statement stmt = conn.createStatement();
stmt.execute("drop database if exists test");
stmt.close();
conn.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
}

52
src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/UtilsTest.java
View File

@ -3,6 +3,8 @@ package com.taosdata.jdbc.utils;
import org.junit.Assert;
import org.junit.Test;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Stream;
public class UtilsTest {
@ -32,7 +34,7 @@ public class UtilsTest {
}
@Test
public void getNativeSqlReplaceQuestionMarks() {
public void lowerCase() {
// given
String nativeSql = "insert into ?.? (ts, temperature, humidity) using ?.? tags(?,?) values(now, ?, ?)";
Object[] parameters = Stream.of("test", "t1", "test", "weather", "beijing", 1, 12.2, 4).toArray();
@ -46,7 +48,7 @@ public class UtilsTest {
}
@Test
public void getNativeSqlReplaceQuestionMarks2() {
public void upperCase() {
// given
String nativeSql = "INSERT INTO ? (TS,CURRENT,VOLTAGE,PHASE) USING METERS TAGS (?) VALUES (?,?,?,?)";
Object[] parameters = Stream.of("d1", 1, 123, 3.14, 220, 4).toArray();
@ -59,9 +61,49 @@ public class UtilsTest {
Assert.assertEquals(expected, actual);
}
@Test
public void multiValues() {
// given
String nativeSql = "INSERT INTO ? (TS,CURRENT,VOLTAGE,PHASE) USING METERS TAGS (?) VALUES (?,?,?,?),(?,?,?,?)";
Object[] parameters = Stream.of("d1", 1, 100, 3.14, "abc", 4, 200, 3.1415, "xyz", 5).toArray();
// when
String actual = Utils.getNativeSql(nativeSql, parameters);
// then
String expected = "INSERT INTO d1 (TS,CURRENT,VOLTAGE,PHASE) USING METERS TAGS (1) VALUES (100,3.14,'abc',4),(200,3.1415,'xyz',5)";
Assert.assertEquals(expected, actual);
}
@Test
public void getNativeSqlReplaceNothing() {
public void lineTerminator() {
// given
String nativeSql = "INSERT INTO ? (TS,CURRENT,VOLTAGE,PHASE) USING METERS TAGS (?) VALUES (?,?,\r\n?,?),(?,?,?,?)";
Object[] parameters = Stream.of("d1", 1, 100, 3.14, "abc", 4, 200, 3.1415, "xyz", 5).toArray();
// when
String actual = Utils.getNativeSql(nativeSql, parameters);
// then
String expected = "INSERT INTO d1 (TS,CURRENT,VOLTAGE,PHASE) USING METERS TAGS (1) VALUES (100,3.14,\r\n'abc',4),(200,3.1415,'xyz',5)";
Assert.assertEquals(expected, actual);
}
@Test
public void lineTerminatorAndMultiValues() {
String nativeSql = "INSERT Into ? TAGS(?) VALUES(?,?,\r\n?,?),(?,? ,\r\n?,?) t? tags (?) Values (?,?,?\r\n,?),(?,?,?,?) t? Tags(?) values (?,?,?,?) , (?,?,?,?)";
Object[] parameters = Stream.of("t1", "abc", 100, 1.1, "xxx", "xxx", 200, 2.2, "xxx", "xxx", 2, "bcd", 300, 3.3, "xxx", "xxx", 400, 4.4, "xxx", "xxx", 3, "cde", 500, 5.5, "xxx", "xxx", 600, 6.6, "xxx", "xxx").toArray();
// when
String actual = Utils.getNativeSql(nativeSql, parameters);
// then
String expected = "INSERT Into t1 TAGS('abc') VALUES(100,1.1,\r\n'xxx','xxx'),(200,2.2 ,\r\n'xxx','xxx') t2 tags ('bcd') Values (300,3.3,'xxx'\r\n,'xxx'),(400,4.4,'xxx','xxx') t3 Tags('cde') values (500,5.5,'xxx','xxx') , (600,6.6,'xxx','xxx')";
Assert.assertEquals(expected, actual);
}
@Test
public void replaceNothing() {
// given
String nativeSql = "insert into test.t1 (ts, temperature, humidity) using test.weather tags('beijing',1) values(now, 12.2, 4)";
@ -73,7 +115,7 @@ public class UtilsTest {
}
@Test
public void getNativeSqlReplaceNothing2() {
public void replaceNothing2() {
// given
String nativeSql = "insert into test.t1 (ts, temperature, humidity) using test.weather tags('beijing',1) values(now, 12.2, 4)";
Object[] parameters = Stream.of("test", "t1", "test", "weather", "beijing", 1, 12.2, 4).toArray();
@ -86,7 +128,7 @@ public class UtilsTest {
}
@Test
public void getNativeSqlReplaceNothing3() {
public void replaceNothing3() {
// given
String nativeSql = "insert into ?.? (ts, temperature, humidity) using ?.? tags(?,?) values(now, ?, ?)";

2
src/dnode/CMakeLists.txt
View File

@ -18,7 +18,7 @@ ELSE ()
ENDIF ()
ADD_EXECUTABLE(taosd ${SRC})
TARGET_LINK_LIBRARIES(taosd mnode monitor http tsdb twal vnode cJson lz4 balance sync ${LINK_JEMALLOC})
TARGET_LINK_LIBRARIES(taosd mnode monitor http tsdb twal vnode cJson lua lz4 balance sync ${LINK_JEMALLOC})
IF (TD_SOMODE_STATIC)
TARGET_LINK_LIBRARIES(taosd taos_static)

4
src/dnode/src/dnodeMain.c
View File

@ -40,6 +40,7 @@
#include "dnodeShell.h"
#include "dnodeTelemetry.h"
#include "module.h"
#include "qScript.h"
#include "mnode.h"
#include "tscompression.h"
@ -85,6 +86,7 @@ static SStep tsDnodeSteps[] = {
{"dnode-shell", dnodeInitShell, dnodeCleanupShell},
{"dnode-statustmr", dnodeInitStatusTimer,dnodeCleanupStatusTimer},
{"dnode-telemetry", dnodeInitTelemetry, dnodeCleanupTelemetry},
{"dnode-script", scriptEnvPoolInit, scriptEnvPoolCleanup},
};
static SStep tsDnodeCompactSteps[] = {
@ -271,7 +273,7 @@ static int32_t dnodeInitStorage() {
return -1;
}
}
//TODO(dengyihao): no need to init here
//TODO(dengyihao): no need to init here
if (dnodeCreateDir(tsMnodeDir) < 0) {
dError("failed to create dir: %s, reason: %s", tsMnodeDir, strerror(errno));
return -1;

3
src/dnode/src/dnodeShell.c
View File

@ -48,9 +48,11 @@ int32_t dnodeInitShell() {
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_DNODE] = dnodeDispatchToMWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DB] = dnodeDispatchToMWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_TP] = dnodeDispatchToMWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_FUNCTION] = dnodeDispatchToMWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_DB] = dnodeDispatchToMWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_SYNC_DB] = dnodeDispatchToMWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_TP] = dnodeDispatchToMWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_FUNCTION] = dnodeDispatchToMWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_DB] = dnodeDispatchToMWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_TP] = dnodeDispatchToMWriteQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_TABLE]= dnodeDispatchToMWriteQueue;
@ -72,6 +74,7 @@ int32_t dnodeInitShell() {
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_TABLES_META] = dnodeDispatchToMReadQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_SHOW] = dnodeDispatchToMReadQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_RETRIEVE] = dnodeDispatchToMReadQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_RETRIEVE_FUNC] = dnodeDispatchToMReadQueue;
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_NETWORK_TEST] = dnodeSendStartupStep;

8
src/dnode/src/dnodeVWrite.c
View File

@ -202,12 +202,12 @@ static void *dnodeProcessVWriteQueue(void *wparam) {
for (int32_t i = 0; i < numOfMsgs; ++i) {
taosGetQitem(pWorker->qall, &qtype, (void **)&pWrite);
dTrace("msg:%p, app:%p type:%s will be processed in vwrite queue, qtype:%s hver:%" PRIu64, pWrite,
pWrite->rpcMsg.ahandle, taosMsg[pWrite->pHead.msgType], qtypeStr[qtype], pWrite->pHead.version);
pWrite->rpcMsg.ahandle, taosMsg[pWrite->walHead.msgType], qtypeStr[qtype], pWrite->walHead.version);
pWrite->code = vnodeProcessWrite(pVnode, &pWrite->pHead, qtype, pWrite);
pWrite->code = vnodeProcessWrite(pVnode, &pWrite->walHead, qtype, pWrite);
if (pWrite->code <= 0) atomic_add_fetch_32(&pWrite->processedCount, 1);
if (pWrite->code > 0) pWrite->code = 0;
if (pWrite->code == 0 && pWrite->pHead.msgType != TSDB_MSG_TYPE_SUBMIT) forceFsync = true;
if (pWrite->code == 0 && pWrite->walHead.msgType != TSDB_MSG_TYPE_SUBMIT) forceFsync = true;
dTrace("msg:%p is processed in vwrite queue, code:0x%x", pWrite, pWrite->code);
}
@ -222,7 +222,7 @@ static void *dnodeProcessVWriteQueue(void *wparam) {
dnodeSendRpcVWriteRsp(pVnode, pWrite, pWrite->code);
} else {
if (qtype == TAOS_QTYPE_FWD) {
vnodeConfirmForward(pVnode, pWrite->pHead.version, pWrite->code, pWrite->pHead.msgType != TSDB_MSG_TYPE_SUBMIT);
vnodeConfirmForward(pVnode, pWrite->walHead.version, pWrite->code, pWrite->walHead.msgType != TSDB_MSG_TYPE_SUBMIT);
}
if (pWrite->rspRet.rsp) {
rpcFreeCont(pWrite->rspRet.rsp);

2
src/inc/query.h
View File

@ -28,7 +28,7 @@ typedef void* qinfo_t;
* @param qinfo
* @return
*/
int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryTableMsg, qinfo_t* qinfo, uint64_t *qId);
int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryTableMsg, qinfo_t* qinfo, uint64_t qId);
/**

8
src/inc/taosdef.h
View File

@ -186,6 +186,10 @@ do { \
#define TSDB_NODE_NAME_LEN 64
#define TSDB_TABLE_NAME_LEN 193 // it is a null-terminated string
#define TSDB_DB_NAME_LEN 33
#define TSDB_FUNC_NAME_LEN 65
#define TSDB_FUNC_CODE_LEN (65535 - 512)
#define TSDB_FUNC_BUF_SIZE 512
#define TSDB_TYPE_STR_MAX_LEN 32
#define TSDB_TABLE_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN + TSDB_TABLE_NAME_LEN)
#define TSDB_COL_NAME_LEN 65
#define TSDB_MAX_SAVED_SQL_LEN TSDB_MAX_COLUMNS * 64
@ -332,6 +336,10 @@ do { \
#define TSDB_QUERY_TYPE_NON_TYPE 0x00u // none type
#define TSDB_QUERY_TYPE_FREE_RESOURCE 0x01u // free qhandle at vnode
#define TSDB_UDF_TYPE_SCALAR 1
#define TSDB_UDF_TYPE_AGGREGATE 2
/*
* 1. ordinary sub query for select * from super_table
* 2. all sqlobj generated by createSubqueryObj with this flag

11
src/inc/taoserror.h
View File

@ -100,6 +100,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_TSC_DB_NOT_SELECTED TAOS_DEF_ERROR_CODE(0, 0x0217) //"Database not specified or available")
#define TSDB_CODE_TSC_INVALID_TABLE_NAME TAOS_DEF_ERROR_CODE(0, 0x0218) //"Table does not exist")
#define TSDB_CODE_TSC_EXCEED_SQL_LIMIT TAOS_DEF_ERROR_CODE(0, 0x0219) //"SQL statement too long check maxSQLLength config")
#define TSDB_CODE_TSC_FILE_EMPTY TAOS_DEF_ERROR_CODE(0, 0x021A) //"File is empty")
// mnode
#define TSDB_CODE_MND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0300) //"Message not processed")
@ -174,6 +175,13 @@ int32_t* taosGetErrno();
#define TSDB_CODE_MND_INVALID_STABLE_NAME TAOS_DEF_ERROR_CODE(0, 0x036D) //"Super table does not exist")
#define TSDB_CODE_MND_INVALID_CREATE_TABLE_MSG TAOS_DEF_ERROR_CODE(0, 0x036E) //"Invalid create table message")
#define TSDB_CODE_MND_INVALID_FUNC_NAME TAOS_DEF_ERROR_CODE(0, 0x0370) //"Invalid func name")
#define TSDB_CODE_MND_INVALID_FUNC_LEN TAOS_DEF_ERROR_CODE(0, 0x0371) //"Invalid func length")
#define TSDB_CODE_MND_INVALID_FUNC_CODE TAOS_DEF_ERROR_CODE(0, 0x0372) //"Invalid func code")
#define TSDB_CODE_MND_FUNC_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0373) //"Func already exists")
#define TSDB_CODE_MND_INVALID_FUNC TAOS_DEF_ERROR_CODE(0, 0x0374) //"Invalid func")
#define TSDB_CODE_MND_INVALID_FUNC_BUFSIZE TAOS_DEF_ERROR_CODE(0, 0x0375) //"Invalid func bufSize")
#define TSDB_CODE_MND_DB_NOT_SELECTED TAOS_DEF_ERROR_CODE(0, 0x0380) //"Database not specified or available")
#define TSDB_CODE_MND_DB_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0381) //"Database already exists")
#define TSDB_CODE_MND_INVALID_DB_OPTION TAOS_DEF_ERROR_CODE(0, 0x0382) //"Invalid database options")
@ -261,6 +269,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW TAOS_DEF_ERROR_CODE(0, 0x070A) //"Too many time window in query")
#define TSDB_CODE_QRY_NOT_ENOUGH_BUFFER TAOS_DEF_ERROR_CODE(0, 0x070B) //"Query buffer limit has reached")
#define TSDB_CODE_QRY_INCONSISTAN TAOS_DEF_ERROR_CODE(0, 0x070C) //"File inconsistency in replica")
#define TSDB_CODE_QRY_SYS_ERROR TAOS_DEF_ERROR_CODE(0, 0x070D) //"System error")
// grant
@ -395,6 +404,8 @@ int32_t* taosGetErrno();
#define TSDB_CODE_HTTP_OP_VALUE_NULL TAOS_DEF_ERROR_CODE(0, 0x11A5) //"value not find")
#define TSDB_CODE_HTTP_OP_VALUE_TYPE TAOS_DEF_ERROR_CODE(0, 0x11A6) //"value type should be boolean number or string")
#define TSDB_CODE_HTTP_REQUEST_JSON_ERROR TAOS_DEF_ERROR_CODE(0, 0x1F00) //"http request json error")
// odbc
#define TSDB_CODE_ODBC_OOM TAOS_DEF_ERROR_CODE(0, 0x2100) //"out of memory")
#define TSDB_CODE_ODBC_CONV_CHAR_NOT_NUM TAOS_DEF_ERROR_CODE(0, 0x2101) //"convertion not a valid literal input")

53
src/inc/taosmsg.h
View File

@ -77,8 +77,10 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_DROP_USER, "drop-user" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_CREATE_DNODE, "create-dnode" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_DROP_DNODE, "drop-dnode" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_CREATE_DB, "create-db" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_DROP_DB, "drop-db" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_USE_DB, "use-db" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_CREATE_FUNCTION, "create-function" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_DROP_DB, "drop-db" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_DROP_FUNCTION, "drop-function" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_USE_DB, "use-db" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_ALTER_DB, "alter-db" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_SYNC_DB, "sync-db-replica" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_CREATE_TABLE, "create-table" )
@ -96,7 +98,7 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_KILL_STREAM, "kill-stream" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_KILL_CONN, "kill-conn" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_CONFIG_DNODE, "cm-config-dnode" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_HEARTBEAT, "heartbeat" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY8, "dummy8" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_CM_RETRIEVE_FUNC, "retrieve-func" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY9, "dummy9" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY10, "dummy10" )
TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY11, "dummy11" )
@ -153,6 +155,7 @@ enum _mgmt_table {
TSDB_MGMT_TABLE_STREAMTABLES,
TSDB_MGMT_TABLE_CLUSTER,
TSDB_MGMT_TABLE_TP,
TSDB_MGMT_TABLE_FUNCTION,
TSDB_MGMT_TABLE_MAX,
};
@ -507,6 +510,9 @@ typedef struct {
int32_t prevResultLen; // previous result length
int32_t numOfOperator;
int32_t tableScanOperator;// table scan operator. -1 means no scan operator
int32_t udfNum; // number of udf function
int32_t udfContentOffset;
int32_t udfContentLen;
SColumnInfo tableCols[];
} SQueryTableMsg;
@ -571,6 +577,41 @@ typedef struct {
int8_t reserve[5];
} SCreateDbMsg, SAlterDbMsg;
typedef struct {
char name[TSDB_FUNC_NAME_LEN];
char path[PATH_MAX];
int32_t funcType;
uint8_t outputType;
int16_t outputLen;
int32_t bufSize;
int32_t codeLen;
char code[];
} SCreateFuncMsg;
typedef struct {
int32_t num;
char name[];
} SRetrieveFuncMsg;
typedef struct {
char name[TSDB_FUNC_NAME_LEN];
int32_t funcType;
int8_t resType;
int16_t resBytes;
int32_t bufSize;
int32_t len;
char content[];
} SFunctionInfoMsg;
typedef struct {
int32_t num;
char content[];
} SUdfFuncMsg;
typedef struct {
char name[TSDB_FUNC_NAME_LEN];
} SDropFuncMsg;
typedef struct {
char db[TSDB_TABLE_FNAME_LEN];
uint8_t ignoreNotExists;
@ -710,8 +751,10 @@ typedef struct {
} STableInfoMsg;
typedef struct {
uint8_t metaClone; // create local clone of the cached table meta
int32_t numOfVgroups;
int32_t numOfTables;
int32_t numOfUdfs;
char tableNames[];
} SMultiTableInfoMsg;
@ -762,7 +805,11 @@ typedef struct STableMetaMsg {
typedef struct SMultiTableMeta {
int32_t numOfTables;
int32_t numOfVgroup;
int32_t numOfUdf;
int32_t contLen;
uint8_t compressed; // denote if compressed or not
uint32_t rawLen; // size before compress
uint8_t metaClone; // make meta clone after retrieve meta from mnode
char meta[];
} SMultiTableMeta;

2
src/inc/tcq.h
View File

@ -27,7 +27,7 @@ typedef struct {
int32_t vgId;
char user[TSDB_USER_LEN];
char pass[TSDB_KEY_LEN];
char db[TSDB_DB_NAME_LEN];
char db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN];
FCqWrite cqWrite;
} SCqCfg;

291
src/inc/ttokendef.h
View File

@ -62,149 +62,154 @@
#define TK_SHOW 44
#define TK_DATABASES 45
#define TK_TOPICS 46
#define TK_MNODES 47
#define TK_DNODES 48
#define TK_ACCOUNTS 49
#define TK_USERS 50
#define TK_MODULES 51
#define TK_QUERIES 52
#define TK_CONNECTIONS 53
#define TK_STREAMS 54
#define TK_VARIABLES 55
#define TK_SCORES 56
#define TK_GRANTS 57
#define TK_VNODES 58
#define TK_IPTOKEN 59
#define TK_DOT 60
#define TK_CREATE 61
#define TK_TABLE 62
#define TK_STABLE 63
#define TK_DATABASE 64
#define TK_TABLES 65
#define TK_STABLES 66
#define TK_VGROUPS 67
#define TK_DROP 68
#define TK_TOPIC 69
#define TK_DNODE 70
#define TK_USER 71
#define TK_ACCOUNT 72
#define TK_USE 73
#define TK_DESCRIBE 74
#define TK_ALTER 75
#define TK_PASS 76
#define TK_PRIVILEGE 77
#define TK_LOCAL 78
#define TK_COMPACT 79
#define TK_LP 80
#define TK_RP 81
#define TK_IF 82
#define TK_EXISTS 83
#define TK_PPS 84
#define TK_TSERIES 85
#define TK_DBS 86
#define TK_STORAGE 87
#define TK_QTIME 88
#define TK_CONNS 89
#define TK_STATE 90
#define TK_COMMA 91
#define TK_KEEP 92
#define TK_CACHE 93
#define TK_REPLICA 94
#define TK_QUORUM 95
#define TK_DAYS 96
#define TK_MINROWS 97
#define TK_MAXROWS 98
#define TK_BLOCKS 99
#define TK_CTIME 100
#define TK_WAL 101
#define TK_FSYNC 102
#define TK_COMP 103
#define TK_PRECISION 104
#define TK_UPDATE 105
#define TK_CACHELAST 106
#define TK_PARTITIONS 107
#define TK_UNSIGNED 108
#define TK_TAGS 109
#define TK_USING 110
#define TK_AS 111
#define TK_NULL 112
#define TK_NOW 113
#define TK_SELECT 114
#define TK_UNION 115
#define TK_ALL 116
#define TK_DISTINCT 117
#define TK_FROM 118
#define TK_VARIABLE 119
#define TK_INTERVAL 120
#define TK_SESSION 121
#define TK_STATE_WINDOW 122
#define TK_FILL 123
#define TK_SLIDING 124
#define TK_ORDER 125
#define TK_BY 126
#define TK_ASC 127
#define TK_DESC 128
#define TK_GROUP 129
#define TK_HAVING 130
#define TK_LIMIT 131
#define TK_OFFSET 132
#define TK_SLIMIT 133
#define TK_SOFFSET 134
#define TK_WHERE 135
#define TK_RESET 136
#define TK_QUERY 137
#define TK_SYNCDB 138
#define TK_ADD 139
#define TK_COLUMN 140
#define TK_MODIFY 141
#define TK_TAG 142
#define TK_CHANGE 143
#define TK_SET 144
#define TK_KILL 145
#define TK_CONNECTION 146
#define TK_STREAM 147
#define TK_COLON 148
#define TK_ABORT 149
#define TK_AFTER 150
#define TK_ATTACH 151
#define TK_BEFORE 152
#define TK_BEGIN 153
#define TK_CASCADE 154
#define TK_CLUSTER 155
#define TK_CONFLICT 156
#define TK_COPY 157
#define TK_DEFERRED 158
#define TK_DELIMITERS 159
#define TK_DETACH 160
#define TK_EACH 161
#define TK_END 162
#define TK_EXPLAIN 163
#define TK_FAIL 164
#define TK_FOR 165
#define TK_IGNORE 166
#define TK_IMMEDIATE 167
#define TK_INITIALLY 168
#define TK_INSTEAD 169
#define TK_MATCH 170
#define TK_KEY 171
#define TK_OF 172
#define TK_RAISE 173
#define TK_REPLACE 174
#define TK_RESTRICT 175
#define TK_ROW 176
#define TK_STATEMENT 177
#define TK_TRIGGER 178
#define TK_VIEW 179
#define TK_SEMI 180
#define TK_NONE 181
#define TK_PREV 182
#define TK_LINEAR 183
#define TK_IMPORT 184
#define TK_TBNAME 185
#define TK_JOIN 186
#define TK_INSERT 187
#define TK_INTO 188
#define TK_VALUES 189
#define TK_FUNCTIONS 47
#define TK_MNODES 48
#define TK_DNODES 49
#define TK_ACCOUNTS 50
#define TK_USERS 51
#define TK_MODULES 52
#define TK_QUERIES 53
#define TK_CONNECTIONS 54
#define TK_STREAMS 55
#define TK_VARIABLES 56
#define TK_SCORES 57
#define TK_GRANTS 58
#define TK_VNODES 59
#define TK_IPTOKEN 60
#define TK_DOT 61
#define TK_CREATE 62
#define TK_TABLE 63
#define TK_STABLE 64
#define TK_DATABASE 65
#define TK_TABLES 66
#define TK_STABLES 67
#define TK_VGROUPS 68
#define TK_DROP 69
#define TK_TOPIC 70
#define TK_FUNCTION 71
#define TK_DNODE 72
#define TK_USER 73
#define TK_ACCOUNT 74
#define TK_USE 75
#define TK_DESCRIBE 76
#define TK_ALTER 77
#define TK_PASS 78
#define TK_PRIVILEGE 79
#define TK_LOCAL 80
#define TK_COMPACT 81
#define TK_LP 82
#define TK_RP 83
#define TK_IF 84
#define TK_EXISTS 85
#define TK_AS 86
#define TK_OUTPUTTYPE 87
#define TK_AGGREGATE 88
#define TK_BUFSIZE 89
#define TK_PPS 90
#define TK_TSERIES 91
#define TK_DBS 92
#define TK_STORAGE 93
#define TK_QTIME 94
#define TK_CONNS 95
#define TK_STATE 96
#define TK_COMMA 97
#define TK_KEEP 98
#define TK_CACHE 99
#define TK_REPLICA 100
#define TK_QUORUM 101
#define TK_DAYS 102
#define TK_MINROWS 103
#define TK_MAXROWS 104
#define TK_BLOCKS 105
#define TK_CTIME 106
#define TK_WAL 107
#define TK_FSYNC 108
#define TK_COMP 109
#define TK_PRECISION 110
#define TK_UPDATE 111
#define TK_CACHELAST 112
#define TK_PARTITIONS 113
#define TK_UNSIGNED 114
#define TK_TAGS 115
#define TK_USING 116
#define TK_NULL 117
#define TK_NOW 118
#define TK_SELECT 119
#define TK_UNION 120
#define TK_ALL 121
#define TK_DISTINCT 122
#define TK_FROM 123
#define TK_VARIABLE 124
#define TK_INTERVAL 125
#define TK_SESSION 126
#define TK_STATE_WINDOW 127
#define TK_FILL 128
#define TK_SLIDING 129
#define TK_ORDER 130
#define TK_BY 131
#define TK_ASC 132
#define TK_DESC 133
#define TK_GROUP 134
#define TK_HAVING 135
#define TK_LIMIT 136
#define TK_OFFSET 137
#define TK_SLIMIT 138
#define TK_SOFFSET 139
#define TK_WHERE 140
#define TK_RESET 141
#define TK_QUERY 142
#define TK_SYNCDB 143
#define TK_ADD 144
#define TK_COLUMN 145
#define TK_MODIFY 146
#define TK_TAG 147
#define TK_CHANGE 148
#define TK_SET 149
#define TK_KILL 150
#define TK_CONNECTION 151
#define TK_STREAM 152
#define TK_COLON 153
#define TK_ABORT 154
#define TK_AFTER 155
#define TK_ATTACH 156
#define TK_BEFORE 157
#define TK_BEGIN 158
#define TK_CASCADE 159
#define TK_CLUSTER 160
#define TK_CONFLICT 161
#define TK_COPY 162
#define TK_DEFERRED 163
#define TK_DELIMITERS 164
#define TK_DETACH 165
#define TK_EACH 166
#define TK_END 167
#define TK_EXPLAIN 168
#define TK_FAIL 169
#define TK_FOR 170
#define TK_IGNORE 171
#define TK_IMMEDIATE 172
#define TK_INITIALLY 173
#define TK_INSTEAD 174
#define TK_MATCH 175
#define TK_KEY 176
#define TK_OF 177
#define TK_RAISE 178
#define TK_REPLACE 179
#define TK_RESTRICT 180
#define TK_ROW 181
#define TK_STATEMENT 182
#define TK_TRIGGER 183
#define TK_VIEW 184
#define TK_SEMI 185
#define TK_NONE 186
#define TK_PREV 187
#define TK_LINEAR 188
#define TK_IMPORT 189
#define TK_TBNAME 190
#define TK_JOIN 191
#define TK_INSERT 192
#define TK_INTO 193
#define TK_VALUES 194
#define TK_SPACE 300

4
src/inc/ttype.h
View File

@ -28,6 +28,10 @@ typedef struct tstr {
#define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT) (_len))
#define IS_VAR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_BINARY) || ((t) == TSDB_DATA_TYPE_NCHAR))
#define varDataNetLen(v) (htons(((VarDataLenT *)(v))[0]))
#define varDataNetTLen(v) (sizeof(VarDataLenT) + varDataNetLen(v))
// this data type is internally used only in 'in' query to hold the values
#define TSDB_DATA_TYPE_ARRAY (1000)

2
src/inc/vnode.h
View File

@ -49,7 +49,7 @@ typedef struct {
SRpcMsg rpcMsg;
SRspRet rspRet;
char reserveForSync[24];
SWalHead pHead;
SWalHead walHead;
} SVWriteMsg;
// vnodeStatus

4
src/kit/shell/CMakeLists.txt
View File

@ -19,9 +19,9 @@ ELSE ()
ENDIF ()
IF (TD_SOMODE_STATIC)
TARGET_LINK_LIBRARIES(shell taos_static ${LINK_JEMALLOC})
TARGET_LINK_LIBRARIES(shell taos_static lua ${LINK_JEMALLOC})
ELSE ()
TARGET_LINK_LIBRARIES(shell taos ${LINK_JEMALLOC})
TARGET_LINK_LIBRARIES(shell taos lua ${LINK_JEMALLOC})
ENDIF ()
SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME taos)

10
src/kit/taosdemo/CMakeLists.txt
View File

@ -67,7 +67,7 @@ IF (TD_LINUX)
ADD_EXECUTABLE(taosdemo ${SRC})
IF (TD_SOMODE_STATIC)
TARGET_LINK_LIBRARIES(taosdemo taos_static cJson ${LINK_JEMALLOC})
TARGET_LINK_LIBRARIES(taosdemo taos_static cJson lua ${LINK_JEMALLOC})
ELSE ()
TARGET_LINK_LIBRARIES(taosdemo taos cJson ${LINK_JEMALLOC})
ENDIF ()
@ -76,9 +76,9 @@ ELSEIF (TD_WINDOWS)
ADD_EXECUTABLE(taosdemo ${SRC})
SET_SOURCE_FILES_PROPERTIES(./taosdemo.c PROPERTIES COMPILE_FLAGS -w)
IF (TD_SOMODE_STATIC)
TARGET_LINK_LIBRARIES(taosdemo taos_static cJson)
TARGET_LINK_LIBRARIES(taosdemo taos_static cJson lua)
ELSE ()
TARGET_LINK_LIBRARIES(taosdemo taos cJson)
TARGET_LINK_LIBRARIES(taosdemo taos cJson lua)
ENDIF ()
ELSEIF (TD_DARWIN)
# missing a few dependencies, such as <argp.h>
@ -86,9 +86,9 @@ ELSEIF (TD_DARWIN)
ADD_EXECUTABLE(taosdemo ${SRC})
IF (TD_SOMODE_STATIC)
TARGET_LINK_LIBRARIES(taosdemo taos_static cJson)
TARGET_LINK_LIBRARIES(taosdemo taos_static cJson lua)
ELSE ()
TARGET_LINK_LIBRARIES(taosdemo taos cJson)
TARGET_LINK_LIBRARIES(taosdemo taos cJson lua)
ENDIF ()
ENDIF ()

3411
src/kit/taosdemo/taosdemo.c
View File

File diff suppressed because it is too large Load Diff

19
src/mnode/inc/mnodeDef.h
View File

@ -214,6 +214,23 @@ typedef struct SUserObj {
struct SAcctObj * pAcct;
} SUserObj;
typedef struct SFuncObj {
char name[TSDB_FUNC_NAME_LEN];
char path[128];
int32_t contLen;
char cont[TSDB_FUNC_CODE_LEN];
int32_t funcType;
int32_t bufSize;
int64_t createdTime;
uint8_t resType;
int16_t resBytes;
int64_t sig; // partial md5 sign
int16_t type; // [lua script|so|js]
int8_t reserved[64];
int8_t updateEnd[4];
int32_t refCount;
} SFuncObj;
typedef struct {
int64_t totalStorage; // Total storage wrtten from this account
int64_t compStorage; // Compressed storage on disk
@ -258,7 +275,7 @@ typedef struct {
void * pIter;
void ** ppShow;
int16_t offset[TSDB_MAX_COLUMNS];
int16_t bytes[TSDB_MAX_COLUMNS];
int32_t bytes[TSDB_MAX_COLUMNS];
int32_t numOfReads;
int8_t maxReplica;
int8_t reserved0[1];

40
src/mnode/inc/mnodeFunc.h Normal file
View File

@ -0,0 +1,40 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_MNODE_FUNC_H
#define TDENGINE_MNODE_FUNC_H
#ifdef __cplusplus
extern "C" {
#endif
#include "mnodeDef.h"
int32_t mnodeInitFuncs();
void mnodeCleanupFuncs();
SFuncObj *mnodeGetFunc(char *name);
void * mnodeGetNextFunc(void *pIter, SFuncObj **pFunc);
void mnodeCancelGetNextFunc(void *pIter);
void mnodeIncFuncRef(SFuncObj *pFunc);
void mnodeDecFuncRef(SFuncObj *pFunc);
int32_t mnodeCreateFunc(SAcctObj *pAcct, char *name, int32_t codeLen, char *code, char *path, uint8_t outputType, int16_t outputLen, int32_t funcType, int32_t bufSize, SMnodeMsg *pMsg);
#ifdef __cplusplus
}
#endif
#endif

5
src/mnode/inc/mnodeSdb.h
View File

@ -33,7 +33,8 @@ typedef enum {
SDB_TABLE_VGROUP = 6,
SDB_TABLE_STABLE = 7,
SDB_TABLE_CTABLE = 8,
SDB_TABLE_MAX = 9
SDB_TABLE_FUNC = 9,
SDB_TABLE_MAX = 10
} ESdbTable;
typedef enum {
@ -112,4 +113,4 @@ int32_t mnodeCompactWal();
}
#endif
#endif
#endif

3
src/mnode/src/mnodeDb.c
View File

@ -554,6 +554,9 @@ void mnodeCleanupDbs() {
tsDbSdb = NULL;
}
static int32_t mnodeGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
int32_t cols = 0;

485
src/mnode/src/mnodeFunc.c Normal file
View File

@ -0,0 +1,485 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "trpc.h"
#include "tutil.h"
#include "tglobal.h"
#include "tgrant.h"
#include "tdataformat.h"
#include "tkey.h"
#include "mnode.h"
#include "dnode.h"
#include "mnodeDef.h"
#include "mnodeInt.h"
#include "mnodeAcct.h"
#include "mnodeUser.h"
#include "mnodeMnode.h"
#include "mnodeSdb.h"
#include "mnodeShow.h"
#include "mnodeFunc.h"
#include "mnodeWrite.h"
#include "mnodeRead.h"
#include "mnodePeer.h"
int64_t tsFuncRid = -1;
static void * tsFuncSdb = NULL;
static int32_t tsFuncUpdateSize = 0;
static int32_t mnodeGetFuncMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
static int32_t mnodeRetrieveFuncs(SShowObj *pShow, char *data, int32_t rows, void *pConn);
static int32_t mnodeProcessRetrieveFuncImplMsg(SMnodeMsg *pMsg);
static int32_t mnodeProcessCreateFuncMsg(SMnodeMsg *pMsg);
static int32_t mnodeProcessDropFuncMsg(SMnodeMsg *pMsg);
static int32_t mnodeFuncActionDestroy(SSdbRow *pRow) {
tfree(pRow->pObj);
return TSDB_CODE_SUCCESS;
}
static int32_t mnodeFuncActionInsert(SSdbRow *pRow) {
SFuncObj *pFunc = pRow->pObj;
mTrace("func:%s, contLen: %d, insert into sdb", pFunc->name, pFunc->contLen);
return TSDB_CODE_SUCCESS;
}
static int32_t mnodeFuncActionDelete(SSdbRow *pRow) {
SFuncObj *pFunc = pRow->pObj;
mTrace("func:%s, length: %d, delete from sdb", pFunc->name, pFunc->contLen);
return TSDB_CODE_SUCCESS;
}
static int32_t mnodeFuncActionUpdate(SSdbRow *pRow) {
SFuncObj *pFunc = pRow->pObj;
SFuncObj *pSaved = mnodeGetFunc(pFunc->name);
if (pFunc != pSaved) {
memcpy(pSaved, pFunc, tsFuncUpdateSize);
free(pFunc);
}
mnodeDecFuncRef(pSaved);
return TSDB_CODE_SUCCESS;
}
static int32_t mnodeFuncActionEncode(SSdbRow *pRow) {
SFuncObj *pFunc = pRow->pObj;
memcpy(pRow->rowData, pFunc, tsFuncUpdateSize);
pRow->rowSize = tsFuncUpdateSize;
return TSDB_CODE_SUCCESS;
}
static int32_t mnodeFuncActionDecode(SSdbRow *pRow) {
SFuncObj *pFunc = (SFuncObj *)calloc(1, sizeof(SFuncObj));
if (pFunc == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY;
memcpy(pFunc, pRow->rowData, tsFuncUpdateSize);
pRow->pObj = pFunc;
return TSDB_CODE_SUCCESS;
}
static int32_t mnodeFuncActionRestored() {
int64_t numOfRows = sdbGetNumOfRows(tsFuncSdb);
if (numOfRows <= 0 && dnodeIsFirstDeploy()) {
mInfo("dnode first deploy, func restored.");
}
return TSDB_CODE_SUCCESS;
}
int32_t mnodeInitFuncs() {
SFuncObj tObj;
tsFuncUpdateSize = (int32_t)((int8_t *)tObj.updateEnd - (int8_t *)&tObj);
SSdbTableDesc desc = {
.id = SDB_TABLE_FUNC,
.name = "funcs",
.hashSessions = TSDB_DEFAULT_USERS_HASH_SIZE,
.maxRowSize = tsFuncUpdateSize,
.refCountPos = (int32_t)((int8_t *)(&tObj.refCount) - (int8_t *)&tObj),
.keyType = SDB_KEY_STRING,
.fpInsert = mnodeFuncActionInsert,
.fpDelete = mnodeFuncActionDelete,
.fpUpdate = mnodeFuncActionUpdate,
.fpEncode = mnodeFuncActionEncode,
.fpDecode = mnodeFuncActionDecode,
.fpDestroy = mnodeFuncActionDestroy,
.fpRestored = mnodeFuncActionRestored
};
tsFuncRid = sdbOpenTable(&desc);
tsFuncSdb = sdbGetTableByRid(tsFuncRid);
if (tsFuncSdb == NULL) {
mError("table:%s, failed to create hash", desc.name);
return -1;
}
mnodeAddWriteMsgHandle(TSDB_MSG_TYPE_CM_CREATE_FUNCTION, mnodeProcessCreateFuncMsg);
mnodeAddWriteMsgHandle(TSDB_MSG_TYPE_CM_DROP_FUNCTION, mnodeProcessDropFuncMsg);
mnodeAddReadMsgHandle(TSDB_MSG_TYPE_CM_RETRIEVE_FUNC, mnodeProcessRetrieveFuncImplMsg);
mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_FUNCTION, mnodeGetFuncMeta);
mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_FUNCTION, mnodeRetrieveFuncs);
mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_FUNCTION, mnodeCancelGetNextFunc);
mDebug("table:%s, hash is created", desc.name);
return 0;
}
void mnodeCleanupFuncs() {
sdbCloseTable(tsFuncRid);
tsFuncSdb = NULL;
}
SFuncObj *mnodeGetFunc(char *name) {
return (SFuncObj *)sdbGetRow(tsFuncSdb, name);
}
void *mnodeGetNextFunc(void *pIter, SFuncObj **pFunc) {
return sdbFetchRow(tsFuncSdb, pIter, (void **)pFunc);
}
void mnodeCancelGetNextFunc(void *pIter) {
sdbFreeIter(tsFuncSdb, pIter);
}
void mnodeIncFuncRef(SFuncObj *pFunc) {
sdbIncRef(tsFuncSdb, pFunc);
}
void mnodeDecFuncRef(SFuncObj *pFunc) {
sdbDecRef(tsFuncSdb, pFunc);
}
/*
static int32_t mnodeUpdateFunc(SFuncObj *pFunc, void *pMsg) {
SSdbRow row = {
.type = SDB_OPER_GLOBAL,
.pTable = tsFuncSdb,
.pObj = pFunc,
.pMsg = pMsg
};
int32_t code = sdbUpdateRow(&row);
if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
mError("func:%s, failed to alter by %s, reason:%s", pFunc->name, mnodeGetUserFromMsg(pMsg), tstrerror(code));
} else {
mLInfo("func:%s, is altered by %s", pFunc->name, mnodeGetUserFromMsg(pMsg));
}
return code;
}
*/
int32_t mnodeCreateFunc(SAcctObj *pAcct, char *name, int32_t codeLen, char *codeScript, char *path, uint8_t outputType, int16_t outputLen, int32_t funcType, int32_t bufSize, SMnodeMsg *pMsg) {
if (grantCheck(TSDB_GRANT_TIME) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_GRANT_EXPIRED;
}
if (!pMsg->pUser->writeAuth) {
return TSDB_CODE_MND_NO_RIGHTS;
}
int32_t code = acctCheck(pAcct, ACCT_GRANT_USER);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
code = grantCheck(TSDB_GRANT_USER);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
if (name[0] == 0) {
return TSDB_CODE_MND_INVALID_FUNC_NAME;
}
if (codeScript[0] == 0) {
return TSDB_CODE_MND_INVALID_FUNC_CODE;
}
if (codeLen < 0 || codeLen > TSDB_FUNC_CODE_LEN) {
return TSDB_CODE_MND_INVALID_FUNC_CODE;
}
if (bufSize < 0 || bufSize > TSDB_FUNC_BUF_SIZE) {
return TSDB_CODE_MND_INVALID_FUNC_BUFSIZE;
}
SFuncObj *pFunc = mnodeGetFunc(name);
if (pFunc != NULL) {
mDebug("func:%s, is already there", name);
mnodeDecFuncRef(pFunc);
return TSDB_CODE_MND_FUNC_ALREADY_EXIST;
}
pFunc = calloc(1, sizeof(SFuncObj));
tstrncpy(pFunc->name, name, TSDB_FUNC_NAME_LEN);
tstrncpy(pFunc->path, path, tListLen(pFunc->path));
memcpy(pFunc->cont, codeScript, codeLen);
pFunc->contLen = codeLen;
pFunc->createdTime = taosGetTimestampMs();
pFunc->resType = outputType;
pFunc->resBytes = outputLen;
pFunc->funcType = funcType;
pFunc->bufSize = bufSize;
pFunc->sig = 0;
pFunc->type = 1; //lua script, refactor
SSdbRow row = {
.type = SDB_OPER_GLOBAL,
.pTable = tsFuncSdb,
.pObj = pFunc,
.rowSize = sizeof(SFuncObj),
.pMsg = pMsg
};
code = sdbInsertRow(&row);
if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
mError("func:%s, failed to create by %s, reason:%s", pFunc->name, mnodeGetUserFromMsg(pMsg), tstrerror(code));
tfree(pFunc);
} else {
mLInfo("func:%s, is created by %s", pFunc->name, mnodeGetUserFromMsg(pMsg));
}
return code;
}
static int32_t mnodeDropFunc(SFuncObj *pFunc, void *pMsg) {
SSdbRow row = {
.type = SDB_OPER_GLOBAL,
.pTable = tsFuncSdb,
.pObj = pFunc,
.pMsg = pMsg
};
int32_t code = sdbDeleteRow(&row);
if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) {
mError("func:%s, failed to drop by %s, reason:%s", pFunc->name, mnodeGetUserFromMsg(pMsg), tstrerror(code));
} else {
mLInfo("func:%s, is dropped by %s", pFunc->name, mnodeGetUserFromMsg(pMsg));
}
return code;
}
static int32_t mnodeGetFuncsNum() {
return (int32_t)sdbGetNumOfRows(tsFuncSdb);
}
static int32_t mnodeGetFuncMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
SUserObj *pUser = mnodeGetUserFromConn(pConn);
if (pUser == NULL) {
return TSDB_CODE_MND_NO_USER_FROM_CONN;
}
int32_t cols = 0;
SSchema *pSchema = pMeta->schema;
pShow->bytes[cols] = TSDB_FUNC_NAME_LEN + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "name");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = PATH_MAX + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "path");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 4;
pSchema[cols].type = TSDB_DATA_TYPE_INT;
strcpy(pSchema[cols].name, "aggregate");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = TSDB_TYPE_STR_MAX_LEN + VARSTR_HEADER_SIZE;
pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema[cols].name, "outputtype");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 8;
pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP;
strcpy(pSchema[cols].name, "create_time");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 4;
pSchema[cols].type = TSDB_DATA_TYPE_INT;
strcpy(pSchema[cols].name, "code_len");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pShow->bytes[cols] = 4;
pSchema[cols].type = TSDB_DATA_TYPE_INT;
strcpy(pSchema[cols].name, "bufsize");
pSchema[cols].bytes = htons(pShow->bytes[cols]);
cols++;
pMeta->numOfColumns = htons(cols);
strcpy(pMeta->tableFname, "show funcs");
pShow->numOfColumns = cols;
pShow->offset[0] = 0;
for (int32_t i = 1; i < cols; ++i) {
pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1];
}
pShow->numOfRows = mnodeGetFuncsNum();
pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
mnodeDecUserRef(pUser);
return 0;
}
static void* mnodeGenTypeStr(char *buf, int32_t buflen, uint8_t type, int16_t len) {
char *msg = "unknown";
if (type >= sizeof(tDataTypes)/sizeof(tDataTypes[0])) {
return msg;
}
if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BINARY) {
int32_t bytes = len > 0 ? (int)(len - VARSTR_HEADER_SIZE) : len;
snprintf(buf, buflen - 1, "%s(%d)", tDataTypes[type].name, type == TSDB_DATA_TYPE_NCHAR ? bytes/4 : bytes);
buf[buflen - 1] = 0;
return buf;
}
return tDataTypes[type].name;
}
static int32_t mnodeRetrieveFuncs(SShowObj *pShow, char *data, int32_t rows, void *pConn) {
int32_t numOfRows = 0;
SFuncObj *pFunc = NULL;
int32_t cols = 0;
char *pWrite;
char buf[TSDB_TYPE_STR_MAX_LEN];
while (numOfRows < rows) {
pShow->pIter = mnodeGetNextFunc(pShow->pIter, &pFunc);
if (pFunc == NULL) break;
cols = 0;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pFunc->name, pShow->bytes[cols]);
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pFunc->path, pShow->bytes[cols]);
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int32_t *)pWrite = pFunc->funcType == TSDB_UDF_TYPE_AGGREGATE ? 1 : 0;
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
STR_WITH_MAXSIZE_TO_VARSTR(pWrite, mnodeGenTypeStr(buf, TSDB_TYPE_STR_MAX_LEN, pFunc->resType, pFunc->resBytes), pShow->bytes[cols]);
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int64_t *)pWrite = pFunc->createdTime;
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int32_t *)pWrite = pFunc->contLen;
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int32_t *)pWrite = pFunc->bufSize;
cols++;
numOfRows++;
mnodeDecFuncRef(pFunc);
}
mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow);
pShow->numOfReads += numOfRows;
return numOfRows;
}
static int32_t mnodeProcessCreateFuncMsg(SMnodeMsg *pMsg) {
SCreateFuncMsg *pCreate = pMsg->rpcMsg.pCont;
pCreate->codeLen = htonl(pCreate->codeLen);
pCreate->outputLen = htons(pCreate->outputLen);
pCreate->funcType = htonl(pCreate->funcType);
pCreate->bufSize = htonl(pCreate->bufSize);
return mnodeCreateFunc(pMsg->pUser->pAcct, pCreate->name, pCreate->codeLen, pCreate->code, pCreate->path, pCreate->outputType, pCreate->outputLen, pCreate->funcType, pCreate->bufSize, pMsg);
}
static int32_t mnodeProcessDropFuncMsg(SMnodeMsg *pMsg) {
SDropFuncMsg *pDrop = pMsg->rpcMsg.pCont;
SFuncObj *pFunc = mnodeGetFunc(pDrop->name);
if (pFunc == NULL) {
return TSDB_CODE_MND_INVALID_FUNC;
}
return mnodeDropFunc(pFunc, pMsg);
}
static int32_t mnodeProcessRetrieveFuncImplMsg(SMnodeMsg *pMsg) {
SRetrieveFuncMsg *pInfo = pMsg->rpcMsg.pCont;
pInfo->num = htonl(pInfo->num);
int32_t t = sizeof(SUdfFuncMsg) + (sizeof(SFunctionInfoMsg) + TSDB_FUNC_CODE_LEN) * pInfo->num + 16384;
SUdfFuncMsg *pFuncMsg = rpcMallocCont(t);
pFuncMsg->num = htonl(pInfo->num);
char* pOutput = pFuncMsg->content;
tstr* name = (tstr*) pInfo->name;
for(int32_t i = 0; i < pInfo->num; ++i) {
char buf[TSDB_FUNC_NAME_LEN] = {0};
tstrncpy(buf, name->data, htons(name->len) + 1);
SFuncObj* pFuncObj = mnodeGetFunc(buf);
if (pFuncObj == NULL) {
mError("function %s does not exist", buf);
return TSDB_CODE_MND_INVALID_FUNC;
}
SFunctionInfoMsg* pFuncInfo = (SFunctionInfoMsg*) pOutput;
strcpy(pFuncInfo->name, buf);
pFuncInfo->len = htonl(pFuncObj->contLen);
memcpy(pFuncInfo->content, pFuncObj->cont, pFuncObj->contLen);
pFuncInfo->funcType = htonl(pFuncObj->funcType);
pFuncInfo->resType = pFuncObj->resType;
pFuncInfo->resBytes = htons(pFuncObj->resBytes);
pFuncInfo->bufSize = htonl(pFuncObj->bufSize);
pOutput += sizeof(SFunctionInfoMsg) + pFuncObj->contLen;
name =(tstr *)((char *)name + sizeof(*name) + htons(name->len));
}
pMsg->rpcRsp.rsp = pFuncMsg;
pMsg->rpcRsp.len = (int32_t)(pOutput - (char*)pFuncMsg);
return TSDB_CODE_SUCCESS;
}

2
src/mnode/src/mnodeMain.c
View File

@ -32,6 +32,7 @@
#include "mnodeSdb.h"
#include "mnodeVgroup.h"
#include "mnodeUser.h"
#include "mnodeFunc.h"
#include "mnodeTable.h"
#include "mnodeCluster.h"
#include "mnodeShow.h"
@ -46,6 +47,7 @@ static SStep tsMnodeSteps[] = {
{"cluster", mnodeInitCluster, mnodeCleanupCluster},
{"accts", mnodeInitAccts, mnodeCleanupAccts},
{"users", mnodeInitUsers, mnodeCleanupUsers},
{"funcs", mnodeInitFuncs, mnodeCleanupFuncs},
{"dnodes", mnodeInitDnodes, mnodeCleanupDnodes},
{"dbs", mnodeInitDbs, mnodeCleanupDbs},
{"vgroups", mnodeInitVgroups, mnodeCleanupVgroups},

125
src/mnode/src/mnodeTable.c
View File

@ -42,6 +42,7 @@
#include "mnodeWrite.h"
#include "mnodeRead.h"
#include "mnodePeer.h"
#include "mnodeFunc.h"
#define ALTER_CTABLE_RETRY_TIMES 3
#define CREATE_CTABLE_RETRY_TIMES 10
@ -104,6 +105,20 @@ static void mnodeDestroyChildTable(SCTableObj *pTable) {
tfree(pTable);
}
static char* mnodeGetTableShowPattern(SShowObj *pShow) {
char* pattern = NULL;
if (pShow != NULL && pShow->payloadLen > 0) {
pattern = (char*)malloc(pShow->payloadLen + 1);
if (pattern == NULL) {
terrno = TSDB_CODE_QRY_OUT_OF_MEMORY;
return NULL;
}
memcpy(pattern, pShow->payload, pShow->payloadLen);
pattern[pShow->payloadLen] = 0;
}
return pattern;
}
static int32_t mnodeChildTableActionDestroy(SSdbRow *pRow) {
mnodeDestroyChildTable(pRow->pObj);
return TSDB_CODE_SUCCESS;
@ -1620,6 +1635,11 @@ int32_t mnodeRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows,
SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER;
char stableName[TSDB_TABLE_NAME_LEN] = {0};
char* pattern = mnodeGetTableShowPattern(pShow);
if (pShow->payloadLen > 0 && pattern == NULL) {
return 0;
}
while (numOfRows < rows) {
pShow->pIter = mnodeGetNextSuperTable(pShow->pIter, &pTable);
if (pTable == NULL) break;
@ -1631,7 +1651,7 @@ int32_t mnodeRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows,
memset(stableName, 0, tListLen(stableName));
mnodeExtractTableName(pTable->info.tableId, stableName);
if (pShow->payloadLen > 0 && patternMatch(pShow->payload, stableName, sizeof(stableName) - 1, &info) != TSDB_PATTERN_MATCH) {
if (pShow->payloadLen > 0 && patternMatch(pattern, stableName, sizeof(stableName) - 1, &info) != TSDB_PATTERN_MATCH) {
mnodeDecTableRef(pTable);
continue;
}
@ -1671,6 +1691,7 @@ int32_t mnodeRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows,
mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow);
mnodeDecDbRef(pDb);
free(pattern);
return numOfRows;
}
@ -2892,7 +2913,7 @@ static SMultiTableMeta* ensureMsgBufferSpace(SMultiTableMeta *pMultiMeta, SArray
(*totalMallocLen) *= 2;
}
pMultiMeta = rpcReallocCont(pMultiMeta, *totalMallocLen);
pMultiMeta = realloc(pMultiMeta, *totalMallocLen);
if (pMultiMeta == NULL) {
return NULL;
}
@ -2906,6 +2927,7 @@ static int32_t mnodeProcessMultiTableMetaMsg(SMnodeMsg *pMsg) {
pInfo->numOfTables = htonl(pInfo->numOfTables);
pInfo->numOfVgroups = htonl(pInfo->numOfVgroups);
pInfo->numOfUdfs = htonl(pInfo->numOfUdfs);
int32_t contLen = pMsg->rpcMsg.contLen - sizeof(SMultiTableInfoMsg);
@ -2914,17 +2936,17 @@ static int32_t mnodeProcessMultiTableMetaMsg(SMnodeMsg *pMsg) {
char* str = strndup(pInfo->tableNames, contLen);
char** nameList = strsplit(str, ",", &num);
SArray* pList = taosArrayInit(4, POINTER_BYTES);
SMultiTableMeta *pMultiMeta = NULL;
if (num != pInfo->numOfTables + pInfo->numOfVgroups) {
SMultiTableMeta *pMultiMeta = NULL;
if (num != pInfo->numOfTables + pInfo->numOfVgroups + pInfo->numOfUdfs) {
mError("msg:%p, app:%p, failed to get multi-tableMeta, msg inconsistent", pMsg, pMsg->rpcMsg.ahandle);
code = TSDB_CODE_MND_INVALID_TABLE_NAME;
goto _end;
}
// first malloc 80KB, subsequent reallocation will expand the size as twice of the original size
int32_t totalMallocLen = sizeof(STableMetaMsg) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16);
pMultiMeta = rpcMallocCont(totalMallocLen);
int32_t totalMallocLen = sizeof(SMultiTableMeta) + sizeof(STableMetaMsg) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16);
pMultiMeta = calloc(1, totalMallocLen);
if (pMultiMeta == NULL) {
code = TSDB_CODE_MND_OUT_OF_MEMORY;
goto _end;
@ -2957,7 +2979,7 @@ static int32_t mnodeProcessMultiTableMetaMsg(SMnodeMsg *pMsg) {
int remain = totalMallocLen - pMultiMeta->contLen;
if (remain <= sizeof(STableMetaMsg) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16)) {
totalMallocLen *= 2;
pMultiMeta = rpcReallocCont(pMultiMeta, totalMallocLen);
pMultiMeta = realloc(pMultiMeta, totalMallocLen);
if (pMultiMeta == NULL) {
mnodeDecTableRef(pMsg->pTable);
code = TSDB_CODE_MND_OUT_OF_MEMORY;
@ -2991,8 +3013,10 @@ static int32_t mnodeProcessMultiTableMetaMsg(SMnodeMsg *pMsg) {
}
}
int32_t tableNum = pInfo->numOfTables + pInfo->numOfVgroups;
// add the additional super table names that needs the vgroup info
for(;t < num; ++t) {
for(;t < tableNum; ++t) {
taosArrayPush(pList, &nameList[t]);
}
@ -3023,20 +3047,77 @@ static int32_t mnodeProcessMultiTableMetaMsg(SMnodeMsg *pMsg) {
pMultiMeta->contLen = (int32_t) (msg - (char*) pMultiMeta);
pMultiMeta->numOfTables = htonl(pMultiMeta->numOfTables);
// add the user-defined-function information
for(int32_t i = 0; i < pInfo->numOfUdfs; ++i, ++t) {
char buf[TSDB_FUNC_NAME_LEN] = {0};
strcpy(buf, nameList[t]);
SFuncObj* pFuncObj = mnodeGetFunc(buf);
if (pFuncObj == NULL) {
mError("function %s does not exist", buf);
code = TSDB_CODE_MND_INVALID_FUNC;
goto _end;
}
SFunctionInfoMsg* pFuncInfo = (SFunctionInfoMsg*) msg;
strcpy(pFuncInfo->name, buf);
pFuncInfo->len = htonl(pFuncObj->contLen);
memcpy(pFuncInfo->content, pFuncObj->cont, pFuncObj->contLen);
pFuncInfo->funcType = htonl(pFuncObj->funcType);
pFuncInfo->resType = pFuncObj->resType;
pFuncInfo->resBytes = htons(pFuncObj->resBytes);
pFuncInfo->bufSize = htonl(pFuncObj->bufSize);
msg += sizeof(SFunctionInfoMsg) + pFuncObj->contLen;
}
pMultiMeta->contLen = (int32_t) (msg - (char*) pMultiMeta);
pMultiMeta->numOfUdf = htonl(pInfo->numOfUdfs);
pMsg->rpcRsp.rsp = pMultiMeta;
pMsg->rpcRsp.len = pMultiMeta->contLen;
code = TSDB_CODE_SUCCESS;
char* tmp = rpcMallocCont(pMultiMeta->contLen + 2);
if (tmp == NULL) {
code = TSDB_CODE_MND_OUT_OF_MEMORY;
goto _end;
}
int32_t dataLen = (int32_t)pMultiMeta->contLen - sizeof(SMultiTableMeta);
int32_t len = tsCompressString(pMultiMeta->meta, dataLen, 1, tmp + sizeof(SMultiTableMeta), (int32_t)dataLen + 2,
ONE_STAGE_COMP, NULL, 0);
pMultiMeta->metaClone = pInfo->metaClone;
pMultiMeta->rawLen = pMultiMeta->contLen;
if (len == -1 || len >= dataLen + 2) { // compress failed, do not compress this binary data
pMultiMeta->compressed = 0;
memcpy(tmp, pMultiMeta, sizeof(SMultiTableMeta) + pMultiMeta->contLen);
} else {
pMultiMeta->compressed = 1;
pMultiMeta->contLen = sizeof(SMultiTableMeta) + len;
// copy the header and the compressed payload
memcpy(tmp, pMultiMeta, sizeof(SMultiTableMeta));
}
pMsg->rpcRsp.rsp = tmp;
pMsg->rpcRsp.len = pMultiMeta->contLen;
SMultiTableMeta* p = (SMultiTableMeta*) tmp;
mDebug("multiTable info build completed, original:%d, compressed:%d, comp:%d", p->rawLen, p->contLen, p->compressed);
_end:
tfree(str);
tfree(nameList);
taosArrayDestroy(pList);
pMsg->pTable = NULL;
pMsg->pVgroup = NULL;
if (code != TSDB_CODE_SUCCESS) {
rpcFreeCont(pMultiMeta);
}
tfree(pMultiMeta);
return code;
}
@ -3132,15 +3213,9 @@ static int32_t mnodeRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows
char prefix[64] = {0};
int32_t prefixLen = (int32_t)tableIdPrefix(pDb->name, prefix, 64);
char* pattern = NULL;
if (pShow->payloadLen > 0) {
pattern = (char*)malloc(pShow->payloadLen + 1);
if (pattern == NULL) {
terrno = TSDB_CODE_QRY_OUT_OF_MEMORY;
return 0;
}
memcpy(pattern, pShow->payload, pShow->payloadLen);
pattern[pShow->payloadLen] = 0;
char* pattern = mnodeGetTableShowPattern(pShow);
if (pShow->payloadLen > 0 && pattern == NULL) {
return 0;
}
while (numOfRows < rows) {
@ -3372,6 +3447,11 @@ static int32_t mnodeRetrieveStreamTables(SShowObj *pShow, char *data, int32_t ro
strcat(prefix, TS_PATH_DELIMITER);
int32_t prefixLen = (int32_t)strlen(prefix);
char* pattern = mnodeGetTableShowPattern(pShow);
if (pShow->payloadLen > 0 && pattern == NULL) {
return 0;
}
while (numOfRows < rows) {
pShow->pIter = mnodeGetNextChildTable(pShow->pIter, &pTable);
if (pTable == NULL) break;
@ -3387,7 +3467,7 @@ static int32_t mnodeRetrieveStreamTables(SShowObj *pShow, char *data, int32_t ro
// pattern compare for table name
mnodeExtractTableName(pTable->info.tableId, tableName);
if (pShow->payloadLen > 0 && patternMatch(pShow->payload, tableName, sizeof(tableName) - 1, &info) != TSDB_PATTERN_MATCH) {
if (pShow->payloadLen > 0 && patternMatch(pattern, tableName, sizeof(tableName) - 1, &info) != TSDB_PATTERN_MATCH) {
mnodeDecTableRef(pTable);
continue;
}
@ -3419,6 +3499,7 @@ static int32_t mnodeRetrieveStreamTables(SShowObj *pShow, char *data, int32_t ro
mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow);
mnodeDecDbRef(pDb);
free(pattern);
return numOfRows;
}

1
src/os/inc/os.h
View File

@ -37,6 +37,7 @@ extern "C" {
#include "osSysinfo.h"
#include "osTime.h"
#include "osTimer.h"
#include "osSystem.h"
void osInit();

0
src/os/inc/osArm32.h Normal file
View File

0
src/os/inc/osArm64.h Normal file
View File

3
src/os/inc/osInc.h
View File

@ -76,7 +76,8 @@ extern "C" {
#include <dispatch/dispatch.h>
#include "osEok.h"
#else
#include <argp.h>
#include <argp.h>
#include <dlfcn.h>
#include <endian.h>
#include <linux/sysctl.h>
#include <poll.h>

0
src/os/inc/osLinux32.h Normal file
View File

0
src/os/inc/osLinux64.h Normal file
View File

31
src/os/inc/osSystem.h Normal file
View File

@ -0,0 +1,31 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_OS_SYSTEM_H
#define TDENGINE_OS_SYSTEM_H
#ifdef __cplusplus
extern "C" {
#endif
void* taosLoadDll(const char *filename);
void* taosLoadSym(void* handle, char* name);
void taosCloseDll(void *handle);
#ifdef __cplusplus
}
#endif
#endif

32
src/os/src/darwin/darwinSystem.c Normal file
View File

@ -0,0 +1,32 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "tglobal.h"
#include "tulog.h"
void* taosLoadDll(const char *filename) {
return NULL;
}
void* taosLoadSym(void* handle, char* name) {
return NULL;
}
void taosCloseDll(void *handle) {
}

2
src/os/src/linux/CMakeLists.txt
View File

@ -4,4 +4,4 @@ PROJECT(TDengine)
AUX_SOURCE_DIRECTORY(. SRC)
ADD_LIBRARY(oslinux ${SRC})
TARGET_LINK_LIBRARIES(oslinux m rt z)
TARGET_LINK_LIBRARIES(oslinux m rt z dl)

54
src/os/src/linux/osSystem.c Normal file
View File

@ -0,0 +1,54 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "tconfig.h"
#include "tglobal.h"
#include "tulog.h"
void* taosLoadDll(const char *filename) {
void *handle = dlopen (filename, RTLD_LAZY);
if (!handle) {
uError("load dll:%s failed, error:%s", filename, dlerror());
return NULL;
}
uDebug("dll %s loaded", filename);
return handle;
}
void* taosLoadSym(void* handle, char* name) {
void* sym = dlsym(handle, name);
char* error = NULL;
if ((error = dlerror()) != NULL) {
uWarn("load sym:%s failed, error:%s", name, dlerror());
return NULL;
}
uDebug("sym %s loaded", name)
return sym;
}
void taosCloseDll(void *handle) {
if (handle) {
dlclose(handle);
}
}

32
src/os/src/windows/wSystem.c Normal file
View File

@ -0,0 +1,32 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "tglobal.h"
#include "tulog.h"
void* taosLoadDll(const char *filename) {
return NULL;
}
void* taosLoadSym(void* handle, char* name) {
return NULL;
}
void taosCloseDll(void *handle) {
}

2
src/plugins/http/inc/httpParser.h
View File

@ -17,7 +17,7 @@
#define HTTP_PARSER_H
#include "httpGzip.h"
#define HTTP_MAX_URL 5 // http url stack size
#define HTTP_MAX_URL 6 // http url stack size
#define HTTP_CODE_CONTINUE 100
#define HTTP_CODE_SWITCHING_PROTOCOL 101

3
src/plugins/http/inc/httpSql.h
View File

@ -35,4 +35,7 @@ void httpTrimTableName(char *name);
int32_t httpShrinkTableName(HttpContext *pContext, int32_t pos, char *name);
char * httpGetCmdsString(HttpContext *pContext, int32_t pos);
int32_t httpCheckAllocEscapeSql(char *oldSql, char **newSql);
void httpCheckFreeEscapedSql(char *oldSql, char *newSql);
#endif

33
src/plugins/http/src/httpGcHandle.c
View File

@ -176,6 +176,16 @@ bool gcProcessQueryRequest(HttpContext* pContext) {
return false;
}
#define ESCAPE_ERROR_PROC(code, context, root) \
do { \
if (code != TSDB_CODE_SUCCESS) { \
httpSendErrorResp(context, code); \
\
cJSON_Delete(root); \
return false; \
} \
} while (0)
for (int32_t i = 0; i < size; ++i) {
cJSON* query = cJSON_GetArrayItem(root, i);
if (query == NULL) continue;
@ -186,7 +196,14 @@ bool gcProcessQueryRequest(HttpContext* pContext) {
continue;
}
int32_t refIdBuffer = httpAddToSqlCmdBuffer(pContext, refId->valuestring);
char *newStr = NULL;
int32_t retCode = 0;
retCode = httpCheckAllocEscapeSql(refId->valuestring, &newStr);
ESCAPE_ERROR_PROC(retCode, pContext, root);
int32_t refIdBuffer = httpAddToSqlCmdBuffer(pContext, newStr);
httpCheckFreeEscapedSql(refId->valuestring, newStr);
if (refIdBuffer == -1) {
httpWarn("context:%p, fd:%d, user:%s, refId buffer is full", pContext, pContext->fd, pContext->user);
break;
@ -195,7 +212,11 @@ bool gcProcessQueryRequest(HttpContext* pContext) {
cJSON* alias = cJSON_GetObjectItem(query, "alias");
int32_t aliasBuffer = -1;
if (!(alias == NULL || alias->valuestring == NULL || strlen(alias->valuestring) == 0)) {
aliasBuffer = httpAddToSqlCmdBuffer(pContext, alias->valuestring);
retCode = httpCheckAllocEscapeSql(alias->valuestring, &newStr);
ESCAPE_ERROR_PROC(retCode, pContext, root);
aliasBuffer = httpAddToSqlCmdBuffer(pContext, newStr);
httpCheckFreeEscapedSql(alias->valuestring, newStr);
if (aliasBuffer == -1) {
httpWarn("context:%p, fd:%d, user:%s, alias buffer is full", pContext, pContext->fd, pContext->user);
break;
@ -211,7 +232,11 @@ bool gcProcessQueryRequest(HttpContext* pContext) {
continue;
}
int32_t sqlBuffer = httpAddToSqlCmdBuffer(pContext, sql->valuestring);
retCode = httpCheckAllocEscapeSql(sql->valuestring, &newStr);
ESCAPE_ERROR_PROC(retCode, pContext, root);
int32_t sqlBuffer = httpAddToSqlCmdBuffer(pContext, newStr);
httpCheckFreeEscapedSql(sql->valuestring, newStr);
if (sqlBuffer == -1) {
httpWarn("context:%p, fd:%d, user:%s, sql buffer is full", pContext, pContext->fd, pContext->user);
break;
@ -237,6 +262,8 @@ bool gcProcessQueryRequest(HttpContext* pContext) {
}
}
#undef ESCAPE_ERROR_PROC
pContext->reqType = HTTP_REQTYPE_MULTI_SQL;
pContext->encodeMethod = &gcQueryMethod;
pContext->multiCmds->pos = 0;

86
src/plugins/http/src/httpRestHandle.c
View File

@ -119,6 +119,90 @@ bool restProcessSqlRequest(HttpContext* pContext, int32_t timestampFmt) {
return true;
}
#define REST_FUNC_URL_POS 2
#define REST_OUTP_URL_POS 3
#define REST_AGGR_URL_POS 4
#define REST_BUFF_URL_POS 5
#define HTTP_FUNC_LEN 32
#define HTTP_OUTP_LEN 16
#define HTTP_AGGR_LEN 2
#define HTTP_BUFF_LEN 32
static int udfSaveFile(const char *fname, const char *content, long len) {
int fd = open(fname, O_WRONLY | O_CREAT | O_EXCL | O_BINARY, 0755);
if (fd < 0)
return -1;
if (taosWrite(fd, (void *)content, len) < 0)
return -1;
return 0;
}
static bool restProcessUdfRequest(HttpContext* pContext) {
HttpParser* pParser = pContext->parser;
if (pParser->path[REST_FUNC_URL_POS].pos >= HTTP_FUNC_LEN || pParser->path[REST_FUNC_URL_POS].pos <= 0) {
return false;
}
if (pParser->path[REST_OUTP_URL_POS].pos >= HTTP_OUTP_LEN || pParser->path[REST_OUTP_URL_POS].pos <= 0) {
return false;
}
if (pParser->path[REST_AGGR_URL_POS].pos >= HTTP_AGGR_LEN || pParser->path[REST_AGGR_URL_POS].pos <= 0) {
return false;
}
if (pParser->path[REST_BUFF_URL_POS].pos >= HTTP_BUFF_LEN || pParser->path[REST_BUFF_URL_POS].pos <= 0) {
return false;
}
char* sql = pContext->parser->body.str;
int len = pContext->parser->body.pos;
if (sql == NULL) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_SQL_INPUT);
return false;
}
char udfDir[256] = {0};
char buf[64] = "udf-";
char funcName[64] = {0};
int aggr = 0;
char outputType[16] = {0};
char buffSize[32] = {0};
tstrncpy(funcName, pParser->path[REST_FUNC_URL_POS].str, HTTP_FUNC_LEN);
tstrncpy(buf + 4, funcName, HTTP_FUNC_LEN);
if (pParser->path[REST_AGGR_URL_POS].str[0] != '0') {
aggr = 1;
}
tstrncpy(outputType, pParser->path[REST_OUTP_URL_POS].str, HTTP_OUTP_LEN);
tstrncpy(buffSize, pParser->path[REST_BUFF_URL_POS].str, HTTP_BUFF_LEN);
taosGetTmpfilePath(funcName, udfDir);
udfSaveFile(udfDir, sql, len);
tfree(sql);
pContext->parser->body.str = malloc(1024);
sql = pContext->parser->body.str;
int sqlLen = sprintf(sql, "create %s function %s as \"%s\" outputtype %s bufsize %s",
aggr == 1 ? "aggregate" : " ", funcName, udfDir, outputType, buffSize);
pContext->parser->body.pos = sqlLen;
pContext->parser->body.size = sqlLen + 1;
HttpSqlCmd* cmd = &(pContext->singleCmd);
cmd->nativSql = sql;
pContext->reqType = HTTP_REQTYPE_SINGLE_SQL;
pContext->encodeMethod = &restEncodeSqlLocalTimeStringMethod;
return true;
}
bool restProcessRequest(struct HttpContext* pContext) {
if (httpUrlMatch(pContext, REST_ACTION_URL_POS, "login")) {
restGetUserFromUrl(pContext);
@ -138,6 +222,8 @@ bool restProcessRequest(struct HttpContext* pContext) {
return restProcessSqlRequest(pContext, REST_TIMESTAMP_FMT_UTC_STRING);
} else if (httpUrlMatch(pContext, REST_ACTION_URL_POS, "login")) {
return restProcessLoginRequest(pContext);
} else if (httpUrlMatch(pContext, REST_ACTION_URL_POS, "udf")) {
return restProcessUdfRequest(pContext);
} else {
}

62
src/plugins/http/src/httpSql.c
View File

@ -423,3 +423,65 @@ void httpProcessRequest(HttpContext *pContext) {
httpExecCmd(pContext);
}
}
int32_t httpCheckAllocEscapeSql(char *oldSql, char **newSql)
{
char *pos;
if (oldSql == NULL || newSql == NULL) {
return TSDB_CODE_SUCCESS;
}
/* bad sql clause */
pos = strstr(oldSql, "%%");
if (pos) {
httpError("bad sql:%s", oldSql);
return TSDB_CODE_HTTP_REQUEST_JSON_ERROR;
}
pos = strchr(oldSql, '%');
if (pos == NULL) {
httpDebug("sql:%s", oldSql);
*newSql = oldSql;
return TSDB_CODE_SUCCESS;
}
*newSql = (char *) calloc(1, (strlen(oldSql) << 1) + 1);
if (newSql == NULL) {
httpError("failed to allocate for new sql, old sql:%s", oldSql);
return TSDB_CODE_HTTP_NO_ENOUGH_MEMORY;
}
char *src = oldSql;
char *dst = *newSql;
size_t sqlLen = strlen(src);
while (1) {
memcpy(dst, src, pos - src + 1);
dst += pos - src + 1;
*dst++ = '%';
if (pos + 1 >= oldSql + sqlLen) {
break;
}
src = ++pos;
pos = strchr(pos, '%');
if (pos == NULL) {
memcpy(dst, src, strlen(src));
break;
}
}
return TSDB_CODE_SUCCESS;
}
void httpCheckFreeEscapedSql(char *oldSql, char *newSql)
{
if (oldSql && newSql) {
if (oldSql != newSql) {
free(newSql);
}
}
}

13
src/plugins/http/src/httpTgHandle.c
View File

@ -610,7 +610,18 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) {
// stable tag for detail
for (int32_t i = 0; i < orderTagsLen; ++i) {
cJSON *tag = orderedTags[i];
stable_cmd->tagNames[i] = table_cmd->tagNames[i] = httpAddToSqlCmdBuffer(pContext, tag->string);
char *tagStr = NULL;
int32_t retCode = httpCheckAllocEscapeSql(tag->string, &tagStr);
if (retCode != TSDB_CODE_SUCCESS) {
httpSendErrorResp(pContext, retCode);
return false;
}
stable_cmd->tagNames[i] = table_cmd->tagNames[i] = httpAddToSqlCmdBuffer(pContext, tagStr);
httpCheckFreeEscapedSql(tag->string, tagStr);
if (tag->type == cJSON_String)
stable_cmd->tagValues[i] = table_cmd->tagValues[i] = httpAddToSqlCmdBuffer(pContext, "'%s'", tag->valuestring);

6
src/query/CMakeLists.txt
View File

@ -8,14 +8,14 @@ INCLUDE_DIRECTORIES(inc)
AUX_SOURCE_DIRECTORY(src SRC)
ADD_LIBRARY(query ${SRC})
SET_SOURCE_FILES_PROPERTIES(src/sql.c PROPERTIES COMPILE_FLAGS -w)
TARGET_LINK_LIBRARIES(query tsdb tutil)
TARGET_LINK_LIBRARIES(query tsdb tutil lua)
IF (TD_LINUX)
TARGET_LINK_LIBRARIES(query m rt)
TARGET_LINK_LIBRARIES(query m rt lua)
ADD_SUBDIRECTORY(tests)
ENDIF ()
IF (TD_DARWIN)
TARGET_LINK_LIBRARIES(query m)
TARGET_LINK_LIBRARIES(query m lua)
ADD_SUBDIRECTORY(tests)
ENDIF ()

11
src/query/inc/qAggMain.h
View File

@ -27,6 +27,7 @@ extern "C" {
#include "trpc.h"
#include "tvariant.h"
#include "tsdb.h"
#include "qUdf.h"
#define TSDB_FUNC_INVALID_ID -1
#define TSDB_FUNC_COUNT 0
@ -201,10 +202,8 @@ typedef struct SAggFunctionInfo {
int8_t stableFuncId; // transfer function for super table query
uint16_t status;
bool (*init)(SQLFunctionCtx *pCtx); // setup the execute environment
bool (*init)(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultCellInfo); // setup the execute environment
void (*xFunction)(SQLFunctionCtx *pCtx); // blocks version function
// void (*xFunctionF)(SQLFunctionCtx *pCtx, int32_t position); // single-row function version, todo merge with blockwise function
// finalizer must be called after all xFunction has been executed to generated final result.
void (*xFinalize)(SQLFunctionCtx *pCtx);
@ -216,7 +215,7 @@ typedef struct SAggFunctionInfo {
#define GET_RES_INFO(ctx) ((ctx)->resultInfo)
int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionId, int32_t param, int16_t *type,
int16_t *len, int32_t *interBytes, int16_t extLength, bool isSuperTable);
int16_t *len, int32_t *interBytes, int16_t extLength, bool isSuperTable, SUdfInfo* pUdfInfo);
int32_t isValidFunction(const char* name, int32_t len);
#define IS_STREAM_QUERY_VALID(x) (((x)&TSDB_FUNCSTATE_STREAM) != 0)
@ -262,9 +261,9 @@ bool topbot_datablock_filter(SQLFunctionCtx *pCtx, const char *minval, const cha
static FORCE_INLINE void initResultInfo(SResultRowCellInfo *pResInfo, int32_t bufLen) {
pResInfo->initialized = true; // the this struct has been initialized flag
pResInfo->complete = false;
pResInfo->complete = false;
pResInfo->hasResult = false;
pResInfo->numOfRes = 0;
pResInfo->numOfRes = 0;
memset(GET_ROWCELL_INTERBUF(pResInfo), 0, bufLen);
}

18
src/query/inc/qExecutor.h
View File

@ -29,6 +29,7 @@
#include "tarray.h"
#include "tlockfree.h"
#include "tsdb.h"
#include "qUdf.h"
struct SColumnFilterElem;
typedef bool (*__filter_func_t)(struct SColumnFilterElem* pFilter, const char* val1, const char* val2, int16_t type);
@ -257,6 +258,7 @@ typedef struct SQueryAttr {
SMemRef memRef;
STableGroupInfo tableGroupInfo; // table <tid, last_key> list SArray<STableKeyInfo>
int32_t vgId;
SArray *pUdfInfo; // no need to free
} SQueryAttr;
typedef SSDataBlock* (*__operator_fn_t)(void* param, bool* newgroup);
@ -297,6 +299,7 @@ typedef struct SQueryRuntimeEnv {
STableQueryInfo *current;
SRspResultInfo resultInfo;
SHashObj *pTableRetrieveTsMap;
SUdfInfo *pUdfInfo;
} SQueryRuntimeEnv;
enum {
@ -392,6 +395,7 @@ typedef struct SQueryParam {
SGroupbyExpr *pGroupbyExpr;
int32_t tableScanOperator;
SArray *pOperator;
SUdfInfo *pUdfInfo;
} SQueryParam;
typedef struct STableScanInfo {
@ -531,6 +535,7 @@ typedef struct SMultiwayMergeInfo {
bool hasPrev;
bool groupMix;
SArray *udfInfo;
} SMultiwayMergeInfo;
void appendUpstream(SOperatorInfo* p, SOperatorInfo* pUpstream);
@ -553,8 +558,8 @@ SOperatorInfo* createDistinctOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperat
SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv);
SOperatorInfo* createMultiwaySortOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput,
int32_t numOfRows, void* merger, bool groupMix);
SOperatorInfo* createGlobalAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, void* param, SArray* pUdfInfo);
SOperatorInfo* createStatewindowOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput);
SOperatorInfo* createGlobalAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, void* param);
SOperatorInfo* createSLimitOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, void* merger);
SOperatorInfo* createFilterOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr,
int32_t numOfOutput, SColumnInfo* pCols, int32_t numOfFilter);
@ -578,18 +583,19 @@ void setInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlo
int32_t getNumOfResult(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput);
void finalizeQueryResult(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset);
void updateOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity, int32_t numOfInputRows);
void clearOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity);
void freeParam(SQueryParam *param);
int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SQueryParam* param);
int32_t createQueryFunc(SQueriedTableInfo* pTableInfo, int32_t numOfOutput, SExprInfo** pExprInfo,
SSqlExpr** pExprMsg, SColumnInfo* pTagCols, int32_t queryType, void* pMsg);
SSqlExpr** pExprMsg, SColumnInfo* pTagCols, int32_t queryType, void* pMsg, SUdfInfo* pUdfInfo);
int32_t createIndirectQueryFuncExprFromMsg(SQueryTableMsg *pQueryMsg, int32_t numOfOutput, SExprInfo **pExprInfo,
SSqlExpr **pExpr, SExprInfo *prevExpr);
SSqlExpr **pExpr, SExprInfo *prevExpr, SUdfInfo *pUdfInfo);
SGroupbyExpr *createGroupbyExprFromMsg(SQueryTableMsg *pQueryMsg, SColIndex *pColIndex, int32_t *code);
SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SGroupbyExpr *pGroupbyExpr, SExprInfo *pExprs,
SExprInfo *pSecExprs, STableGroupInfo *pTableGroupInfo, SColumnInfo* pTagCols, int32_t vgId, char* sql, uint64_t *qId);
SExprInfo *pSecExprs, STableGroupInfo *pTableGroupInfo, SColumnInfo* pTagCols, int32_t vgId, char* sql, uint64_t qId, SUdfInfo* pUdfInfo);
int32_t initQInfo(STsBufInfo* pTsBufInfo, void* tsdb, void* sourceOptr, SQInfo* pQInfo, SQueryParam* param, char* start,
int32_t prevResultLen, void* merger);
@ -608,6 +614,8 @@ bool doBuildResCheck(SQInfo* pQInfo);
void setQueryStatus(SQueryRuntimeEnv *pRuntimeEnv, int8_t status);
bool onlyQueryTags(SQueryAttr* pQueryAttr);
void destroyUdfInfo(SUdfInfo* pUdfInfo);
bool isValidQInfo(void *param);
int32_t doDumpQueryResult(SQInfo *pQInfo, char *data);
@ -625,4 +633,6 @@ void freeQueryAttr(SQueryAttr *pQuery);
int32_t getMaximumIdleDurationSec();
void doInvokeUdf(SUdfInfo* pUdfInfo, SQLFunctionCtx *pCtx, int32_t idx, int32_t type);
#endif // TDENGINE_QEXECUTOR_H

82
src/query/inc/qScript.h Normal file
View File

@ -0,0 +1,82 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_QSCRIPT_H
#define TDENGINE_QSCRIPT_H
#include <lua.h>
#include <lauxlib.h>
#include <lualib.h>
#include "tutil.h"
#include "hash.h"
#include "tlist.h"
#include "qUdf.h"
#define MAX_FUNC_NAME 64
#define USER_FUNC_NAME "funcName"
#define USER_FUNC_NAME_LIMIT 48
enum ScriptState {
SCRIPT_STATE_INIT,
SCRIPT_STATE_ADD,
SCRIPT_STATE_MERGE,
SCRIPT_STATE_FINALIZE
};
typedef struct {
SHashObj *funcId; //func already registed in lua_env, may be no use
lua_State *lua_state; // lua env
} ScriptEnv;
typedef struct ScriptCtx {
char funcName[USER_FUNC_NAME_LIMIT];
int8_t state;
ScriptEnv *pEnv;
int8_t isAgg; // agg function or not
// init value of udf script
int8_t resType;
int16_t resBytes;
int32_t numOfOutput;
int32_t offset;
} ScriptCtx;
int taosLoadScriptInit(void *pInit);
void taosLoadScriptNormal(void *pInit, char *pInput, int16_t iType, int16_t iBytes, int32_t numOfRows,
int64_t *ptsList, int64_t key, char* pOutput, char *ptsOutput, int32_t *numOfOutput, int16_t oType, int16_t oBytes);
void taosLoadScriptFinalize(void *pInit, int64_t key, char *pOutput, int32_t *output);
void taosLoadScriptMerge(void *pCtx, char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput);
void taosLoadScriptDestroy(void *pInit);
typedef struct {
SList *scriptEnvs; //
int32_t mSize; // pool limit
int32_t cSize; // current available size
pthread_mutex_t mutex;
} ScriptEnvPool;
ScriptCtx* createScriptCtx(char *str, int8_t resType, int16_t resBytes);
void destroyScriptCtx(void *pScriptCtx);
int32_t scriptEnvPoolInit();
void scriptEnvPoolCleanup();
bool isValidScript(char *script, int32_t len);
#endif //TDENGINE_QSCRIPT_H

20
src/query/inc/qSqlparser.h
View File

@ -182,6 +182,15 @@ typedef struct SCreateDbInfo {
int16_t partitions;
} SCreateDbInfo;
typedef struct SCreateFuncInfo {
SStrToken name;
SStrToken path;
int32_t type;
int32_t bufSize;
TAOS_FIELD output;
} SCreateFuncInfo;
typedef struct SCreateAcctInfo {
int32_t maxUsers;
int32_t maxDbs;
@ -214,10 +223,11 @@ typedef struct SMiscInfo {
int16_t tableType;
SUserInfo user;
union {
SCreateDbInfo dbOpt;
SCreateAcctInfo acctOpt;
SShowInfo showOpt;
SStrToken id;
SCreateDbInfo dbOpt;
SCreateAcctInfo acctOpt;
SCreateFuncInfo funcOpt;
SShowInfo showOpt;
SStrToken id;
};
} SMiscInfo;
@ -226,6 +236,7 @@ typedef struct SSqlInfo {
bool valid;
SArray *list; // todo refactor
char msg[256];
SArray *funcs;
union {
SCreateTableSql *pCreateTableInfo;
SAlterTableInfo *pAlterInfo;
@ -271,6 +282,7 @@ SRelationInfo *addSubqueryElem(SRelationInfo* pRelationInfo, SArray* pSub, SStrT
// sql expr leaf node
tSqlExpr *tSqlExprCreateIdValue(SStrToken *pToken, int32_t optrType);
tSqlExpr *tSqlExprCreateFunction(SArray *pParam, SStrToken *pFuncToken, SStrToken *endToken, int32_t optType);
SArray *tStrTokenAppend(SArray *pList, SStrToken *pToken);
tSqlExpr *tSqlExprCreate(tSqlExpr *pLeft, tSqlExpr *pRight, int32_t optrType);
tSqlExpr *tSqlExprClone(tSqlExpr *pSrc);

14
src/query/inc/qTableMeta.h
View File

@ -106,11 +106,14 @@ typedef struct SQueryInfo {
STagCond tagCond;
SOrderVal order;
int16_t fillType; // final result fill type
int16_t numOfTables;
STableMetaInfo **pTableMetaInfo;
struct STSBuf *tsBuf;
int16_t fillType; // final result fill type
int64_t * fillVal; // default value for fill
int32_t numOfFillVal; // fill value size
char * msg; // pointer to the pCmd->payload to keep error message temporarily
int64_t clauseLimit; // limit for current sub clause
@ -122,8 +125,12 @@ typedef struct SQueryInfo {
int32_t round; // 0/1/....
int32_t bufLen;
char* buf;
struct SQInfo* pQInfo; // global merge operator
struct SQueryAttr* pQueryAttr; // query object
bool udfCopy;
SArray *pUdfInfo;
struct SQInfo *pQInfo; // global merge operator
struct SQueryAttr *pQueryAttr; // query object
struct SQueryInfo *sibling; // sibling
SArray *pUpstream; // SArray<struct SQueryInfo>
@ -138,6 +145,7 @@ typedef struct SQueryInfo {
bool onlyTagQuery;
bool orderProjectQuery;
bool stateWindow;
bool globalMerge;
} SQueryInfo;
/**

74
src/query/inc/qUdf.h Normal file
View File

@ -0,0 +1,74 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_QUDF_H
#define TDENGINE_QUDF_H
enum { TSDB_UDF_FUNC_NORMAL = 0, TSDB_UDF_FUNC_INIT, TSDB_UDF_FUNC_FINALIZE, TSDB_UDF_FUNC_MERGE, TSDB_UDF_FUNC_DESTROY, TSDB_UDF_FUNC_MAX_NUM };
typedef struct SUdfInit{
int32_t maybe_null; /* 1 if function can return NULL */
uint32_t decimals; /* for real functions */
uint64_t length; /* For string functions */
char *ptr; /* free pointer for function data */
int32_t const_item; /* 0 if result is independent of arguments */
// script like lua/javascript
void* script_ctx;
void (*destroyCtxFunc)(void *script_ctx);
} SUdfInit;
typedef struct SUdfInfo {
int32_t functionId; // system assigned function id
int32_t funcType; // scalar function or aggregate function
int8_t resType; // result type
int16_t resBytes; // result byte
int32_t contLen; // content length
int32_t bufSize; //interbuf size
char *name; // function name
void *handle; // handle loaded in mem
void *funcs[TSDB_UDF_FUNC_MAX_NUM]; // function ptr
// for script like lua/javascript only
int isScript;
void *pScriptCtx;
SUdfInit init;
char *content;
char *path;
} SUdfInfo;
//script
typedef int32_t (*scriptInitFunc)(void *pCtx);
typedef void (*scriptNormalFunc)(void *pCtx, char* data, int16_t iType, int16_t iBytes, int32_t numOfRows,
int64_t* ptList, int64_t key, char* dataOutput, char* tsOutput, int32_t* numOfOutput, int16_t oType, int16_t oBytes);
typedef void (*scriptFinalizeFunc)(void *pCtx, int64_t key, char* dataOutput, int32_t* numOfOutput);
typedef void (*scriptMergeFunc)(void *pCtx, char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput);
typedef void (*scriptDestroyFunc)(void* pCtx);
// dynamic lib
typedef void (*udfNormalFunc)(char* data, int16_t itype, int16_t iBytes, int32_t numOfRows, int64_t* ts, char* dataOutput, char* interBuf,
char* tsOutput, int32_t* numOfOutput, int16_t oType, int16_t oBytes, SUdfInit* buf);
typedef int32_t (*udfInitFunc)(SUdfInit* data);
typedef void (*udfFinalizeFunc)(char* dataOutput, char *interBuf, int32_t* numOfOutput, SUdfInit* buf);
typedef void (*udfMergeFunc)(char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput, SUdfInit* buf);
typedef void (*udfDestroyFunc)(SUdfInit* buf);
#endif // TDENGINE_QUDF_H

13
src/query/inc/qUtil.h
View File

@ -24,7 +24,18 @@
memcpy((_k) + sizeof(uint64_t), (_ori), (_len)); \
} while (0)
#define SET_RES_EXT_WINDOW_KEY(_k, _ori, _len, _uid, _buf) \
do { \
assert(sizeof(_uid) == sizeof(uint64_t)); \
*(void **)(_k) = (_buf); \
*(uint64_t *)((_k) + POINTER_BYTES) = (_uid); \
memcpy((_k) + POINTER_BYTES + sizeof(uint64_t), (_ori), (_len)); \
} while (0)
#define GET_RES_WINDOW_KEY_LEN(_l) ((_l) + sizeof(uint64_t))
#define GET_RES_EXT_WINDOW_KEY_LEN(_l) ((_l) + sizeof(uint64_t) + POINTER_BYTES)
#define GET_QID(_r) (((SQInfo*)((_r)->qinfo))->qId)
#define curTimeWindowIndex(_winres) ((_winres)->curIndex)
@ -93,4 +104,6 @@ int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo);
int32_t mergeIntoGroupResult(SGroupResInfo* pGroupResInfo, SQueryRuntimeEnv *pRuntimeEnv, int32_t* offset);
int32_t initUdfInfo(SUdfInfo* pUdfInfo);
#endif // TDENGINE_QUERYUTIL_H

11
src/query/inc/sql.y
View File

@ -65,6 +65,7 @@ program ::= cmd. {}
//////////////////////////////////THE SHOW STATEMENT///////////////////////////////////////////
cmd ::= SHOW DATABASES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_DB, 0, 0);}
cmd ::= SHOW TOPICS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_TP, 0, 0);}
cmd ::= SHOW FUNCTIONS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_FUNCTION, 0, 0);}
cmd ::= SHOW MNODES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_MNODE, 0, 0);}
cmd ::= SHOW DNODES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_DNODE, 0, 0);}
cmd ::= SHOW ACCOUNTS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_ACCT, 0, 0);}
@ -147,6 +148,7 @@ cmd ::= DROP STABLE ifexists(Y) ids(X) cpxName(Z). {
cmd ::= DROP DATABASE ifexists(Y) ids(X). { setDropDbTableInfo(pInfo, TSDB_SQL_DROP_DB, &X, &Y, TSDB_DB_TYPE_DEFAULT, -1); }
cmd ::= DROP TOPIC ifexists(Y) ids(X). { setDropDbTableInfo(pInfo, TSDB_SQL_DROP_DB, &X, &Y, TSDB_DB_TYPE_TOPIC, -1); }
cmd ::= DROP FUNCTION ids(X). { setDropFuncInfo(pInfo, TSDB_SQL_DROP_FUNCTION, &X); }
cmd ::= DROP DNODE ids(X). { setDCLSqlElems(pInfo, TSDB_SQL_DROP_DNODE, 1, &X); }
cmd ::= DROP USER ids(X). { setDCLSqlElems(pInfo, TSDB_SQL_DROP_USER, 1, &X); }
@ -200,8 +202,13 @@ cmd ::= CREATE ACCOUNT ids(X) PASS ids(Y) acct_optr(Z).
{ setCreateAcctSql(pInfo, TSDB_SQL_CREATE_ACCT, &X, &Y, &Z);}
cmd ::= CREATE DATABASE ifnotexists(Z) ids(X) db_optr(Y). { setCreateDbInfo(pInfo, TSDB_SQL_CREATE_DB, &X, &Y, &Z);}
cmd ::= CREATE TOPIC ifnotexists(Z) ids(X) topic_optr(Y). { setCreateDbInfo(pInfo, TSDB_SQL_CREATE_DB, &X, &Y, &Z);}
cmd ::= CREATE FUNCTION ids(X) AS ids(Y) OUTPUTTYPE typename(Z) bufsize(B). { setCreateFuncInfo(pInfo, TSDB_SQL_CREATE_FUNCTION, &X, &Y, &Z, &B, 1);}
cmd ::= CREATE AGGREGATE FUNCTION ids(X) AS ids(Y) OUTPUTTYPE typename(Z) bufsize(B). { setCreateFuncInfo(pInfo, TSDB_SQL_CREATE_FUNCTION, &X, &Y, &Z, &B, 2);}
cmd ::= CREATE USER ids(X) PASS ids(Y). { setCreateUserSql(pInfo, &X, &Y);}
bufsize(Y) ::= . { Y.n = 0; }
bufsize(Y) ::= BUFSIZE INTEGER(X). { Y = X; }
pps(Y) ::= . { Y.n = 0; }
pps(Y) ::= PPS INTEGER(X). { Y = X; }
@ -704,10 +711,10 @@ expr(A) ::= BOOL(X). { A = tSqlExprCreateIdValue(&X, TK_BOOL);}
expr(A) ::= NULL(X). { A = tSqlExprCreateIdValue(&X, TK_NULL);}
// ordinary functions: min(x), max(x), top(k, 20)
expr(A) ::= ID(X) LP exprlist(Y) RP(E). { A = tSqlExprCreateFunction(Y, &X, &E, X.type); }
expr(A) ::= ID(X) LP exprlist(Y) RP(E). { tStrTokenAppend(pInfo->funcs, &X); A = tSqlExprCreateFunction(Y, &X, &E, X.type); }
// for parsing sql functions with wildcard for parameters. e.g., count(*)/first(*)/last(*) operation
expr(A) ::= ID(X) LP STAR RP(Y). { A = tSqlExprCreateFunction(NULL, &X, &Y, X.type); }
expr(A) ::= ID(X) LP STAR RP(Y). { tStrTokenAppend(pInfo->funcs, &X); A = tSqlExprCreateFunction(NULL, &X, &Y, X.type); }
// is (not) null expression
expr(A) ::= expr(X) IS NULL. {A = tSqlExprCreate(X, NULL, TK_ISNULL);}

286
src/query/src/qAggMain.c
View File

@ -27,6 +27,7 @@
#include "qPercentile.h"
#include "qTsbuf.h"
#include "queryLog.h"
#include "qUdf.h"
#define GET_INPUT_DATA_LIST(x) ((char *)((x)->pInput))
#define GET_INPUT_DATA(x, y) (GET_INPUT_DATA_LIST(x) + (y) * (x)->inputBytes)
@ -58,7 +59,7 @@
for (int32_t _i = 0; _i < (ctx)->tagInfo.numOfTagCols; ++_i) { \
SQLFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[_i]; \
if (__ctx->functionId == TSDB_FUNC_TS_DUMMY) { \
__ctx->tag.i64 = (ts); \
__ctx->tag.i64 = (ts); \
__ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; \
} \
aAggs[TSDB_FUNC_TAG].xFunction(__ctx); \
@ -169,12 +170,13 @@ typedef struct SDerivInfo {
} SDerivInfo;
int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionId, int32_t param, int16_t *type,
int16_t *bytes, int32_t *interBytes, int16_t extLength, bool isSuperTable) {
int16_t *bytes, int32_t *interBytes, int16_t extLength, bool isSuperTable, SUdfInfo* pUdfInfo) {
if (!isValidDataType(dataType)) {
qError("Illegal data type %d or data type length %d", dataType, dataBytes);
return TSDB_CODE_TSC_INVALID_OPERATION;
}
if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TS_DUMMY || functionId == TSDB_FUNC_TAG_DUMMY ||
functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_PRJ || functionId == TSDB_FUNC_TAGPRJ ||
functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_INTERP) {
@ -234,6 +236,20 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI
}
if (isSuperTable) {
if (functionId < 0) {
if (pUdfInfo->bufSize > 0) {
*type = TSDB_DATA_TYPE_BINARY;
*bytes = pUdfInfo->bufSize;
*interBytes = *bytes;
} else {
*type = pUdfInfo->resType;
*bytes = pUdfInfo->resBytes;
*interBytes = *bytes;
}
return TSDB_CODE_SUCCESS;
}
if (functionId == TSDB_FUNC_MIN || functionId == TSDB_FUNC_MAX) {
*type = TSDB_DATA_TYPE_BINARY;
*bytes = (int16_t)(dataBytes + DATA_SET_FLAG_SIZE);
@ -288,7 +304,7 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI
return TSDB_CODE_SUCCESS;
}
}
if (functionId == TSDB_FUNC_SUM) {
if (IS_SIGNED_NUMERIC_TYPE(dataType)) {
*type = TSDB_DATA_TYPE_BIGINT;
@ -313,7 +329,20 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI
*interBytes = sizeof(STwaInfo);
return TSDB_CODE_SUCCESS;
}
if (functionId < 0) {
*type = pUdfInfo->resType;
*bytes = pUdfInfo->resBytes;
if (pUdfInfo->bufSize > 0) {
*interBytes = pUdfInfo->bufSize;
} else {
*interBytes = *bytes;
}
return TSDB_CODE_SUCCESS;
}
if (functionId == TSDB_FUNC_AVG) {
*type = TSDB_DATA_TYPE_DOUBLE;
*bytes = sizeof(double);
@ -390,14 +419,13 @@ int32_t isValidFunction(const char* name, int32_t len) {
return -1;
}
static bool function_setup(SQLFunctionCtx *pCtx) {
SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
if (pResInfo->initialized) {
static bool function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultInfo) {
if (pResultInfo->initialized) {
return false;
}
memset(pCtx->pOutput, 0, (size_t)pCtx->outputBytes);
initResultInfo(pResInfo, pCtx->interBufBytes);
initResultInfo(pResultInfo, pCtx->interBufBytes);
return true;
}
@ -520,7 +548,7 @@ int32_t noDataRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) {
if ((ctx)->hasNull && isNull((char *)&(list)[i], tsdbType)) { \
continue; \
} \
TSKEY key = GET_TS_DATA(ctx, i); \
TSKEY key = (ctx)->ptsList != NULL? GET_TS_DATA(ctx, i):0; \
UPDATE_DATA(ctx, val, (list)[i], num, sign, key); \
}
@ -985,8 +1013,8 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin,
}
}
static bool min_func_setup(SQLFunctionCtx *pCtx) {
if (!function_setup(pCtx)) {
static bool min_func_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultInfo) {
if (!function_setup(pCtx, pResultInfo)) {
return false; // not initialized since it has been initialized
}
@ -1030,8 +1058,8 @@ static bool min_func_setup(SQLFunctionCtx *pCtx) {
return true;
}
static bool max_func_setup(SQLFunctionCtx *pCtx) {
if (!function_setup(pCtx)) {
static bool max_func_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultInfo) {
if (!function_setup(pCtx, pResultInfo)) {
return false; // not initialized since it has been initialized
}
@ -1435,8 +1463,8 @@ static void stddev_dst_finalizer(SQLFunctionCtx *pCtx) {
}
//////////////////////////////////////////////////////////////////////////////////////
static bool first_last_function_setup(SQLFunctionCtx *pCtx) {
if (!function_setup(pCtx)) {
static bool first_last_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) {
if (!function_setup(pCtx, pResInfo)) {
return false;
}
@ -1463,10 +1491,11 @@ static void first_function(SQLFunctionCtx *pCtx) {
}
memcpy(pCtx->pOutput, data, pCtx->inputBytes);
TSKEY k = GET_TS_DATA(pCtx, i);
DO_UPDATE_TAG_COLUMNS(pCtx, k);
if (pCtx->ptsList != NULL) {
TSKEY k = GET_TS_DATA(pCtx, i);
DO_UPDATE_TAG_COLUMNS(pCtx, k);
}
SResultRowCellInfo *pInfo = GET_RES_INFO(pCtx);
pInfo->hasResult = DATA_SET_FLAG;
pInfo->complete = true;
@ -1605,7 +1634,7 @@ static void last_function(SQLFunctionCtx *pCtx) {
break;
}
}
SET_VAL(pCtx, notNullElems, 1);
}
@ -1758,6 +1787,49 @@ static void valuePairAssign(tValuePair *dst, int16_t type, const char *val, int6
memcpy((dst)->pTags, (src)->pTags, (size_t)(__l)); \
} while (0)
static int32_t topBotComparFn(const void *p1, const void *p2, const void *param)
{
uint16_t type = *(uint16_t *) param;
tValuePair *val1 = *(tValuePair **) p1;
tValuePair *val2 = *(tValuePair **) p2;
if (IS_SIGNED_NUMERIC_TYPE(type)) {
if (val1->v.i64 == val2->v.i64) {
return 0;
}
return (val1->v.i64 > val2->v.i64) ? 1 : -1;
} else if (IS_UNSIGNED_NUMERIC_TYPE(type)) {
if (val1->v.u64 == val2->v.u64) {
return 0;
}
return (val1->v.u64 > val2->v.u64) ? 1 : -1;
}
if (val1->v.dKey == val2->v.dKey) {
return 0;
}
return (val1->v.dKey > val2->v.dKey) ? 1 : -1;
}
static void topBotSwapFn(void *dst, void *src, const void *param)
{
char tag[32768];
tValuePair temp;
uint16_t tagLen = *(uint16_t *) param;
tValuePair *vdst = *(tValuePair **) dst;
tValuePair *vsrc = *(tValuePair **) src;
memset(tag, 0, sizeof(tag));
temp.pTags = tag;
VALUEPAIRASSIGN(&temp, vdst, tagLen);
VALUEPAIRASSIGN(vdst, vsrc, tagLen);
VALUEPAIRASSIGN(vsrc, &temp, tagLen);
}
static void do_top_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, int64_t ts, uint16_t type,
SExtTagsInfo *pTagInfo, char *pTags, int16_t stage) {
tVariant val = {0};
@ -1765,61 +1837,19 @@ static void do_top_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData,
tValuePair **pList = pInfo->res;
assert(pList != NULL);
if (pInfo->num < maxLen) {
if (pInfo->num == 0 ||
(IS_SIGNED_NUMERIC_TYPE(type) && val.i64 >= pList[pInfo->num - 1]->v.i64) ||
(IS_UNSIGNED_NUMERIC_TYPE(type) && val.u64 >= pList[pInfo->num - 1]->v.u64) ||
(IS_FLOAT_TYPE(type) && val.dKey >= pList[pInfo->num - 1]->v.dKey)) {
valuePairAssign(pList[pInfo->num], type, (const char*)&val.i64, ts, pTags, pTagInfo, stage);
} else {
int32_t i = pInfo->num - 1;
if (IS_SIGNED_NUMERIC_TYPE(type)) {
while (i >= 0 && pList[i]->v.i64 > val.i64) {
VALUEPAIRASSIGN(pList[i + 1], pList[i], pTagInfo->tagsLen);
i -= 1;
}
} else if (IS_UNSIGNED_NUMERIC_TYPE(type)) {
while (i >= 0 && pList[i]->v.u64 > val.u64) {
VALUEPAIRASSIGN(pList[i + 1], pList[i], pTagInfo->tagsLen);
i -= 1;
}
} else {
while (i >= 0 && pList[i]->v.dKey > val.dKey) {
VALUEPAIRASSIGN(pList[i + 1], pList[i], pTagInfo->tagsLen);
i -= 1;
}
}
valuePairAssign(pList[i + 1], type, (const char*) &val.i64, ts, pTags, pTagInfo, stage);
}
valuePairAssign(pList[pInfo->num], type, (const char *)&val.i64, ts, pTags, pTagInfo, stage);
taosheapsort((void *) pList, sizeof(tValuePair **), pInfo->num + 1, (const void *) &type, topBotComparFn, (const void *) &pTagInfo->tagsLen, topBotSwapFn, 0);
pInfo->num++;
} else {
int32_t i = 0;
if ((IS_SIGNED_NUMERIC_TYPE(type) && val.i64 > pList[0]->v.i64) ||
(IS_UNSIGNED_NUMERIC_TYPE(type) && val.u64 > pList[0]->v.u64) ||
(IS_FLOAT_TYPE(type) && val.dKey > pList[0]->v.dKey)) {
// find the appropriate the slot position
if (IS_SIGNED_NUMERIC_TYPE(type)) {
while (i + 1 < maxLen && pList[i + 1]->v.i64 < val.i64) {
VALUEPAIRASSIGN(pList[i], pList[i + 1], pTagInfo->tagsLen);
i += 1;
}
} if (IS_UNSIGNED_NUMERIC_TYPE(type)) {
while (i + 1 < maxLen && pList[i + 1]->v.u64 < val.u64) {
VALUEPAIRASSIGN(pList[i], pList[i + 1], pTagInfo->tagsLen);
i += 1;
}
} else {
while (i + 1 < maxLen && pList[i + 1]->v.dKey < val.dKey) {
VALUEPAIRASSIGN(pList[i], pList[i + 1], pTagInfo->tagsLen);
i += 1;
}
}
valuePairAssign(pList[i], type, (const char *)&val.i64, ts, pTags, pTagInfo, stage);
valuePairAssign(pList[0], type, (const char *)&val.i64, ts, pTags, pTagInfo, stage);
taosheapadjust((void *) pList, sizeof(tValuePair **), 0, maxLen - 1, (const void *) &type, topBotComparFn, (const void *) &pTagInfo->tagsLen, topBotSwapFn, 0);
}
}
}
@ -1833,57 +1863,17 @@ static void do_bottom_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pDa
assert(pList != NULL);
if (pInfo->num < maxLen) {
if (pInfo->num == 0) {
valuePairAssign(pList[pInfo->num], type, (const char*) &val.i64, ts, pTags, pTagInfo, stage);
} else {
int32_t i = pInfo->num - 1;
if (IS_SIGNED_NUMERIC_TYPE(type)) {
while (i >= 0 && pList[i]->v.i64 < val.i64) {
VALUEPAIRASSIGN(pList[i + 1], pList[i], pTagInfo->tagsLen);
i -= 1;
}
} else if (IS_UNSIGNED_NUMERIC_TYPE(type)) {
while (i >= 0 && pList[i]->v.u64 < val.u64) {
VALUEPAIRASSIGN(pList[i + 1], pList[i], pTagInfo->tagsLen);
i -= 1;
}
} else {
while (i >= 0 && pList[i]->v.dKey < val.dKey) {
VALUEPAIRASSIGN(pList[i + 1], pList[i], pTagInfo->tagsLen);
i -= 1;
}
}
valuePairAssign(pList[i + 1], type, (const char*)&val.i64, ts, pTags, pTagInfo, stage);
}
valuePairAssign(pList[pInfo->num], type, (const char *)&val.i64, ts, pTags, pTagInfo, stage);
taosheapsort((void *) pList, sizeof(tValuePair **), pInfo->num + 1, (const void *) &type, topBotComparFn, (const void *) &pTagInfo->tagsLen, topBotSwapFn, 1);
pInfo->num++;
} else {
int32_t i = 0;
if ((IS_SIGNED_NUMERIC_TYPE(type) && val.i64 < pList[0]->v.i64) ||
(IS_UNSIGNED_NUMERIC_TYPE(type) && val.u64 < pList[0]->v.u64) ||
(IS_FLOAT_TYPE(type) && val.dKey < pList[0]->v.dKey)) {
// find the appropriate the slot position
if (IS_SIGNED_NUMERIC_TYPE(type)) {
while (i + 1 < maxLen && pList[i + 1]->v.i64 > val.i64) {
VALUEPAIRASSIGN(pList[i], pList[i + 1], pTagInfo->tagsLen);
i += 1;
}
} if (IS_UNSIGNED_NUMERIC_TYPE(type)) {
while (i + 1 < maxLen && pList[i + 1]->v.u64 > val.u64) {
VALUEPAIRASSIGN(pList[i], pList[i + 1], pTagInfo->tagsLen);
i += 1;
}
} else {
while (i + 1 < maxLen && pList[i + 1]->v.dKey > val.dKey) {
VALUEPAIRASSIGN(pList[i], pList[i + 1], pTagInfo->tagsLen);
i += 1;
}
}
valuePairAssign(pList[i], type, (const char*)&val.i64, ts, pTags, pTagInfo, stage);
valuePairAssign(pList[0], type, (const char *)&val.i64, ts, pTags, pTagInfo, stage);
taosheapadjust((void *) pList, sizeof(tValuePair **), 0, maxLen - 1, (const void *) &type, topBotComparFn, (const void *) &pTagInfo->tagsLen, topBotSwapFn, 1);
}
}
}
@ -2045,7 +2035,7 @@ static void buildTopBotStruct(STopBotInfo *pTopBotInfo, SQLFunctionCtx *pCtx) {
char *tmp = (char *)pTopBotInfo + sizeof(STopBotInfo);
pTopBotInfo->res = (tValuePair**) tmp;
tmp += POINTER_BYTES * pCtx->param[0].i64;
size_t size = sizeof(tValuePair) + pCtx->tagInfo.tagsLen;
for (int32_t i = 0; i < pCtx->param[0].i64; ++i) {
@ -2111,14 +2101,13 @@ bool topbot_datablock_filter(SQLFunctionCtx *pCtx, const char *minval, const cha
}
}
static bool top_bottom_function_setup(SQLFunctionCtx *pCtx) {
if (!function_setup(pCtx)) {
static bool top_bottom_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) {
if (!function_setup(pCtx, pResInfo)) {
return false;
}
STopBotInfo *pInfo = getTopBotOutputInfo(pCtx);
buildTopBotStruct(pInfo, pCtx);
return true;
}
@ -2267,14 +2256,13 @@ static void top_bottom_func_finalizer(SQLFunctionCtx *pCtx) {
}
///////////////////////////////////////////////////////////////////////////////////////////////
static bool percentile_function_setup(SQLFunctionCtx *pCtx) {
if (!function_setup(pCtx)) {
static bool percentile_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultInfo) {
if (!function_setup(pCtx, pResultInfo)) {
return false;
}
// in the first round, get the min-max value of all involved data
SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
SPercentileInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
SPercentileInfo *pInfo = GET_ROWCELL_INTERBUF(pResultInfo);
SET_DOUBLE_VAL(&pInfo->minval, DBL_MAX);
SET_DOUBLE_VAL(&pInfo->maxval, -DBL_MAX);
pInfo->numOfElems = 0;
@ -2405,8 +2393,8 @@ static SAPercentileInfo *getAPerctInfo(SQLFunctionCtx *pCtx) {
return pInfo;
}
static bool apercentile_function_setup(SQLFunctionCtx *pCtx) {
if (!function_setup(pCtx)) {
static bool apercentile_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultInfo) {
if (!function_setup(pCtx, pResultInfo)) {
return false;
}
@ -2515,12 +2503,11 @@ static void apercentile_finalizer(SQLFunctionCtx *pCtx) {
}
/////////////////////////////////////////////////////////////////////////////////
static bool leastsquares_function_setup(SQLFunctionCtx *pCtx) {
if (!function_setup(pCtx)) {
static bool leastsquares_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) {
if (!function_setup(pCtx, pResInfo)) {
return false;
}
SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx);
SLeastsquaresInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
// 2*3 matrix
@ -2744,8 +2731,8 @@ enum {
INITIAL_VALUE_NOT_ASSIGNED = 0,
};
static bool diff_function_setup(SQLFunctionCtx *pCtx) {
if (!function_setup(pCtx)) {
static bool diff_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) {
if (!function_setup(pCtx, pResInfo)) {
return false;
}
@ -2754,14 +2741,13 @@ static bool diff_function_setup(SQLFunctionCtx *pCtx) {
return false;
}
static bool deriv_function_setup(SQLFunctionCtx *pCtx) {
if (!function_setup(pCtx)) {
static bool deriv_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResultInfo) {
if (!function_setup(pCtx, pResultInfo)) {
return false;
}
// diff function require the value is set to -1
SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
SDerivInfo* pDerivInfo = GET_ROWCELL_INTERBUF(pResInfo);
SDerivInfo* pDerivInfo = GET_ROWCELL_INTERBUF(pResultInfo);
pDerivInfo->ignoreNegative = pCtx->param[1].i64;
pDerivInfo->prevTs = -1;
@ -3168,12 +3154,12 @@ static void arithmetic_function(SQLFunctionCtx *pCtx) {
}
/////////////////////////////////////////////////////////////////////////////////
static bool spread_function_setup(SQLFunctionCtx *pCtx) {
if (!function_setup(pCtx)) {
static bool spread_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) {
if (!function_setup(pCtx, pResInfo)) {
return false;
}
SSpreadInfo *pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
SSpreadInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
// this is the server-side setup function in client-side, the secondary merge do not need this procedure
if (pCtx->currentStage == MERGE_STAGE) {
@ -3327,12 +3313,10 @@ void spread_function_finalizer(SQLFunctionCtx *pCtx) {
* param[2]: end time
* @param pCtx
*/
static bool twa_function_setup(SQLFunctionCtx *pCtx) {
if (!function_setup(pCtx)) {
static bool twa_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) {
if (!function_setup(pCtx, pResInfo)) {
return false;
}
SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
pInfo->p.key = INT64_MIN;
@ -3764,14 +3748,12 @@ static void interp_function(SQLFunctionCtx *pCtx) {
}
}
static bool ts_comp_function_setup(SQLFunctionCtx *pCtx) {
if (!function_setup(pCtx)) {
static bool ts_comp_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) {
if (!function_setup(pCtx, pResInfo)) {
return false; // not initialized since it has been initialized
}
SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
STSCompInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
pInfo->pTSBuf = tsBufCreate(false, pCtx->order);
pInfo->pTSBuf->tsOrder = pCtx->order;
return true;
@ -3854,13 +3836,11 @@ static double do_calc_rate(const SRateInfo* pRateInfo, double tickPerSec) {
return (duration > 0)? ((double)diff) / (duration/tickPerSec):0.0;
}
static bool rate_function_setup(SQLFunctionCtx *pCtx) {
if (!function_setup(pCtx)) {
static bool rate_function_setup(SQLFunctionCtx *pCtx, SResultRowCellInfo* pResInfo) {
if (!function_setup(pCtx, pResInfo)) {
return false;
}
SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
SRateInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo);
pInfo->correctionValue = 0;
pInfo->firstKey = INT64_MIN;
@ -3958,7 +3938,7 @@ static void irate_function(SQLFunctionCtx *pCtx) {
int32_t notNullElems = 0;
SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo);
TSKEY *primaryKey = GET_TS_LIST(pCtx);
for (int32_t i = pCtx->size - 1; i >= 0; --i) {
char *pData = GET_INPUT_DATA(pCtx, i);
if (pCtx->hasNull && isNull(pData, pCtx->inputType)) {

497
src/query/src/qExecutor.c
View File

@ -29,10 +29,11 @@
#include "ttype.h"
#include "tcompare.h"
#include "tscompression.h"
#include "qScript.h"
#define IS_MASTER_SCAN(runtime) ((runtime)->scanFlag == MASTER_SCAN)
#define IS_REVERSE_SCAN(runtime) ((runtime)->scanFlag == REVERSE_SCAN)
#define IS_REPEAT_SCAN(runtime) ((runtime)->scanFlag == REPEAT_SCAN)
#define IS_REPEAT_SCAN(runtime) ((runtime)->scanFlag == REPEAT_SCAN)
#define SET_MASTER_SCAN_FLAG(runtime) ((runtime)->scanFlag = MASTER_SCAN)
#define SET_REVERSE_SCAN_FLAG(runtime) ((runtime)->scanFlag = REVERSE_SCAN)
@ -212,24 +213,24 @@ SArray* getOrderCheckColumns(SQueryAttr* pQuery);
typedef struct SRowCompSupporter {
SQueryRuntimeEnv *pRuntimeEnv;
SQueryRuntimeEnv *pRuntimeEnv;
int16_t dataOffset;
__compar_fn_t comFunc;
__compar_fn_t comFunc;
} SRowCompSupporter;
static int compareRowData(const void *a, const void *b, const void *userData) {
const SResultRow *pRow1 = (const SResultRow *)a;
const SResultRow *pRow1 = (const SResultRow *)a;
const SResultRow *pRow2 = (const SResultRow *)b;
SRowCompSupporter *supporter = (SRowCompSupporter *)userData;
SQueryRuntimeEnv* pRuntimeEnv = supporter->pRuntimeEnv;
SRowCompSupporter *supporter = (SRowCompSupporter *)userData;
SQueryRuntimeEnv* pRuntimeEnv = supporter->pRuntimeEnv;
tFilePage *page1 = getResBufPage(pRuntimeEnv->pResultBuf, pRow1->pageId);
tFilePage *page2 = getResBufPage(pRuntimeEnv->pResultBuf, pRow2->pageId);
int16_t offset = supporter->dataOffset;
int16_t offset = supporter->dataOffset;
char *in1 = getPosInResultPage(pRuntimeEnv->pQueryAttr, page1, pRow1->offset, offset);
char *in2 = getPosInResultPage(pRuntimeEnv->pQueryAttr, page2, pRow2->offset, offset);
char *in2 = getPosInResultPage(pRuntimeEnv->pQueryAttr, page2, pRow2->offset, offset);
return (in1 != NULL && in2 != NULL) ? supporter->comFunc(in1, in2) : 0;
}
@ -250,7 +251,7 @@ static void sortGroupResByOrderList(SGroupResInfo *pGroupResInfo, SQueryRuntimeE
bool found = false;
int16_t dataOffset = 0;
for (int32_t j = 0; j < pDataBlock->info.numOfCols; ++j) {
SColumnInfoData* pColInfoData = (SColumnInfoData *)taosArrayGet(pDataBlock->pDataBlock, j);
if (orderId == j) {
@ -259,15 +260,15 @@ static void sortGroupResByOrderList(SGroupResInfo *pGroupResInfo, SQueryRuntimeE
}
dataOffset += pColInfoData->info.bytes;
}
}
if (found == false) {
return;
}
int16_t type = pRuntimeEnv->pQueryAttr->pExpr1[orderId].base.resType;
SRowCompSupporter support = {.pRuntimeEnv = pRuntimeEnv, .dataOffset = dataOffset, .comFunc = getComparFunc(type, 0)};
SRowCompSupporter support = {.pRuntimeEnv = pRuntimeEnv, .dataOffset = dataOffset, .comFunc = getComparFunc(type, 0)};
taosArraySortPWithExt(pGroupResInfo->pRows, compareRowData, &support);
}
@ -433,8 +434,8 @@ static SResultRow* doSetResultOutBufByKey(SQueryRuntimeEnv* pRuntimeEnv, SResult
existed = (pResultRowInfo->pResult[0] == (*p1));
pResultRowInfo->curPos = 0;
} else { // check if current pResultRowInfo contains the existed pResultRow
SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, tid);
int64_t* index = taosHashGet(pRuntimeEnv->pResultRowListSet, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
SET_RES_EXT_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, tid, pResultRowInfo);
int64_t* index = taosHashGet(pRuntimeEnv->pResultRowListSet, pRuntimeEnv->keyBuf, GET_RES_EXT_WINDOW_KEY_LEN(bytes));
if (index != NULL) {
pResultRowInfo->curPos = (int32_t) *index;
existed = true;
@ -471,8 +472,8 @@ static SResultRow* doSetResultOutBufByKey(SQueryRuntimeEnv* pRuntimeEnv, SResult
pResultRowInfo->pResult[pResultRowInfo->size++] = pResult;
int64_t index = pResultRowInfo->curPos;
SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, tid);
taosHashPut(pRuntimeEnv->pResultRowListSet, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), &index, POINTER_BYTES);
SET_RES_EXT_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, tid, pResultRowInfo);
taosHashPut(pRuntimeEnv->pResultRowListSet, pRuntimeEnv->keyBuf, GET_RES_EXT_WINDOW_KEY_LEN(bytes), &index, POINTER_BYTES);
}
// too many time window in query
@ -780,6 +781,79 @@ static int32_t getNumOfRowsInTimeWindow(SQueryRuntimeEnv* pRuntimeEnv, SDataBloc
return num;
}
void doInvokeUdf(SUdfInfo* pUdfInfo, SQLFunctionCtx *pCtx, int32_t idx, int32_t type) {
int32_t output = 0;
if (pUdfInfo == NULL || pUdfInfo->funcs[type] == NULL) {
qError("empty udf function, type:%d", type);
return;
}
qDebug("invoke udf function:%s,%p", pUdfInfo->name, pUdfInfo->funcs[type]);
switch (type) {
case TSDB_UDF_FUNC_NORMAL:
if (pUdfInfo->isScript) {
(*(scriptNormalFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_NORMAL])(pUdfInfo->pScriptCtx,
(char *)pCtx->pInput + idx * pCtx->inputType, pCtx->inputType, pCtx->inputBytes, pCtx->size, pCtx->ptsList, pCtx->startTs, pCtx->pOutput,
(char *)pCtx->ptsOutputBuf, &output, pCtx->outputType, pCtx->outputBytes);
} else {
SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
void *interBuf = (void *)GET_ROWCELL_INTERBUF(pResInfo);
(*(udfNormalFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_NORMAL])((char *)pCtx->pInput + idx * pCtx->inputType, pCtx->inputType, pCtx->inputBytes, pCtx->size, pCtx->ptsList,
pCtx->pOutput, interBuf, (char *)pCtx->ptsOutputBuf, &output, pCtx->outputType, pCtx->outputBytes, &pUdfInfo->init);
}
if (pUdfInfo->funcType == TSDB_UDF_TYPE_AGGREGATE) {
pCtx->resultInfo->numOfRes = output;
} else {
pCtx->resultInfo->numOfRes += output;
}
if (pCtx->resultInfo->numOfRes > 0) {
pCtx->resultInfo->hasResult = DATA_SET_FLAG;
}
break;
case TSDB_UDF_FUNC_MERGE:
if (pUdfInfo->isScript) {
(*(scriptMergeFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_MERGE])(pUdfInfo->pScriptCtx, pCtx->pInput, pCtx->size, pCtx->pOutput, &output);
} else {
(*(udfMergeFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_MERGE])(pCtx->pInput, pCtx->size, pCtx->pOutput, &output, &pUdfInfo->init);
}
// set the output value exist
pCtx->resultInfo->numOfRes = output;
if (output > 0) {
pCtx->resultInfo->hasResult = DATA_SET_FLAG;
}
break;
case TSDB_UDF_FUNC_FINALIZE: {
SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
void *interBuf = (void *)GET_ROWCELL_INTERBUF(pResInfo);
if (pUdfInfo->isScript) {
(*(scriptFinalizeFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE])(pUdfInfo->pScriptCtx, pCtx->startTs, pCtx->pOutput, &output);
} else {
(*(udfFinalizeFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE])(pCtx->pOutput, interBuf, &output, &pUdfInfo->init);
}
// set the output value exist
pCtx->resultInfo->numOfRes = output;
if (output > 0) {
pCtx->resultInfo->hasResult = DATA_SET_FLAG;
}
break;
}
}
return;
}
static void doApplyFunctions(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, STimeWindow* pWin, int32_t offset,
int32_t forwardStep, TSKEY* tsCol, int32_t numOfTotal, int32_t numOfOutput) {
SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr;
@ -789,7 +863,7 @@ static void doApplyFunctions(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx
pCtx[k].size = forwardStep;
pCtx[k].startTs = pWin->skey;
// keep it temprarily
// keep it temporarialy
char* start = pCtx[k].pInput;
int32_t pos = (QUERY_IS_ASC_QUERY(pQueryAttr)) ? offset : offset - (forwardStep - 1);
@ -807,8 +881,14 @@ static void doApplyFunctions(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx
pCtx[k].preAggVals.isSet = false;
}
int32_t functionId = pCtx[k].functionId;
if (functionNeedToExecute(pRuntimeEnv, &pCtx[k])) {
aAggs[pCtx[k].functionId].xFunction(&pCtx[k]);
if (functionId < 0) { // load the script and exec, pRuntimeEnv->pUdfInfo
SUdfInfo* pUdfInfo = pRuntimeEnv->pUdfInfo;
doInvokeUdf(pUdfInfo, &pCtx[k], 0, TSDB_UDF_FUNC_NORMAL);
} else {
aAggs[functionId].xFunction(&pCtx[k]);
}
}
// restore it
@ -951,7 +1031,9 @@ static TSKEY getStartTsKey(SQueryAttr* pQueryAttr, STimeWindow* win, const TSKEY
static void setArithParams(SArithmeticSupport* sas, SExprInfo *pExprInfo, SSDataBlock* pSDataBlock) {
sas->numOfCols = (int32_t) pSDataBlock->info.numOfCols;
sas->pExprInfo = pExprInfo;
if (sas->colList != NULL) {
return;
}
sas->colList = calloc(1, pSDataBlock->info.numOfCols*sizeof(SColumnInfo));
for(int32_t i = 0; i < sas->numOfCols; ++i) {
SColumnInfoData* pColData = taosArrayGet(pSDataBlock->pDataBlock, i);
@ -1016,6 +1098,13 @@ static void doSetInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx,
pCtx[i].pInput = p->pData;
assert(p->info.colId == pColIndex->colId && pCtx[i].inputType == p->info.type);
if (pCtx[i].functionId < 0) {
SColumnInfoData* tsInfo = taosArrayGet(pBlock->pDataBlock, 0);
pCtx[i].ptsList = (int64_t*) tsInfo->pData;
continue;
}
uint32_t status = aAggs[pCtx[i].functionId].status;
if ((status & (TSDB_FUNCSTATE_SELECTIVITY | TSDB_FUNCSTATE_NEED_TS)) != 0) {
SColumnInfoData* tsInfo = taosArrayGet(pBlock->pDataBlock, 0);
@ -1048,7 +1137,14 @@ static void doAggregateImpl(SOperatorInfo* pOperator, TSKEY startTs, SQLFunction
for (int32_t k = 0; k < pOperator->numOfOutput; ++k) {
if (functionNeedToExecute(pRuntimeEnv, &pCtx[k])) {
pCtx[k].startTs = startTs;// this can be set during create the struct
aAggs[pCtx[k].functionId].xFunction(&pCtx[k]);
int32_t functionId = pCtx[k].functionId;
if (functionId < 0) {
SUdfInfo* pUdfInfo = pRuntimeEnv->pUdfInfo;
doInvokeUdf(pUdfInfo, &pCtx[k], 0, TSDB_UDF_FUNC_NORMAL);
} else {
aAggs[functionId].xFunction(&pCtx[k]);
}
}
}
}
@ -1064,7 +1160,15 @@ static void projectApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx
pCtx[k].order = TSDB_ORDER_ASC;
}
aAggs[pCtx[k].functionId].xFunction(&pCtx[k]);
pCtx[k].startTs = pQueryAttr->window.skey;
if (pCtx[k].functionId < 0) {
// load the script and exec
SUdfInfo* pUdfInfo = pRuntimeEnv->pUdfInfo;
doInvokeUdf(pUdfInfo, &pCtx[k], 0, TSDB_UDF_FUNC_NORMAL);
} else {
aAggs[pCtx[k].functionId].xFunction(&pCtx[k]);
}
}
}
@ -1790,7 +1894,7 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOf
pRuntimeEnv->pResultRowHashTable = taosHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
pRuntimeEnv->pResultRowListSet = taosHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
pRuntimeEnv->keyBuf = malloc(pQueryAttr->maxTableColumnWidth + sizeof(int64_t));
pRuntimeEnv->keyBuf = malloc(pQueryAttr->maxTableColumnWidth + sizeof(int64_t) + POINTER_BYTES);
pRuntimeEnv->pool = initResultRowPool(getResultRowSize(pRuntimeEnv));
pRuntimeEnv->prevRow = malloc(POINTER_BYTES * pQueryAttr->numOfCols + pQueryAttr->srcRowSize);
@ -1951,7 +2055,7 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOf
case OP_GlobalAggregate: {
pRuntimeEnv->proot = createGlobalAggregateOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr3,
pQueryAttr->numOfExpr3, merger);
pQueryAttr->numOfExpr3, merger, pQueryAttr->pUdfInfo);
break;
}
@ -2022,6 +2126,8 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) {
tfree(pRuntimeEnv->sasArray);
}
destroyUdfInfo(pRuntimeEnv->pUdfInfo);
destroyResultBuf(pRuntimeEnv->pResultBuf);
doFreeQueryHandle(pRuntimeEnv);
@ -2232,7 +2338,7 @@ static bool onlyFirstQuery(SQueryAttr *pQueryAttr) { return onlyOneQueryType(pQu
static bool onlyLastQuery(SQueryAttr *pQueryAttr) { return onlyOneQueryType(pQueryAttr, TSDB_FUNC_LAST, TSDB_FUNC_LAST_DST); }
static bool notContainSessionOrStateWindow(SQueryAttr *pQueryAttr) { return !(pQueryAttr->sw.gap > 0 || pQueryAttr->stateWindow); }
static bool notContainSessionOrStateWindow(SQueryAttr *pQueryAttr) { return !(pQueryAttr->sw.gap > 0 || pQueryAttr->stateWindow); }
static int32_t updateBlockLoadStatus(SQueryAttr *pQuery, int32_t status) {
bool hasFirstLastFunc = false;
@ -2268,10 +2374,11 @@ static int32_t updateBlockLoadStatus(SQueryAttr *pQuery, int32_t status) {
return status;
}
static void doExchangeTimeWindow(SQInfo* pQInfo, STimeWindow* win) {
SQueryAttr* pQueryAttr = &pQInfo->query;
size_t t = taosArrayGetSize(pQueryAttr->tableGroupInfo.pGroupList);
for(int32_t i = 0; i < t; ++i) {
static void doUpdateLastKey(SQueryAttr* pQueryAttr) {
STimeWindow* win = &pQueryAttr->window;
size_t num = taosArrayGetSize(pQueryAttr->tableGroupInfo.pGroupList);
for(int32_t i = 0; i < num; ++i) {
SArray* p1 = taosArrayGetP(pQueryAttr->tableGroupInfo.pGroupList, i);
size_t len = taosArrayGetSize(p1);
@ -2286,7 +2393,7 @@ static void doExchangeTimeWindow(SQInfo* pQInfo, STimeWindow* win) {
}
}
static void changeExecuteScanOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bool stableQuery) {
static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bool stableQuery) {
SQueryAttr* pQueryAttr = pQInfo->runtimeEnv.pQueryAttr;
// in case of point-interpolation query, use asc order scan
@ -2303,6 +2410,7 @@ static void changeExecuteScanOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bo
SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY);
}
pQueryAttr->needReverseScan = false;
return;
}
@ -2312,7 +2420,8 @@ static void changeExecuteScanOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bo
SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY);
}
doExchangeTimeWindow(pQInfo, &pQueryAttr->window);
pQueryAttr->needReverseScan = false;
doUpdateLastKey(pQueryAttr);
return;
}
@ -2333,20 +2442,22 @@ static void changeExecuteScanOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bo
pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey);
SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY);
doExchangeTimeWindow(pQInfo, &pQueryAttr->window);
doUpdateLastKey(pQueryAttr);
}
pQueryAttr->order.order = TSDB_ORDER_ASC;
pQueryAttr->needReverseScan = false;
} else if (onlyLastQuery(pQueryAttr) && notContainSessionOrStateWindow(pQueryAttr)) {
if (QUERY_IS_ASC_QUERY(pQueryAttr)) {
qDebug(msg, pQInfo, "only-last", pQueryAttr->order.order, TSDB_ORDER_DESC, pQueryAttr->window.skey,
pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey);
SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY);
doExchangeTimeWindow(pQInfo, &pQueryAttr->window);
doUpdateLastKey(pQueryAttr);
}
pQueryAttr->order.order = TSDB_ORDER_DESC;
pQueryAttr->needReverseScan = false;
}
} else { // interval query
@ -2357,20 +2468,22 @@ static void changeExecuteScanOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bo
pQueryAttr->window.skey, pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey);
SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY);
doExchangeTimeWindow(pQInfo, &pQueryAttr->window);
doUpdateLastKey(pQueryAttr);
}
pQueryAttr->order.order = TSDB_ORDER_ASC;
pQueryAttr->needReverseScan = false;
} else if (onlyLastQuery(pQueryAttr)) {
if (QUERY_IS_ASC_QUERY(pQueryAttr)) {
qDebug(msg, pQInfo, "only-last stable", pQueryAttr->order.order, TSDB_ORDER_DESC,
pQueryAttr->window.skey, pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey);
SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY);
doExchangeTimeWindow(pQInfo, &pQueryAttr->window);
doUpdateLastKey(pQueryAttr);
}
pQueryAttr->order.order = TSDB_ORDER_DESC;
pQueryAttr->needReverseScan = false;
}
}
}
@ -2388,9 +2501,6 @@ static void getIntermediateBufInfo(SQueryRuntimeEnv* pRuntimeEnv, int32_t* ps, i
while(((*rowsize) * MIN_ROWS_PER_PAGE) > (*ps) - overhead) {
*ps = ((*ps) << 1u);
}
// pRuntimeEnv->numOfRowsPerPage = ((*ps) - sizeof(tFilePage)) / (*rowsize);
// assert(pRuntimeEnv->numOfRowsPerPage <= MAX_ROWS_PER_RESBUF_PAGE);
}
#define IS_PREFILTER_TYPE(_t) ((_t) != TSDB_DATA_TYPE_BINARY && (_t) != TSDB_DATA_TYPE_NCHAR)
@ -2438,14 +2548,14 @@ static bool doFilterByBlockStatistics(SQueryRuntimeEnv* pRuntimeEnv, SDataStatis
}
SDataStatis* pDataBlockst = &pDataStatis[index];
if (pFilterInfo->info.type == TSDB_DATA_TYPE_FLOAT) {
float minval = (float)(*(double *)(&pDataBlockst->min));
float maxval = (float)(*(double *)(&pDataBlockst->max));
for (int32_t i = 0; i < pFilterInfo->numOfFilters; ++i) {
if (pFilterInfo->pFilters[i].filterInfo.lowerRelOptr == TSDB_RELATION_IN) {
continue;
continue;
}
ret &= pFilterInfo->pFilters[i].fp(&pFilterInfo->pFilters[i], (char *)&minval, (char *)&maxval, TSDB_DATA_TYPE_FLOAT);
if (ret == false) {
@ -2455,9 +2565,9 @@ static bool doFilterByBlockStatistics(SQueryRuntimeEnv* pRuntimeEnv, SDataStatis
} else {
for (int32_t i = 0; i < pFilterInfo->numOfFilters; ++i) {
if (pFilterInfo->pFilters[i].filterInfo.lowerRelOptr == TSDB_RELATION_IN) {
continue;
continue;
}
ret &= pFilterInfo->pFilters[i].fp(&pFilterInfo->pFilters[i], (char *)&pDataBlockst->min, (char *)&pDataBlockst->max, pFilterInfo->info.type);
ret &= pFilterInfo->pFilters[i].fp(&pFilterInfo->pFilters[i], (char *)&pDataBlockst->min, (char *)&pDataBlockst->max, pFilterInfo->info.type);
if (ret == false) {
return false;
}
@ -2697,9 +2807,14 @@ static uint32_t doFilterByBlockTimeWindow(STableScanInfo* pTableScanInfo, SSData
int32_t colId = pTableScanInfo->pExpr[i].base.colInfo.colId;
// group by + first/last should not apply the first/last block filter
status |= aAggs[functionId].dataReqFunc(&pTableScanInfo->pCtx[i], &pBlock->info.window, colId);
if ((status & BLK_DATA_ALL_NEEDED) == BLK_DATA_ALL_NEEDED) {
if (functionId < 0) {
status |= BLK_DATA_ALL_NEEDED;
return status;
} else {
status |= aAggs[functionId].dataReqFunc(&pTableScanInfo->pCtx[i], &pBlock->info.window, colId);
if ((status & BLK_DATA_ALL_NEEDED) == BLK_DATA_ALL_NEEDED) {
return status;
}
}
}
@ -3004,8 +3119,8 @@ void setTagValue(SOperatorInfo* pOperatorInfo, void *pTable, SQLFunctionCtx* pCt
doSetTagValueInParam(pTable, pLocalExprInfo->base.colInfo.colId, &pCtx[idx].tag, pLocalExprInfo->base.resType,
pLocalExprInfo->base.resBytes);
if (IS_NUMERIC_TYPE(pLocalExprInfo->base.resType)
|| pLocalExprInfo->base.resType == TSDB_DATA_TYPE_BOOL
if (IS_NUMERIC_TYPE(pLocalExprInfo->base.resType)
|| pLocalExprInfo->base.resType == TSDB_DATA_TYPE_BOOL
|| pLocalExprInfo->base.resType == TSDB_DATA_TYPE_TIMESTAMP) {
memcpy(pRuntimeEnv->tagVal + offset, &pCtx[idx].tag.i64, pLocalExprInfo->base.resBytes);
} else {
@ -3293,6 +3408,21 @@ void updateOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity, int32_t numOf
}
}
void clearOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity) {
SSDataBlock* pDataBlock = pBInfo->pRes;
for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) {
SColumnInfoData *pColInfo = taosArrayGet(pDataBlock->pDataBlock, i);
int32_t functionId = pBInfo->pCtx[i].functionId;
if (functionId < 0) {
memset(pBInfo->pCtx[i].pOutput, 0, pColInfo->info.bytes * (*bufCapacity));
}
}
}
void initCtxOutputBuffer(SQLFunctionCtx* pCtx, int32_t size) {
for (int32_t j = 0; j < size; ++j) {
SResultRowCellInfo* pResInfo = GET_RES_INFO(&pCtx[j]);
@ -3300,7 +3430,11 @@ void initCtxOutputBuffer(SQLFunctionCtx* pCtx, int32_t size) {
continue;
}
aAggs[pCtx[j].functionId].init(&pCtx[j]);
if (pCtx[j].functionId < 0) { // todo udf initialization
continue;
} else {
aAggs[pCtx[j].functionId].init(&pCtx[j], pCtx[j].resultInfo);
}
}
}
@ -3355,9 +3489,15 @@ void finalizeQueryResult(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SResult
setResultOutputBuf(pRuntimeEnv, buf, pCtx, numOfOutput, rowCellInfoOffset);
for (int32_t j = 0; j < numOfOutput; ++j) {
aAggs[pCtx[j].functionId].xFinalize(&pCtx[j]);
pCtx[j].startTs = buf->win.skey;
if (pCtx[j].functionId < 0) {
doInvokeUdf(pRuntimeEnv->pUdfInfo, &pCtx[j], 0, TSDB_UDF_FUNC_FINALIZE);
} else {
aAggs[pCtx[j].functionId].xFinalize(&pCtx[j]);
}
}
/*
* set the number of output results for group by normal columns, the number of output rows usually is 1 except
* the top and bottom query
@ -3367,7 +3507,11 @@ void finalizeQueryResult(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SResult
} else {
for (int32_t j = 0; j < numOfOutput; ++j) {
aAggs[pCtx[j].functionId].xFinalize(&pCtx[j]);
if (pCtx[j].functionId < 0) {
doInvokeUdf(pRuntimeEnv->pUdfInfo, &pCtx[j], 0, TSDB_UDF_FUNC_FINALIZE);
} else {
aAggs[pCtx[j].functionId].xFinalize(&pCtx[j]);
}
}
}
}
@ -3453,12 +3597,16 @@ void setResultRowOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pRe
offset += pCtx[i].outputBytes;
int32_t functionId = pCtx[i].functionId;
if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_DERIVATIVE) {
if (functionId < 0) {
continue;
}
if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) {
pCtx[i].ptsOutputBuf = pCtx[0].pOutput;
}
if (!pResInfo->initialized) {
aAggs[functionId].init(&pCtx[i]);
aAggs[functionId].init(&pCtx[i], pResInfo);
}
}
}
@ -3945,7 +4093,7 @@ void calculateOperatorProfResults(SQInfo* pQInfo) {
qDebug("QInfo:0x%"PRIx64" operator prof results hash is null", pQInfo->qId);
return;
}
SArray* opStack = taosArrayInit(32, sizeof(SOperatorStackItem));
if (opStack == NULL) {
return;
@ -4382,7 +4530,7 @@ int32_t doInitQInfo(SQInfo* pQInfo, STSBuf* pTsBuf, void* tsdb, void* sourceOptr
break;
}
case OP_DataBlocksOptScan: {
pRuntimeEnv->proot = createDataBlocksOptScanInfo(pRuntimeEnv->pQueryHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr), 1);
pRuntimeEnv->proot = createDataBlocksOptScanInfo(pRuntimeEnv->pQueryHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr), pQueryAttr->needReverseScan? 1:0);
break;
}
case OP_TableScan: {
@ -4420,8 +4568,10 @@ int32_t doInitQInfo(SQInfo* pQInfo, STSBuf* pTsBuf, void* tsdb, void* sourceOptr
if (pQInfo->summary.queryProfEvents == NULL) {
qDebug("QInfo:0x%"PRIx64" failed to allocate query prof events array", pQInfo->qId);
}
pQInfo->summary.operatorProfResults =
taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_TINYINT), true, HASH_NO_LOCK);
if (pQInfo->summary.operatorProfResults == NULL) {
qDebug("QInfo:0x%"PRIx64" failed to allocate operator prof results hash", pQInfo->qId);
}
@ -4795,7 +4945,7 @@ void setTableScanFilterOperatorInfo(STableScanInfo* pTableScanInfo, SOperatorInf
pTableScanInfo->pResultRowInfo = &pInfo->binfo.resultRowInfo;
pTableScanInfo->rowCellInfoOffset = pInfo->binfo.rowCellInfoOffset;
} else if (pDownstream->operatorType == OP_StateWindow) {
SStateWindowOperatorInfo* pInfo = pDownstream->info;
SStateWindowOperatorInfo* pInfo = pDownstream->info;
pTableScanInfo->pCtx = pInfo->binfo.pCtx;
pTableScanInfo->pResultRowInfo = &pInfo->binfo.resultRowInfo;
@ -4814,7 +4964,6 @@ SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, SQueryRuntime
pInfo->reverseTimes = reverseTime;
pInfo->current = 0;
pInfo->order = pRuntimeEnv->pQueryAttr->order.order;
// pInfo->prevGroupId = -1;
SOperatorInfo* pOptr = calloc(1, sizeof(SOperatorInfo));
pOptr->name = "DataBlocksOptimizedScanOperator";
@ -4916,7 +5065,7 @@ static void destroySlimitOperatorInfo(void* param, int32_t numOfOutput) {
}
SOperatorInfo* createGlobalAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream,
SExprInfo* pExpr, int32_t numOfOutput, void* param) {
SExprInfo* pExpr, int32_t numOfOutput, void* param, SArray* pUdfInfo) {
SMultiwayMergeInfo* pInfo = calloc(1, sizeof(SMultiwayMergeInfo));
pInfo->resultRowFactor =
@ -4926,6 +5075,7 @@ SOperatorInfo* createGlobalAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv,
pInfo->pMerge = param;
pInfo->bufCapacity = 4096;
pInfo->udfInfo = pUdfInfo;
pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, pInfo->bufCapacity * pInfo->resultRowFactor);
pInfo->binfo.pCtx = createSQLFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset);
@ -5455,7 +5605,7 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI
TSKEY* tsList = (TSKEY*)pTsColInfoData->pData;
if (IS_REPEAT_SCAN(pRuntimeEnv) && !pInfo->reptScan) {
pInfo->reptScan = true;
tfree(pInfo->prevData);
tfree(pInfo->prevData);
}
pInfo->numOfRows = 0;
@ -5465,17 +5615,17 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI
continue;
}
if (pInfo->prevData == NULL) {
pInfo->prevData = malloc(bytes);
pInfo->prevData = malloc(bytes);
memcpy(pInfo->prevData, val, bytes);
pInfo->numOfRows = 1;
pInfo->curWindow.skey = tsList[j];
pInfo->curWindow.ekey = tsList[j];
pInfo->start = j;
pInfo->start = j;
} else if (memcmp(pInfo->prevData, val, bytes) == 0) {
pInfo->curWindow.ekey = tsList[j];
pInfo->numOfRows += 1;
//pInfo->start = j;
//pInfo->start = j;
if (j == 0 && pInfo->start != 0) {
pInfo->numOfRows = 1;
pInfo->start = 0;
@ -5497,7 +5647,7 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI
memcpy(pInfo->prevData, val, bytes);
pInfo->numOfRows = 1;
pInfo->start = j;
}
}
@ -5550,10 +5700,10 @@ static SSDataBlock* doStateWindowAgg(void *param, bool* newgroup) {
setInputDataBlock(pOperator, pBInfo->pCtx, pBlock, pQueryAttr->order.order);
if (pWindowInfo->colIndex == -1) {
pWindowInfo->colIndex = getGroupbyColumnIndex(pRuntimeEnv->pQueryAttr->pGroupbyExpr, pBlock);
}
}
doStateWindowAggImpl(pOperator, pWindowInfo, pBlock);
}
// restore the value
pQueryAttr->order.order = order;
pQueryAttr->window = win;
@ -5571,7 +5721,7 @@ static SSDataBlock* doStateWindowAgg(void *param, bool* newgroup) {
}
return pBInfo->pRes->info.rows == 0? NULL:pBInfo->pRes;
}
}
static SSDataBlock* doSessionWindowAgg(void* param, bool* newgroup) {
SOperatorInfo* pOperator = (SOperatorInfo*) param;
if (pOperator->status == OP_EXEC_DONE) {
@ -5686,7 +5836,7 @@ static SSDataBlock* hashGroupbyAggregate(void* param, bool* newgroup) {
sortGroupResByOrderList(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pInfo->binfo.pRes);
}
toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pInfo->binfo.pRes);
if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) {
pOperator->status = OP_EXEC_DONE;
}
@ -5979,8 +6129,13 @@ SColumnInfo* extractColumnFilterInfo(SExprInfo* pExpr, int32_t numOfOutput, int3
pCols[i].colId = pExpr[i].base.resColId;
pCols[i].flist.numOfFilters = pExpr[i].base.flist.numOfFilters;
pCols[i].flist.filterInfo = calloc(pCols[i].flist.numOfFilters, sizeof(SColumnFilterInfo));
memcpy(pCols[i].flist.filterInfo, pExpr[i].base.flist.filterInfo, pCols[i].flist.numOfFilters * sizeof(SColumnFilterInfo));
if (pCols[i].flist.numOfFilters != 0) {
pCols[i].flist.filterInfo = calloc(pCols[i].flist.numOfFilters, sizeof(SColumnFilterInfo));
memcpy(pCols[i].flist.filterInfo, pExpr[i].base.flist.filterInfo, pCols[i].flist.numOfFilters * sizeof(SColumnFilterInfo));
} else {
// avoid runtime error
pCols[i].flist.filterInfo = NULL;
}
}
assert(numOfFilter > 0);
@ -6076,8 +6231,8 @@ SOperatorInfo* createStatewindowOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOpe
pOperator->cleanup = destroyStateWindowOperatorInfo;
appendUpstream(pOperator, upstream);
return pOperator;
return pOperator;
}
SOperatorInfo* createSWindowOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) {
SSWindowOperatorInfo* pInfo = calloc(1, sizeof(SSWindowOperatorInfo));
@ -6087,7 +6242,7 @@ SOperatorInfo* createSWindowOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperato
initResultRowInfo(&pInfo->binfo.resultRowInfo, 8, TSDB_DATA_TYPE_INT);
pInfo->prevTs = INT64_MIN;
pInfo->reptScan = false;
pInfo->reptScan = false;
SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo));
pOperator->name = "SessionWindowAggOperator";
@ -6439,10 +6594,10 @@ static SSDataBlock* hashDistinct(void* param, bool* newgroup) {
if (isNull(val, type)) {
continue;
}
int dummy;
void* res = taosHashGet(pInfo->pSet, val, bytes);
if (res == NULL) {
taosHashPut(pInfo->pSet, val, bytes, NULL, 0);
taosHashPut(pInfo->pSet, val, bytes, &dummy, sizeof(dummy));
char* start = pResultColInfoData->pData + bytes * pInfo->pRes->info.rows;
memcpy(start, val, bytes);
pRes->info.rows += 1;
@ -6678,6 +6833,9 @@ int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SQueryParam* param) {
pQueryMsg->sw.primaryColId = htonl(pQueryMsg->sw.primaryColId);
pQueryMsg->tableScanOperator = htonl(pQueryMsg->tableScanOperator);
pQueryMsg->numOfOperator = htonl(pQueryMsg->numOfOperator);
pQueryMsg->udfContentOffset = htonl(pQueryMsg->udfContentOffset);
pQueryMsg->udfContentLen = htonl(pQueryMsg->udfContentLen);
pQueryMsg->udfNum = htonl(pQueryMsg->udfNum);
// query msg safety check
if (!validateQueryMsg(pQueryMsg)) {
@ -6921,6 +7079,33 @@ int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SQueryParam* param) {
pMsg += sizeof(int32_t);
}
if (pQueryMsg->udfContentLen > 0) {
param->pUdfInfo = calloc(1, sizeof(SUdfInfo));
param->pUdfInfo->contLen = pQueryMsg->udfContentLen;
pMsg = (char*)pQueryMsg + pQueryMsg->udfContentOffset;
param->pUdfInfo->resType = *(int8_t*) pMsg;
pMsg += sizeof(int8_t);
param->pUdfInfo->resBytes = htons(*(int16_t*)pMsg);
pMsg += sizeof(int16_t);
tstr* name = (tstr*)(pMsg);
param->pUdfInfo->name = strndup(name->data, name->len);
pMsg += varDataTLen(name);
param->pUdfInfo->funcType = htonl(*(int32_t*)pMsg);
pMsg += sizeof(int32_t);
param->pUdfInfo->bufSize = htonl(*(int32_t*)pMsg);
pMsg += sizeof(int32_t);
param->pUdfInfo->content = malloc(pQueryMsg->udfContentLen);
memcpy(param->pUdfInfo->content, pMsg, pQueryMsg->udfContentLen);
pMsg += pQueryMsg->udfContentLen;
}
param->sql = strndup(pMsg, pQueryMsg->sqlstrLen);
SQueriedTableInfo info = { .numOfTags = pQueryMsg->numOfTags, .numOfCols = pQueryMsg->numOfCols, .colList = pQueryMsg->tableCols};
@ -7011,7 +7196,7 @@ static int32_t updateOutputBufForTopBotQuery(SQueriedTableInfo* pTableInfo, SCol
} else {
SColumnInfo* pCol = &pTableInfo->colList[j];
int32_t ret = getResultDataInfo(pCol->type, pCol->bytes, functId, (int32_t)pExprs[i].base.param[0].i64,
&pExprs[i].base.resType, &pExprs[i].base.resBytes, &pExprs[i].base.interBytes, tagLen, superTable);
&pExprs[i].base.resType, &pExprs[i].base.resBytes, &pExprs[i].base.interBytes, tagLen, superTable, NULL);
assert(ret == TSDB_CODE_SUCCESS);
}
}
@ -7020,12 +7205,138 @@ static int32_t updateOutputBufForTopBotQuery(SQueriedTableInfo* pTableInfo, SCol
return TSDB_CODE_SUCCESS;
}
void destroyUdfInfo(SUdfInfo* pUdfInfo) {
if (pUdfInfo == NULL) {
return;
}
if (pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY]) {
if (pUdfInfo->isScript) {
(*(scriptDestroyFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY])(pUdfInfo->pScriptCtx);
tfree(pUdfInfo->content);
}else{
(*(udfDestroyFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY])(&pUdfInfo->init);
}
}
tfree(pUdfInfo->name);
if (pUdfInfo->path) {
unlink(pUdfInfo->path);
}
tfree(pUdfInfo->path);
tfree(pUdfInfo->content);
taosCloseDll(pUdfInfo->handle);
tfree(pUdfInfo);
}
static char* getUdfFuncName(char* name, int type) {
char* funcname = calloc(1, TSDB_FUNCTIONS_NAME_MAX_LENGTH + 10);
switch (type) {
case TSDB_UDF_FUNC_NORMAL:
strcpy(funcname, name);
break;
case TSDB_UDF_FUNC_INIT:
sprintf(funcname, "%s_init", name);
break;
case TSDB_UDF_FUNC_FINALIZE:
sprintf(funcname, "%s_finalize", name);
break;
case TSDB_UDF_FUNC_MERGE:
sprintf(funcname, "%s_merge", name);
break;
case TSDB_UDF_FUNC_DESTROY:
sprintf(funcname, "%s_destroy", name);
break;
default:
assert(0);
break;
}
return funcname;
}
int32_t initUdfInfo(SUdfInfo* pUdfInfo) {
if (pUdfInfo == NULL) {
return TSDB_CODE_SUCCESS;
}
//qError("script len: %d", pUdfInfo->contLen);
if (isValidScript(pUdfInfo->content, pUdfInfo->contLen)) {
pUdfInfo->isScript = 1;
pUdfInfo->pScriptCtx = createScriptCtx(pUdfInfo->content, pUdfInfo->resType, pUdfInfo->resBytes);
if (pUdfInfo->pScriptCtx == NULL) {
return TSDB_CODE_QRY_SYS_ERROR;
}
tfree(pUdfInfo->content);
pUdfInfo->funcs[TSDB_UDF_FUNC_INIT] = taosLoadScriptInit;
if (pUdfInfo->funcs[TSDB_UDF_FUNC_INIT] == NULL
|| (*(scriptInitFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_INIT])(pUdfInfo->pScriptCtx) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_QRY_SYS_ERROR;
}
pUdfInfo->funcs[TSDB_UDF_FUNC_NORMAL] = taosLoadScriptNormal;
if (pUdfInfo->funcType == TSDB_UDF_TYPE_AGGREGATE) {
pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE] = taosLoadScriptFinalize;
pUdfInfo->funcs[TSDB_UDF_FUNC_MERGE] = taosLoadScriptMerge;
}
pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY] = taosLoadScriptDestroy;
} else {
char path[PATH_MAX] = {0};
taosGetTmpfilePath("script", path);
FILE* file = fopen(path, "w+");
// TODO check for failure of flush to disk
/*size_t t = */ fwrite(pUdfInfo->content, pUdfInfo->contLen, 1, file);
fclose(file);
tfree(pUdfInfo->content);
pUdfInfo->path = strdup(path);
pUdfInfo->handle = taosLoadDll(path);
if (NULL == pUdfInfo->handle) {
return TSDB_CODE_QRY_SYS_ERROR;
}
pUdfInfo->funcs[TSDB_UDF_FUNC_NORMAL] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(pUdfInfo->name, TSDB_UDF_FUNC_NORMAL));
if (NULL == pUdfInfo->funcs[TSDB_UDF_FUNC_NORMAL]) {
return TSDB_CODE_QRY_SYS_ERROR;
}
pUdfInfo->funcs[TSDB_UDF_FUNC_INIT] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(pUdfInfo->name, TSDB_UDF_FUNC_INIT));
if (pUdfInfo->funcType == TSDB_UDF_TYPE_AGGREGATE) {
pUdfInfo->funcs[TSDB_UDF_FUNC_FINALIZE] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(pUdfInfo->name, TSDB_UDF_FUNC_FINALIZE));
pUdfInfo->funcs[TSDB_UDF_FUNC_MERGE] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(pUdfInfo->name, TSDB_UDF_FUNC_MERGE));
}
pUdfInfo->funcs[TSDB_UDF_FUNC_DESTROY] = taosLoadSym(pUdfInfo->handle, getUdfFuncName(pUdfInfo->name, TSDB_UDF_FUNC_DESTROY));
if (pUdfInfo->funcs[TSDB_UDF_FUNC_INIT]) {
return (*(udfInitFunc)pUdfInfo->funcs[TSDB_UDF_FUNC_INIT])(&pUdfInfo->init);
}
}
return TSDB_CODE_SUCCESS;
}
// TODO tag length should be passed from client, refactor
int32_t createQueryFunc(SQueriedTableInfo* pTableInfo, int32_t numOfOutput, SExprInfo** pExprInfo,
SSqlExpr** pExprMsg, SColumnInfo* pTagCols, int32_t queryType, void* pMsg) {
SSqlExpr** pExprMsg, SColumnInfo* pTagCols, int32_t queryType, void* pMsg, SUdfInfo* pUdfInfo) {
*pExprInfo = NULL;
int32_t code = TSDB_CODE_SUCCESS;
code = initUdfInfo(pUdfInfo);
if (code) {
return code;
}
SExprInfo *pExprs = (SExprInfo *)calloc(numOfOutput, sizeof(SExprInfo));
if (pExprs == NULL) {
return TSDB_CODE_QRY_OUT_OF_MEMORY;
@ -7114,7 +7425,7 @@ int32_t createQueryFunc(SQueriedTableInfo* pTableInfo, int32_t numOfOutput, SExp
// todo remove it
if (getResultDataInfo(type, bytes, pExprs[i].base.functionId, param, &pExprs[i].base.resType, &pExprs[i].base.resBytes,
&pExprs[i].base.interBytes, 0, isSuperTable) != TSDB_CODE_SUCCESS) {
&pExprs[i].base.interBytes, 0, isSuperTable, pUdfInfo) != TSDB_CODE_SUCCESS) {
tfree(pExprs);
return TSDB_CODE_QRY_INVALID_MSG;
}
@ -7135,7 +7446,7 @@ int32_t createQueryFunc(SQueriedTableInfo* pTableInfo, int32_t numOfOutput, SExp
// todo refactor
int32_t createIndirectQueryFuncExprFromMsg(SQueryTableMsg* pQueryMsg, int32_t numOfOutput, SExprInfo** pExprInfo,
SSqlExpr** pExpr, SExprInfo* prevExpr) {
SSqlExpr** pExpr, SExprInfo* prevExpr, SUdfInfo *pUdfInfo) {
*pExprInfo = NULL;
int32_t code = TSDB_CODE_SUCCESS;
@ -7180,7 +7491,7 @@ int32_t createIndirectQueryFuncExprFromMsg(SQueryTableMsg* pQueryMsg, int32_t nu
int32_t param = (int32_t)pExprs[i].base.param[0].i64;
if (getResultDataInfo(type, bytes, pExprs[i].base.functionId, param, &pExprs[i].base.resType, &pExprs[i].base.resBytes,
&pExprs[i].base.interBytes, 0, isSuperTable) != TSDB_CODE_SUCCESS) {
&pExprs[i].base.interBytes, 0, isSuperTable, pUdfInfo) != TSDB_CODE_SUCCESS) {
tfree(pExprs);
return TSDB_CODE_QRY_INVALID_MSG;
}
@ -7366,7 +7677,7 @@ FORCE_INLINE bool checkQIdEqual(void *qHandle, uint64_t qId) {
SQInfo* createQInfoImpl(SQueryTableMsg* pQueryMsg, SGroupbyExpr* pGroupbyExpr, SExprInfo* pExprs,
SExprInfo* pSecExprs, STableGroupInfo* pTableGroupInfo, SColumnInfo* pTagCols, int32_t vgId,
char* sql, uint64_t *qId) {
char* sql, uint64_t qId, SUdfInfo* pUdfInfo) {
int16_t numOfCols = pQueryMsg->numOfCols;
int16_t numOfOutput = pQueryMsg->numOfOutput;
@ -7375,7 +7686,9 @@ SQInfo* createQInfoImpl(SQueryTableMsg* pQueryMsg, SGroupbyExpr* pGroupbyExpr, S
goto _cleanup_qinfo;
}
pQInfo->qId = *qId;
pQInfo->qId = qId;
pQInfo->runtimeEnv.pUdfInfo = pUdfInfo;
// to make sure third party won't overwrite this structure
pQInfo->signature = pQInfo;
@ -7400,6 +7713,7 @@ SQInfo* createQInfoImpl(SQueryTableMsg* pQueryMsg, SGroupbyExpr* pGroupbyExpr, S
pQueryAttr->prjInfo.vgroupLimit = pQueryMsg->vgroupLimit;
pQueryAttr->prjInfo.ts = (pQueryMsg->order == TSDB_ORDER_ASC)? INT64_MIN:INT64_MAX;
pQueryAttr->sw = pQueryMsg->sw;
pQueryAttr->vgId = vgId;
pQueryAttr->stableQuery = pQueryMsg->stableQuery;
pQueryAttr->topBotQuery = pQueryMsg->topBotQuery;
@ -7432,7 +7746,6 @@ SQInfo* createQInfoImpl(SQueryTableMsg* pQueryMsg, SGroupbyExpr* pGroupbyExpr, S
}
}
// calculate the result row size
for (int16_t col = 0; col < numOfOutput; ++col) {
assert(pExprs[col].base.resBytes > 0);
pQueryAttr->resultRowSize += pExprs[col].base.resBytes;
@ -7453,6 +7766,20 @@ SQInfo* createQInfoImpl(SQueryTableMsg* pQueryMsg, SGroupbyExpr* pGroupbyExpr, S
goto _cleanup;
}
if (pSecExprs != NULL) {
int32_t resultRowSize = 0;
// calculate the result row size
for (int16_t col = 0; col < pQueryAttr->numOfExpr2; ++col) {
assert(pSecExprs[col].base.resBytes > 0);
resultRowSize += pSecExprs[col].base.resBytes;
}
if (resultRowSize > pQueryAttr->resultRowSize) {
pQueryAttr->resultRowSize = resultRowSize;
}
}
if (pQueryAttr->fillType != TSDB_FILL_NONE) {
pQueryAttr->fillVal = malloc(sizeof(int64_t) * pQueryAttr->numOfOutput);
if (pQueryAttr->fillVal == NULL) {
@ -7485,7 +7812,7 @@ SQInfo* createQInfoImpl(SQueryTableMsg* pQueryMsg, SGroupbyExpr* pGroupbyExpr, S
tsem_init(&pQInfo->ready, 0, 0);
pQueryAttr->window = pQueryMsg->window;
changeExecuteScanOrder(pQInfo, pQueryMsg, pQueryAttr->stableQuery);
updateDataCheckOrder(pQInfo, pQueryMsg, pQueryAttr->stableQuery);
SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv;
STimeWindow window = pQueryAttr->window;
@ -7598,8 +7925,8 @@ int32_t initQInfo(STsBufInfo* pTsBufInfo, void* tsdb, void* sourceOptr, SQInfo*
SArray* prevResult = NULL;
if (prevResultLen > 0) {
prevResult = interResFromBinary(param->prevResult, prevResultLen);
prevResult = interResFromBinary(param->prevResult, prevResultLen);
pRuntimeEnv->prevResult = prevResult;
}

443
src/query/src/qScript.c Normal file
View File

@ -0,0 +1,443 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "os.h"
#include "qScript.h"
#include "ttype.h"
#include "tstrbuild.h"
#include "queryLog.h"
#include "ttokendef.h"
static ScriptEnvPool *pool = NULL;
static ScriptEnv* getScriptEnvFromPool();
static void addScriptEnvToPool(ScriptEnv *pEnv);
static lua_State* createLuaEnv();
static void destroyLuaEnv(lua_State *state);
static void destroyScriptEnv(ScriptEnv *pEnv);
static void luaValueToTaosType(lua_State *lua, char *interBuf, int32_t *numOfOutput, int16_t oType, int16_t oBytes);
static void taosValueToLuaType(lua_State *lua, int32_t type, char *val);
static bool hasBaseFuncDefinedInScript(lua_State *lua, const char *funcPrefix, int32_t len);
static int userlib_exampleFunc(lua_State *lua) {
double op1 = luaL_checknumber(lua,1);
double op2 = luaL_checknumber(lua,2);
lua_pushnumber(lua, op1 * op2);
return 1;
}
void luaRegisterLibFunc(lua_State *lua) {
lua_register(lua, "exampleFunc", userlib_exampleFunc);
}
void luaLoadLib(lua_State *lua, const char *libname, lua_CFunction luafunc) {
lua_pushcfunction(lua, luafunc);
lua_pushstring(lua, libname);
lua_call(lua, 1, 0);
}
LUALIB_API int (luaopen_cjson) (lua_State *L);
LUALIB_API int (luaopen_struct) (lua_State *L);
LUALIB_API int (luaopen_cmsgpack) (lua_State *L);
LUALIB_API int (luaopen_bit) (lua_State *L);
static void luaLoadLibraries(lua_State *lua) {
luaLoadLib(lua, "", luaopen_base);
luaLoadLib(lua, LUA_TABLIBNAME, luaopen_table);
luaLoadLib(lua, LUA_STRLIBNAME, luaopen_string);
luaLoadLib(lua, LUA_MATHLIBNAME, luaopen_math);
luaLoadLib(lua, LUA_DBLIBNAME, luaopen_debug);
}
static void luaRemoveUnsupportedFunctions(lua_State *lua) {
lua_pushnil(lua);
lua_setglobal(lua,"loadfile");
lua_pushnil(lua);
lua_setglobal(lua,"dofile");
}
void taosValueToLuaType(lua_State *lua, int32_t type, char *val) {
//TODO(dengyihao): handle more data type
if (IS_SIGNED_NUMERIC_TYPE(type) || type == TSDB_DATA_TYPE_BOOL) {
int64_t v;
GET_TYPED_DATA(v, int64_t, type, val);
lua_pushnumber(lua, (lua_Number)v);
} else if (IS_UNSIGNED_NUMERIC_TYPE(type)) {
uint64_t v;
GET_TYPED_DATA(v, uint64_t, type, val);
lua_pushnumber(lua,(lua_Number)v);
} else if (IS_FLOAT_TYPE(type)) {
double v;
GET_TYPED_DATA(v, double, type, val);
lua_pushnumber(lua,v);
} else if (type == TSDB_DATA_TYPE_BINARY) {
lua_pushlstring(lua, val, varDataLen(val));
} else if (type == TSDB_DATA_TYPE_NCHAR) {
}
}
int taosLoadScriptInit(void* pInit) {
ScriptCtx *pCtx = pInit;
char funcName[MAX_FUNC_NAME] = {0};
sprintf(funcName, "%s_init", pCtx->funcName);
lua_State* lua = pCtx->pEnv->lua_state;
lua_getglobal(lua, funcName);
if (lua_pcall(lua, 0, -1, 0)) {
lua_pop(lua, -1);
}
lua_setglobal(lua, "global");
return 0;
}
void taosLoadScriptNormal(void *pInit, char *pInput, int16_t iType, int16_t iBytes, int32_t numOfRows,
int64_t *ptsList, int64_t key, char* pOutput, char *ptsOutput, int32_t *numOfOutput, int16_t oType, int16_t oBytes) {
ScriptCtx* pCtx = pInit;
char funcName[MAX_FUNC_NAME] = {0};
sprintf(funcName, "%s_add", pCtx->funcName);
lua_State* lua = pCtx->pEnv->lua_state;
lua_getglobal(lua, funcName);
// first param of script;
lua_newtable(lua);
int32_t offset = 0;
for (int32_t i = 0; i < numOfRows; i++) {
taosValueToLuaType(lua, iType, pInput + offset);
lua_rawseti(lua, -2, i+1);
offset += iBytes;
}
int isGlobalState = false;
lua_getglobal(lua, "global");
if (lua_istable(lua, -1)) {
isGlobalState = true;
}
lua_pushnumber(lua, (lua_Number)key);
// do call lua script
if (lua_pcall(lua, 3, 1, 0) != 0) {
qError("SCRIPT ERROR: %s", lua_tostring(lua, -1));
lua_pop(lua, -1);
return;
}
int tNumOfOutput = 0;
if (isGlobalState == false) {
luaValueToTaosType(lua, pOutput, &tNumOfOutput, oType, oBytes);
} else {
lua_setglobal(lua, "global");
}
*numOfOutput = tNumOfOutput;
}
void taosLoadScriptMerge(void *pInit, char* data, int32_t numOfRows, char* pOutput, int32_t* numOfOutput) {
ScriptCtx *pCtx = pInit;
char funcName[MAX_FUNC_NAME] = {0};
sprintf(funcName, "%s_merge", pCtx->funcName);
lua_State* lua = pCtx->pEnv->lua_state;
lua_getglobal(lua, funcName);
if (!lua_isfunction(lua, -1)) {
qError("SCRIPT ERROR: %s", lua_tostring(lua, -1));
return;
}
lua_getglobal(lua, "global");
if (lua_pcall(lua, 1, 1, 0) != 0) {
qError("SCRIPT ERROR: %s", lua_tostring(lua, -1));
lua_pop(lua, -1);
return;
}
int tNumOfOutput = 0;
luaValueToTaosType(lua, pOutput, &tNumOfOutput, pCtx->resType, pCtx->resBytes);
*numOfOutput = tNumOfOutput;
}
//do not support agg now
void taosLoadScriptFinalize(void *pInit,int64_t key, char *pOutput, int32_t* numOfOutput) {
ScriptCtx *pCtx = pInit;
char funcName[MAX_FUNC_NAME] = {0};
sprintf(funcName, "%s_finalize", pCtx->funcName);
lua_State* lua = pCtx->pEnv->lua_state;
lua_getglobal(lua, funcName);
if (!lua_isfunction(lua, -1)) {
qError("SCRIPT ERROR: %s", lua_tostring(lua, -1));
return;
}
lua_getglobal(lua, "global");
lua_pushnumber(lua, (lua_Number)key);
if (lua_pcall(lua, 2, 2, 0) != 0) {
qError("SCRIPT ERROR: %s", lua_tostring(lua, -1));
lua_pop(lua, -1);
return;
}
lua_setglobal(lua, "global");
int tNumOfOutput = 0;
luaValueToTaosType(lua, pOutput, &tNumOfOutput, pCtx->resType, pCtx->resBytes);
*numOfOutput = tNumOfOutput;
}
void taosLoadScriptDestroy(void *pInit) {
destroyScriptCtx(pInit);
}
ScriptCtx* createScriptCtx(char *script, int8_t resType, int16_t resBytes) {
ScriptCtx *pCtx = (ScriptCtx *)calloc(1, sizeof(ScriptCtx));
pCtx->state = SCRIPT_STATE_INIT;
pCtx->pEnv = getScriptEnvFromPool(); //
pCtx->resType = resType;
pCtx->resBytes = resBytes;
if (pCtx->pEnv == NULL) {
destroyScriptCtx(pCtx);
return NULL;
}
lua_State *lua = pCtx->pEnv->lua_state;
if (luaL_dostring(lua, script)) {
lua_pop(lua, 1);
qError("dynamic load script failed");
destroyScriptCtx(pCtx);
return NULL;
}
lua_getglobal(lua, USER_FUNC_NAME);
const char *name = lua_tostring(lua, -1);
if (name == NULL) {
lua_pop(lua, 1);
qError("SCRIPT ERROR: invalid script");
destroyScriptCtx(pCtx);
return NULL;
}
memcpy(pCtx->funcName, name, strlen(name));
lua_pop(lua, 1);
return pCtx;
}
void destroyScriptCtx(void *pCtx) {
if (pCtx == NULL) return;
addScriptEnvToPool(((ScriptCtx *)pCtx)->pEnv);
free(pCtx);
}
void luaValueToTaosType(lua_State *lua, char *interBuf, int32_t *numOfOutput, int16_t oType, int16_t oBytes) {
int t = lua_type(lua,-1);
int32_t sz = 0;
switch (t) {
case LUA_TSTRING:
//TODO(yihaodeng): handle str type
{
const char *v = lua_tostring(lua, -1);
memcpy(interBuf, v, strlen(v));
sz = 1;
}
break;
case LUA_TBOOLEAN:
{
double v = lua_tonumber(lua, -1);
memcpy(interBuf, (char *)&v, oBytes);
sz = 1;
}
break;
case LUA_TNUMBER:
{
if (oType == TSDB_DATA_TYPE_FLOAT) {
float v = (float)lua_tonumber(lua, -1);
memcpy(interBuf, (char *)&v, oBytes);
sz = 1;
} else if (oType == TSDB_DATA_TYPE_DOUBLE) {
double v = (double)lua_tonumber(lua, -1);
memcpy(interBuf, (char *)&v, oBytes);
sz = 1;
} else if (oType == TSDB_DATA_TYPE_BIGINT) {
int64_t v = (int64_t)lua_tonumber(lua, -1);
memcpy(interBuf, (char *)&v, oBytes);
sz = 1;
} else if (oType <= TSDB_DATA_TYPE_INT) {
int32_t v = (int32_t)lua_tonumber(lua, -1);
memcpy(interBuf, (char *)&v, oBytes);
sz = 1;
}
}
break;
case LUA_TTABLE:
{
lua_pushnil(lua);
int32_t offset = 0;
while(lua_next(lua, -2)) {
int32_t v = (int32_t)lua_tonumber(lua, -1);
memcpy(interBuf + offset, (char *)&v, oBytes);
offset += oBytes;
lua_pop(lua, 1);
sz += 1;
}
}
break;
default:
setNull(interBuf, oType, oBytes);
sz = 1;
break;
}
lua_pop(lua,1); // pop ret value from script
*numOfOutput = sz;
}
/*
*Initialize the scripting environment.
*/
lua_State* createLuaEnv() {
lua_State *lua = lua_open();
luaLoadLibraries(lua);
luaRemoveUnsupportedFunctions(lua);
// register func in external lib
luaRegisterLibFunc(lua);
{
char *errh_func = "local dbg = debug\n"
"function __taos__err__handler(err)\n"
" local i = dbg.getinfo(2,'nSl')\n"
" if i and i.what == 'C' then\n"
" i = dbg.getinfo(3,'nSl')\n"
" end\n"
" if i then\n"
" return i.source .. ':' .. i.currentline .. ': ' .. err\n"
" else\n"
" return err\n"
" end\n"
"end\n";
luaL_loadbuffer(lua,errh_func,strlen(errh_func),"@err_handler_def");
lua_pcall(lua,0,0,0);
}
return lua;
}
void destroyLuaEnv(lua_State *lua) {
lua_close(lua);
}
int32_t scriptEnvPoolInit() {
const int size = 10; // configure or not
pool = malloc(sizeof(ScriptEnvPool));
pthread_mutex_init(&pool->mutex, NULL);
pool->scriptEnvs = tdListNew(sizeof(ScriptEnv *));
for (int i = 0; i < size; i++) {
ScriptEnv *env = malloc(sizeof(ScriptEnv));
env->funcId = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK);;
env->lua_state = createLuaEnv();
tdListAppend(pool->scriptEnvs, (void *)(&env));
}
pool->mSize = size;
pool->cSize = size;
return 0;
}
void scriptEnvPoolCleanup() {
if (pool == NULL) return;
SListNode *pNode = NULL;
while ((pNode = tdListPopHead(pool->scriptEnvs)) != NULL) {
ScriptEnv *pEnv = NULL;
tdListNodeGetData(pool->scriptEnvs, pNode, (void *)(&pEnv));
destroyScriptEnv(pEnv);
listNodeFree(pNode);
}
tdListFree(pool->scriptEnvs);
pthread_mutex_destroy(&pool->mutex);
free(pool);
}
void destroyScriptEnv(ScriptEnv *pEnv) {
destroyLuaEnv(pEnv->lua_state);
taosHashCleanup(pEnv->funcId);
free(pEnv);
}
ScriptEnv* getScriptEnvFromPool() {
ScriptEnv *pEnv = NULL;
pthread_mutex_lock(&pool->mutex);
if (pool->cSize <= 0) {
pthread_mutex_unlock(&pool->mutex);
return NULL;
}
SListNode *pNode = tdListPopHead(pool->scriptEnvs);
tdListNodeGetData(pool->scriptEnvs, pNode, (void *)(&pEnv));
listNodeFree(pNode);
pool->cSize--;
pthread_mutex_unlock(&pool->mutex);
return pEnv;
}
void addScriptEnvToPool(ScriptEnv *pEnv) {
if (pEnv == NULL) {
return;
}
pthread_mutex_lock(&pool->mutex);
lua_settop(pEnv->lua_state, 0);
tdListAppend(pool->scriptEnvs, (void *)(&pEnv));
pool->cSize++;
pthread_mutex_unlock(&pool->mutex);
}
bool hasBaseFuncDefinedInScript(lua_State *lua, const char *funcPrefix, int32_t len) {
bool ret = true;
char funcName[MAX_FUNC_NAME];
memcpy(funcName, funcPrefix, len);
const char *base[] = {"_init", "_add"};
for (int i = 0; (i < sizeof(base)/sizeof(base[0])) && (ret == true); i++) {
memcpy(funcName + len, base[i], strlen(base[i]));
memset(funcName + len + strlen(base[i]), 0, MAX_FUNC_NAME - len - strlen(base[i]));
lua_getglobal(lua, funcName);
ret = lua_isfunction(lua, -1); // exsit function or not
lua_pop(lua, 1);
}
return ret;
}
bool isValidScript(char *script, int32_t len) {
ScriptEnv *pEnv = getScriptEnvFromPool(); //
if (pEnv == NULL) {
return false;
}
lua_State *lua = pEnv->lua_state;
if (len < strlen(script)) {
script[len] = 0;
}
if (luaL_dostring(lua, script)) {
lua_pop(lua, 1);
addScriptEnvToPool(pEnv);
qError("error at %s and %d", script, (int)(strlen(script)));
return false;
}
lua_getglobal(lua, USER_FUNC_NAME);
const char *name = lua_tostring(lua, -1);
if (name == NULL || strlen(name) >= USER_FUNC_NAME_LIMIT) {
lua_pop(lua, 1);
addScriptEnvToPool(pEnv);
qError("error at %s name: %s, len = %d", script, name, (int)(strlen(name)));
return false;
}
bool ret = hasBaseFuncDefinedInScript(lua, name, (int32_t)strlen(name));
lua_pop(lua, 1); // pop
addScriptEnvToPool(pEnv);
return ret;
}

51
src/query/src/qSqlParser.c
View File

@ -28,6 +28,7 @@ SSqlInfo qSqlParse(const char *pStr) {
SSqlInfo sqlInfo = {0};
sqlInfo.valid = true;
sqlInfo.funcs = taosArrayInit(4, sizeof(SStrToken));
int32_t i = 0;
while (1) {
@ -120,6 +121,19 @@ void tSqlExprListDestroy(SArray *pList) {
taosArrayDestroyEx(pList, freeExprElem);
}
SArray *tStrTokenAppend(SArray *pList, SStrToken *pToken) {
if (pList == NULL) {
pList = taosArrayInit(4, sizeof(tVariantListItem));
}
if (pToken) {
taosArrayPush(pList, pToken);
}
return pList;
}
tSqlExpr *tSqlExprCreateIdValue(SStrToken *pToken, int32_t optrType) {
tSqlExpr *pSqlExpr = calloc(1, sizeof(tSqlExpr));
@ -146,7 +160,8 @@ tSqlExpr *tSqlExprCreateIdValue(SStrToken *pToken, int32_t optrType) {
pSqlExpr->type = SQL_NODE_VALUE;
pSqlExpr->flags |= 1 << EXPR_FLAG_NS_TIMESTAMP;
} else if (optrType == TK_VARIABLE) {
// use nanosecond by default TODO set value after getting database precision
// use nanosecond by default
// TODO set value after getting database precision
int32_t ret = parseAbsoluteDuration(pToken->z, pToken->n, &pSqlExpr->value.i64, TSDB_TIME_PRECISION_NANO);
if (ret != TSDB_CODE_SUCCESS) {
terrno = TSDB_CODE_TSC_SQL_SYNTAX_ERROR;
@ -922,8 +937,8 @@ void* destroyCreateTableSql(SCreateTableSql* pCreate) {
}
void SqlInfoDestroy(SSqlInfo *pInfo) {
if (pInfo == NULL) return;
if (pInfo == NULL) return;;
taosArrayDestroy(pInfo->funcs);
if (pInfo->type == TSDB_SQL_SELECT) {
destroyAllSqlNode(pInfo->list);
} else if (pInfo->type == TSDB_SQL_CREATE_TABLE) {
@ -1018,6 +1033,18 @@ void setDropDbTableInfo(SSqlInfo *pInfo, int32_t type, SStrToken* pToken, SStrTo
pInfo->pMiscInfo->tableType = tableType;
}
void setDropFuncInfo(SSqlInfo *pInfo, int32_t type, SStrToken* pToken) {
pInfo->type = type;
if (pInfo->pMiscInfo == NULL) {
pInfo->pMiscInfo = (SMiscInfo *)calloc(1, sizeof(SMiscInfo));
pInfo->pMiscInfo->a = taosArrayInit(4, sizeof(SStrToken));
}
taosArrayPush(pInfo->pMiscInfo->a, pToken);
}
void setShowOptions(SSqlInfo *pInfo, int32_t type, SStrToken* prefix, SStrToken* pPatterns) {
if (pInfo->pMiscInfo == NULL) {
pInfo->pMiscInfo = calloc(1, sizeof(SMiscInfo));
@ -1052,6 +1079,24 @@ void setCreateDbInfo(SSqlInfo *pInfo, int32_t type, SStrToken *pToken, SCreateDb
pInfo->pMiscInfo->dbOpt.ignoreExists = pIgExists->n; // sql.y has: ifnotexists(X) ::= IF NOT EXISTS. {X.n = 1;}
}
void setCreateFuncInfo(SSqlInfo *pInfo, int32_t type, SStrToken *pName, SStrToken *pPath, TAOS_FIELD *output, SStrToken* bufSize, int32_t funcType) {
pInfo->type = type;
if (pInfo->pMiscInfo == NULL) {
pInfo->pMiscInfo = calloc(1, sizeof(SMiscInfo));
}
pInfo->pMiscInfo->funcOpt.name = *pName;
pInfo->pMiscInfo->funcOpt.path = *pPath;
pInfo->pMiscInfo->funcOpt.output = *output;
pInfo->pMiscInfo->funcOpt.type = funcType;
if (bufSize->n > 0) {
pInfo->pMiscInfo->funcOpt.bufSize = strtol(bufSize->z, NULL, 10);
} else {
pInfo->pMiscInfo->funcOpt.bufSize = 0;
}
}
void setCreateAcctSql(SSqlInfo *pInfo, int32_t type, SStrToken *pName, SStrToken *pPwd, SCreateAcctInfo *pAcctInfo) {
pInfo->type = type;
if (pInfo->pMiscInfo == NULL) {

9
src/query/src/qTsbuf.c
View File

@ -267,6 +267,10 @@ static void writeDataToDisk(STSBuf* pTSBuf) {
if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR) {
metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f);
metaLen += (int32_t)fwrite(pBlock->tag.pz, 1, (size_t)pBlock->tag.nLen, pTSBuf->f);
} else if (pBlock->tag.nType == TSDB_DATA_TYPE_FLOAT) {
metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f);
float tfloat = (float)pBlock->tag.dKey;
metaLen += (int32_t)fwrite(&tfloat, 1, (size_t) pBlock->tag.nLen, pTSBuf->f);
} else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) {
metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f);
metaLen += (int32_t)fwrite(&pBlock->tag.i64, 1, (size_t) pBlock->tag.nLen, pTSBuf->f);
@ -351,6 +355,11 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) {
sz = fread(pBlock->tag.pz, (size_t)pBlock->tag.nLen, 1, pTSBuf->f);
UNUSED(sz);
} else if (pBlock->tag.nType == TSDB_DATA_TYPE_FLOAT) {
float tfloat = 0;
sz = fread(&tfloat, (size_t) pBlock->tag.nLen, 1, pTSBuf->f);
pBlock->tag.dKey = (double)tfloat;
UNUSED(sz);
} else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { //TODO check the return value
sz = fread(&pBlock->tag.i64, (size_t) pBlock->tag.nLen, 1, pTSBuf->f);
UNUSED(sz);

16
src/query/src/queryMain.c
View File

@ -68,7 +68,7 @@ void freeParam(SQueryParam *param) {
tfree(param->prevResult);
}
int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qinfo_t* pQInfo, uint64_t *qId) {
int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qinfo_t* pQInfo, uint64_t qId) {
assert(pQueryMsg != NULL && tsdb != NULL);
int32_t code = TSDB_CODE_SUCCESS;
@ -93,12 +93,12 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qi
SQueriedTableInfo info = { .numOfTags = pQueryMsg->numOfTags, .numOfCols = pQueryMsg->numOfCols, .colList = pQueryMsg->tableCols};
if ((code = createQueryFunc(&info, pQueryMsg->numOfOutput, &param.pExprs, param.pExpr, param.pTagColumnInfo,
pQueryMsg->queryType, pQueryMsg)) != TSDB_CODE_SUCCESS) {
pQueryMsg->queryType, pQueryMsg, param.pUdfInfo)) != TSDB_CODE_SUCCESS) {
goto _over;
}
if (param.pSecExpr != NULL) {
if ((code = createIndirectQueryFuncExprFromMsg(pQueryMsg, pQueryMsg->secondStageOutput, &param.pSecExprs, param.pSecExpr, param.pExprs)) != TSDB_CODE_SUCCESS) {
if ((code = createIndirectQueryFuncExprFromMsg(pQueryMsg, pQueryMsg->secondStageOutput, &param.pSecExprs, param.pSecExpr, param.pExprs, param.pUdfInfo)) != TSDB_CODE_SUCCESS) {
goto _over;
}
}
@ -162,18 +162,14 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qi
assert(pQueryMsg->stableQuery == isSTableQuery);
(*pQInfo) = createQInfoImpl(pQueryMsg, param.pGroupbyExpr, param.pExprs, param.pSecExprs, &tableGroupInfo,
param.pTagColumnInfo, vgId, param.sql, qId);
param.pTagColumnInfo, vgId, param.sql, qId, param.pUdfInfo);
param.sql = NULL;
param.pExprs = NULL;
param.pSecExprs = NULL;
param.pGroupbyExpr = NULL;
param.pTagColumnInfo = NULL;
if ((*pQInfo) == NULL) {
code = TSDB_CODE_QRY_OUT_OF_MEMORY;
goto _over;
}
param.pUdfInfo = NULL;
code = initQInfo(&pQueryMsg->tsBuf, tsdb, NULL, *pQInfo, &param, (char*)pQueryMsg, pQueryMsg->prevResultLen, NULL);
@ -182,6 +178,8 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qi
taosArrayDestroy(param.pGroupbyExpr->columnInfo);
}
destroyUdfInfo(param.pUdfInfo);
taosArrayDestroy(param.pTableIdList);
param.pTableIdList = NULL;

3304
src/query/src/sql.c
View File

File diff suppressed because it is too large Load Diff

20
src/tsdb/src/tsdbRead.c
View File

@ -98,6 +98,8 @@ typedef struct SIOCostSummary {
int64_t blockLoadTime;
int64_t statisInfoLoadTime;
int64_t checkForNextTime;
int64_t headFileLoad;
int64_t headFileLoadTime;
} SIOCostSummary;
typedef struct STsdbQueryHandle {
@ -1045,15 +1047,21 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo
int32_t code = TSDB_CODE_SUCCESS;
*numOfBlocks = 0;
pQueryHandle->cost.headFileLoad += 1;
int64_t s = taosGetTimestampUs();
size_t numOfTables = 0;
if (pQueryHandle->loadType == BLOCK_LOAD_TABLE_SEQ_ORDER) {
code = loadBlockInfo(pQueryHandle, pQueryHandle->activeIndex, numOfBlocks);
code = loadBlockInfo(pQueryHandle, pQueryHandle->activeIndex, numOfBlocks);
} else if (pQueryHandle->loadType == BLOCK_LOAD_OFFSET_SEQ_ORDER) {
numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo);
for (int32_t i = 0; i < numOfTables; ++i) {
code = loadBlockInfo(pQueryHandle, i, numOfBlocks);
if (code != TSDB_CODE_SUCCESS) {
int64_t e = taosGetTimestampUs();
pQueryHandle->cost.headFileLoadTime += (e - s);
return code;
}
}
@ -1061,6 +1069,8 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo
assert(0);
}
int64_t e = taosGetTimestampUs();
pQueryHandle->cost.headFileLoadTime += (e - s);
return code;
}
@ -3688,6 +3698,10 @@ static void* doFreeColumnInfoData(SArray* pColumnInfoData) {
}
static void* destroyTableCheckInfo(SArray* pTableCheckInfo) {
if (pTableCheckInfo == NULL) {
return NULL;
}
size_t size = taosArrayGetSize(pTableCheckInfo);
for (int32_t i = 0; i < size; ++i) {
STableCheckInfo* p = taosArrayGet(pTableCheckInfo, i);
@ -3731,8 +3745,8 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) {
pQueryHandle->next = doFreeColumnInfoData(pQueryHandle->next);
SIOCostSummary* pCost = &pQueryHandle->cost;
tsdbDebug("%p :io-cost summary: statis-info:%"PRId64" us, datablock:%" PRId64" us, check data:%"PRId64" us, 0x%"PRIx64,
pQueryHandle, pCost->statisInfoLoadTime, pCost->blockLoadTime, pCost->checkForNextTime, pQueryHandle->qId);
tsdbDebug("%p :io-cost summary: head-file read cnt:%"PRIu64", head-file time:%"PRIu64" us, statis-info:%"PRId64" us, datablock:%" PRId64" us, check data:%"PRId64" us, 0x%"PRIx64,
pQueryHandle, pCost->headFileLoad, pCost->headFileLoadTime, pCost->statisInfoLoadTime, pCost->blockLoadTime, pCost->checkForNextTime, pQueryHandle->qId);
tfree(pQueryHandle);
}

33
src/util/inc/talgo.h
View File

@ -34,6 +34,7 @@ typedef int (*__compar_fn_t) (const void *, const void *);
#define elePtrAt(base, size, idx) (void *)((char *)(base) + (size) * (idx))
typedef int32_t (*__ext_compar_fn_t)(const void *p1, const void *p2, const void *param);
typedef void (*__ext_swap_fn_t)(void *p1, void *p2, const void *param);
/**
* quick sort, with the compare function requiring additional parameters support
@ -59,6 +60,38 @@ void taosqsort(void *src, size_t numOfElem, size_t size, const void* param, __ex
*/
void *taosbsearch(const void *key, const void *base, size_t nmemb, size_t size, __compar_fn_t fn, int flags);
/**
* adjust heap
*
* @param base: the start address of array
* @param size: size of every item in array
* @param start: the first index
* @param end: the last index
* @param parcompar: parameters for compare function
* @param compar: user defined compare function
* @param parswap: parameters for swap function
* @param swap: user defined swap function, the default swap function doswap will be used if swap is NULL
* @param maxroot: if heap is max root heap
* @return
*/
void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const void *parcompar, __ext_compar_fn_t compar, const void *parswap, __ext_swap_fn_t swap, bool maxroot);
/**
* sort heap to make sure it is a max/min root heap
*
* @param base: the start address of array
* @param size: size of every item in array
* @param len: the length of array
* @param parcompar: parameters for compare function
* @param compar: user defined compare function
* @param parswap: parameters for swap function
* @param swap: user defined swap function, the default swap function doswap will be used if swap is NULL
* @param maxroot: if heap is max root heap
* @return
*/
void taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar, __ext_compar_fn_t compar, const void *parswap, __ext_swap_fn_t swap, bool maxroot);
#ifdef __cplusplus
}
#endif

86
src/util/src/talgo.c
View File

@ -225,3 +225,89 @@ void * taosbsearch(const void *key, const void *base, size_t nmemb, size_t size,
return NULL;
}
void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const void *parcompar, __ext_compar_fn_t compar, const void *parswap, __ext_swap_fn_t swap, bool maxroot)
{
int32_t parent;
int32_t child;
char *buf;
if (base && size > 0 && compar) {
parent = start;
child = 2 * parent + 1;
if (swap == NULL) {
buf = calloc(1, size);
if (buf == NULL) {
return;
}
}
if (maxroot) {
while (child <= end) {
if (child + 1 <= end && (*compar)(elePtrAt(base, size, child), elePtrAt(base, size, child + 1), parcompar) < 0) {
child++;
}
if ((*compar)(elePtrAt(base, size, parent), elePtrAt(base, size, child), parcompar) > 0) {
break;
}
if (swap == NULL) {
doswap(elePtrAt(base, size, parent), elePtrAt(base, size, child), size, buf);
} else {
(*swap)(elePtrAt(base, size, parent), elePtrAt(base, size, child), parswap);
}
parent = child;
child = 2 * parent + 1;
}
} else {
while (child <= end) {
if (child + 1 <= end && (*compar)(elePtrAt(base, size, child), elePtrAt(base, size, child + 1), parcompar) > 0) {
child++;
}
if ((*compar)(elePtrAt(base, size, parent), elePtrAt(base, size, child), parcompar) < 0) {
break;
}
if (swap == NULL) {
doswap(elePtrAt(base, size, parent), elePtrAt(base, size, child), size, buf);
} else {
(*swap)(elePtrAt(base, size, parent), elePtrAt(base, size, child), parswap);
}
parent = child;
child = 2 * parent + 1;
}
}
if (swap == NULL) {
tfree(buf);
}
}
}
void taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar, __ext_compar_fn_t compar, const void *parswap, __ext_swap_fn_t swap, bool maxroot)
{
int32_t i;
if (base && size > 0) {
for (i = len / 2 - 1; i >= 0; i--) {
taosheapadjust(base, size, i, len - 1, parcompar, compar, parswap, swap, maxroot);
}
}
/*
char *buf = calloc(1, size);
for (i = len - 1; i > 0; i--) {
doswap(elePtrAt(base, size, 0), elePtrAt(base, size, i));
taosheapadjust(base, size, 0, i - 1, parcompar, compar, parswap, swap, maxroot);
}
tfree(buf);
*/
}

13
src/util/src/terror.c
View File

@ -109,11 +109,12 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_SQL_SYNTAX_ERROR, "Syntax error in SQL")
TAOS_DEFINE_ERROR(TSDB_CODE_TSC_DB_NOT_SELECTED, "Database not specified or available")
TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_TABLE_NAME, "Table does not exist")
TAOS_DEFINE_ERROR(TSDB_CODE_TSC_EXCEED_SQL_LIMIT, "SQL statement too long, check maxSQLLength config")
TAOS_DEFINE_ERROR(TSDB_CODE_TSC_FILE_EMPTY, "File is empty")
// mnode
TAOS_DEFINE_ERROR(TSDB_CODE_MND_MSG_NOT_PROCESSED, "Message not processed")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACTION_IN_PROGRESS, "Message is progressing")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACTION_NEED_REPROCESSED, "Messag need to be reprocessed")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_ACTION_NEED_REPROCESSED, "Message need to be reprocessed")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_RIGHTS, "Insufficient privilege for operation")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_APP_ERROR, "Unexpected generic error in mnode")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CONNECTION, "Invalid message connection")
@ -183,6 +184,13 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_FIELD_NOT_EXIST, "Field does not exist"
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STABLE_NAME, "Super table does not exist")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CREATE_TABLE_MSG, "Invalid create table message")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_NAME, "Invalid func name")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_LEN, "Invalid func length")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_CODE, "Invalid func code")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_FUNC_ALREADY_EXIST, "Func already exists")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC, "Invalid func")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_BUFSIZE, "Invalid func bufSize")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_NOT_SELECTED, "Database not specified or available")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_ALREADY_EXIST, "Database already exists")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_OPTION, "Invalid database options")
@ -269,6 +277,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_IN_EXEC, "Multiple retrieval of
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW, "Too many time window in query")
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_ENOUGH_BUFFER, "Query buffer limit has reached")
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INCONSISTAN, "File inconsistance in replica")
TAOS_DEFINE_ERROR(TSDB_CODE_QRY_SYS_ERROR, "System error")
// grant
@ -403,6 +412,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAG_VALUE_TOO_LONG, "tag value can not mor
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_VALUE_NULL, "value not find")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_VALUE_TYPE, "value type should be boolean, number or string")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_REQUEST_JSON_ERROR, "http request json error")
// odbc
TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_OOM, "out of memory")
TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_CHAR_NOT_NUM, "convertion not a valid literal input")

32
src/util/src/tqueue.c
View File

@ -173,10 +173,12 @@ int taosReadAllQitems(taos_queue param, taos_qall p2) {
STaosQueue *queue = (STaosQueue *)param;
STaosQall *qall = (STaosQall *)p2;
int code = 0;
bool empty;
pthread_mutex_lock(&queue->mutex);
if (queue->head) {
empty = queue->head == NULL;
if (!empty) {
memset(qall, 0, sizeof(STaosQall));
qall->current = queue->head;
qall->start = queue->head;
@ -188,11 +190,17 @@ int taosReadAllQitems(taos_queue param, taos_qall p2) {
queue->tail = NULL;
queue->numOfItems = 0;
if (queue->qset) atomic_sub_fetch_32(&queue->qset->numOfItems, qall->numOfItems);
}
}
pthread_mutex_unlock(&queue->mutex);
return code;
// if source queue is empty, we set destination qall to empty too.
if (empty) {
qall->current = NULL;
qall->start = NULL;
qall->numOfItems = 0;
}
return code;
}
int taosGetQitem(taos_qall param, int *type, void **pitem) {
@ -423,10 +431,22 @@ int taosReadAllQitemsFromQset(taos_qset param, taos_qall p2, void **phandle) {
int taosGetQueueItemsNumber(taos_queue param) {
STaosQueue *queue = (STaosQueue *)param;
return queue->numOfItems;
if (!queue) return 0;
int num;
pthread_mutex_lock(&queue->mutex);
num = queue->numOfItems;
pthread_mutex_unlock(&queue->mutex);
return num;
}
int taosGetQsetItemsNumber(taos_qset param) {
STaosQset *qset = (STaosQset *)param;
return qset->numOfItems;
if (!qset) return 0;
int num = 0;
pthread_mutex_lock(&qset->mutex);
num = qset->numOfItems;
pthread_mutex_unlock(&qset->mutex);
return num;
}

1
src/util/src/tskiplist.c
View File

@ -546,7 +546,6 @@ static FORCE_INLINE int32_t getSkipListNodeRandomHeight(SSkipList *pSkipList) {
const uint32_t factor = 4;
int32_t n = 1;
#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)
while ((rand() % factor) == 0 && n <= pSkipList->maxLevel) {
#else

11
src/util/src/ttokenizer.c
View File

@ -219,8 +219,13 @@ static SKeyword keywordTable[] = {
{"PARTITIONS", TK_PARTITIONS},
{"TOPIC", TK_TOPIC},
{"TOPICS", TK_TOPICS},
{"COMPACT", TK_COMPACT},
{"MODIFY", TK_MODIFY}
{"COMPACT", TK_COMPACT},
{"MODIFY", TK_MODIFY},
{"FUNCTION", TK_FUNCTION},
{"FUNCTIONS", TK_FUNCTIONS},
{"OUTPUTTYPE", TK_OUTPUTTYPE},
{"AGGREGATE", TK_AGGREGATE},
{"BUFSIZE", TK_BUFSIZE},
};
static const char isIdChar[] = {
@ -269,7 +274,7 @@ static int32_t tKeywordCode(const char* z, int n) {
if (keywordHashTable == NULL) {
return TK_ILLEGAL;
}
SKeyword** pKey = (SKeyword**)taosHashGet(keywordHashTable, key, n);
return (pKey != NULL)? (*pKey)->type:TK_ID;
}

20
src/vnode/src/vnodeMain.c
View File

@ -47,9 +47,6 @@ int32_t vnodeCreate(SCreateVnodeMsg *pVnodeCfg) {
return terrno;
}
char rootDir[TSDB_FILENAME_LEN] = {0};
sprintf(rootDir, "%s/vnode%d", tsVnodeDir, pVnodeCfg->cfg.vgId);
char vnodeDir[TSDB_FILENAME_LEN] = "\0";
snprintf(vnodeDir, TSDB_FILENAME_LEN, "/vnode/vnode%d", pVnodeCfg->cfg.vgId);
if (tfsMkdir(vnodeDir) < 0) {
@ -63,23 +60,6 @@ int32_t vnodeCreate(SCreateVnodeMsg *pVnodeCfg) {
return code;
}
// STsdbCfg tsdbCfg = {0};
// tsdbCfg.tsdbId = pVnodeCfg->cfg.vgId;
// tsdbCfg.cacheBlockSize = pVnodeCfg->cfg.cacheBlockSize;
// tsdbCfg.totalBlocks = pVnodeCfg->cfg.totalBlocks;
// tsdbCfg.daysPerFile = pVnodeCfg->cfg.daysPerFile;
// tsdbCfg.keep = pVnodeCfg->cfg.daysToKeep;
// tsdbCfg.keep1 = pVnodeCfg->cfg.daysToKeep1;
// tsdbCfg.keep2 = pVnodeCfg->cfg.daysToKeep2;
// tsdbCfg.minRowsPerFileBlock = pVnodeCfg->cfg.minRowsPerFileBlock;
// tsdbCfg.maxRowsPerFileBlock = pVnodeCfg->cfg.maxRowsPerFileBlock;
// tsdbCfg.precision = pVnodeCfg->cfg.precision;
// tsdbCfg.compression = pVnodeCfg->cfg.compression;
// tsdbCfg.update = pVnodeCfg->cfg.update;
// tsdbCfg.cacheLastRow = pVnodeCfg->cfg.cacheLastRow;
// char tsdbDir[TSDB_FILENAME_LEN] = {0};
// sprintf(tsdbDir, "vnode/vnode%d/tsdb", pVnodeCfg->cfg.vgId);
if (tsdbCreateRepo(pVnodeCfg->cfg.vgId) < 0) {
vError("vgId:%d, failed to create tsdb in vnode, reason:%s", pVnodeCfg->cfg.vgId, tstrerror(terrno));
return TSDB_CODE_VND_INIT_FAILED;

2
src/vnode/src/vnodeRead.c
View File

@ -231,7 +231,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead) {
if (contLen != 0) {
qinfo_t pQInfo = NULL;
uint64_t qId = genQueryId();
code = qCreateQueryInfo(pVnode->tsdb, pVnode->vgId, pQueryTableMsg, &pQInfo, &qId);
code = qCreateQueryInfo(pVnode->tsdb, pVnode->vgId, pQueryTableMsg, &pQInfo, qId);
SQueryTableRsp *pRsp = (SQueryTableRsp *)rpcMallocCont(sizeof(SQueryTableRsp));
pRsp->code = code;

10
src/vnode/src/vnodeWrite.c
View File

@ -90,7 +90,7 @@ int32_t vnodeProcessWrite(void *vparam, void *wparam, int32_t qtype, void *rpara
// forward to peers, even it is WAL/FWD, it shall be called to update version in sync
int32_t syncCode = 0;
bool force = (pWrite == NULL ? false : pWrite->pHead.msgType != TSDB_MSG_TYPE_SUBMIT);
bool force = (pWrite == NULL ? false : pWrite->walHead.msgType != TSDB_MSG_TYPE_SUBMIT);
syncCode = syncForwardToPeer(pVnode->sync, pHead, pWrite, qtype, force);
if (syncCode < 0) {
pHead->version = 0;
@ -237,7 +237,7 @@ static SVWriteMsg *vnodeBuildVWriteMsg(SVnodeObj *pVnode, SWalHead *pHead, int32
return NULL;
}
int32_t size = sizeof(SVWriteMsg) + sizeof(SWalHead) + pHead->len;
int32_t size = sizeof(SVWriteMsg) + pHead->len;
SVWriteMsg *pWrite = taosAllocateQitem(size);
if (pWrite == NULL) {
terrno = TSDB_CODE_VND_OUT_OF_MEMORY;
@ -248,7 +248,7 @@ static SVWriteMsg *vnodeBuildVWriteMsg(SVnodeObj *pVnode, SWalHead *pHead, int32
pWrite->rpcMsg = *pRpcMsg;
}
memcpy(&pWrite->pHead, pHead, sizeof(SWalHead) + pHead->len);
memcpy(&pWrite->walHead, pHead, sizeof(SWalHead) + pHead->len);
pWrite->pVnode = pVnode;
pWrite->qtype = qtype;
@ -286,7 +286,7 @@ static int32_t vnodeWriteToWQueueImp(SVWriteMsg *pWrite) {
}
int32_t queued = atomic_add_fetch_32(&pVnode->queuedWMsg, 1);
int64_t queuedSize = atomic_add_fetch_64(&pVnode->queuedWMsgSize, pWrite->pHead.len);
int64_t queuedSize = atomic_add_fetch_64(&pVnode->queuedWMsgSize, pWrite->walHead.len);
if (queued > MAX_QUEUED_MSG_NUM || queuedSize > MAX_QUEUED_MSG_SIZE) {
int32_t ms = (queued / MAX_QUEUED_MSG_NUM) * 10 + 3;
@ -330,7 +330,7 @@ void vnodeFreeFromWQueue(void *vparam, SVWriteMsg *pWrite) {
SVnodeObj *pVnode = vparam;
if (pVnode) {
int32_t queued = atomic_sub_fetch_32(&pVnode->queuedWMsg, 1);
int64_t queuedSize = atomic_sub_fetch_64(&pVnode->queuedWMsgSize, pWrite->pHead.len);
int64_t queuedSize = atomic_sub_fetch_64(&pVnode->queuedWMsgSize, pWrite->walHead.len);
vTrace("vgId:%d, msg:%p, app:%p, free from vwqueue, queued:%d size:%" PRId64, pVnode->vgId, pWrite,
pWrite->rpcMsg.ahandle, queued, queuedSize);

20
src/wal/src/walMgmt.c
View File

@ -36,9 +36,16 @@ static int32_t walInitObj(SWal *pWal);
static void walFreeObj(void *pWal);
int32_t walInit() {
int32_t code = 0;
tsWal.refId = taosOpenRef(TSDB_MIN_VNODES, walFreeObj);
int32_t code = walCreateThread();
code = pthread_mutex_init(&tsWal.mutex, NULL);
if (code) {
wError("failed to init wal mutex since %s", tstrerror(code));
return code;
}
code = walCreateThread();
if (code != TSDB_CODE_SUCCESS) {
wError("failed to init wal module since %s", tstrerror(code));
return code;
@ -51,6 +58,7 @@ int32_t walInit() {
void walCleanUp() {
walStopThread();
taosCloseRef(tsWal.refId);
pthread_mutex_destroy(&tsWal.mutex);
wInfo("wal module is cleaned up");
}
@ -183,10 +191,15 @@ static void walFsyncAll() {
}
static void *walThreadFunc(void *param) {
int stop = 0;
while (1) {
walUpdateSeq();
walFsyncAll();
if (tsWal.stop) break;
pthread_mutex_lock(&tsWal.mutex);
stop = tsWal.stop;
pthread_mutex_unlock(&tsWal.mutex);
if (stop) break;
}
return NULL;
@ -209,7 +222,10 @@ static int32_t walCreateThread() {
}
static void walStopThread() {
pthread_mutex_lock(&tsWal.mutex);
tsWal.stop = 1;
pthread_mutex_unlock(&tsWal.mutex);
if (taosCheckPthreadValid(tsWal.thread)) {
pthread_join(tsWal.thread, NULL);
}

2
tests/comparisonTest/cassandra/cassandratest/pom.xml
View File

@ -75,7 +75,7 @@
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.11</version>
<version>4.13.1</version>
<scope>test</scope>
</dependency>

4
tests/comparisonTest/opentsdb/opentsdbtest/pom.xml
View File

@ -87,14 +87,14 @@
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.11</version>
<version>4.13.1</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>29.0-jre</version>
<version>30.0-jre</version>
</dependency>
<dependency>

2
tests/examples/JDBC/SpringJdbcTemplate/pom.xml
View File

@ -40,7 +40,7 @@
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.13</version>
<version>4.13.1</version>
<scope>test</scope>
</dependency>

13
tests/examples/JDBC/connectionPools/pom.xml
View File

@ -4,6 +4,11 @@
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<properties>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
</properties>
<groupId>com.taosdata.demo</groupId>
<artifactId>connectionPools</artifactId>
<version>1.0-SNAPSHOT</version>
@ -46,9 +51,9 @@
</dependency>
<!-- log4j -->
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<version>1.2.17</version>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
<version>2.14.1</version>
</dependency>
<!-- proxool -->
<dependency>
@ -108,4 +113,4 @@
</plugins>
</build>
</project>
</project>

Some files were not shown because too many files have changed in this diff Show More