Merge branch 'develop' into feature/update

This commit is contained in:
Hongze Cheng 2020-09-17 14:09:48 +08:00
commit 29318b10f4
156 changed files with 6328 additions and 3728 deletions

View File

@ -2,7 +2,6 @@
[![Build status](https://ci.appveyor.com/api/projects/status/kf3pwh2or5afsgl9/branch/master?svg=true)](https://ci.appveyor.com/project/sangshuduo/tdengine-2n8ge/branch/master)
[![Coverage Status](https://coveralls.io/repos/github/taosdata/TDengine/badge.svg?branch=develop)](https://coveralls.io/github/taosdata/TDengine?branch=develop)
[![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/4201/badge)](https://bestpractices.coreinfrastructure.org/projects/4201)
[![Docker Pulls](https://img.shields.io/docker/pulls/tdengine/tdengine)](https://hub.docker.com/repository/docker/tdengine/tdengine)
[![tdengine](https://snapcraft.io//tdengine/badge.svg)](https://snapcraft.io/tdengine)
[![TDengine](TDenginelogo.png)](https://www.taosdata.com)
@ -92,6 +91,8 @@ cd TDengine
## Build TDengine
### On Linux platform
```bash
mkdir debug && cd debug
cmake .. && cmake --build .
@ -109,6 +110,34 @@ aarch32:
cmake .. -DCPUTYPE=aarch32 && cmake --build .
```
### On Windows platform
If you use the Visual Studio 2013, please open a command window by executing "cmd.exe".
Please specify "x86_amd64" for 64 bits Windows or specify "x86" is for 32 bits Windows when you execute vcvarsall.bat.
```
mkdir debug && cd debug
"C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat" < x86_amd64 | x86 >
cmake .. -G "NMake Makefiles"
nmake
```
If you use the Visual Studio 2019, please open a command window by executing "cmd.exe".
Please specify "x64" for 64 bits Windows or specify "x86" is for 32 bits Windows when you execute vcvarsall.bat.
```
mkdir debug && cd debug
"c:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvarsall.bat" < x64 | x86 >
cmake .. -G "NMake Makefiles"
nmake
```
Or, you can open a command window by clicking Visual Studio 2019 menu "Tools -> Command Line -> Developer Command Prompt" or "Tools -> Command Line -> Developer PowerShell" then execute commands as follows:
```
mkdir debug && cd debug
cmake .. -G "NMake Makefiles"
nmake
```
# Quick Run
# Quick Run
To quickly start a TDengine server after building, run the command below in terminal:
```cmd

35
alert/cmd/alert/install_driver.sh Normal file → Executable file
View File

@ -9,9 +9,7 @@ set -e
script_dir=$(dirname $(readlink -f "$0"))
# Dynamic directory
lib_link_dir="/usr/lib"
#install main path
install_main_dir="/usr/local/taos"
lib64_link_dir="/usr/lib64"
# Color setting
RED='\033[0;31m'
@ -25,24 +23,23 @@ if command -v sudo > /dev/null; then
csudo="sudo"
fi
function clean_driver() {
${csudo} rm -f /usr/lib/libtaos.so || :
}
function install_driver() {
echo -e "${GREEN}Start to install TDengine client driver ...${NC}"
#create install main dir and all sub dir
${csudo} mkdir -p ${install_main_dir}
${csudo} mkdir -p ${install_main_dir}/driver
${csudo} rm -f ${lib_link_dir}/libtaos.* || :
${csudo} cp -rf ${script_dir}/driver/* ${install_main_dir}/driver && ${csudo} chmod 777 ${install_main_dir}/driver/*
${csudo} ln -s ${install_main_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1
${csudo} ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so
echo
echo -e "\033[44;32;1mTDengine client driver is successfully installed!${NC}"
if [[ -d ${lib_link_dir} && ! -e ${lib_link_dir}/libtaos.so ]]; then
echo -e "${GREEN}Start to install TDengine client driver ...${NC}"
${csudo} ln -s ${script_dir}/driver/libtaos.* ${lib_link_dir}/libtaos.so.1 || :
${csudo} ln -s ${lib_link_dir}/libtaos.so.1 ${lib_link_dir}/libtaos.so || :
if [[ -d ${lib64_link_dir} && ! -e ${lib64_link_dir}/libtaos.so ]]; then
${csudo} ln -s ${script_dir}/driver/libtaos.* ${lib64_link_dir}/libtaos.so.1 || :
${csudo} ln -s ${lib64_link_dir}/libtaos.so.1 ${lib64_link_dir}/libtaos.so || :
fi
echo
echo -e "${GREEN}TDengine client driver is successfully installed!${NC}"
else
echo -e "${GREEN}TDengine client driver already exists, Please confirm whether the alert version matches the client driver version!${NC}"
fi
}
install_driver

View File

@ -119,7 +119,7 @@ WantedBy=multi-user.target
return nil
}
const version = "TDengine alert v2.0.0.1"
var version = "2.0.0.1s"
func main() {
var (
@ -133,7 +133,7 @@ func main() {
flag.Parse()
if showVersion {
fmt.Println(version)
fmt.Println("TDengine alert v" + version)
return
}

View File

@ -6,9 +6,9 @@ set -e
# set parameters by default value
cpuType=amd64 # [armv6l | arm64 | amd64 | 386]
osType=linux # [linux | darwin | windows]
version=""
declare -A archMap=(["armv6l"]="arm" ["arm64"]="arm64" ["amd64"]="x64" ["386"]="x86")
while getopts "h:c:o:" arg
while getopts "h:c:o:n:" arg
do
case $arg in
c)
@ -19,6 +19,10 @@ do
#echo "osType=$OPTARG"
osType=$(echo $OPTARG)
;;
n)
#echo "version=$OPTARG"
version=$(echo $OPTARG)
;;
h)
echo "Usage: `basename $0` -c [armv6l | arm64 | amd64 | 386] -o [linux | darwin | windows]"
exit 0
@ -30,18 +34,27 @@ do
esac
done
if [ "$version" == "" ]; then
echo "Please input the correct version!"
exit 1
fi
startdir=$(pwd)
scriptdir=$(dirname $(readlink -f $0))
cd ${scriptdir}/cmd/alert
version=$(grep 'const version =' main.go | awk '{print $NF}')
version=${version%\"}
version=${version:1}
echo "cpuType=${cpuType}"
echo "osType=${osType}"
echo "version=${version}"
GOOS=${osType} GOARCH=${cpuType} go build
GOOS=${osType} GOARCH=${cpuType} go build -ldflags '-X main.version='${version}
mkdir -p TDengine-alert/driver
cp alert alert.cfg install_driver.sh ./TDengine-alert/.
cp ../../../debug/build/lib/libtaos.so.${version} ./TDengine-alert/driver/.
chmod 777 ./TDengine-alert/install_driver.sh
tar -I 'gzip -9' -cf ${startdir}/TDengine-alert-${version}-${osType^}-${archMap[${cpuType}]}.tar.gz TDengine-alert/
rm -rf ./TDengine-alert
tar -I 'gzip -9' -cf ${startdir}/TDengine-alert-${version}-${osType^}-${archMap[${cpuType}]}.tar.gz alert alert.cfg install_driver.sh driver/

View File

@ -100,7 +100,7 @@ IF (TD_LINUX)
ENDIF ()
SET(DEBUG_FLAGS "-O0 -DDEBUG")
SET(RELEASE_FLAGS "-O0")
SET(RELEASE_FLAGS "-O0 -Wno-unused-variable -Wunused-but-set-variable")
IF (${COVER} MATCHES "true")
MESSAGE(STATUS "Test coverage mode, add extra flags")

View File

@ -13,7 +13,7 @@ ELSEIF (TD_WINDOWS)
IF (NOT TD_GODLL)
#INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/go DESTINATION connector)
#INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/grafana DESTINATION connector)
#INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/python DESTINATION connector)
INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/python DESTINATION connector)
INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/tests/examples DESTINATION .)
INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/packaging/cfg DESTINATION .)
INSTALL(FILES ${TD_COMMUNITY_DIR}/src/inc/taos.h DESTINATION include)

View File

@ -11,7 +11,7 @@ TDengine能够与开源数据可视化系统[Grafana](https://www.grafana.com/)
### 配置Grafana
TDengine的Grafana插件在安装包的/usr/local/taos/connector/grafana目录下。
TDengine的Grafana插件在安装包的/usr/local/taos/connector/grafanaplugin目录下。
以CentOS 7.2操作系统为例将tdengine目录拷贝到/var/lib/grafana/plugins目录下重新启动grafana即可。

View File

@ -56,4 +56,4 @@ INSERT INTO d1001 USING METERS TAGS ("Beijng.Chaoyang", 2) VALUES (now, 10.2, 21
## 多列模型 vs 单列模型
TDengine支持多列模型只要物理量是一个数据采集点同时采集的时间戳一致这些量就可以作为不同列放在一张超级表里。但还有一种极限的设计单列模型每个采集的物理量都单独建表因此每种类型的物理量都单独建立一超级表。比如电流、电压、相位就建三张超级表。
TDengine建议尽可能采用多列模型因为插入效率以及存储效率更高。但对于有些场景一个采集点的采集量的种类经常变化这个时候如果采用多列模型就需要频繁修改超级表的结构定义让应用变的复杂这个时候采用单列模型会显得简单。
TDengine建议尽可能采用多列模型因为插入效率以及存储效率更高。但对于有些场景一个采集点的采集量的种类经常变化这个时候如果采用多列模型就需要频繁修改超级表的结构定义让应用变的复杂这个时候采用单列模型会显得简单。

View File

@ -78,6 +78,18 @@ taos> SELECT SUM(current) FROM meters INTERVAL(1s);
2018-10-03 14:38:16.000 | 36.000000000 |
Query OK, 5 row(s) in set (0.001538s)
```
降采样操作也支持时间偏移,比如:将所有智能电表采集的电流值每秒钟求和,但要求每个时间窗口从 500 毫秒开始
```mysql
taos> SELECT SUM(current) FROM meters INTERVAL(1s, 500a);
ts | sum(current) |
======================================================
2018-10-03 14:38:04.500 | 11.189999809 |
2018-10-03 14:38:05.500 | 31.900000572 |
2018-10-03 14:38:06.500 | 11.600000000 |
2018-10-03 14:38:15.500 | 12.300000381 |
2018-10-03 14:38:16.500 | 35.000000000 |
Query OK, 5 row(s) in set (0.001521s)
```
物联网场景里,每个数据采集点采集数据的时间是难同步的,但很多分析算法(比如FFT)需要把采集的数据严格按照时间等间隔的对齐在很多系统里需要应用自己写程序来处理但使用TDengine的降采样操作就轻松解决。如果一个时间间隔里没有采集的数据TDengine还提供插值计算的功能。

View File

@ -148,7 +148,7 @@ INSERT INTO <tb1_name> USING <stb1_name> TAGS (<tag1_value1>, ...) VALUES (<fiel
SELECT function<field_name>,…
FROM <stable_name>
WHERE <tag_name> <[=|<=|>=|<>] values..> ([AND|OR] …)
INTERVAL (<time range>)
INTERVAL (<interval> [, offset])
GROUP BY <tag_name>, <tag_name>
ORDER BY <tag_name> <asc|desc>
SLIMIT <group_limit>

View File

@ -33,8 +33,7 @@ taos> DESCRIBE meters;
- 内部函数now是服务器的当前时间
- 插入记录时如果时间戳为now插入数据时使用服务器当前时间
- Epoch Time: 时间戳也可以是一个长整数表示从1970-01-01 08:00:00.000开始的毫秒数
- 时间可以加减,比如 now-2h表明查询时刻向前推2个小时(最近2小时)。数字后面的时间单位a(毫秒), s(秒), m(分), h(小时), d(天)w(周), n(月), y(年)。比如select * from t1 where ts > now-2w and ts <= now-1w, 表示查询两周前整整一周的数据
- TDengine暂不支持时间窗口按照自然年和自然月切分。Where条件中的时间窗口单位的换算关系如下interval(1y) 等效于 interval(365d), interval(1n) 等效于 interval(30d), interval(1w) 等效于 interval(7d)
- 时间可以加减,比如 now-2h表明查询时刻向前推2个小时(最近2小时)。 数字后面的时间单位可以是 a(毫秒)、s(秒)、 m(分)、h(小时)、d(天)、w(周)。 比如select * from t1 where ts > now-2w and ts <= now-1w, 表示查询两周前整整一周的数据。 在指定降频操作(down sampling)的时间窗口(interval)时,时间单位还可以使用 n(自然月) 和 y(自然年)。
TDengine缺省的时间戳是毫秒精度但通过修改配置参数enableMicrosecond就可支持微秒。
@ -110,6 +109,7 @@ TDengine缺省的时间戳是毫秒精度但通过修改配置参数enableMic
**Tips**: 以上所有参数修改后都可以用show databases来确认是否修改成功。
- **显示系统所有数据库**
```mysql
SHOW DATABASES;
@ -298,7 +298,7 @@ TDengine缺省的时间戳是毫秒精度但通过修改配置参数enableMic
SELECT select_expr [, select_expr ...]
FROM {tb_name_list}
[WHERE where_condition]
[INTERVAL [interval_offset,] interval_val]
[INTERVAL (interval_val [, interval_offset])]
[FILL fill_val]
[SLIDING fill_val]
[GROUP BY col_list]
@ -971,17 +971,17 @@ TDengine支持按时间段进行聚合可以将表中数据按照时间段进
```mysql
SELECT function_list FROM tb_name
[WHERE where_condition]
INTERVAL (interval)
INTERVAL (interval [, offset])
[FILL ({NONE | VALUE | PREV | NULL | LINEAR})]
SELECT function_list FROM stb_name
[WHERE where_condition]
INTERVAL (interval)
INTERVAL (interval [, offset])
[FILL ({ VALUE | PREV | NULL | LINEAR})]
[GROUP BY tags]
```
- 聚合时间段的长度由关键词INTERVAL指定最短时间间隔10毫秒10a。聚合查询中能够同时执行的聚合和选择函数仅限于单个输出的函数count、avg、sum 、stddev、leastsquares、percentile、min、max、first、last不能使用具有多行输出结果的函数例如top、bottom、diff以及四则运算
- 聚合时间段的长度由关键词INTERVAL指定最短时间间隔10毫秒10a,并且支持偏移(偏移必须小于间隔)。聚合查询中能够同时执行的聚合和选择函数仅限于单个输出的函数count、avg、sum 、stddev、leastsquares、percentile、min、max、first、last不能使用具有多行输出结果的函数例如top、bottom、diff以及四则运算
- WHERE语句可以指定查询的起止时间和其他过滤条件
- FILL语句指定某一时间区间数据缺失的情况下的填充模式。填充模式包括以下几种
1. 不进行填充NONE(默认填充模式)。

View File

@ -1,7 +1,7 @@
# TDengine 2.0 错误码以及对应的十进制码
| Code | bit | error code | 错误描述 | 十进制错误码 |
| 状态码 | 模 | 错误码(十六进制) | 错误描述 | 错误码(十进制) |
|-----------------------| :---: | :---------: | :------------------------ | ---------------- |
|TSDB_CODE_RPC_ACTION_IN_PROGRESS| 0 | 0x0001| "Action in progress"| -2147483647|
|TSDB_CODE_RPC_AUTH_REQUIRED| 0 | 0x0002 | "Authentication required"| -2147483646|
@ -87,7 +87,7 @@
|TSDB_CODE_MND_INVALID_ACCT_OPTION| 0 | 0x0342 | "Invalid account options"| -2147482814|
|TSDB_CODE_MND_USER_ALREADY_EXIST| 0 | 0x0350 | "User already exists"| -2147482800|
|TSDB_CODE_MND_INVALID_USER |0 | 0x0351 | "Invalid user" |-2147482799|
|TSDB_CODE_MND_INVALID_USER_FORMAT| |0 |0x0352 |"Invalid user format" |-2147482798|
|TSDB_CODE_MND_INVALID_USER_FORMAT| 0 |0x0352 |"Invalid user format" |-2147482798|
|TSDB_CODE_MND_INVALID_PASS_FORMAT| 0| 0x0353 | "Invalid password format"| -2147482797|
|TSDB_CODE_MND_NO_USER_FROM_CONN| 0 | 0x0354 | "Can not get user from conn"| -2147482796|
|TSDB_CODE_MND_TOO_MANY_USERS| 0 | 0x0355| "Too many users"| -2147482795|
@ -107,7 +107,7 @@
|TSDB_CODE_MND_DB_NOT_SELECTED| 0 | 0x0380 | "Database not specified or available"| -2147482752|
|TSDB_CODE_MND_DB_ALREADY_EXIST| 0 | 0x0381 | "Database already exists"| -2147482751|
|TSDB_CODE_MND_INVALID_DB_OPTION| 0 | 0x0382 | "Invalid database options"| -2147482750|
|TSDB_CODE_MND_INVALID_DB| |0 | 0x0383 | "Invalid database name"| -2147482749|
|TSDB_CODE_MND_INVALID_DB| 0 | 0x0383 | "Invalid database name"| -2147482749|
|TSDB_CODE_MND_MONITOR_DB_FORBIDDEN| 0 | 0x0384 | "Cannot delete monitor database"| -2147482748|
|TSDB_CODE_MND_TOO_MANY_DATABASES| 0| 0x0385 | "Too many databases for account"| -2147482747|
|TSDB_CODE_MND_DB_IN_DROPPING| 0 | 0x0386| "Database not available" |-2147482746|

View File

@ -1,48 +1,69 @@
# TDengine 集群安装、管理
多个taosd的运行实例可以组成一个集群以保证TDengine的高可靠运行并提供水平扩展能力。要了解TDengine 2.0的集群管理需要对集群的基本概念有所了解请看TDengine 2.0整体架构一章。而且在安装集群之前,请按照[《立即开始》](https://www.taosdata.com/cn/getting-started20/)一章安装并体验单节点功能。
多个TDengine服务器也就是多个taosd的运行实例可以组成一个集群以保证TDengine的高可靠运行并提供水平扩展能力。要了解TDengine 2.0的集群管理需要对集群的基本概念有所了解请看TDengine 2.0整体架构一章。而且在安装集群之前,请按照[《立即开始》](https://www.taosdata.com/cn/getting-started20/)一章安装并体验单节点功能。
集群的每个节点是由End Point来唯一标识的End Point是由FQDN(Fully Qualified Domain Name)外加Port组成比如 h1.taosdata.com:6030。一般FQDN就是服务器的hostname可通过Linux命令`hostname -f`获取,FQDN配置参考[一篇文章说清楚TDengine的FQDN](https://www.taosdata.com/blog/2020/09/11/1824.html)。端口是这个节点对外服务的端口号缺省是6030但可以通过taos.cfg里配置参数serverPort进行修改。一个节点可能配置了多个hostname, TDengine会自动获取第一个但也可以通过taos.cfg里配置参数fqdn进行指定。如果习惯IP地址直接访问可以将参数fqdn设置为本节点的IP地址。
集群的每个数据节点是由End Point来唯一标识的End Point是由FQDN(Fully Qualified Domain Name)外加Port组成比如 h1.taosdata.com:6030。一般FQDN就是服务器的hostname可通过Linux命令`hostname -f`获取,FQDN配置参考[一篇文章说清楚TDengine的FQDN](https://www.taosdata.com/blog/2020/09/11/1824.html)。端口是这个数据节点对外服务的端口号缺省是6030但可以通过taos.cfg里配置参数serverPort进行修改。一个物理节点可能配置了多个hostname, TDengine会自动获取第一个但也可以通过taos.cfg里配置参数fqdn进行指定。如果习惯IP地址直接访问可以将参数fqdn设置为本节点的IP地址。
TDengine的集群管理极其简单除添加和删除节点需要人工干预之外其他全部是自动完成最大程度的降低了运维的工作量。本章对集群管理的操作做详细的描述。
## 准备工作
**第一步**如果搭建集群的节点中存有之前的测试数据、装过1.X的版本或者装过其他版本的TDengine请先将其删除并清空所有数据具体步骤请参考博客[《TDengine多种安装包的安装和卸载》](https://www.taosdata.com/blog/2019/08/09/566.html )
**第零步**规划集群所有物理节点的FQDN将规划好的FQDN分别添加到每个物理节点的/etc/hostname修改每个物理节点的/etc/hosts将所有集群物理节点的IP与FQDN的对应添加好【如部署了DNS请联系网络管理员在DNS上做好相关配置】
**第一步**如果搭建集群的物理节点中存有之前的测试数据、装过1.X的版本或者装过其他版本的TDengine请先将其删除并清空所有数据具体步骤请参考博客[《TDengine多种安装包的安装和卸载》](https://www.taosdata.com/blog/2019/08/09/566.html )
**注意1**因为FQDN的信息会写进文件如果之前没有配置或者更改FQDN且启动了TDengine。请一定在确保数据无用或者备份的前提下清理一下之前的数据rm -rf /var/lib/taos/
**注意2**客户端也需要配置确保它可以正确解析每个节点的FQDN配置不管是通过DNS服务还是 Host 文件。
**第二步**建议关闭防火墙至少保证端口6030 - 6042的TCP和UDP端口都是开放的。**强烈建议**先关闭防火墙,集群搭建完毕之后,再来配置端口;
**第二步**:建议关闭所有物理节点的防火墙至少保证端口6030 - 6042的TCP和UDP端口都是开放的。**强烈建议**先关闭防火墙,集群搭建完毕之后,再来配置端口;
**第三步**在所有节点安装TDengine且版本必须是一致的**但不要启动taosd**
**第三步**在所有节点安装TDengine且版本必须是一致的**但不要启动taosd**。安装时提示输入是否要加入一个已经存在的TDengine集群时第一个物理节点直接回车创建新集群后续物理节点则输入该集群任何一个在线的物理节点的FQDN:端口号(默认6030)
**第四步**:检查、配置所有节点的FQDN
**第四步**:检查所有数据节点,以及应用所在物理节点的网络设置
1. 每个节点上执行命令`hostname -f`查看和确认所有节点的hostname是不相同的
2. 每个节点上执行`ping host`, 其中host是其他节点的hostname, 看能否ping通其它节点; 如果不能ping通需要检查网络设置, 或/etc/hosts文件或DNS的配置。如果无法ping通是无法组成集群的
3. 每个节点的FQDN就是输出的hostname外加端口号比如h1.taosdata.com:6030
1. 每个物理节点上执行命令`hostname -f`查看和确认所有节点的hostname是不相同的(应用驱动所在节点无需做此项检查)
2. 每个物理节点上执行`ping host`, 其中host是其他物理节点的hostname, 看能否ping通其它物理节点; 如果不能ping通需要检查网络设置, 或/etc/hosts文件(Windows系统默认路径为C:\Windows\system32\drivers\etc\hosts)或DNS的配置。如果无法ping通是无法组成集群的
3. 每个数据节点的End Point就是输出的hostname外加端口号比如h1.taosdata.com:6030
**第五步**修改TDengine的配置文件所有节点的文件/etc/taos/taos.cfg都需要修改。假设准备启动的第一个节点End Point为 h1.taosdata.com:6030, 那么以下几个参数与集群相关
**第五步**修改TDengine的配置文件所有节点的文件/etc/taos/taos.cfg都需要修改。假设准备启动的第一个数据节点End Point为 h1.taosdata.com:6030, 其与集群配置相关参数如下
```
// firstEp 集群中所有节点的配置都是一致的,对其第一次访问后,就获得了整个集群的信息
// firstEp 是每个数据节点首次启动后连接的第一个数据节点
firstEp h1.taosdata.com:6030
// 配置本节点的FQDN如果本机只有一个hostname, 无需配置
// 配置本数据节点的FQDN如果本机只有一个hostname, 无需配置
fqdn h1.taosdata.com
// 配置本节点的端口号缺省是6030
// 配置本数据节点的端口号缺省是6030
serverPort 6030
// 服务端节点数为偶数的时候需要配置请参考《Arbitrator的使用》的部分
arbitrator ha.taosdata.com:6042
```
一定要修改的参数是firstEp, 其他参数可不做任何修改,除非你很清楚为什么要修改。
一定要修改的参数是firstEp和fqdn, 其他参数可不做任何修改,除非你很清楚为什么要修改。
## 启动第一个节点
**加入到集群中的数据节点dnode涉及集群相关的下表11项参数必须完全相同否则不能成功加入到集群中。**
| **#** | **配置参数名称** | **含义** |
| ----- | ------------------ | ---------------------------------------- |
| 1 | numOfMnodes | 系统中管理节点个数 |
| 2 | mnodeEqualVnodeNum | 一个mnode等同于vnode消耗的个数 |
| 3 | offlineThreshold | dnode离线阈值超过该时间将导致Dnode离线 |
| 4 | statusInterval | dnode向mnode报告状态时长 |
| 5 | arbitrator | 系统中裁决器的end point |
| 6 | timezone | 时区 |
| 7 | locale | 系统区位信息及编码格式 |
| 8 | charset | 字符集编码 |
| 9 | balance | 是否启动负载均衡 |
| 10 | maxTablesPerVnode | 每个vnode中能够创建的最大表个数 |
| 11 | maxVgroupsPerDb | 每个DB中 能够使用的最大vnode个数 |
## 启动第一个数据节点
按照[《立即开始》](https://www.taosdata.com/cn/getting-started20/)里的指示启动第一个数据节点例如h1.taosdata.com然后执行taos, 启动taos shell从shell里执行命令"show dnodes;",如下所示:
按照[《立即开始》](https://www.taosdata.com/cn/getting-started20/)里的指示启动第一个节点h1.taosdata.com然后执行taos, 启动taos shell从shell里执行命令"show dnodes;",如下所示:
```
Welcome to the TDengine shell from Linux, Client Version:2.0.0.0
Copyright (c) 2017 by TAOS Data, Inc. All rights reserved.
@ -55,74 +76,86 @@ Query OK, 1 row(s) in set (0.006385s)
taos>
```
上述命令里可以看到这个刚启动的这个节点的End Point是h1.taos.com:6030
## 启动后续节点
上述命令里可以看到这个刚启动的这个数据节点的End Point是h1.taos.com:6030就是这个新集群的firstEP。
将后续的节点添加到现有集群,具体有以下几步:
## 启动后续数据节点
1. 按照["立即开始“](https://www.taosdata.com/cn/getting-started/)一章的方法在每个节点启动taosd。
将后续的数据节点添加到现有集群,具体有以下几步:
2. 在第一个节点使用CLI程序taos, 登录进TDengine系统, 执行命令:
1. 按照["立即开始“](https://www.taosdata.com/cn/getting-started/)一章的方法在每个物理节点启动taosd
```
CREATE DNODE "h2.taos.com:6030"
```
2. 在第一个数据节点使用CLI程序taos, 登录进TDengine系统, 执行命令:
将新节点的End Point (准备工作中第四步获知的) 添加进集群的EP列表。**"fqdn:port"需要用双引号引起来**否则出错。请注意将示例的“h2.taos.com:6030" 替换为这个新节点的End Point。
```
CREATE DNODE "h2.taos.com:6030"
```
将新数据节点的End Point (准备工作中第四步获知的) 添加进集群的EP列表。**"fqdn:port"需要用双引号引起来**否则出错。请注意将示例的“h2.taos.com:6030" 替换为这个新数据节点的End Point。
3. 然后执行命令
```
SHOW DNODES
```
```
SHOW DNODES
```
查看新节点是否被成功加入。如果该被加入的节点处于离线状态,请做两个检查
- 查看该节点的taosd是否正常工作如果没有正常运行需要先检查为什么
- 查看该节点taosd日志文件taosdlog.0里前面几行日志(一般在/var/log/taos目录)看日志里输出的该节点fqdn以及端口号是否为刚添加的End Point。如果不一致需要将正确的End Point添加进去。
查看新节点是否被成功加入。如果该被加入的数据节点处于离线状态,请做两个检查
按照上述步骤可以源源不断的将新的节点加入到集群。
- 查看该数据节点的taosd是否正常工作如果没有正常运行需要先检查为什么
- 查看该数据节点taosd日志文件taosdlog.0里前面几行日志(一般在/var/log/taos目录)看日志里输出的该数据节点fqdn以及端口号是否为刚添加的End Point。如果不一致需要将正确的End Point添加进去。
按照上述步骤可以源源不断的将新的数据节点加入到集群。
**提示:**
- firstEp这个参数仅仅在该节点第一次加入集群时有作用加入集群后该节点会保存最新的mnode的End Point列表不再依赖这两个参数。
- 两个没有配置firstEp参数的dnode启动后会独立运行起来。这个时候无法将其中一个节点加入到另外一个节点形成集群。**无法将两个独立的集群合并成为新的集群**。
- 任何已经加入集群在线的数据节点都可以作为后续待加入节点的firstEP。
- firstEp这个参数仅仅在该数据节点首次加入集群时有作用加入集群后该数据节点会保存最新的mnode的End Point列表不再依赖这个参数。
- 两个没有配置firstEp参数的数据节点dnode启动后会独立运行起来。这个时候无法将其中一个数据节点加入到另外一个数据节点形成集群。**无法将两个独立的集群合并成为新的集群**。
## 节点管理
## 数据节点管理
### 添加数据节点
### 添加节点
执行CLI程序taos, 使用root账号登录进系统, 执行:
```
CREATE DNODE "fqdn:port"
```
将新节点的End Point添加进集群的EP列表。**"fqdn:port"需要用双引号引起来**否则出错。一个节点对外服务的fqdn和port可以通过配置文件taos.cfg进行配置缺省是自动获取。
### 删除节点
将新数据节点的End Point添加进集群的EP列表。**"fqdn:port"需要用双引号引起来**否则出错。一个数据节点对外服务的fqdn和port可以通过配置文件taos.cfg进行配置缺省是自动获取。【强烈不建议用自动获取方式来配置FQDN可能导致生成的数据节点的End Point不是所期望的】
### 删除数据节点
执行CLI程序taos, 使用root账号登录进TDengine系统执行
```
DROP DNODE "fqdn:port";
```
其中fqdn是被删除的节点的FQDNport是其对外服务器的端口号
### 查看节点
### 查看数据节点
执行CLI程序taos,使用root账号登录进TDengine系统执行
```
SHOW DNODES;
```
它将列出集群中所有的dnode,每个dnode的fqdn:port, 状态(ready, offline等vnode数目还未使用的vnode数目等信息。在添加或删除一个节点后可以使用该命令查看。
它将列出集群中所有的dnode,每个dnode的fqdn:port, 状态(ready, offline等vnode数目还未使用的vnode数目等信息。在添加或删除一个数据节点后可以使用该命令查看。
### 查看虚拟节点组
为充分利用多核技术并提供scalability数据需要分片处理。因此TDengine会将一个DB的数据切分成多份存放在多个vnode里。这些vnode可能分布在多个dnode里这样就实现了水平扩展。一个vnode仅仅属于一个DB但一个DB可以有多个vnode。vnode的是mnode根据当前系统资源的情况自动进行分配的无需任何人工干预。
为充分利用多核技术并提供scalability数据需要分片处理。因此TDengine会将一个DB的数据切分成多份存放在多个vnode里。这些vnode可能分布在多个数据节点dnode里这样就实现了水平扩展。一个vnode仅仅属于一个DB但一个DB可以有多个vnode。vnode的是mnode根据当前系统资源的情况自动进行分配的无需任何人工干预。
执行CLI程序taos,使用root账号登录进TDengine系统执行
```
SHOW VGROUPS;
```
## vnode的高可用性
TDengine通过多副本的机制来提供系统的高可用性包括vnode和mnode的高可用性。
vnode的副本数是与DB关联的一个集群里可以有多个DB根据运营的需求每个DB可以配置不同的副本数。创建数据库时通过参数replica 指定副本数缺省为1。如果副本数为1系统的可靠性无法保证只要数据所在的节点宕机就将无法提供服务。集群的节点数必须大于等于副本数否则创建表时将返回错误“more dnodes are needed"。比如下面的命令将创建副本数为3的数据库demo
@ -130,21 +163,25 @@ vnode的副本数是与DB关联的一个集群里可以有多个DB根据
```
CREATE DATABASE demo replica 3;
```
一个DB里的数据会被切片分到多个vnode groupvnode group里的vnode数目就是DB的副本数同一个vnode group里各vnode的数据是完全一致的。为保证高可用性vnode group里的vnode一定要分布在不同的dnode里实际部署时需要在不同的物理机上只要一个vgroup里超过半数的vnode处于工作状态这个vgroup就能正常的对外服务。
一个dnode里可能有多个DB的数据因此一个dnode离线时可能会影响到多个DB。如果一个vnode group里的一半或一半以上的vnode不工作那么该vnode group就无法对外服务无法插入或读取数据这样会影响到它所属的DB的一部分表的读写操作
一个DB里的数据会被切片分到多个vnode groupvnode group里的vnode数目就是DB的副本数同一个vnode group里各vnode的数据是完全一致的。为保证高可用性vnode group里的vnode一定要分布在不同的数据节点dnode里实际部署时需要在不同的物理机上只要一个vgroup里超过半数的vnode处于工作状态这个vgroup就能正常的对外服务
因为vnode的引入无法简单的给出结论“集群中过半dnode工作集群就应该工作”。但是对于简单的情形很好下结论。比如副本数为3只有三个dnode那如果仅有一个节点不工作整个集群还是可以正常工作的但如果有两个节点不工作那整个集群就无法正常工作了。
一个数据节点dnode里可能有多个DB的数据因此一个dnode离线时可能会影响到多个DB。如果一个vnode group里的一半或一半以上的vnode不工作那么该vnode group就无法对外服务无法插入或读取数据这样会影响到它所属的DB的一部分表的读写操作。
因为vnode的引入无法简单的给出结论“集群中过半数据节点dnode工作集群就应该工作”。但是对于简单的情形很好下结论。比如副本数为3只有三个dnode那如果仅有一个节点不工作整个集群还是可以正常工作的但如果有两个数据节点不工作那整个集群就无法正常工作了。
## Mnode的高可用性
TDengine集群是由mnode (taosd的一个模块逻辑节点) 负责管理的为保证mnode的高可用可以配置多个mnode副本副本数由系统配置参数numOfMnodes决定有效范围为1-3。为保证元数据的强一致性mnode副本之间是通过同步的方式进行数据复制的。
一个集群有多个dnode, 但一个dnode至多运行一个mnode实例。多个dnode情况下哪个dnode可以作为mnode呢这是完全由系统根据整个系统资源情况自动指定的。用户可通过CLI程序taos在TDengine的console里执行如下命令
TDengine集群是由mnode (taosd的一个模块管理节点) 负责管理的为保证mnode的高可用可以配置多个mnode副本副本数由系统配置参数numOfMnodes决定有效范围为1-3。为保证元数据的强一致性mnode副本之间是通过同步的方式进行数据复制的。
一个集群有多个数据节点dnode, 但一个dnode至多运行一个mnode实例。多个dnode情况下哪个dnode可以作为mnode呢这是完全由系统根据整个系统资源情况自动指定的。用户可通过CLI程序taos在TDengine的console里执行如下命令
```
SHOW MNODES;
```
来查看mnode列表该列表将列出mnode所处的dnode的End Point和角色(master, slave, unsynced 或offline)。
当集群中第一个节点启动时该节点一定会运行一个mnode实例否则该dnode无法正常工作因为一个系统是必须有至少一个mnode的。如果numOfMnodes配置为2启动第二个dnode时该dnode也将运行一个mnode实例。
当集群中第一个数据节点启动时,该数据节点一定会运行一个mnode实例否则该数据节点dnode无法正常工作因为一个系统是必须有至少一个mnode的。如果numOfMnodes配置为2启动第二个dnode时该dnode也将运行一个mnode实例。
为保证mnode服务的高可用性numOfMnodes必须设置为2或更大。因为mnode保存的元数据必须是强一致的如果numOfMnodes大于2复制参数quorum自动设为2也就是说至少要保证有两个副本写入数据成功才通知客户端应用写入成功。
@ -154,22 +191,25 @@ SHOW MNODES;
有三种情况,将触发负载均衡,而且都无需人工干预。
- 当一个新节点添加进集群时,系统将自动触发负载均衡,一些节点上的数据将被自动转移到新节点上,无需任何人工干预。
- 当一个节点从集群中移除时,系统将自动把该节点上的数据转移到其他节点,无需任何人工干预。
- 如果一个节点过热数据量过大系统将自动进行负载均衡将该节点的一些vnode自动挪到其他节点。
- 当一个新数据节点添加进集群时,系统将自动触发负载均衡,一些节点上的数据将被自动转移到新数据节点上,无需任何人工干预。
- 当一个数据节点从集群中移除时,系统将自动把该数据节点上的数据转移到其他数据节点,无需任何人工干预。
- 如果一个数据节点过热(数据量过大),系统将自动进行负载均衡,将该数据节点的一些vnode自动挪到其他节点。
当上述三种情况发生时,系统将启动一各个节点的负载计算,从而决定如何挪动。
当上述三种情况发生时,系统将启动一各个数据节点的负载计算,从而决定如何挪动。
##节点离线处理
如果一个节点离线TDengine集群将自动检测到。有如下两种情况
- 改节点离线超过一定时间taos.cfg里配置参数offlineThreshold控制时长),系统将自动把该节点删除,产生系统报警信息,触发负载均衡流程。如果该被删除的节点重现上线时,它将无法加入集群,需要系统管理员重新将其添加进集群才会开始工作。
**【提示】负载均衡由参数balance控制它决定是否启动自动负载均衡。**
## 数据节点离线处理
如果一个数据节点离线TDengine集群将自动检测到。有如下两种情况
- 该数据节点离线超过一定时间taos.cfg里配置参数offlineThreshold控制时长),系统将自动把该数据节点删除,产生系统报警信息,触发负载均衡流程。如果该被删除的数据节点重现上线时,它将无法加入集群,需要系统管理员重新将其添加进集群才会开始工作。
- 离线后在offlineThreshold的时长内重新上线系统将自动启动数据恢复流程等数据完全恢复后该节点将开始正常工作。
**注意:**如果一个虚拟节点组包括mnode组里每个节点都处于离线或unsynced状态必须等该虚拟节点组里的所有节点都上线、都能交换状态信息后才能选出Master该虚拟节点组才能对外提供服务。比如整个集群有3个节点副本数为3如果3个节点都宕机然后2个节点重启是无法工作的只有等3个节点都重启成功才能对外服务。
**注意:**如果一个虚拟节点组包括mnode组所归属的每个数据节点都处于离线或unsynced状态必须等该虚拟节点组里的所有数据节点都上线、都能交换状态信息后才能选出Master该虚拟节点组才能对外提供服务。比如整个集群有3个数据节点副本数为3如果3个数据节点都宕机然后2个数据节点重启是无法工作的只有等3个数据节点都重启成功,才能对外服务。
## Arbitrator的使用
如果副本数为偶数当一个vnode group里一半或超过一半的vnode不工作时是无法从中选出master的。同理一半或超过一半的mnode不工作时是无法选出mnode的master的因为存在“split brain”问题。为解决这个问题TDengine引入了arbitrator的概念。Arbitrator模拟一个vnode或mnode在工作但只简单的负责网络连接不处理任何数据插入或访问。只要包含arbitrator在内超过半数的vnode或mnode工作那么该vnode group或mnode组就可以正常的提供数据插入或查询服务。比如对于副本数为2的情形如果一个节点A离线但另外一个节点B正常而且能连接到arbitrator, 那么节点B就能正常工作。
TDengine安装包里带有一个执行程序tarbitrator, 找任何一台Linux服务器运行它即可。该程序对系统资源几乎没有要求只需要保证有网络连接即可。该应用的命令行参数`-p`可以指定其对外服务的端口号缺省是6042。配置每个taosd实例时可以在配置文件taos.cfg里将参数arbitrator设置为arbitrator的End Point。如果该参数配置了当副本数为偶数数系统将自动连接配置的arbitrator。

View File

@ -69,17 +69,13 @@ typedef struct SJoinSupporter {
SSubqueryState* pState;
SSqlObj* pObj; // parent SqlObj
int32_t subqueryIndex; // index of sub query
char intervalTimeUnit;
char slidingTimeUnit;
int64_t intervalTime; // interval time
int64_t slidingTime; // sliding time
SInterval interval;
SLimitVal limit; // limit info
uint64_t uid; // query meter uid
uint64_t uid; // query table uid
SArray* colList; // previous query information, no need to use this attribute, and the corresponding attribution
SArray* exprList;
SFieldInfo fieldsInfo;
STagCond tagCond;
SSqlGroupbyExpr groupbyExpr;
struct STSBuf* pTSBuf; // the TSBuf struct that holds the compressed timestamp array
FILE* f; // temporary file in order to create TSBuf
char path[PATH_MAX]; // temporary file path, todo dynamic allocate memory

View File

@ -29,6 +29,7 @@ extern "C" {
#include "tglobal.h"
#include "tsqlfunction.h"
#include "tutil.h"
#include "tcache.h"
#include "qExecutor.h"
#include "qSqlparser.h"
@ -225,12 +226,8 @@ typedef struct SQueryInfo {
int16_t command; // the command may be different for each subclause, so keep it seperately.
uint32_t type; // query/insert type
// TODO refactor
char intervalTimeUnit;
char slidingTimeUnit;
STimeWindow window; // query time window
int64_t intervalTime; // aggregation time window range
int64_t slidingTime; // sliding window in mseconds
int64_t intervalOffset;// start offset of each time window
SInterval interval;
int32_t tz; // query client timezone
SSqlGroupbyExpr groupbyExpr; // group by tags info
@ -333,6 +330,7 @@ typedef struct STscObj {
struct SSqlStream *streamList;
void* pDnodeConn;
pthread_mutex_t mutex;
T_REF_DECLARE()
} STscObj;
typedef struct SSqlObj {
@ -359,6 +357,8 @@ typedef struct SSqlObj {
uint16_t numOfSubs;
struct SSqlObj **pSubs;
struct SSqlObj * prev, *next;
struct SSqlObj **self;
} SSqlObj;
typedef struct SSqlStream {
@ -366,8 +366,6 @@ typedef struct SSqlStream {
uint32_t streamId;
char listed;
bool isProject;
char intervalTimeUnit;
char slidingTimeUnit;
int16_t precision;
int64_t num; // number of computing count
@ -381,8 +379,7 @@ typedef struct SSqlStream {
int64_t ctime; // stream created time
int64_t stime; // stream next executed time
int64_t etime; // stream end query time, when time is larger then etime, the stream will be closed
int64_t intervalTime;
int64_t slidingTime;
SInterval interval;
void * pTimer;
void (*fp)();
@ -413,7 +410,6 @@ int32_t tscTansformSQLFuncForSTableQuery(SQueryInfo *pQueryInfo);
void tscRestoreSQLFuncForSTableQuery(SQueryInfo *pQueryInfo);
int32_t tscCreateResPointerInfo(SSqlRes *pRes, SQueryInfo *pQueryInfo);
void tscDestroyResPointerInfo(SSqlRes *pRes);
void tscResetSqlCmdObj(SSqlCmd *pCmd, bool removeFromCache);
@ -425,17 +421,19 @@ void tscFreeSqlResult(SSqlObj *pSql);
/**
* only free part of resources allocated during query.
* TODO remove it later
* Note: this function is multi-thread safe.
* @param pObj
*/
void tscPartiallyFreeSqlObj(SSqlObj *pObj);
void tscPartiallyFreeSqlObj(SSqlObj *pSql);
/**
* free sql object, release allocated resource
* @param pObj Free metric/meta information, dynamically allocated payload, and
* response buffer, object itself
* @param pObj
*/
void tscFreeSqlObj(SSqlObj *pObj);
void tscFreeSqlObj(SSqlObj *pSql);
void tscFreeSqlObjInCache(void *pSql);
void tscCloseTscObj(STscObj *pObj);
@ -451,9 +449,6 @@ void tscInitResObjForLocalQuery(SSqlObj *pObj, int32_t numOfRes, int32_t rowLen)
bool tscIsUpdateQuery(SSqlObj* pSql);
bool tscHasReachLimitation(SQueryInfo *pQueryInfo, SSqlRes *pRes);
// todo remove this function.
bool tscResultsetFetchCompleted(TAOS_RES *result);
char *tscGetErrorMsgPayload(SSqlCmd *pCmd);
int32_t tscInvalidSQLErrMsg(char *msg, const char *additionalInfo, const char *sql);
@ -468,7 +463,7 @@ static FORCE_INLINE void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pField
int32_t type = pInfo->pSqlExpr->resType;
int32_t bytes = pInfo->pSqlExpr->resBytes;
char* pData = pRes->data + pInfo->pSqlExpr->offset * pRes->numOfRows + bytes * pRes->row;
char* pData = pRes->data + (int32_t)(pInfo->pSqlExpr->offset * pRes->numOfRows + bytes * pRes->row);
// user defined constant value output columns
if (TSDB_COL_IS_UD_COL(pInfo->pSqlExpr->colInfo.flag)) {
@ -502,7 +497,8 @@ static FORCE_INLINE void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pField
}
}
extern void * tscCacheHandle;
extern SCacheObj* tscMetaCache;
extern SCacheObj* tscObjCache;
extern void * tscTmr;
extern void * tscQhandle;
extern int tscKeepConn[];

View File

@ -18,6 +18,7 @@
#include "tnote.h"
#include "trpc.h"
#include "tcache.h"
#include "tscLog.h"
#include "tscSubquery.h"
#include "tscLocalMerge.h"
@ -40,6 +41,8 @@ static void tscAsyncFetchRowsProxy(void *param, TAOS_RES *tres, int numOfRows);
static void tscAsyncFetchSingleRowProxy(void *param, TAOS_RES *tres, int numOfRows);
void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const char* sqlstr, size_t sqlLen) {
SSqlCmd* pCmd = &pSql->cmd;
pSql->signature = pSql;
pSql->param = param;
pSql->pTscObj = pObj;
@ -48,6 +51,11 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const
pSql->fp = fp;
pSql->fetchFp = fp;
uint64_t handle = (uint64_t) pSql;
pSql->self = taosCachePut(tscObjCache, &handle, sizeof(uint64_t), &pSql, sizeof(uint64_t), 2*3600*1000);
T_REF_INC(pSql->pTscObj);
pSql->sqlstr = calloc(1, sqlLen + 1);
if (pSql->sqlstr == NULL) {
tscError("%p failed to malloc sql string buffer", pSql);
@ -59,7 +67,7 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const
strntolower(pSql->sqlstr, sqlstr, (int32_t)sqlLen);
tscDebugL("%p SQL: %s", pSql, pSql->sqlstr);
pSql->cmd.curSql = pSql->sqlstr;
pCmd->curSql = pSql->sqlstr;
int32_t code = tsParseSql(pSql, true);
if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) return;
@ -69,7 +77,7 @@ void doAsyncQuery(STscObj* pObj, SSqlObj* pSql, void (*fp)(), void* param, const
tscQueueAsyncRes(pSql);
return;
}
tscDoQuery(pSql);
}

View File

@ -117,6 +117,10 @@ typedef struct SFirstLastInfo {
typedef struct SFirstLastInfo SLastrowInfo;
typedef struct SPercentileInfo {
tMemBucket *pMemBucket;
int32_t stage;
double minval;
double maxval;
int64_t numOfElems;
} SPercentileInfo;
typedef struct STopBotInfo {
@ -302,7 +306,7 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI
} else if (functionId == TSDB_FUNC_PERCT) {
*type = (int16_t)TSDB_DATA_TYPE_DOUBLE;
*bytes = (int16_t)sizeof(double);
*interBytes = (int16_t)sizeof(double);
*interBytes = (int16_t)sizeof(SPercentileInfo);
} else if (functionId == TSDB_FUNC_LEASTSQR) {
*type = TSDB_DATA_TYPE_BINARY;
*bytes = TSDB_AVG_FUNCTION_INTER_BUFFER_SIZE; // string
@ -2428,12 +2432,14 @@ static bool percentile_function_setup(SQLFunctionCtx *pCtx) {
if (!function_setup(pCtx)) {
return false;
}
SResultInfo *pResInfo = GET_RES_INFO(pCtx);
((SPercentileInfo *)(pResInfo->interResultBuf))->pMemBucket =
tMemBucketCreate(pCtx->inputBytes, pCtx->inputType);
// in the first round, get the min-max value of all involved data
SResultInfo *pResInfo = GET_RES_INFO(pCtx);
SPercentileInfo *pInfo = pResInfo->interResultBuf;
pInfo->minval = DBL_MAX;
pInfo->maxval = -DBL_MAX;
pInfo->numOfElems = 0;
return true;
}
@ -2442,7 +2448,65 @@ static void percentile_function(SQLFunctionCtx *pCtx) {
SResultInfo * pResInfo = GET_RES_INFO(pCtx);
SPercentileInfo *pInfo = pResInfo->interResultBuf;
// the first stage, only acquire the min/max value
if (pInfo->stage == 0) {
if (pCtx->preAggVals.isSet) {
if (pInfo->minval > pCtx->preAggVals.statis.min) {
pInfo->minval = (double)pCtx->preAggVals.statis.min;
}
if (pInfo->maxval < pCtx->preAggVals.statis.max) {
pInfo->maxval = (double)pCtx->preAggVals.statis.max;
}
pInfo->numOfElems += (pCtx->size - pCtx->preAggVals.statis.numOfNull);
} else {
for (int32_t i = 0; i < pCtx->size; ++i) {
char *data = GET_INPUT_CHAR_INDEX(pCtx, i);
if (pCtx->hasNull && isNull(data, pCtx->inputType)) {
continue;
}
// TODO extract functions
double v = 0;
switch (pCtx->inputType) {
case TSDB_DATA_TYPE_TINYINT:
v = GET_INT8_VAL(data);
break;
case TSDB_DATA_TYPE_SMALLINT:
v = GET_INT16_VAL(data);
break;
case TSDB_DATA_TYPE_BIGINT:
v = (double)(GET_INT64_VAL(data));
break;
case TSDB_DATA_TYPE_FLOAT:
v = GET_FLOAT_VAL(data);
break;
case TSDB_DATA_TYPE_DOUBLE:
v = GET_DOUBLE_VAL(data);
break;
default:
v = GET_INT32_VAL(data);
break;
}
if (v < pInfo->minval) {
pInfo->minval = v;
}
if (v > pInfo->maxval) {
pInfo->maxval = v;
}
pInfo->numOfElems += 1;
}
}
return;
}
// the second stage, calculate the true percentile value
for (int32_t i = 0; i < pCtx->size; ++i) {
char *data = GET_INPUT_CHAR_INDEX(pCtx, i);
if (pCtx->hasNull && isNull(data, pCtx->inputType)) {
@ -2462,10 +2526,47 @@ static void percentile_function_f(SQLFunctionCtx *pCtx, int32_t index) {
if (pCtx->hasNull && isNull(pData, pCtx->inputType)) {
return;
}
SResultInfo *pResInfo = GET_RES_INFO(pCtx);
SPercentileInfo *pInfo = (SPercentileInfo *)pResInfo->interResultBuf;
if (pInfo->stage == 0) {
// TODO extract functions
double v = 0;
switch (pCtx->inputType) {
case TSDB_DATA_TYPE_TINYINT:
v = GET_INT8_VAL(pData);
break;
case TSDB_DATA_TYPE_SMALLINT:
v = GET_INT16_VAL(pData);
break;
case TSDB_DATA_TYPE_BIGINT:
v = (double)(GET_INT64_VAL(pData));
break;
case TSDB_DATA_TYPE_FLOAT:
v = GET_FLOAT_VAL(pData);
break;
case TSDB_DATA_TYPE_DOUBLE:
v = GET_DOUBLE_VAL(pData);
break;
default:
v = GET_INT32_VAL(pData);
break;
}
if (v < pInfo->minval) {
pInfo->minval = v;
}
if (v > pInfo->maxval) {
pInfo->maxval = v;
}
pInfo->numOfElems += 1;
return;
}
tMemBucketPut(pInfo->pMemBucket, pData, 1);
SET_VAL(pCtx, 1, 1);
@ -2488,6 +2589,23 @@ static void percentile_finalizer(SQLFunctionCtx *pCtx) {
doFinalizer(pCtx);
}
static void percentile_next_step(SQLFunctionCtx *pCtx) {
SResultInfo * pResInfo = GET_RES_INFO(pCtx);
SPercentileInfo *pInfo = pResInfo->interResultBuf;
if (pInfo->stage == 0) {
// all data are null, set it completed
if (pInfo->numOfElems == 0) {
pResInfo->complete = true;
}
pInfo->stage += 1;
pInfo->pMemBucket = tMemBucketCreate(pCtx->inputBytes, pCtx->inputType, pInfo->minval, pInfo->maxval);
} else {
pResInfo->complete = true;
}
}
//////////////////////////////////////////////////////////////////////////////////
static SAPercentileInfo *getAPerctInfo(SQLFunctionCtx *pCtx) {
SResultInfo *pResInfo = GET_RES_INFO(pCtx);
@ -4513,7 +4631,7 @@ SQLAggFuncElem aAggs[] = {{
percentile_function_setup,
percentile_function,
percentile_function_f,
no_next_step,
percentile_next_step,
percentile_finalizer,
noop1,
noop1,

View File

@ -16,7 +16,6 @@
#include "os.h"
#include "taosmsg.h"
#include "qExtbuffer.h"
#include "taosdef.h"
#include "tcache.h"
#include "tname.h"
@ -430,7 +429,7 @@ int tscProcessLocalCmd(SSqlObj *pSql) {
pRes->qhandle = 0x1;
pRes->numOfRows = 0;
} else if (pCmd->command == TSDB_SQL_RESET_CACHE) {
taosCacheEmpty(tscCacheHandle);
taosCacheEmpty(tscMetaCache);
pRes->code = TSDB_CODE_SUCCESS;
} else if (pCmd->command == TSDB_SQL_SERV_VERSION) {
pRes->code = tscProcessServerVer(pSql);

View File

@ -368,13 +368,12 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd
STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta);
TSKEY stime = (pQueryInfo->order.order == TSDB_ORDER_ASC)? pQueryInfo->window.skey : pQueryInfo->window.ekey;
int64_t revisedSTime =
taosGetIntervalStartTimestamp(stime, pQueryInfo->slidingTime, pQueryInfo->intervalTime, pQueryInfo->slidingTimeUnit, tinfo.precision);
int64_t revisedSTime = taosTimeTruncate(stime, &pQueryInfo->interval, tinfo.precision);
if (pQueryInfo->fillType != TSDB_FILL_NONE) {
SFillColInfo* pFillCol = createFillColInfo(pQueryInfo);
pReducer->pFillInfo = taosInitFillInfo(pQueryInfo->order.order, revisedSTime, pQueryInfo->groupbyExpr.numOfGroupCols,
4096, (int32_t)numOfCols, pQueryInfo->slidingTime, pQueryInfo->slidingTimeUnit,
4096, (int32_t)numOfCols, pQueryInfo->interval.sliding, pQueryInfo->interval.slidingUnit,
tinfo.precision, pQueryInfo->fillType, pFillCol);
}
}
@ -472,10 +471,8 @@ void tscDestroyLocalReducer(SSqlObj *pSql) {
return;
}
tscDebug("%p start to free local reducer", pSql);
SSqlRes *pRes = &(pSql->res);
if (pRes->pLocalReducer == NULL) {
tscDebug("%p local reducer has been freed, abort", pSql);
return;
}
@ -553,7 +550,7 @@ static int32_t createOrderDescriptor(tOrderDescriptor **pOrderDesc, SSqlCmd *pCm
}
// primary timestamp column is involved in final result
if (pQueryInfo->intervalTime != 0 || tscOrderedProjectionQueryOnSTable(pQueryInfo, 0)) {
if (pQueryInfo->interval.interval != 0 || tscOrderedProjectionQueryOnSTable(pQueryInfo, 0)) {
numOfGroupByCols++;
}
@ -570,7 +567,7 @@ static int32_t createOrderDescriptor(tOrderDescriptor **pOrderDesc, SSqlCmd *pCm
orderIdx[i] = startCols++;
}
if (pQueryInfo->intervalTime != 0) {
if (pQueryInfo->interval.interval != 0) {
// the first column is the timestamp, handles queries like "interval(10m) group by tags"
orderIdx[numOfGroupByCols - 1] = PRIMARYKEY_TIMESTAMP_COL_INDEX;
}
@ -614,12 +611,12 @@ bool isSameGroup(SSqlCmd *pCmd, SLocalReducer *pReducer, char *pPrev, tFilePage
* super table interval query
* if the order columns is the primary timestamp, all result data belongs to one group
*/
assert(pQueryInfo->intervalTime > 0);
assert(pQueryInfo->interval.interval > 0);
if (numOfCols == 1) {
return true;
}
} else { // simple group by query
assert(pQueryInfo->intervalTime == 0);
assert(pQueryInfo->interval.interval == 0);
}
// only one row exists
@ -827,8 +824,7 @@ void savePrevRecordAndSetupFillInfo(SLocalReducer *pLocalReducer, SQueryInfo *pQ
if (pFillInfo != NULL) {
int64_t stime = (pQueryInfo->window.skey < pQueryInfo->window.ekey) ? pQueryInfo->window.skey : pQueryInfo->window.ekey;
int64_t revisedSTime =
taosGetIntervalStartTimestamp(stime, pQueryInfo->slidingTime, pQueryInfo->intervalTime, pQueryInfo->slidingTimeUnit, tinfo.precision);
int64_t revisedSTime = taosTimeTruncate(stime, &pQueryInfo->interval, tinfo.precision);
taosResetFillInfo(pFillInfo, revisedSTime);
}
@ -841,7 +837,7 @@ void savePrevRecordAndSetupFillInfo(SLocalReducer *pLocalReducer, SQueryInfo *pQ
}
static void genFinalResWithoutFill(SSqlRes* pRes, SLocalReducer *pLocalReducer, SQueryInfo* pQueryInfo) {
assert(pQueryInfo->intervalTime == 0 || pQueryInfo->fillType == TSDB_FILL_NONE);
assert(pQueryInfo->interval.interval == 0 || pQueryInfo->fillType == TSDB_FILL_NONE);
tFilePage * pBeforeFillData = pLocalReducer->pResultBuf;
@ -1222,7 +1218,7 @@ bool genFinalResults(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool noMoreCur
#endif
// no interval query, no fill operation
if (pQueryInfo->intervalTime == 0 || pQueryInfo->fillType == TSDB_FILL_NONE) {
if (pQueryInfo->interval.interval == 0 || pQueryInfo->fillType == TSDB_FILL_NONE) {
genFinalResWithoutFill(pRes, pLocalReducer, pQueryInfo);
} else {
SFillInfo* pFillInfo = pLocalReducer->pFillInfo;
@ -1260,13 +1256,10 @@ static void resetEnvForNewResultset(SSqlRes *pRes, SSqlCmd *pCmd, SLocalReducer
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0);
STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta);
int8_t precision = tinfo.precision;
// for group result interpolation, do not return if not data is generated
if (pQueryInfo->fillType != TSDB_FILL_NONE) {
TSKEY skey = (pQueryInfo->order.order == TSDB_ORDER_ASC)? pQueryInfo->window.skey:pQueryInfo->window.ekey;//MIN(pQueryInfo->window.skey, pQueryInfo->window.ekey);
int64_t newTime =
taosGetIntervalStartTimestamp(skey, pQueryInfo->slidingTime, pQueryInfo->intervalTime, pQueryInfo->slidingTimeUnit, precision);
int64_t newTime = taosTimeTruncate(skey, &pQueryInfo->interval, tinfo.precision);
taosResetFillInfo(pLocalReducer->pFillInfo, newTime);
}
}

View File

@ -142,7 +142,7 @@ int tsParseTime(SStrToken *pToken, int64_t *time, char **next, char *error, int1
return tscInvalidSQLErrMsg(error, "value expected in timestamp", sToken.z);
}
if (getTimestampInUsFromStr(valueToken.z, valueToken.n, &interval) != TSDB_CODE_SUCCESS) {
if (parseAbsoluteDuration(valueToken.z, valueToken.n, &interval) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_SQL;
}
@ -526,7 +526,7 @@ int tsParseValues(char **str, STableDataBlocks *pDataBlock, STableMeta *pTableMe
int32_t index = 0;
SStrToken sToken;
int16_t numOfRows = 0;
int32_t numOfRows = 0;
SSchema *pSchema = tscGetTableSchema(pTableMeta);
STableComInfo tinfo = tscGetTableInfo(pTableMeta);

View File

@ -259,11 +259,11 @@ int tscBuildQueryStreamDesc(void *pMsg, STscObj *pObj) {
pSdesc->num = htobe64(pStream->num);
pSdesc->useconds = htobe64(pStream->useconds);
pSdesc->stime = htobe64(pStream->stime - pStream->intervalTime);
pSdesc->stime = htobe64(pStream->stime - pStream->interval.interval);
pSdesc->ctime = htobe64(pStream->ctime);
pSdesc->slidingTime = htobe64(pStream->slidingTime);
pSdesc->interval = htobe64(pStream->intervalTime);
pSdesc->slidingTime = htobe64(pStream->interval.sliding);
pSdesc->interval = htobe64(pStream->interval.interval);
pHeartbeat->numOfStreams++;
pSdesc++;

View File

@ -81,6 +81,7 @@ static void setColumnOffsetValueInResultset(SQueryInfo* pQueryInfo);
static int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* pCmd);
static int32_t parseIntervalClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql);
static int32_t parseOffsetClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql);
static int32_t parseSlidingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql);
static int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExprItem* pItem);
@ -92,7 +93,6 @@ static int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQueryS
static int32_t tsRewriteFieldNameIfNecessary(SSqlCmd* pCmd, SQueryInfo* pQueryInfo);
static int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo);
static int32_t validateSqlFunctionInStreamSql(SSqlCmd* pCmd, SQueryInfo* pQueryInfo);
static int32_t arithmeticExprToString(tSQLExpr* pExpr, char** exprString);
static int32_t validateFunctionsInIntervalOrGroupbyQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo);
static int32_t validateArithmeticSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryInfo* pQueryInfo, SColumnList* pList, int32_t* type);
static int32_t validateEp(char* ep);
@ -103,6 +103,7 @@ static int32_t setKillInfo(SSqlObj* pSql, struct SSqlInfo* pInfo, int32_t killTy
static bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField);
static bool hasTimestampForPointInterpQuery(SQueryInfo* pQueryInfo);
static bool hasNormalColumnFilter(SQueryInfo* pQueryInfo);
static int32_t parseLimitClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t index, SQuerySQL* pQuerySql, SSqlObj* pSql);
static int32_t parseCreateDBOptions(SSqlCmd* pCmd, SCreateDBInfo* pCreateDbSql);
@ -360,8 +361,9 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
}
// additional msg has been attached already
if (tscSetTableFullName(pTableMetaInfo, pToken, pSql) != TSDB_CODE_SUCCESS) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
return TSDB_CODE_TSC_INVALID_SQL;
}
return tscGetTableMeta(pSql, pTableMetaInfo);
@ -594,24 +596,28 @@ int32_t parseIntervalClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQ
// interval is not null
SStrToken* t = &pQuerySql->interval;
if (parseDuration(t->z, t->n, &pQueryInfo->intervalTime, &pQueryInfo->intervalTimeUnit) != TSDB_CODE_SUCCESS) {
if (parseNatualDuration(t->z, t->n, &pQueryInfo->interval.interval, &pQueryInfo->interval.intervalUnit) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_SQL;
}
if (pQueryInfo->intervalTimeUnit != 'n' && pQueryInfo->intervalTimeUnit != 'y') {
if (pQueryInfo->interval.intervalUnit != 'n' && pQueryInfo->interval.intervalUnit != 'y') {
// if the unit of time window value is millisecond, change the value from microsecond
if (tinfo.precision == TSDB_TIME_PRECISION_MILLI) {
pQueryInfo->intervalTime = pQueryInfo->intervalTime / 1000;
pQueryInfo->interval.interval = pQueryInfo->interval.interval / 1000;
}
// interval cannot be less than 10 milliseconds
if (pQueryInfo->intervalTime < tsMinIntervalTime) {
if (pQueryInfo->interval.interval < tsMinIntervalTime) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
}
}
// for top/bottom + interval query, we do not add additional timestamp column in the front
if (isTopBottomQuery(pQueryInfo)) {
if (parseOffsetClause(pCmd, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_SQL;
}
if (parseSlidingClause(pCmd, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_SQL;
}
@ -635,7 +641,7 @@ int32_t parseIntervalClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQ
* check invalid SQL:
* select tbname, tags_fields from super_table_name interval(1s)
*/
if (tscQueryTags(pQueryInfo) && pQueryInfo->intervalTime > 0) {
if (tscQueryTags(pQueryInfo) && pQueryInfo->interval.interval > 0) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
}
@ -661,6 +667,10 @@ int32_t parseIntervalClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQ
SColumnIndex index = {tableIndex, PRIMARYKEY_TIMESTAMP_COL_INDEX};
tscAddSpecialColumnForSelect(pQueryInfo, 0, TSDB_FUNC_TS, &index, &s, TSDB_COL_NORMAL);
if (parseOffsetClause(pCmd, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_SQL;
}
if (parseSlidingClause(pCmd, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_SQL;
}
@ -668,6 +678,57 @@ int32_t parseIntervalClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQ
return TSDB_CODE_SUCCESS;
}
int32_t parseOffsetClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) {
const char* msg1 = "interval offset cannot be negative";
const char* msg2 = "interval offset should be shorter than interval";
const char* msg3 = "cannot use 'year' as offset when interval is 'month'";
STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta);
SStrToken* t = &pQuerySql->offset;
if (t->n == 0) {
pQueryInfo->interval.offsetUnit = pQueryInfo->interval.intervalUnit;
pQueryInfo->interval.offset = 0;
return TSDB_CODE_SUCCESS;
}
if (parseNatualDuration(t->z, t->n, &pQueryInfo->interval.offset, &pQueryInfo->interval.offsetUnit) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_SQL;
}
if (pQueryInfo->interval.offset < 0) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
}
if (pQueryInfo->interval.offsetUnit != 'n' && pQueryInfo->interval.offsetUnit != 'y') {
// if the unit of time window value is millisecond, change the value from microsecond
if (tinfo.precision == TSDB_TIME_PRECISION_MILLI) {
pQueryInfo->interval.offset = pQueryInfo->interval.offset / 1000;
}
if (pQueryInfo->interval.intervalUnit != 'n' && pQueryInfo->interval.intervalUnit != 'y') {
if (pQueryInfo->interval.offset >= pQueryInfo->interval.interval) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
}
}
} else if (pQueryInfo->interval.offsetUnit == pQueryInfo->interval.intervalUnit) {
if (pQueryInfo->interval.offset >= pQueryInfo->interval.interval) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
}
} else if (pQueryInfo->interval.intervalUnit == 'n' && pQueryInfo->interval.offsetUnit == 'y') {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
} else if (pQueryInfo->interval.intervalUnit == 'y' && pQueryInfo->interval.offsetUnit == 'n') {
if (pQueryInfo->interval.interval * 12 <= pQueryInfo->interval.offset) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
}
} else {
// TODO: offset should be shorter than interval, but how to check
// conflicts like 30days offset and 1 month interval
}
return TSDB_CODE_SUCCESS;
}
int32_t parseSlidingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) {
const char* msg0 = "sliding value too small";
const char* msg1 = "sliding value no larger than the interval value";
@ -681,29 +742,29 @@ int32_t parseSlidingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu
SStrToken* pSliding = &pQuerySql->sliding;
if (pSliding->n == 0) {
pQueryInfo->slidingTimeUnit = pQueryInfo->intervalTimeUnit;
pQueryInfo->slidingTime = pQueryInfo->intervalTime;
pQueryInfo->interval.slidingUnit = pQueryInfo->interval.intervalUnit;
pQueryInfo->interval.sliding = pQueryInfo->interval.interval;
return TSDB_CODE_SUCCESS;
}
if (pQueryInfo->intervalTimeUnit == 'n' || pQueryInfo->intervalTimeUnit == 'y') {
if (pQueryInfo->interval.intervalUnit == 'n' || pQueryInfo->interval.intervalUnit == 'y') {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
}
getTimestampInUsFromStr(pSliding->z, pSliding->n, &pQueryInfo->slidingTime);
parseAbsoluteDuration(pSliding->z, pSliding->n, &pQueryInfo->interval.sliding);
if (tinfo.precision == TSDB_TIME_PRECISION_MILLI) {
pQueryInfo->slidingTime /= 1000;
pQueryInfo->interval.sliding /= 1000;
}
if (pQueryInfo->slidingTime < tsMinSlidingTime) {
if (pQueryInfo->interval.sliding < tsMinSlidingTime) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg0);
}
if (pQueryInfo->slidingTime > pQueryInfo->intervalTime) {
if (pQueryInfo->interval.sliding > pQueryInfo->interval.interval) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
}
if ((pQueryInfo->intervalTime != 0) && (pQueryInfo->intervalTime/pQueryInfo->slidingTime > INTERVAL_SLIDING_FACTOR)) {
if ((pQueryInfo->interval.interval != 0) && (pQueryInfo->interval.interval/pQueryInfo->interval.sliding > INTERVAL_SLIDING_FACTOR)) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
}
@ -712,8 +773,7 @@ int32_t parseSlidingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu
int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SStrToken* pzTableName, SSqlObj* pSql) {
const char* msg1 = "name too long";
const char* msg2 = "invalid db name";
const char *msg = msg1;
const char* msg2 = "current database name is invalid";
SSqlCmd* pCmd = &pSql->cmd;
int32_t code = TSDB_CODE_SUCCESS;
@ -728,17 +788,24 @@ int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SStrToken* pzTableNa
if (hasSpecifyDB(pzTableName)) {
// db has been specified in sql string so we ignore current db path
code = setObjFullName(pTableMetaInfo->name, getAccountId(pSql), NULL, pzTableName, NULL);
if (code != 0) {
invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
}
} else { // get current DB name first, then set it into path
SStrToken t = {0};
getCurrentDBName(pSql, &t);
if (t.n == 0) {
msg = msg2;
invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
}
code = setObjFullName(pTableMetaInfo->name, NULL, &t, pzTableName, NULL);
if (code != 0) {
invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
}
}
if (code != TSDB_CODE_SUCCESS) {
invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg);
free(oldName);
taosTFree(oldName);
return code;
}
@ -1181,13 +1248,14 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t
} END_TRY
len = tbufTell(&bw);
char* c = tbufGetData(&bw, true);
char* c = tbufGetData(&bw, false);
// set the serialized binary string as the parameter of arithmetic expression
addExprParams(pExpr, c, TSDB_DATA_TYPE_BINARY, (int32_t)len, index.tableIndex);
insertResultField(pQueryInfo, exprIndex, &columnList, sizeof(double), TSDB_DATA_TYPE_DOUBLE, pExpr->aliasName, pExpr);
tbufCloseWriter(&bw);
taosArrayDestroy(colList);
tExprTreeDestroy(&pNode, NULL);
} else {
@ -1331,7 +1399,7 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel
/*
* transfer sql functions that need secondary merge into another format
* in dealing with metric queries such as: count/first/last
* in dealing with super table queries such as: count/first/last
*/
if (isSTable) {
tscTansformSQLFuncForSTableQuery(pQueryInfo);
@ -1516,16 +1584,16 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t
static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSchema* pSchema, SConvertFunc cvtFunc, char* aliasName,
int32_t resColIdx, SColumnIndex* pColIndex) {
const char* msg1 = "not support column types";
int16_t type = 0;
int16_t bytes = 0;
char columnName[TSDB_COL_NAME_LEN] = {0};
const char* msg1 = "not support column types";
char columnName[TSDB_COL_NAME_LEN] = {0};
int32_t functionID = cvtFunc.execFuncId;
if (functionID == TSDB_FUNC_SPREAD) {
if (pSchema[pColIndex->columnIndex].type == TSDB_DATA_TYPE_BINARY ||
pSchema[pColIndex->columnIndex].type == TSDB_DATA_TYPE_NCHAR ||
pSchema[pColIndex->columnIndex].type == TSDB_DATA_TYPE_BOOL) {
int32_t t1 = pSchema[pColIndex->columnIndex].type;
if (t1 == TSDB_DATA_TYPE_BINARY || t1 == TSDB_DATA_TYPE_NCHAR || t1 == TSDB_DATA_TYPE_BOOL) {
invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
return -1;
} else {
@ -1542,8 +1610,7 @@ static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SS
} else {
getRevisedName(columnName, cvtFunc.originFuncId, sizeof(columnName) - 1, pSchema[pColIndex->columnIndex].name);
}
SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, functionID, pColIndex, type, bytes, bytes, false);
tstrncpy(pExpr->aliasName, columnName, sizeof(pExpr->aliasName));
@ -1800,10 +1867,13 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
if (changeFunctionID(optr, &functionID) != TSDB_CODE_SUCCESS) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg9);
}
// NOTE: has time range condition or normal column filter condition, the last_row query will be transferred to last query
SConvertFunc cvtFunc = {.originFuncId = functionID, .execFuncId = functionID};
if (functionID == TSDB_FUNC_LAST_ROW && TSWINDOW_IS_EQUAL(pQueryInfo->window,TSWINDOW_INITIALIZER)) {
if (functionID == TSDB_FUNC_LAST_ROW && ((!TSWINDOW_IS_EQUAL(pQueryInfo->window, TSWINDOW_INITIALIZER)) || (hasNormalColumnFilter(pQueryInfo)))) {
cvtFunc.execFuncId = TSDB_FUNC_LAST;
}
if (!requireAllFields) {
if (pItem->pNode->pParam->nExpr < 1) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
@ -2591,7 +2661,7 @@ static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery) {
continue;
}
if (functionId == TSDB_FUNC_PRJ && pExpr1->colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) {
if (functionId == TSDB_FUNC_PRJ && (pExpr1->colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX || TSDB_COL_IS_UD_COL(pExpr1->colInfo.flag))) {
continue;
}
@ -2803,6 +2873,12 @@ static int32_t doExtractColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo,
case TK_LIKE:
pColumnFilter->lowerRelOptr = TSDB_RELATION_LIKE;
break;
case TK_ISNULL:
pColumnFilter->lowerRelOptr = TSDB_RELATION_ISNULL;
break;
case TK_NOTNULL:
pColumnFilter->lowerRelOptr = TSDB_RELATION_NOTNULL;
break;
default:
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg);
}
@ -2848,19 +2924,19 @@ static int32_t tSQLExprNodeToString(tSQLExpr* pExpr, char** str) {
return TSDB_CODE_SUCCESS;
}
// pExpr->nSQLOptr == 0 while handling "is null" query
static bool isExprLeafNode(tSQLExpr* pExpr) {
return (pExpr->pRight == NULL && pExpr->pLeft == NULL) &&
(pExpr->nSQLOptr == TK_ID || (pExpr->nSQLOptr >= TK_BOOL && pExpr->nSQLOptr <= TK_NCHAR) ||
pExpr->nSQLOptr == TK_SET);
(pExpr->nSQLOptr == 0 || pExpr->nSQLOptr == TK_ID || (pExpr->nSQLOptr >= TK_BOOL && pExpr->nSQLOptr <= TK_NCHAR) || pExpr->nSQLOptr == TK_SET);
}
static bool isExprDirectParentOfLeaftNode(tSQLExpr* pExpr) {
static bool isExprDirectParentOfLeafNode(tSQLExpr* pExpr) {
return (pExpr->pLeft != NULL && pExpr->pRight != NULL) &&
(isExprLeafNode(pExpr->pLeft) && isExprLeafNode(pExpr->pRight));
}
static int32_t tSQLExprLeafToString(tSQLExpr* pExpr, bool addParentheses, char** output) {
if (!isExprDirectParentOfLeaftNode(pExpr)) {
if (!isExprDirectParentOfLeafNode(pExpr)) {
return TSDB_CODE_TSC_INVALID_SQL;
}
@ -3071,7 +3147,7 @@ static int32_t getTagCondString(tSQLExpr* pExpr, char** str) {
return TSDB_CODE_SUCCESS;
}
if (!isExprDirectParentOfLeaftNode(pExpr)) {
if (!isExprDirectParentOfLeafNode(pExpr)) {
*(*str) = '(';
*str += 1;
@ -3127,7 +3203,7 @@ static int32_t getColumnQueryCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQ
return TSDB_CODE_SUCCESS;
}
if (!isExprDirectParentOfLeaftNode(pExpr)) { // internal node
if (!isExprDirectParentOfLeafNode(pExpr)) { // internal node
int32_t ret = getColumnQueryCondInfo(pCmd, pQueryInfo, pExpr->pLeft, pExpr->nSQLOptr);
if (ret != TSDB_CODE_SUCCESS) {
return ret;
@ -3153,7 +3229,7 @@ static int32_t getJoinCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr*
return TSDB_CODE_SUCCESS;
}
if (!isExprDirectParentOfLeaftNode(pExpr)) {
if (!isExprDirectParentOfLeafNode(pExpr)) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
}
@ -3225,19 +3301,6 @@ int32_t doArithmeticExprToString(tSQLExpr* pExpr, char** exprString) {
return TSDB_CODE_SUCCESS;
}
static UNUSED_FUNC int32_t arithmeticExprToString(tSQLExpr* pExpr, char** str) {
char* start = *str;
int32_t code = doArithmeticExprToString(pExpr, str);
if (code == TSDB_CODE_SUCCESS) { // remove out the parenthesis
int32_t len = (int32_t)strlen(start);
memmove(start, start + 1, len - 2);
start[len - 2] = 0;
}
return code;
}
static int32_t validateSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryInfo* pQueryInfo, SColumnList* pList, int32_t* type) {
if (pExpr->nSQLOptr == TK_ID) {
if (*type == NON_ARITHMEIC_EXPR) {
@ -3485,7 +3548,7 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQL
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
}
assert(isExprDirectParentOfLeaftNode(*pExpr));
assert(isExprDirectParentOfLeafNode(*pExpr));
STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex);
STableMeta* pTableMeta = pTableMetaInfo->pTableMeta;
@ -3531,7 +3594,7 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQL
}
}
// in case of in operator, keep it in a seperate attribute
// in case of in operator, keep it in a seprate attribute
if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) {
if (!validTableNameOptr(*pExpr)) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg7);
@ -3552,7 +3615,7 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQL
*type = TSQL_EXPR_TBNAME;
*pExpr = NULL;
} else {
if (pRight->nSQLOptr == TK_ID) { // join on tag columns for stable query
if (pRight != NULL && pRight->nSQLOptr == TK_ID) { // join on tag columns for stable query
if (!validateJoinExprNode(pCmd, pQueryInfo, *pExpr, &index)) {
return TSDB_CODE_TSC_INVALID_SQL;
}
@ -3605,7 +3668,7 @@ int32_t getQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr** pExpr
int32_t leftType = -1;
int32_t rightType = -1;
if (!isExprDirectParentOfLeaftNode(*pExpr)) {
if (!isExprDirectParentOfLeafNode(*pExpr)) {
int32_t ret = getQueryCondExpr(pCmd, pQueryInfo, &(*pExpr)->pLeft, pCondExpr, &leftType, (*pExpr)->nSQLOptr);
if (ret != TSDB_CODE_SUCCESS) {
return ret;
@ -3636,7 +3699,7 @@ int32_t getQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr** pExpr
}
static void doCompactQueryExpr(tSQLExpr** pExpr) {
if (*pExpr == NULL || isExprDirectParentOfLeaftNode(*pExpr)) {
if (*pExpr == NULL || isExprDirectParentOfLeafNode(*pExpr)) {
return;
}
@ -3667,7 +3730,7 @@ static void doCompactQueryExpr(tSQLExpr** pExpr) {
}
static void doExtractExprForSTable(SSqlCmd* pCmd, tSQLExpr** pExpr, SQueryInfo* pQueryInfo, tSQLExpr** pOut, int32_t tableIndex) {
if (isExprDirectParentOfLeaftNode(*pExpr)) {
if (isExprDirectParentOfLeafNode(*pExpr)) {
tSQLExpr* pLeft = (*pExpr)->pLeft;
SColumnIndex index = COLUMN_INDEX_INITIALIZER;
@ -3826,7 +3889,7 @@ static int32_t getTimeRangeFromExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLE
return TSDB_CODE_SUCCESS;
}
if (!isExprDirectParentOfLeaftNode(pExpr)) {
if (!isExprDirectParentOfLeafNode(pExpr)) {
if (pExpr->nSQLOptr == TK_OR) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
}
@ -3996,7 +4059,6 @@ int32_t parseWhereClause(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, SSqlObj* pSql
const char* msg2 = "invalid filter expression";
int32_t ret = TSDB_CODE_SUCCESS;
pQueryInfo->window = TSWINDOW_INITIALIZER;
// tags query condition may be larger than 512bytes, therefore, we need to prepare enough large space
SStringBuilder sb; memset(&sb, 0, sizeof(sb));
@ -4714,9 +4776,9 @@ int32_t validateSqlFunctionInStreamSql(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) {
const char* msg0 = "sample interval can not be less than 10ms.";
const char* msg1 = "functions not allowed in select clause";
if (pQueryInfo->intervalTime != 0 && pQueryInfo->intervalTime < 10 &&
pQueryInfo->intervalTimeUnit != 'n' &&
pQueryInfo->intervalTimeUnit != 'y') {
if (pQueryInfo->interval.interval != 0 && pQueryInfo->interval.interval < 10 &&
pQueryInfo->interval.intervalUnit != 'n' &&
pQueryInfo->interval.intervalUnit != 'y') {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg0);
}
@ -5244,7 +5306,8 @@ static void doUpdateSqlFunctionForTagPrj(SQueryInfo* pQueryInfo) {
for (int32_t i = 0; i < size; ++i) {
SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i);
if (pExpr->functionId != TSDB_FUNC_TAG_DUMMY && pExpr->functionId != TSDB_FUNC_TS_DUMMY) {
if ((pExpr->functionId != TSDB_FUNC_TAG_DUMMY && pExpr->functionId != TSDB_FUNC_TS_DUMMY) &&
!(pExpr->functionId == TSDB_FUNC_PRJ && TSDB_COL_IS_UD_COL(pExpr->colInfo.flag))) {
SSchema* pColSchema = &pSchema[pExpr->colInfo.colIndex];
getResultDataInfo(pColSchema->type, pColSchema->bytes, pExpr->functionId, (int32_t)pExpr->param[0].i64Key, &pExpr->resType,
&pExpr->resBytes, &pExpr->interBytes, tagLength, true);
@ -5252,12 +5315,13 @@ static void doUpdateSqlFunctionForTagPrj(SQueryInfo* pQueryInfo) {
}
}
static void doUpdateSqlFunctionForColPrj(SQueryInfo* pQueryInfo) {
static int32_t doUpdateSqlFunctionForColPrj(SQueryInfo* pQueryInfo) {
size_t size = taosArrayGetSize(pQueryInfo->exprList);
for (int32_t i = 0; i < size; ++i) {
SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i);
if (pExpr->functionId == TSDB_FUNC_PRJ) {
if (pExpr->functionId == TSDB_FUNC_PRJ && (!TSDB_COL_IS_UD_COL(pExpr->colInfo.flag))) {
bool qualifiedCol = false;
for (int32_t j = 0; j < pQueryInfo->groupbyExpr.numOfGroupCols; ++j) {
SColIndex* pColIndex = taosArrayGet(pQueryInfo->groupbyExpr.columnInfo, j);
@ -5270,9 +5334,14 @@ static void doUpdateSqlFunctionForColPrj(SQueryInfo* pQueryInfo) {
}
}
assert(qualifiedCol);
// it is not a tag column/tbname column/user-defined column, return error
if (!qualifiedCol) {
return TSDB_CODE_TSC_INVALID_SQL;
}
}
}
return TSDB_CODE_SUCCESS;
}
static bool tagColumnInGroupby(SSqlGroupbyExpr* pGroupbyExpr, int16_t columnId) {
@ -5346,16 +5415,23 @@ static int32_t checkUpdateTagPrjFunctions(SQueryInfo* pQueryInfo, SSqlCmd* pCmd)
const char* msg1 = "only one selectivity function allowed in presence of tags function";
const char* msg3 = "aggregation function should not be mixed up with projection";
bool tagColExists = false;
bool tagTsColExists = false;
int16_t numOfSelectivity = 0;
int16_t numOfAggregation = 0;
// todo is 0??
STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
bool isSTable = UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo);
if (!isSTable) {
return TSDB_CODE_SUCCESS;
}
size_t numOfExprs = taosArrayGetSize(pQueryInfo->exprList);
for (int32_t i = 0; i < numOfExprs; ++i) {
SSqlExpr* pExpr = taosArrayGetP(pQueryInfo->exprList, i);
if (pExpr->functionId == TSDB_FUNC_TAGPRJ ||
(pExpr->functionId == TSDB_FUNC_PRJ && pExpr->colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX)) {
tagColExists = true;
tagTsColExists = true; // selectivity + ts/tag column
break;
}
}
@ -5376,7 +5452,7 @@ static int32_t checkUpdateTagPrjFunctions(SQueryInfo* pQueryInfo, SSqlCmd* pCmd)
}
}
if (tagColExists) { // check if the selectivity function exists
if (tagTsColExists) { // check if the selectivity function exists
// When the tag projection function on tag column that is not in the group by clause, aggregation function and
// selectivity function exist in select clause is not allowed.
if (numOfAggregation > 0) {
@ -5388,26 +5464,36 @@ static int32_t checkUpdateTagPrjFunctions(SQueryInfo* pQueryInfo, SSqlCmd* pCmd)
*/
if (numOfSelectivity == 1) {
doUpdateSqlFunctionForTagPrj(pQueryInfo);
doUpdateSqlFunctionForColPrj(pQueryInfo);
int32_t code = doUpdateSqlFunctionForColPrj(pQueryInfo);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
} else if (numOfSelectivity > 1) {
/*
* If more than one selectivity functions exist, all the selectivity functions must be last_row.
* Otherwise, return with error code.
*/
for (int32_t i = 0; i < numOfExprs; ++i) {
int16_t functionId = tscSqlExprGet(pQueryInfo, i)->functionId;
if (functionId == TSDB_FUNC_TAGPRJ) {
SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i);
int16_t functionId = pExpr->functionId;
if (functionId == TSDB_FUNC_TAGPRJ || (aAggs[functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) == 0) {
continue;
}
if (((aAggs[functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0) && (functionId != TSDB_FUNC_LAST_ROW)) {
if ((functionId == TSDB_FUNC_LAST_ROW) ||
(functionId == TSDB_FUNC_LAST_DST && (pExpr->colInfo.flag & TSDB_COL_NULL) != 0)) {
// do nothing
} else {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
}
}
doUpdateSqlFunctionForTagPrj(pQueryInfo);
doUpdateSqlFunctionForColPrj(pQueryInfo);
int32_t code = doUpdateSqlFunctionForColPrj(pQueryInfo);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
}
} else {
if ((pQueryInfo->type & TSDB_QUERY_TYPE_PROJECTION_QUERY) != 0) {
@ -5418,7 +5504,10 @@ static int32_t checkUpdateTagPrjFunctions(SQueryInfo* pQueryInfo, SSqlCmd* pCmd)
if (numOfAggregation > 0 || numOfSelectivity > 0) {
// clear the projection type flag
pQueryInfo->type &= (~TSDB_QUERY_TYPE_PROJECTION_QUERY);
doUpdateSqlFunctionForColPrj(pQueryInfo);
int32_t code = doUpdateSqlFunctionForColPrj(pQueryInfo);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
}
}
}
@ -5474,7 +5563,7 @@ static int32_t doAddGroupbyColumnsOnDemand(SSqlCmd* pCmd, SQueryInfo* pQueryInfo
insertResultField(pQueryInfo, (int32_t)size, &ids, bytes, (int8_t)type, name, pExpr);
} else {
// if this query is "group by" normal column, interval is not allowed
if (pQueryInfo->intervalTime > 0) {
if (pQueryInfo->interval.interval > 0) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
}
@ -5507,7 +5596,7 @@ int32_t doFunctionsCompatibleCheck(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) {
// only retrieve tags, group by is not supportted
if (tscQueryTags(pQueryInfo)) {
if (pQueryInfo->groupbyExpr.numOfGroupCols > 0 || pQueryInfo->intervalTime > 0) {
if (pQueryInfo->groupbyExpr.numOfGroupCols > 0 || pQueryInfo->interval.interval > 0) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4);
} else {
return TSDB_CODE_SUCCESS;
@ -5959,7 +6048,7 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) {
if (parseIntervalClause(pCmd, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_SQL;
} else {
if ((pQueryInfo->intervalTime > 0) &&
if ((pQueryInfo->interval.interval > 0) &&
(validateFunctionsInIntervalOrGroupbyQuery(pCmd, pQueryInfo) != TSDB_CODE_SUCCESS)) {
return TSDB_CODE_TSC_INVALID_SQL;
}
@ -5989,7 +6078,7 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) {
* not here.
*/
if (pQuerySql->fillType != NULL) {
if (pQueryInfo->intervalTime == 0) {
if (pQueryInfo->interval.interval == 0) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
}
@ -6128,33 +6217,9 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) {
return TSDB_CODE_TSC_INVALID_SQL;
}
int32_t joinQuery = (pQuerySql->from != NULL && pQuerySql->from->nExpr > 2);
if (pQuerySql->pWhere) {
pQueryInfo->window = TSWINDOW_INITIALIZER;
}
if (parseSelectClause(pCmd, index, pQuerySql->pSelection, isSTable, joinQuery) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_SQL;
}
// set interval value
if (parseIntervalClause(pCmd, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_SQL;
} else {
if ((pQueryInfo->intervalTime > 0) &&
(validateFunctionsInIntervalOrGroupbyQuery(pCmd, pQueryInfo) != TSDB_CODE_SUCCESS)) {
return TSDB_CODE_TSC_INVALID_SQL;
}
}
// set order by info
if (parseOrderbyClause(pCmd, pQueryInfo, pQuerySql, tscGetTableSchema(pTableMetaInfo->pTableMeta)) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_SQL;
}
// set where info
STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta);
if (pQuerySql->pWhere != NULL) {
if (parseWhereClause(pQueryInfo, &pQuerySql->pWhere, pSql) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_SQL;
@ -6166,12 +6231,32 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) {
pQueryInfo->window.ekey = pQueryInfo->window.ekey / 1000;
}
} else { // set the time rang
pQueryInfo->window = TSWINDOW_INITIALIZER;
if (pQuerySql->from->nExpr > 2) { // it is a join query, no wher clause is not allowed.
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "condition missing for join query ");
}
}
int32_t joinQuery = (pQuerySql->from != NULL && pQuerySql->from->nExpr > 2);
if (parseSelectClause(pCmd, index, pQuerySql->pSelection, isSTable, joinQuery) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_SQL;
}
// set interval value
if (parseIntervalClause(pCmd, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_SQL;
} else {
if ((pQueryInfo->interval.interval > 0) &&
(validateFunctionsInIntervalOrGroupbyQuery(pCmd, pQueryInfo) != TSDB_CODE_SUCCESS)) {
return TSDB_CODE_TSC_INVALID_SQL;
}
}
// set order by info
if (parseOrderbyClause(pCmd, pQueryInfo, pQuerySql, tscGetTableSchema(pTableMetaInfo->pTableMeta)) != TSDB_CODE_SUCCESS) {
return TSDB_CODE_TSC_INVALID_SQL;
}
// user does not specified the query time window, twa is not allowed in such case.
if ((pQueryInfo->window.skey == INT64_MIN || pQueryInfo->window.ekey == INT64_MAX ||
(pQueryInfo->window.ekey == INT64_MAX / 1000 && tinfo.precision == TSDB_TIME_PRECISION_MILLI)) && tscIsTWAQuery(pQueryInfo)) {
@ -6212,14 +6297,14 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) {
* the columns may be increased due to group by operation
*/
if (pQuerySql->fillType != NULL) {
if (pQueryInfo->intervalTime == 0 && (!tscIsPointInterpQuery(pQueryInfo))) {
if (pQueryInfo->interval.interval == 0 && (!tscIsPointInterpQuery(pQueryInfo))) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5);
}
if (pQueryInfo->intervalTime > 0 && pQueryInfo->intervalTimeUnit != 'n' && pQueryInfo->intervalTimeUnit != 'y') {
if (pQueryInfo->interval.interval > 0 && pQueryInfo->interval.intervalUnit != 'n' && pQueryInfo->interval.intervalUnit != 'y') {
int64_t timeRange = ABS(pQueryInfo->window.skey - pQueryInfo->window.ekey);
// number of result is not greater than 10,000,000
if ((timeRange == 0) || (timeRange / pQueryInfo->intervalTime) > MAX_INTERVAL_TIME_WINDOW) {
if ((timeRange == 0) || (timeRange / pQueryInfo->interval.interval) > MAX_INTERVAL_TIME_WINDOW) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6);
}
}
@ -6250,6 +6335,11 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pS
return ret;
}
}
if (pSqlExpr->pLeft == NULL && pSqlExpr->pRight == NULL && pSqlExpr->nSQLOptr == 0) {
*pExpr = calloc(1, sizeof(tExprNode));
return TSDB_CODE_SUCCESS;
}
if (pSqlExpr->pLeft == NULL) {
if (pSqlExpr->nSQLOptr >= TK_BOOL && pSqlExpr->nSQLOptr <= TK_STRING) {
@ -6322,7 +6412,8 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pS
(*pExpr)->_node.optr = getBinaryExprOptr(&t);
assert((*pExpr)->_node.optr != 0);
// check for dividing by 0
if ((*pExpr)->_node.optr == TSDB_BINARY_OP_DIVIDE) {
if (pRight->nodeType == TSQL_NODE_VALUE) {
if (pRight->pVal->nType == TSDB_DATA_TYPE_INT && pRight->pVal->i64Key == 0) {
@ -6335,7 +6426,7 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pS
// NOTE: binary|nchar data allows the >|< type filter
if ((*pExpr)->_node.optr != TSDB_RELATION_EQUAL && (*pExpr)->_node.optr != TSDB_RELATION_NOT_EQUAL) {
if (pRight->nodeType == TSQL_NODE_VALUE) {
if (pRight != NULL && pRight->nodeType == TSQL_NODE_VALUE) {
if (pRight->pVal->nType == TSDB_DATA_TYPE_BOOL) {
return TSDB_CODE_TSC_INVALID_SQL;
}
@ -6345,3 +6436,15 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pS
return TSDB_CODE_SUCCESS;
}
bool hasNormalColumnFilter(SQueryInfo* pQueryInfo) {
size_t numOfCols = taosArrayGetSize(pQueryInfo->colList);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumn* pCol = taosArrayGetP(pQueryInfo->colList, i);
if (pCol->numOfFilters > 0) {
return true;
}
}
return false;
}

View File

@ -27,10 +27,7 @@
#include "tutil.h"
#include "tlockfree.h"
#define TSC_MGMT_VNODE 999
SRpcCorEpSet tscMgmtEpSet;
SRpcEpSet tscDnodeEpSet;
int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo) = {0};
@ -236,20 +233,27 @@ int tscSendMsgToServer(SSqlObj *pSql) {
}
void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) {
SSqlObj *pSql = (SSqlObj *)rpcMsg->ahandle;
if (pSql == NULL || pSql->signature != pSql) {
tscError("%p sql is already released", pSql);
uint64_t handle = (uint64_t) rpcMsg->ahandle;
void** p = taosCacheAcquireByKey(tscObjCache, &handle, sizeof(uint64_t));
if (p == NULL) {
rpcFreeCont(rpcMsg->pCont);
return;
}
SSqlObj* pSql = *p;
assert(pSql != NULL);
STscObj *pObj = pSql->pTscObj;
SSqlRes *pRes = &pSql->res;
SSqlCmd *pCmd = &pSql->cmd;
assert(*pSql->self == pSql);
if (pObj->signature != pObj) {
tscDebug("%p DB connection is closed, cmd:%d pObj:%p signature:%p", pSql, pCmd->command, pObj, pObj->signature);
tscFreeSqlObj(pSql);
taosCacheRelease(tscObjCache, (void**) &p, true);
rpcFreeCont(rpcMsg->pCont);
return;
}
@ -261,18 +265,21 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) {
tscDebug("%p sqlObj needs to be released or DB connection is closed, cmd:%d type:%d, pObj:%p signature:%p",
pSql, pCmd->command, pQueryInfo->type, pObj, pObj->signature);
tscFreeSqlObj(pSql);
void** p1 = p;
taosCacheRelease(tscObjCache, (void**) &p1, false);
taosCacheRelease(tscObjCache, (void**) &p, true);
rpcFreeCont(rpcMsg->pCont);
return;
}
if (pEpSet) {
if (pEpSet) {
if (!tscEpSetIsEqual(&pSql->epSet, pEpSet)) {
if(pCmd->command < TSDB_SQL_MGMT) {
tscUpdateVgroupInfo(pSql, pEpSet);
if (pCmd->command < TSDB_SQL_MGMT) {
tscUpdateVgroupInfo(pSql, pEpSet);
} else {
tscUpdateMgmtEpSet(pEpSet);
}
}
}
}
@ -294,7 +301,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) {
if (pSql->retry > pSql->maxRetry) {
tscError("%p max retry %d reached, give up", pSql, pSql->maxRetry);
} else {
// wait for a little bit moment and then retry
// wait for a little bit moment and then retry, todo do not sleep in rpc callback thread
if (rpcMsg->code == TSDB_CODE_APP_NOT_READY || rpcMsg->code == TSDB_CODE_VND_INVALID_VGROUP_ID) {
int32_t duration = getWaitingTimeInterval(pSql->retry);
taosMsleep(duration);
@ -304,6 +311,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) {
// if there is an error occurring, proceed to the following error handling procedure.
if (rpcMsg->code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
taosCacheRelease(tscObjCache, (void**) &p, false);
rpcFreeCont(rpcMsg->pCont);
return;
}
@ -365,16 +373,18 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) {
rpcMsg->code = (*tscProcessMsgRsp[pCmd->command])(pSql);
}
bool shouldFree = tscShouldBeFreed(pSql);
if (rpcMsg->code != TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
rpcMsg->code = (pRes->code == TSDB_CODE_SUCCESS) ? (int32_t)pRes->numOfRows : pRes->code;
bool shouldFree = tscShouldBeFreed(pSql);
(*pSql->fp)(pSql->param, pSql, rpcMsg->code);
}
if (shouldFree) {
tscDebug("%p sqlObj is automatically freed", pSql);
tscFreeSqlObj(pSql);
}
void** p1 = p;
taosCacheRelease(tscObjCache, (void**) &p1, false);
if (shouldFree) { // in case of table-meta/vgrouplist query, automatically free it
taosCacheRelease(tscObjCache, (void **)&p, true);
tscDebug("%p sqlObj is automatically freed", pSql);
}
rpcFreeCont(rpcMsg->pCont);
@ -637,8 +647,8 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
return TSDB_CODE_TSC_INVALID_SQL;
}
if (pQueryInfo->intervalTime < 0) {
tscError("%p illegal value of aggregation time interval in query msg: %ld", pSql, pQueryInfo->intervalTime);
if (pQueryInfo->interval.interval < 0) {
tscError("%p illegal value of aggregation time interval in query msg: %ld", pSql, pQueryInfo->interval.interval);
return TSDB_CODE_TSC_INVALID_SQL;
}
@ -665,10 +675,12 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) {
pQueryMsg->limit = htobe64(pQueryInfo->limit.limit);
pQueryMsg->offset = htobe64(pQueryInfo->limit.offset);
pQueryMsg->numOfCols = htons((int16_t)taosArrayGetSize(pQueryInfo->colList));
pQueryMsg->intervalTime = htobe64(pQueryInfo->intervalTime);
pQueryMsg->slidingTime = htobe64(pQueryInfo->slidingTime);
pQueryMsg->intervalTimeUnit = pQueryInfo->intervalTimeUnit;
pQueryMsg->slidingTimeUnit = pQueryInfo->slidingTimeUnit;
pQueryMsg->interval.interval = htobe64(pQueryInfo->interval.interval);
pQueryMsg->interval.sliding = htobe64(pQueryInfo->interval.sliding);
pQueryMsg->interval.offset = htobe64(pQueryInfo->interval.offset);
pQueryMsg->interval.intervalUnit = pQueryInfo->interval.intervalUnit;
pQueryMsg->interval.slidingUnit = pQueryInfo->interval.slidingUnit;
pQueryMsg->interval.offsetUnit = pQueryInfo->interval.offsetUnit;
pQueryMsg->numOfGroupCols = htons(pQueryInfo->groupbyExpr.numOfGroupCols);
pQueryMsg->numOfTags = htonl(numOfTags);
pQueryMsg->tagNameRelType = htons(pQueryInfo->tagCond.relType);
@ -1667,8 +1679,10 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) {
pMetaMsg->contLen = htons(pMetaMsg->contLen);
pMetaMsg->numOfColumns = htons(pMetaMsg->numOfColumns);
if (pMetaMsg->sid < 0 || pMetaMsg->vgroup.numOfEps < 0) {
tscError("invalid meter vgId:%d, sid%d", pMetaMsg->vgroup.numOfEps, pMetaMsg->sid);
if ((pMetaMsg->tableType != TSDB_SUPER_TABLE) &&
(pMetaMsg->sid <= 0 || pMetaMsg->vgroup.vgId < 2 || pMetaMsg->vgroup.numOfEps <= 0)) {
tscError("invalid value in table numOfEps:%d, vgId:%d tid:%d, name:%s", pMetaMsg->vgroup.numOfEps, pMetaMsg->vgroup.vgId,
pMetaMsg->sid, pMetaMsg->tableId);
return TSDB_CODE_TSC_INVALID_VALUE;
}
@ -1708,7 +1722,7 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) {
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0);
assert(pTableMetaInfo->pTableMeta == NULL);
pTableMetaInfo->pTableMeta = (STableMeta *) taosCachePut(tscCacheHandle, pTableMetaInfo->name,
pTableMetaInfo->pTableMeta = (STableMeta *) taosCachePut(tscMetaCache, pTableMetaInfo->name,
strlen(pTableMetaInfo->name), pTableMeta, size, tsTableMetaKeepTimer * 1000);
// todo handle out of memory case
@ -1820,7 +1834,7 @@ int tscProcessMultiMeterMetaRsp(SSqlObj *pSql) {
// int32_t size = (int32_t)(rsp - ((char *)pMeta)); // Consistent with STableMeta in cache
//
// pMeta->index = 0;
// (void)taosCachePut(tscCacheHandle, pMeta->tableId, (char *)pMeta, size, tsTableMetaKeepTimer);
// (void)taosCachePut(tscMetaCache, pMeta->tableId, (char *)pMeta, size, tsTableMetaKeepTimer);
// }
}
@ -1907,12 +1921,14 @@ int tscProcessShowRsp(SSqlObj *pSql) {
key[0] = pCmd->msgType + 'a';
strcpy(key + 1, "showlist");
taosCacheRelease(tscCacheHandle, (void *)&(pTableMetaInfo->pTableMeta), false);
if (pTableMetaInfo->pTableMeta != NULL) {
taosCacheRelease(tscMetaCache, (void *)&(pTableMetaInfo->pTableMeta), false);
}
size_t size = 0;
STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg, &size);
pTableMetaInfo->pTableMeta = taosCachePut(tscCacheHandle, key, strlen(key), (char *)pTableMeta, size,
pTableMetaInfo->pTableMeta = taosCachePut(tscMetaCache, key, strlen(key), (char *)pTableMeta, size,
tsTableMetaKeepTimer * 1000);
SSchema *pTableSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta);
@ -1971,6 +1987,8 @@ static void createHBObj(STscObj* pObj) {
pSql->pTscObj = pObj;
pSql->signature = pSql;
pObj->pHb = pSql;
T_REF_INC(pObj);
tscAddSubqueryInfo(&pObj->pHb->cmd);
tscDebug("%p HB is allocated, pObj:%p", pObj->pHb, pObj);
@ -2000,7 +2018,7 @@ int tscProcessConnectRsp(SSqlObj *pSql) {
createHBObj(pObj);
taosTmrReset(tscProcessActivityTimer, tsShellActivityTimer * 500, pObj, tscTmr, &pObj->pTimer);
// taosTmrReset(tscProcessActivityTimer, tsShellActivityTimer * 500, pObj, tscTmr, &pObj->pTimer);
return 0;
}
@ -2015,14 +2033,14 @@ int tscProcessUseDbRsp(SSqlObj *pSql) {
int tscProcessDropDbRsp(SSqlObj *pSql) {
pSql->pTscObj->db[0] = 0;
taosCacheEmpty(tscCacheHandle);
taosCacheEmpty(tscMetaCache);
return 0;
}
int tscProcessDropTableRsp(SSqlObj *pSql) {
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0);
STableMeta *pTableMeta = taosCacheAcquireByKey(tscCacheHandle, pTableMetaInfo->name, strlen(pTableMetaInfo->name));
STableMeta *pTableMeta = taosCacheAcquireByKey(tscMetaCache, pTableMetaInfo->name, strlen(pTableMetaInfo->name));
if (pTableMeta == NULL) { /* not in cache, abort */
return 0;
}
@ -2035,10 +2053,10 @@ int tscProcessDropTableRsp(SSqlObj *pSql) {
* instead.
*/
tscDebug("%p force release table meta after drop table:%s", pSql, pTableMetaInfo->name);
taosCacheRelease(tscCacheHandle, (void **)&pTableMeta, true);
taosCacheRelease(tscMetaCache, (void **)&pTableMeta, true);
if (pTableMetaInfo->pTableMeta) {
taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pTableMeta), true);
taosCacheRelease(tscMetaCache, (void **)&(pTableMetaInfo->pTableMeta), true);
}
return 0;
@ -2047,21 +2065,21 @@ int tscProcessDropTableRsp(SSqlObj *pSql) {
int tscProcessAlterTableMsgRsp(SSqlObj *pSql) {
STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0);
STableMeta *pTableMeta = taosCacheAcquireByKey(tscCacheHandle, pTableMetaInfo->name, strlen(pTableMetaInfo->name));
STableMeta *pTableMeta = taosCacheAcquireByKey(tscMetaCache, pTableMetaInfo->name, strlen(pTableMetaInfo->name));
if (pTableMeta == NULL) { /* not in cache, abort */
return 0;
}
tscDebug("%p force release metermeta in cache after alter-table: %s", pSql, pTableMetaInfo->name);
taosCacheRelease(tscCacheHandle, (void **)&pTableMeta, true);
taosCacheRelease(tscMetaCache, (void **)&pTableMeta, true);
if (pTableMetaInfo->pTableMeta) {
bool isSuperTable = UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo);
taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pTableMeta), true);
taosCacheRelease(tscMetaCache, (void **)&(pTableMetaInfo->pTableMeta), true);
if (isSuperTable) { // if it is a super table, reset whole query cache
tscDebug("%p reset query cache since table:%s is stable", pSql, pTableMetaInfo->name);
taosCacheEmpty(tscCacheHandle);
taosCacheEmpty(tscMetaCache);
}
}
@ -2146,6 +2164,12 @@ static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInf
pNew->signature = pNew;
pNew->cmd.command = TSDB_SQL_META;
T_REF_INC(pNew->pTscObj);
// TODO add test case on x86 platform
uint64_t adr = (uint64_t) pNew;
pNew->self = taosCachePut(tscObjCache, &adr, sizeof(uint64_t), &pNew, sizeof(uint64_t), 2*60*1000);
tscAddSubqueryInfo(&pNew->cmd);
SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetailSafely(&pNew->cmd, 0);
@ -2182,10 +2206,10 @@ int32_t tscGetTableMeta(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) {
// If this STableMetaInfo owns a table meta, release it first
if (pTableMetaInfo->pTableMeta != NULL) {
taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pTableMeta), false);
taosCacheRelease(tscMetaCache, (void **)&(pTableMetaInfo->pTableMeta), false);
}
pTableMetaInfo->pTableMeta = (STableMeta *)taosCacheAcquireByKey(tscCacheHandle, pTableMetaInfo->name, strlen(pTableMetaInfo->name));
pTableMetaInfo->pTableMeta = (STableMeta *)taosCacheAcquireByKey(tscMetaCache, pTableMetaInfo->name, strlen(pTableMetaInfo->name));
if (pTableMetaInfo->pTableMeta != NULL) {
STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta);
tscDebug("%p retrieve table Meta from cache, the number of columns:%d, numOfTags:%d, %p", pSql, tinfo.numOfColumns,
@ -2220,7 +2244,7 @@ int tscRenewTableMeta(SSqlObj *pSql, int32_t tableIndex) {
tscGetNumOfTags(pTableMeta), tscGetNumOfColumns(pTableMeta), pTableMeta->id.uid, pTableMeta);
}
taosCacheRelease(tscCacheHandle, (void **)&(pTableMetaInfo->pTableMeta), true);
taosCacheRelease(tscMetaCache, (void **)&(pTableMetaInfo->pTableMeta), true);
return getTableMetaFromMgmt(pSql, pTableMetaInfo);
}
@ -2250,7 +2274,8 @@ int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) {
pNew->signature = pNew;
pNew->cmd.command = TSDB_SQL_STABLEVGROUP;
// TODO TEST IT
SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetailSafely(&pNew->cmd, 0);
if (pNewQueryInfo == NULL) {
tscFreeSqlObj(pNew);
@ -2260,7 +2285,7 @@ int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) {
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, clauseIndex);
for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
STableMetaInfo *pMInfo = tscGetMetaInfo(pQueryInfo, i);
STableMeta *pTableMeta = taosCacheAcquireByData(tscCacheHandle, pMInfo->pTableMeta);
STableMeta *pTableMeta = taosCacheAcquireByData(tscMetaCache, pMInfo->pTableMeta);
tscAddTableMetaInfo(pNewQueryInfo, pMInfo->name, pTableMeta, NULL, pMInfo->tagColList);
}
@ -2270,6 +2295,10 @@ int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) {
}
pNewQueryInfo->numOfTables = pQueryInfo->numOfTables;
T_REF_INC(pNew->pTscObj);
uint64_t p = (uint64_t) pNew;
pNew->self = taosCachePut(tscObjCache, &p, sizeof(uint64_t), &pNew, sizeof(uint64_t), 2 * 600 * 1000);
tscDebug("%p new sqlObj:%p to get vgroupInfo, numOfTables:%d", pSql, pNew, pNewQueryInfo->numOfTables);
pNew->fp = tscTableMetaCallBack;

View File

@ -26,6 +26,7 @@
#include "tsclient.h"
#include "ttokendef.h"
#include "tutil.h"
#include "tscProfile.h"
static bool validImpl(const char* str, size_t maxsize) {
if (str == NULL) {
@ -100,6 +101,8 @@ SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con
}
pObj->signature = pObj;
pObj->pDnodeConn = pDnodeConn;
T_REF_INIT_VAL(pObj, 1);
tstrncpy(pObj->user, user, sizeof(pObj->user));
secretEncryptLen = MIN(secretEncryptLen, sizeof(pObj->pass));
@ -132,20 +135,15 @@ SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con
return NULL;
}
pSql->pTscObj = pObj;
pSql->pTscObj = pObj;
pSql->signature = pSql;
pSql->maxRetry = TSDB_MAX_REPLICA;
tsem_init(&pSql->rspSem, 0, 0);
pObj->pDnodeConn = pDnodeConn;
pSql->fp = fp;
pSql->param = param;
if (taos != NULL) {
*taos = pObj;
}
pSql->maxRetry = TSDB_MAX_REPLICA;
pSql->fp = fp;
pSql->param = param;
pSql->cmd.command = TSDB_SQL_CONNECT;
tsem_init(&pSql->rspSem, 0, 0);
if (TSDB_CODE_SUCCESS != tscAllocPayload(&pSql->cmd, TSDB_DEFAULT_PAYLOAD_SIZE)) {
terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
rpcClose(pDnodeConn);
@ -154,7 +152,16 @@ SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pass, con
return NULL;
}
if (taos != NULL) {
*taos = pObj;
}
T_REF_INC(pSql->pTscObj);
uint64_t key = (uint64_t) pSql;
pSql->self = taosCachePut(tscObjCache, &key, sizeof(uint64_t), &pSql, sizeof(uint64_t), 2*3600*1000);
tsInsertHeadSize = sizeof(SMsgDesc) + sizeof(SSubmitMsg);
return pSql;
}
@ -257,6 +264,33 @@ void taos_close(TAOS *taos) {
tscFreeSqlObj(pObj->pHb);
}
// free all sqlObjs created by using this connect before free the STscObj
// while(1) {
// pthread_mutex_lock(&pObj->mutex);
// void* p = pObj->sqlList;
// pthread_mutex_unlock(&pObj->mutex);
//
// if (p == NULL) {
// break;
// }
//
// tscDebug("%p waiting for sqlObj to be freed, %p", pObj, p);
// taosMsleep(100);
//
// // todo fix me!! two threads call taos_free_result will cause problem.
// tscDebug("%p free :%p", pObj, p);
// taos_free_result(p);
// }
int32_t ref = T_REF_DEC(pObj);
assert(ref >= 0);
if (ref > 0) {
tscDebug("%p %d remain sqlObjs, not free tscObj and dnodeConn:%p", pObj, ref, pObj->pDnodeConn);
return;
}
tscDebug("%p all sqlObj are freed, free tscObj and close dnodeConn:%p", pObj, pObj->pDnodeConn);
tscCloseTscObj(pObj);
}
@ -533,57 +567,51 @@ int taos_select_db(TAOS *taos, const char *db) {
}
// send free message to vnode to free qhandle and corresponding resources in vnode
static bool tscKillQueryInVnode(SSqlObj* pSql) {
static UNUSED_FUNC bool tscKillQueryInDnode(SSqlObj* pSql) {
SSqlCmd* pCmd = &pSql->cmd;
SSqlRes* pRes = &pSql->res;
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
if (pRes->code == TSDB_CODE_SUCCESS && pRes->completed == false && !tscIsTwoStageSTableQuery(pQueryInfo, 0) &&
(pCmd->command == TSDB_SQL_SELECT ||
pCmd->command == TSDB_SQL_SHOW ||
pCmd->command == TSDB_SQL_RETRIEVE ||
pCmd->command == TSDB_SQL_FETCH) &&
(pSql->pStream == NULL && pTableMetaInfo->pTableMeta != NULL)) {
pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH;
tscDebug("%p send msg to dnode to free qhandle ASAP, command:%s, ", pSql, sqlCmd[pCmd->command]);
tscProcessSql(pSql);
if (pRes == NULL || pRes->qhandle == 0) {
return true;
}
return false;
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0);
if ((pQueryInfo == NULL) || tscIsTwoStageSTableQuery(pQueryInfo, 0)) {
return true;
}
STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
tscRemoveFromSqlList(pSql);
int32_t cmd = pCmd->command;
if (pRes->code == TSDB_CODE_SUCCESS && pRes->completed == false && pSql->pStream == NULL && (pTableMetaInfo->pTableMeta != NULL) &&
(cmd == TSDB_SQL_SELECT ||
cmd == TSDB_SQL_SHOW ||
cmd == TSDB_SQL_RETRIEVE ||
cmd == TSDB_SQL_FETCH)) {
pQueryInfo->type = TSDB_QUERY_TYPE_FREE_RESOURCE;
pCmd->command = (pCmd->command > TSDB_SQL_MGMT) ? TSDB_SQL_RETRIEVE : TSDB_SQL_FETCH;
tscDebug("%p send msg to dnode to free qhandle ASAP before free sqlObj, command:%s", pSql, sqlCmd[pCmd->command]);
tscProcessSql(pSql);
return false;
}
return true;
}
void taos_free_result(TAOS_RES *res) {
SSqlObj *pSql = (SSqlObj *)res;
SSqlObj* pSql = (SSqlObj*) res;
if (pSql == NULL || pSql->signature != pSql) {
tscDebug("%p sqlObj has been freed", pSql);
return;
}
// The semaphore can not be changed while freeing async sub query objects.
SSqlRes *pRes = &pSql->res;
if (pRes == NULL || pRes->qhandle == 0) {
tscFreeSqlObj(pSql);
tscDebug("%p SqlObj is freed by app, qhandle is null", pSql);
tscError("%p already released sqlObj", res);
return;
}
// set freeFlag to 1 in retrieve message if there are un-retrieved results data in node
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
if (pQueryInfo == NULL) {
tscFreeSqlObj(pSql);
tscDebug("%p SqlObj is freed by app", pSql);
return;
}
pQueryInfo->type = TSDB_QUERY_TYPE_FREE_RESOURCE;
if (!tscKillQueryInVnode(pSql)) {
tscFreeSqlObj(pSql);
tscDebug("%p sqlObj is freed by app", pSql);
bool freeNow = tscKillQueryInDnode(pSql);
if (freeNow) {
tscDebug("%p free sqlObj in cache", pSql);
SSqlObj** p = pSql->self;
taosCacheRelease(tscObjCache, (void**) &p, true);
}
}

View File

@ -51,7 +51,7 @@ static int64_t tscGetRetryDelayTime(SSqlStream* pStream, int64_t slidingTime, in
int64_t retryDelta = (int64_t)(tsStreamCompRetryDelay * retryRangeFactor);
retryDelta = ((rand() % retryDelta) + tsStreamCompRetryDelay) * 1000L;
if (pStream->intervalTimeUnit != 'n' && pStream->intervalTimeUnit != 'y') {
if (pStream->interval.intervalUnit != 'n' && pStream->interval.intervalUnit != 'y') {
// change to ms
if (prec == TSDB_TIME_PRECISION_MICRO) {
slidingTime = slidingTime / 1000;
@ -87,7 +87,7 @@ static void tscProcessStreamLaunchQuery(SSchedMsg *pMsg) {
// failed to get meter/metric meta, retry in 10sec.
if (code != TSDB_CODE_SUCCESS) {
int64_t retryDelayTime = tscGetRetryDelayTime(pStream, pStream->slidingTime, pStream->precision);
int64_t retryDelayTime = tscGetRetryDelayTime(pStream, pStream->interval.sliding, pStream->precision);
tscDebug("%p stream:%p,get metermeta failed, retry in %" PRId64 "ms", pStream->pSql, pStream, retryDelayTime);
tscSetRetryTimer(pStream, pSql, retryDelayTime);
@ -132,15 +132,16 @@ static void tscProcessStreamTimer(void *handle, void *tmrId) {
}
if (etime > pStream->etime) {
etime = pStream->etime;
} else if (pStream->intervalTimeUnit != 'y' && pStream->intervalTimeUnit != 'n') {
etime = pStream->stime + (etime - pStream->stime) / pStream->intervalTime * pStream->intervalTime;
} else if (pStream->interval.intervalUnit != 'y' && pStream->interval.intervalUnit != 'n') {
etime = pStream->stime + (etime - pStream->stime) / pStream->interval.interval * pStream->interval.interval;
} else {
etime = taosGetIntervalStartTimestamp(etime, pStream->slidingTime, pStream->intervalTime, pStream->slidingTimeUnit, pStream->precision);
etime = taosTimeTruncate(etime, &pStream->interval, pStream->precision);
//etime = taosGetIntervalStartTimestamp(etime, pStream->interval.sliding, pStream->interval.sliding, pStream->interval.slidingUnit, pStream->precision);
}
pQueryInfo->window.ekey = etime;
if (pQueryInfo->window.skey >= pQueryInfo->window.ekey) {
int64_t timer = pStream->slidingTime;
if (pStream->intervalTimeUnit == 'y' || pStream->intervalTimeUnit == 'n') {
int64_t timer = pStream->interval.sliding;
if (pStream->interval.intervalUnit == 'y' || pStream->interval.intervalUnit == 'n') {
timer = 86400 * 1000l;
} else if (pStream->precision == TSDB_TIME_PRECISION_MICRO) {
timer /= 1000l;
@ -162,12 +163,12 @@ static void tscProcessStreamTimer(void *handle, void *tmrId) {
static void tscProcessStreamQueryCallback(void *param, TAOS_RES *tres, int numOfRows) {
SSqlStream *pStream = (SSqlStream *)param;
if (tres == NULL || numOfRows < 0) {
int64_t retryDelay = tscGetRetryDelayTime(pStream, pStream->slidingTime, pStream->precision);
int64_t retryDelay = tscGetRetryDelayTime(pStream, pStream->interval.sliding, pStream->precision);
tscError("%p stream:%p, query data failed, code:0x%08x, retry in %" PRId64 "ms", pStream->pSql, pStream, numOfRows,
retryDelay);
STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pStream->pSql->cmd, 0, 0);
taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pTableMeta), true);
taosCacheRelease(tscMetaCache, (void**)&(pTableMetaInfo->pTableMeta), true);
taosTFree(pTableMetaInfo->vgroupList);
tscSetRetryTimer(pStream, pStream->pSql, retryDelay);
@ -223,7 +224,7 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf
SSqlObj * pSql = (SSqlObj *)res;
if (pSql == NULL || numOfRows < 0) {
int64_t retryDelayTime = tscGetRetryDelayTime(pStream, pStream->slidingTime, pStream->precision);
int64_t retryDelayTime = tscGetRetryDelayTime(pStream, pStream->interval.sliding, pStream->precision);
tscError("%p stream:%p, retrieve data failed, code:0x%08x, retry in %" PRId64 "ms", pSql, pStream, numOfRows, retryDelayTime);
tscSetRetryTimer(pStream, pStream->pSql, retryDelayTime);
@ -246,11 +247,7 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf
}
if (!pStream->isProject) {
if (pStream->intervalTimeUnit == 'y' || pStream->intervalTimeUnit == 'n') {
pStream->stime = taosAddNatualInterval(pStream->stime, pStream->slidingTime, pStream->slidingTimeUnit, pStream->precision);
} else {
pStream->stime += pStream->slidingTime;
}
pStream->stime = taosTimeAdd(pStream->stime, pStream->interval.sliding, pStream->interval.slidingUnit, pStream->precision);
}
// actually only one row is returned. this following is not necessary
taos_fetch_rows_a(res, tscProcessStreamRetrieveResult, pStream);
@ -275,7 +272,7 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf
// release the metric/meter meta information reference, so data in cache can be updated
taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pTableMeta), false);
taosCacheRelease(tscMetaCache, (void**)&(pTableMetaInfo->pTableMeta), false);
tscFreeSqlResult(pSql);
taosTFree(pSql->pSubs);
pSql->numOfSubs = 0;
@ -310,7 +307,7 @@ static void tscSetRetryTimer(SSqlStream *pStream, SSqlObj *pSql, int64_t timer)
now + timer, timer, delay, pStream->stime, etime);
} else {
tscDebug("%p stream:%p, next start at %" PRId64 ", in %" PRId64 "ms. delay:%" PRId64 "ms qrange %" PRId64 "-%" PRId64, pStream->pSql, pStream,
pStream->stime, timer, delay, pStream->stime - pStream->intervalTime, pStream->stime - 1);
pStream->stime, timer, delay, pStream->stime - pStream->interval.interval, pStream->stime - 1);
}
pSql->cmd.command = TSDB_SQL_SELECT;
@ -324,12 +321,12 @@ static int64_t getLaunchTimeDelay(const SSqlStream* pStream) {
(pStream->precision == TSDB_TIME_PRECISION_MICRO) ? tsMaxStreamComputDelay * 1000L : tsMaxStreamComputDelay;
int64_t delayDelta = maxDelay;
if (pStream->intervalTimeUnit != 'n' && pStream->intervalTimeUnit != 'y') {
delayDelta = (int64_t)(pStream->slidingTime * tsStreamComputDelayRatio);
if (pStream->interval.intervalUnit != 'n' && pStream->interval.intervalUnit != 'y') {
delayDelta = (int64_t)(pStream->interval.sliding * tsStreamComputDelayRatio);
if (delayDelta > maxDelay) {
delayDelta = maxDelay;
}
int64_t remainTimeWindow = pStream->slidingTime - delayDelta;
int64_t remainTimeWindow = pStream->interval.sliding - delayDelta;
if (maxDelay > remainTimeWindow) {
maxDelay = (int64_t)(remainTimeWindow / 1.5f);
}
@ -337,8 +334,8 @@ static int64_t getLaunchTimeDelay(const SSqlStream* pStream) {
int64_t currentDelay = (rand() % maxDelay); // a random number
currentDelay += delayDelta;
if (pStream->intervalTimeUnit != 'n' && pStream->intervalTimeUnit != 'y') {
assert(currentDelay < pStream->slidingTime);
if (pStream->interval.intervalUnit != 'n' && pStream->interval.intervalUnit != 'y') {
assert(currentDelay < pStream->interval.sliding);
}
return currentDelay;
@ -353,7 +350,7 @@ static void tscSetNextLaunchTimer(SSqlStream *pStream, SSqlObj *pSql) {
* for project query, no mater fetch data successfully or not, next launch will issue
* more than the sliding time window
*/
timer = pStream->slidingTime;
timer = pStream->interval.sliding;
if (pStream->stime > pStream->etime) {
tscDebug("%p stream:%p, stime:%" PRId64 " is larger than end time: %" PRId64 ", stop the stream", pStream->pSql, pStream,
pStream->stime, pStream->etime);
@ -366,7 +363,8 @@ static void tscSetNextLaunchTimer(SSqlStream *pStream, SSqlObj *pSql) {
return;
}
} else {
int64_t stime = taosGetIntervalStartTimestamp(pStream->stime - 1, pStream->intervalTime, pStream->intervalTime, pStream->intervalTimeUnit, pStream->precision);
int64_t stime = taosTimeTruncate(pStream->stime - 1, &pStream->interval, pStream->precision);
//int64_t stime = taosGetIntervalStartTimestamp(pStream->stime - 1, pStream->interval.interval, pStream->interval.interval, pStream->interval.intervalUnit, pStream->precision);
if (stime >= pStream->etime) {
tscDebug("%p stream:%p, stime:%" PRId64 " is larger than end time: %" PRId64 ", stop the stream", pStream->pSql, pStream,
pStream->stime, pStream->etime);
@ -400,43 +398,43 @@ static void tscSetSlidingWindowInfo(SSqlObj *pSql, SSqlStream *pStream) {
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0);
if (pQueryInfo->intervalTimeUnit != 'n' && pQueryInfo->intervalTimeUnit != 'y' && pQueryInfo->intervalTime < minIntervalTime) {
if (pQueryInfo->interval.intervalUnit != 'n' && pQueryInfo->interval.intervalUnit!= 'y' && pQueryInfo->interval.interval < minIntervalTime) {
tscWarn("%p stream:%p, original sample interval:%ld too small, reset to:%" PRId64, pSql, pStream,
pQueryInfo->intervalTime, minIntervalTime);
pQueryInfo->intervalTime = minIntervalTime;
pQueryInfo->interval.interval, minIntervalTime);
pQueryInfo->interval.interval = minIntervalTime;
}
pStream->intervalTimeUnit = pQueryInfo->intervalTimeUnit;
pStream->intervalTime = pQueryInfo->intervalTime; // it shall be derived from sql string
pStream->interval.intervalUnit = pQueryInfo->interval.intervalUnit;
pStream->interval.interval = pQueryInfo->interval.interval; // it shall be derived from sql string
if (pQueryInfo->slidingTime <= 0) {
pQueryInfo->slidingTime = pQueryInfo->intervalTime;
pQueryInfo->slidingTimeUnit = pQueryInfo->intervalTimeUnit;
if (pQueryInfo->interval.sliding <= 0) {
pQueryInfo->interval.sliding = pQueryInfo->interval.interval;
pQueryInfo->interval.slidingUnit = pQueryInfo->interval.intervalUnit;
}
int64_t minSlidingTime =
(pStream->precision == TSDB_TIME_PRECISION_MICRO) ? tsMinSlidingTime * 1000L : tsMinSlidingTime;
if (pQueryInfo->intervalTimeUnit != 'n' && pQueryInfo->intervalTimeUnit != 'y' && pQueryInfo->slidingTime < minSlidingTime) {
if (pQueryInfo->interval.intervalUnit != 'n' && pQueryInfo->interval.intervalUnit!= 'y' && pQueryInfo->interval.sliding < minSlidingTime) {
tscWarn("%p stream:%p, original sliding value:%" PRId64 " too small, reset to:%" PRId64, pSql, pStream,
pQueryInfo->slidingTime, minSlidingTime);
pQueryInfo->interval.sliding, minSlidingTime);
pQueryInfo->slidingTime = minSlidingTime;
pQueryInfo->interval.sliding = minSlidingTime;
}
if (pQueryInfo->slidingTime > pQueryInfo->intervalTime) {
if (pQueryInfo->interval.sliding > pQueryInfo->interval.interval) {
tscWarn("%p stream:%p, sliding value:%" PRId64 " can not be larger than interval range, reset to:%" PRId64, pSql, pStream,
pQueryInfo->slidingTime, pQueryInfo->intervalTime);
pQueryInfo->interval.sliding, pQueryInfo->interval.interval);
pQueryInfo->slidingTime = pQueryInfo->intervalTime;
pQueryInfo->interval.sliding = pQueryInfo->interval.interval;
}
pStream->slidingTimeUnit = pQueryInfo->slidingTimeUnit;
pStream->slidingTime = pQueryInfo->slidingTime;
pStream->interval.slidingUnit = pQueryInfo->interval.slidingUnit;
pStream->interval.sliding = pQueryInfo->interval.sliding;
if (pStream->isProject) {
pQueryInfo->intervalTime = 0; // clear the interval value to avoid the force time window split by query processor
pQueryInfo->slidingTime = 0;
pQueryInfo->interval.interval = 0; // clear the interval value to avoid the force time window split by query processor
pQueryInfo->interval.sliding = 0;
}
}
@ -445,8 +443,8 @@ static int64_t tscGetStreamStartTimestamp(SSqlObj *pSql, SSqlStream *pStream, in
if (pStream->isProject) {
// no data in table, flush all data till now to destination meter, 10sec delay
pStream->intervalTime = tsProjectExecInterval;
pStream->slidingTime = tsProjectExecInterval;
pStream->interval.interval = tsProjectExecInterval;
pStream->interval.sliding = tsProjectExecInterval;
if (stime != 0) { // first projection start from the latest event timestamp
assert(stime >= pQueryInfo->window.skey);
@ -459,12 +457,15 @@ static int64_t tscGetStreamStartTimestamp(SSqlObj *pSql, SSqlStream *pStream, in
stime = pQueryInfo->window.skey;
if (stime == INT64_MIN) {
stime = (int64_t)taosGetTimestamp(pStream->precision);
stime = taosGetIntervalStartTimestamp(stime, pStream->intervalTime, pStream->intervalTime, pStream->intervalTimeUnit, pStream->precision);
stime = taosGetIntervalStartTimestamp(stime - 1, pStream->intervalTime, pStream->intervalTime, pStream->intervalTimeUnit, pStream->precision);
stime = taosTimeTruncate(stime, &pStream->interval, pStream->precision);
stime = taosTimeTruncate(stime - 1, &pStream->interval, pStream->precision);
//stime = taosGetIntervalStartTimestamp(stime, pStream->interval.interval, pStream->interval.interval, pStream->interval.intervalUnit, pStream->precision);
//stime = taosGetIntervalStartTimestamp(stime - 1, pStream->interval.interval, pStream->interval.interval, pStream->interval.intervalUnit, pStream->precision);
tscWarn("%p stream:%p, last timestamp:0, reset to:%" PRId64, pSql, pStream, stime);
}
} else {
int64_t newStime = taosGetIntervalStartTimestamp(stime, pStream->intervalTime, pStream->intervalTime, pStream->intervalTimeUnit, pStream->precision);
//int64_t newStime = taosGetIntervalStartTimestamp(stime, pStream->interval.interval, pStream->interval.interval, pStream->interval.intervalUnit, pStream->precision);
int64_t newStime = taosTimeTruncate(stime, &pStream->interval, pStream->precision);
if (newStime != stime) {
tscWarn("%p stream:%p, last timestamp:%" PRId64 ", reset to:%" PRId64, pSql, pStream, stime, newStime);
stime = newStime;
@ -534,7 +535,7 @@ static void tscCreateStream(void *param, TAOS_RES *res, int code) {
taosTmrReset(tscProcessStreamTimer, (int32_t)starttime, pStream, tscTmr, &pStream->pTimer);
tscDebug("%p stream:%p is opened, query on:%s, interval:%" PRId64 ", sliding:%" PRId64 ", first launched in:%" PRId64 ", sql:%s", pSql,
pStream, pTableMetaInfo->name, pStream->intervalTime, pStream->slidingTime, starttime, pSql->sqlstr);
pStream, pTableMetaInfo->name, pStream->interval.interval, pStream->interval.sliding, starttime, pSql->sqlstr);
}
TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *param, TAOS_RES *, TAOS_ROW row),

View File

@ -92,7 +92,7 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, SJ
STSElem elem2 = tsBufGetElem(pSupporter2->pTSBuf);
#ifdef _DEBUG_VIEW
tscInfo("%" PRId64 ", tags:%d \t %" PRId64 ", tags:%d", elem1.ts, elem1.tag, elem2.ts, elem2.tag);
tscInfo("%" PRId64 ", tags:%"PRId64" \t %" PRId64 ", tags:%"PRId64, elem1.ts, elem1.tag.i64Key, elem2.ts, elem2.tag.i64Key);
#endif
int32_t res = tVariantCompare(&elem1.tag, &elem2.tag);
@ -113,7 +113,7 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, SJ
* in case of stable query, limit/offset is not applied here. the limit/offset is applied to the
* final results which is acquired after the secondry merge of in the client.
*/
if (pLimit->offset == 0 || pQueryInfo->intervalTime > 0 || QUERY_IS_STABLE_QUERY(pQueryInfo->type)) {
if (pLimit->offset == 0 || pQueryInfo->interval.interval > 0 || QUERY_IS_STABLE_QUERY(pQueryInfo->type)) {
if (win->skey > elem1.ts) {
win->skey = elem1.ts;
}
@ -178,10 +178,7 @@ SJoinSupporter* tscCreateJoinSupporter(SSqlObj* pSql, SSubqueryState* pState, in
pSupporter->subqueryIndex = index;
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, pSql->cmd.clauseIndex);
pSupporter->intervalTimeUnit = pQueryInfo->intervalTimeUnit;
pSupporter->slidingTime = pQueryInfo->slidingTimeUnit;
pSupporter->intervalTime = pQueryInfo->intervalTime;
pSupporter->slidingTime = pQueryInfo->slidingTime;
memcpy(&pSupporter->interval, &pQueryInfo->interval, sizeof(pSupporter->interval));
pSupporter->limit = pQueryInfo->limit;
STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, pSql->cmd.clauseIndex, index);
@ -301,28 +298,24 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) {
success = false;
break;
}
tscClearSubqueryInfo(&pNew->cmd);
pSql->pSubs[i] = pNew;
SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0);
pQueryInfo->tsBuf = pTSBuf; // transfer the ownership of timestamp comp-z data to the new created object
// set the second stage sub query for join process
TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_SEC_STAGE);
pQueryInfo->intervalTimeUnit = pSupporter->intervalTimeUnit;
pQueryInfo->slidingTimeUnit = pSupporter->slidingTimeUnit;
pQueryInfo->intervalTime = pSupporter->intervalTime;
pQueryInfo->slidingTime = pSupporter->slidingTime;
pQueryInfo->groupbyExpr = pSupporter->groupbyExpr;
memcpy(&pQueryInfo->interval, &pSupporter->interval, sizeof(pQueryInfo->interval));
tscTagCondCopy(&pQueryInfo->tagCond, &pSupporter->tagCond);
pQueryInfo->colList = pSupporter->colList;
pQueryInfo->exprList = pSupporter->exprList;
pQueryInfo->fieldsInfo = pSupporter->fieldsInfo;
pSupporter->exprList = NULL;
pSupporter->colList = NULL;
memset(&pSupporter->fieldsInfo, 0, sizeof(SFieldInfo));
@ -1214,14 +1207,13 @@ int32_t tscCreateJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter
}
pNew->cmd.numOfCols = 0;
pNewQueryInfo->intervalTime = 0;
pNewQueryInfo->interval.interval = 0;
pSupporter->limit = pNewQueryInfo->limit;
pNewQueryInfo->limit.limit = -1;
pNewQueryInfo->limit.offset = 0;
// backup the data and clear it in the sqlcmd object
pSupporter->groupbyExpr = pNewQueryInfo->groupbyExpr;
memset(&pNewQueryInfo->groupbyExpr, 0, sizeof(SSqlGroupbyExpr));
tscInitQueryInfo(pNewQueryInfo);
@ -1523,9 +1515,9 @@ static void tscFreeSubSqlObj(SRetrieveSupport *trsupport, SSqlObj *pSql) {
SSqlObj *pParentSql = trsupport->pParentSql;
assert(pSql == pParentSql->pSubs[index]);
pParentSql->pSubs[index] = NULL;
taos_free_result(pSql);
// pParentSql->pSubs[index] = NULL;
//
// taos_free_result(pSql);
taosTFree(trsupport->localBuffer);
taosTFree(trsupport);
}
@ -1739,10 +1731,6 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR
assert(tres != NULL);
SSqlObj *pSql = (SSqlObj *)tres;
// if (pSql == NULL) { // sql object has been released in error process, return immediately
// tscDebug("%p subquery has been released, idx:%d, abort", pParentSql, idx);
// return;
// }
SSubqueryState* pState = trsupport->pState;
assert(pState->numOfRemain <= pState->numOfTotal && pState->numOfRemain >= 0 && pParentSql->numOfSubs == pState->numOfTotal);
@ -1918,9 +1906,7 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows)
pParentObj->res.code = pSql->res.code;
}
taos_free_result(tres);
taosTFree(pSupporter);
if (atomic_sub_fetch_32(&pState->numOfRemain, 1) > 0) {
return;
}
@ -1964,28 +1950,27 @@ int32_t tscHandleInsertRetry(SSqlObj* pSql) {
}
int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) {
SSqlRes *pRes = &pSql->res;
SSqlCmd *pCmd = &pSql->cmd;
size_t size = taosArrayGetSize(pCmd->pDataBlocks);
assert(size > 0);
SSqlRes *pRes = &pSql->res;
pSql->numOfSubs = (uint16_t)taosArrayGetSize(pCmd->pDataBlocks);
assert(pSql->numOfSubs > 0);
pRes->code = TSDB_CODE_SUCCESS;
// the number of already initialized subqueries
int32_t numOfSub = 0;
pSql->numOfSubs = (uint16_t)size;
pSql->pSubs = calloc(size, POINTER_BYTES);
SSubqueryState *pState = calloc(1, sizeof(SSubqueryState));
pState->numOfTotal = pSql->numOfSubs;
pState->numOfRemain = pSql->numOfSubs;
pSql->pSubs = calloc(pSql->numOfSubs, POINTER_BYTES);
if (pSql->pSubs == NULL) {
goto _error;
}
tscDebug("%p submit data to %" PRIzu " vnode(s)", pSql, size);
SSubqueryState *pState = calloc(1, sizeof(SSubqueryState));
pState->numOfTotal = pSql->numOfSubs;
pState->numOfRemain = pSql->numOfSubs;
pRes->code = TSDB_CODE_SUCCESS;
tscDebug("%p submit data to %d vnode(s)", pSql, pSql->numOfSubs);
while(numOfSub < pSql->numOfSubs) {
SInsertSupporter* pSupporter = calloc(1, sizeof(SInsertSupporter));
@ -2016,8 +2001,8 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) {
tscDebug("%p sub:%p create subObj success. orderOfSub:%d", pSql, pNew, numOfSub);
numOfSub++;
} else {
tscDebug("%p prepare submit data block failed in async insertion, vnodeIdx:%d, total:%" PRIzu ", code:%s", pSql, numOfSub,
size, tstrerror(pRes->code));
tscDebug("%p prepare submit data block failed in async insertion, vnodeIdx:%d, total:%d, code:%s", pSql, numOfSub,
pSql->numOfSubs, tstrerror(pRes->code));
goto _error;
}
}
@ -2040,11 +2025,6 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) {
return TSDB_CODE_SUCCESS;
_error:
for(int32_t j = 0; j < numOfSub; ++j) {
taosTFree(pSql->pSubs[j]->param);
taos_free_result(pSql->pSubs[j]);
}
taosTFree(pState);
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
@ -2208,7 +2188,7 @@ void **doSetResultRowData(SSqlObj *pSql, bool finalResult) {
}
// primary key column cannot be null in interval query, no need to check
if (i == 0 && pQueryInfo->intervalTime > 0) {
if (i == 0 && pQueryInfo->interval.interval > 0) {
continue;
}
@ -2220,16 +2200,15 @@ void **doSetResultRowData(SSqlObj *pSql, bool finalResult) {
// calculate the result from several other columns
if (pSup->pArithExprInfo != NULL) {
if (pRes->pArithSup == NULL) {
SArithmeticSupport *sas = (SArithmeticSupport *) calloc(1, sizeof(SArithmeticSupport));
sas->offset = 0;
sas->pArithExpr = pSup->pArithExprInfo;
sas->numOfCols = (int32_t)tscSqlExprNumOfExprs(pQueryInfo);
sas->exprList = pQueryInfo->exprList;
sas->data = calloc(sas->numOfCols, POINTER_BYTES);
pRes->pArithSup = sas;
pRes->pArithSup = (SArithmeticSupport*)calloc(1, sizeof(SArithmeticSupport));
}
pRes->pArithSup->offset = 0;
pRes->pArithSup->pArithExpr = pSup->pArithExprInfo;
pRes->pArithSup->numOfCols = (int32_t)tscSqlExprNumOfExprs(pQueryInfo);
pRes->pArithSup->exprList = pQueryInfo->exprList;
pRes->pArithSup->data = calloc(pRes->pArithSup->numOfCols, POINTER_BYTES);
if (pRes->buffer[i] == NULL) {
TAOS_FIELD* field = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i);
pRes->buffer[i] = malloc(field->bytes);

View File

@ -30,7 +30,8 @@
#include "tlocale.h"
// global, not configurable
void * tscCacheHandle;
SCacheObj* tscMetaCache;
SCacheObj* tscObjCache;
void * tscTmr;
void * tscQhandle;
void * tscCheckDiskUsageTmr;
@ -144,8 +145,9 @@ void taos_init_imp(void) {
refreshTime = refreshTime > 10 ? 10 : refreshTime;
refreshTime = refreshTime < 10 ? 10 : refreshTime;
if (tscCacheHandle == NULL) {
tscCacheHandle = taosCacheInit(TSDB_DATA_TYPE_BINARY, refreshTime, false, NULL, "tableMeta");
if (tscMetaCache == NULL) {
tscMetaCache = taosCacheInit(TSDB_DATA_TYPE_BINARY, refreshTime, false, NULL, "tableMeta");
tscObjCache = taosCacheInit(TSDB_DATA_TYPE_BIGINT, refreshTime/2, false, tscFreeSqlObjInCache, "sqlObj");
}
tscDebug("client is initialized successfully");
@ -154,9 +156,12 @@ void taos_init_imp(void) {
void taos_init() { pthread_once(&tscinit, taos_init_imp); }
void taos_cleanup() {
if (tscCacheHandle != NULL) {
taosCacheCleanup(tscCacheHandle);
tscCacheHandle = NULL;
if (tscMetaCache != NULL) {
taosCacheCleanup(tscMetaCache);
tscMetaCache = NULL;
taosCacheCleanup(tscObjCache);
tscObjCache = NULL;
}
if (tscQhandle != NULL) {

View File

@ -252,11 +252,11 @@ int32_t tscCreateResPointerInfo(SSqlRes* pRes, SQueryInfo* pQueryInfo) {
if (pRes->tsrow == NULL) {
int32_t numOfOutput = pQueryInfo->fieldsInfo.numOfOutput;
pRes->numOfCols = numOfOutput;
pRes->tsrow = calloc(numOfOutput, POINTER_BYTES);
pRes->length = calloc(numOfOutput, sizeof(int32_t));
pRes->buffer = calloc(numOfOutput, POINTER_BYTES);
// not enough memory
if (pRes->tsrow == NULL || (pRes->buffer == NULL && pRes->numOfCols > 0)) {
taosTFree(pRes->tsrow);
@ -268,7 +268,7 @@ int32_t tscCreateResPointerInfo(SSqlRes* pRes, SQueryInfo* pQueryInfo) {
return TSDB_CODE_SUCCESS;
}
void tscDestroyResPointerInfo(SSqlRes* pRes) {
static void tscDestroyResPointerInfo(SSqlRes* pRes) {
if (pRes->buffer != NULL) { // free all buffers containing the multibyte string
for (int i = 0; i < pRes->numOfCols; i++) {
taosTFree(pRes->buffer[i]);
@ -344,8 +344,6 @@ void tscPartiallyFreeSqlObj(SSqlObj* pSql) {
}
SSqlCmd* pCmd = &pSql->cmd;
STscObj* pObj = pSql->pTscObj;
int32_t cmd = pCmd->command;
if (cmd < TSDB_SQL_INSERT || cmd == TSDB_SQL_RETRIEVE_LOCALMERGE || cmd == TSDB_SQL_RETRIEVE_EMPTY_RESULT ||
cmd == TSDB_SQL_TABLE_JOIN_RETRIEVE) {
@ -353,26 +351,61 @@ void tscPartiallyFreeSqlObj(SSqlObj* pSql) {
}
// pSql->sqlstr will be used by tscBuildQueryStreamDesc
if (pObj->signature == pObj) {
// if (pObj->signature == pObj) {
//pthread_mutex_lock(&pObj->mutex);
taosTFree(pSql->sqlstr);
//pthread_mutex_unlock(&pObj->mutex);
}
// }
tscFreeSqlResult(pSql);
taosTFree(pSql->pSubs);
pSql->numOfSubs = 0;
pSql->self = 0;
tscResetSqlCmdObj(pCmd, false);
}
static UNUSED_FUNC void tscFreeSubobj(SSqlObj* pSql) {
if (pSql->numOfSubs == 0) {
return;
}
tscDebug("%p start to free sub SqlObj, numOfSub:%d", pSql, pSql->numOfSubs);
for(int32_t i = 0; i < pSql->numOfSubs; ++i) {
tscDebug("%p free sub SqlObj:%p, index:%d", pSql, pSql->pSubs[i], i);
taos_free_result(pSql->pSubs[i]);
pSql->pSubs[i] = NULL;
}
pSql->numOfSubs = 0;
}
/**
* The free operation will cause the pSql to be removed from hash table and free it in
* the function of processmsgfromserver is impossible in this case, since it will fail
* to retrieve pSqlObj in hashtable.
*
* @param pSql
*/
void tscFreeSqlObjInCache(void *pSql) {
assert(pSql != NULL);
SSqlObj** p = (SSqlObj**)pSql;
assert((*p)->self != 0 && (*p)->self == (p));
tscFreeSqlObj(*p);
}
void tscFreeSqlObj(SSqlObj* pSql) {
if (pSql == NULL || pSql->signature != pSql) {
return;
}
tscDebug("%p start to free sql object", pSql);
tscDebug("%p start to free sqlObj", pSql);
STscObj* pTscObj = pSql->pTscObj;
tscFreeSubobj(pSql);
tscPartiallyFreeSqlObj(pSql);
pSql->signature = NULL;
@ -388,6 +421,14 @@ void tscFreeSqlObj(SSqlObj* pSql) {
tsem_destroy(&pSql->rspSem);
free(pSql);
tscDebug("%p free sqlObj completed", pSql);
int32_t ref = T_REF_DEC(pTscObj);
assert(ref >= 0);
if (ref == 0) {
tscCloseTscObj(pTscObj);
}
}
void tscDestroyDataBlock(STableDataBlocks* pDataBlock) {
@ -399,7 +440,10 @@ void tscDestroyDataBlock(STableDataBlocks* pDataBlock) {
taosTFree(pDataBlock->params);
// free the refcount for metermeta
taosCacheRelease(tscCacheHandle, (void**)&(pDataBlock->pTableMeta), false);
if (pDataBlock->pTableMeta != NULL) {
taosCacheRelease(tscMetaCache, (void**)&(pDataBlock->pTableMeta), false);
}
taosTFree(pDataBlock);
}
@ -454,9 +498,12 @@ int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock) {
// set the correct table meta object, the table meta has been locked in pDataBlocks, so it must be in the cache
if (pTableMetaInfo->pTableMeta != pDataBlock->pTableMeta) {
tstrncpy(pTableMetaInfo->name, pDataBlock->tableId, sizeof(pTableMetaInfo->name));
taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pTableMeta), false);
pTableMetaInfo->pTableMeta = taosCacheTransfer(tscCacheHandle, (void**)&pDataBlock->pTableMeta);
if (pTableMetaInfo->pTableMeta != NULL) {
taosCacheRelease(tscMetaCache, (void**)&(pTableMetaInfo->pTableMeta), false);
}
pTableMetaInfo->pTableMeta = taosCacheTransfer(tscMetaCache, (void**)&pDataBlock->pTableMeta);
} else {
assert(strncmp(pTableMetaInfo->name, pDataBlock->tableId, tListLen(pDataBlock->tableId)) == 0);
}
@ -527,7 +574,7 @@ int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOff
* due to operation such as drop database. So here we add the reference count directly instead of invoke
* taosGetDataFromCache, which may return NULL value.
*/
dataBuf->pTableMeta = taosCacheAcquireByData(tscCacheHandle, pTableMeta);
dataBuf->pTableMeta = taosCacheAcquireByData(tscMetaCache, pTableMeta);
assert(initialSize > 0 && pTableMeta != NULL && dataBuf->pTableMeta != NULL);
*dataBlocks = dataBuf;
@ -721,17 +768,19 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SArray* pTableDataBlockList) {
// TODO: all subqueries should be freed correctly before close this connection.
void tscCloseTscObj(STscObj* pObj) {
assert(pObj != NULL);
pObj->signature = NULL;
taosTmrStopA(&(pObj->pTimer));
pthread_mutex_destroy(&pObj->mutex);
void* p = pObj->pDnodeConn;
if (pObj->pDnodeConn != NULL) {
rpcClose(pObj->pDnodeConn);
pObj->pDnodeConn = NULL;
}
tscDebug("%p DB connection is closed, dnodeConn:%p", pObj, pObj->pDnodeConn);
pthread_mutex_destroy(&pObj->mutex);
tscDebug("%p DB connection is closed, dnodeConn:%p", pObj, p);
taosTFree(pObj);
}
@ -1531,8 +1580,8 @@ void tscInitQueryInfo(SQueryInfo* pQueryInfo) {
pQueryInfo->fieldsInfo.pSupportInfo = taosArrayInit(4, sizeof(SFieldSupInfo));
assert(pQueryInfo->exprList == NULL);
pQueryInfo->exprList = taosArrayInit(4, POINTER_BYTES);
pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES);
pQueryInfo->exprList = taosArrayInit(4, POINTER_BYTES);
pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES);
pQueryInfo->udColumnId = TSDB_UD_COLUMN_INDEX;
}
@ -1554,6 +1603,8 @@ int32_t tscAddSubqueryInfo(SSqlCmd* pCmd) {
}
tscInitQueryInfo(pQueryInfo);
pQueryInfo->window = TSWINDOW_INITIALIZER;
pQueryInfo->msg = pCmd->payload; // pointer to the parent error message buffer
pCmd->pQueryInfo[pCmd->numOfClause++] = pQueryInfo;
@ -1665,7 +1716,10 @@ void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache)
return;
}
taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pTableMeta), removeFromCache);
if (pTableMetaInfo->pTableMeta != NULL) {
taosCacheRelease(tscMetaCache, (void**)&(pTableMetaInfo->pTableMeta), removeFromCache);
}
taosTFree(pTableMetaInfo->vgroupList);
tscColumnListDestroy(pTableMetaInfo->tagColList);
@ -1689,6 +1743,8 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cm
}
pNew->pTscObj = pSql->pTscObj;
T_REF_INC(pNew->pTscObj);
pNew->signature = pNew;
SSqlCmd* pCmd = &pNew->cmd;
@ -1719,6 +1775,11 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cm
STableMetaInfo* pMasterTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, pSql->cmd.clauseIndex, 0);
tscAddTableMetaInfo(pQueryInfo, pMasterTableMetaInfo->name, NULL, NULL, NULL);
T_REF_INC(pNew->pTscObj);
uint64_t p = (uint64_t) pNew;
pNew->self = taosCachePut(tscObjCache, &p, sizeof(uint64_t), &pNew, sizeof(uint64_t), 2 * 600 * 1000);
return pNew;
}
@ -1788,6 +1849,7 @@ static void doSetSqlExprAndResultFieldInfo(SQueryInfo* pQueryInfo, SQueryInfo* p
}
assert(matched);
(void)matched;
}
tscFieldInfoUpdateOffset(pNewQueryInfo);
@ -1807,6 +1869,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
pNew->pTscObj = pSql->pTscObj;
pNew->signature = pNew;
T_REF_INC(pNew->pTscObj);
pNew->sqlstr = strdup(pSql->sqlstr);
if (pNew->sqlstr == NULL) {
@ -1837,10 +1900,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex);
pNewQueryInfo->command = pQueryInfo->command;
pNewQueryInfo->intervalTimeUnit = pQueryInfo->intervalTimeUnit;
pNewQueryInfo->slidingTimeUnit = pQueryInfo->slidingTimeUnit;
pNewQueryInfo->intervalTime = pQueryInfo->intervalTime;
pNewQueryInfo->slidingTime = pQueryInfo->slidingTime;
memcpy(&pNewQueryInfo->interval, &pQueryInfo->interval, sizeof(pNewQueryInfo->interval));
pNewQueryInfo->type = pQueryInfo->type;
pNewQueryInfo->window = pQueryInfo->window;
pNewQueryInfo->limit = pQueryInfo->limit;
@ -1911,14 +1971,14 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
STableMetaInfo* pFinalInfo = NULL;
if (pPrevSql == NULL) {
STableMeta* pTableMeta = taosCacheAcquireByData(tscCacheHandle, pTableMetaInfo->pTableMeta); // get by name may failed due to the cache cleanup
STableMeta* pTableMeta = taosCacheAcquireByData(tscMetaCache, pTableMetaInfo->pTableMeta); // get by name may failed due to the cache cleanup
assert(pTableMeta != NULL);
pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pTableMeta, pTableMetaInfo->vgroupList, pTableMetaInfo->tagColList);
} else { // transfer the ownership of pTableMeta to the newly create sql object.
STableMetaInfo* pPrevInfo = tscGetTableMetaInfoFromCmd(&pPrevSql->cmd, pPrevSql->cmd.clauseIndex, 0);
STableMeta* pPrevTableMeta = taosCacheTransfer(tscCacheHandle, (void**)&pPrevInfo->pTableMeta);
STableMeta* pPrevTableMeta = taosCacheTransfer(tscMetaCache, (void**)&pPrevInfo->pTableMeta);
SVgroupsInfo* pVgroupsInfo = pPrevInfo->vgroupList;
pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pPrevTableMeta, pVgroupsInfo, pTableMetaInfo->tagColList);
@ -1958,6 +2018,10 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void
tscDebug("%p new sub insertion: %p, vnodeIdx:%d", pSql, pNew, pTableMetaInfo->vgroupIndex);
}
T_REF_INC(pNew->pTscObj);
uint64_t p = (uint64_t) pNew;
pNew->self = taosCachePut(tscObjCache, &p, sizeof(uint64_t), &pNew, sizeof(uint64_t), 2 * 600 * 10);
return pNew;
_error:
@ -2069,6 +2133,7 @@ int32_t tscSQLSyntaxErrMsg(char* msg, const char* additionalInfo, const char* s
return TSDB_CODE_TSC_SQL_SYNTAX_ERROR;
}
int32_t tscInvalidSQLErrMsg(char* msg, const char* additionalInfo, const char* sql) {
const char* msgFormat1 = "invalid SQL: %s";
const char* msgFormat2 = "invalid SQL: \'%s\' (%s)";
@ -2099,11 +2164,6 @@ bool tscHasReachLimitation(SQueryInfo* pQueryInfo, SSqlRes* pRes) {
return (pQueryInfo->clauseLimit > 0 && pRes->numOfClauseTotal >= pQueryInfo->clauseLimit);
}
bool tscResultsetFetchCompleted(TAOS_RES *result) {
SSqlRes* pRes = result;
return pRes->completed;
}
char* tscGetErrorMsgPayload(SSqlCmd* pCmd) { return pCmd->payload; }
/**

View File

@ -35,8 +35,6 @@ bool tscValidateTableNameLength(size_t len);
SColumnFilterInfo* tscFilterInfoClone(const SColumnFilterInfo* src, int32_t numOfFilters);
int64_t taosAddNatualInterval(int64_t key, int64_t intervalTime, char timeUnit, int16_t precision);
int32_t taosCountNatualInterval(int64_t skey, int64_t ekey, int64_t intervalTime, char timeUnit, int16_t precision);
int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, int64_t intervalTime, char timeUnit, int16_t precision);
// int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, int64_t intervalTime, char timeUnit, int16_t precision);
#endif // TDENGINE_NAME_H

View File

@ -957,17 +957,6 @@ static void doInitGlobalConfig(void) {
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
// http configs
cfg.option = "httpCacheSessions";
cfg.ptr = &tsHttpCacheSessions;
cfg.valType = TAOS_CFG_VTYPE_INT32;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
cfg.minValue = 1;
cfg.maxValue = 100000;
cfg.ptrLength = 0;
cfg.unitType = TAOS_CFG_UTYPE_NONE;
taosInitConfigOption(cfg);
cfg.option = "httpEnableRecordSql";
cfg.ptr = &tsHttpEnableRecordSql;
cfg.valType = TAOS_CFG_VTYPE_INT32;

View File

@ -62,10 +62,9 @@ SSchema tGetUserSpecifiedColumnSchema(tVariant* pVal, SStrToken* exprStr, const
if (name != NULL) {
tstrncpy(s.name, name, sizeof(s.name));
} else {
size_t len = strdequote(exprStr->z);
size_t tlen = MIN(sizeof(s.name), len + 1);
size_t tlen = MIN(sizeof(s.name), exprStr->n + 1);
tstrncpy(s.name, exprStr->z, tlen);
strdequote(s.name);
}
return s;
@ -100,62 +99,7 @@ SColumnFilterInfo* tscFilterInfoClone(const SColumnFilterInfo* src, int32_t numO
return pFilter;
}
int64_t taosAddNatualInterval(int64_t key, int64_t intervalTime, char timeUnit, int16_t precision) {
key /= 1000;
if (precision == TSDB_TIME_PRECISION_MICRO) {
key /= 1000;
}
struct tm tm;
time_t t = (time_t)key;
localtime_r(&t, &tm);
if (timeUnit == 'y') {
intervalTime *= 12;
}
int mon = (int)(tm.tm_year * 12 + tm.tm_mon + intervalTime);
tm.tm_year = mon / 12;
tm.tm_mon = mon % 12;
key = mktime(&tm) * 1000L;
if (precision == TSDB_TIME_PRECISION_MICRO) {
key *= 1000L;
}
return key;
}
int32_t taosCountNatualInterval(int64_t skey, int64_t ekey, int64_t intervalTime, char timeUnit, int16_t precision) {
skey /= 1000;
ekey /= 1000;
if (precision == TSDB_TIME_PRECISION_MICRO) {
skey /= 1000;
ekey /= 1000;
}
if (ekey < skey) {
int64_t tmp = ekey;
ekey = skey;
skey = tmp;
}
struct tm tm;
time_t t = (time_t)skey;
localtime_r(&t, &tm);
int smon = tm.tm_year * 12 + tm.tm_mon;
t = (time_t)ekey;
localtime_r(&t, &tm);
int emon = tm.tm_year * 12 + tm.tm_mon;
if (timeUnit == 'y') {
intervalTime *= 12;
}
return (emon - smon) / (int32_t)intervalTime;
}
#if 0
int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, int64_t intervalTime, char timeUnit, int16_t precision) {
if (slidingTime == 0) {
return startTime;
@ -220,6 +164,8 @@ int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, in
return start;
}
#endif
/*
* tablePrefix.columnName
* extract table name and save it in pTable, with only column name in pToken

View File

@ -540,9 +540,7 @@ void assignVal(char *val, const char *src, int32_t len, int32_t type) {
}
}
void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size) {
char tmpBuf[4096] = {0};
void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf) {
switch (type) {
case TSDB_DATA_TYPE_INT: {
SWAP(*(int32_t *)(pLeft), *(int32_t *)(pRight), int32_t);
@ -575,10 +573,9 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size) {
}
default: {
assert(size <= 4096);
memcpy(tmpBuf, pLeft, size);
memcpy(buf, pLeft, size);
memcpy(pLeft, pRight, size);
memcpy(pRight, tmpBuf, size);
memcpy(pRight, buf, size);
break;
}
}

View File

@ -48,11 +48,6 @@
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>${commons-lang3.version}</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>

View File

@ -14,8 +14,6 @@
*****************************************************************************/
package com.taosdata.jdbc;
import org.apache.commons.lang3.StringUtils;
import java.sql.*;
import java.util.Properties;
import java.util.logging.Logger;
@ -42,9 +40,8 @@ import java.util.logging.Logger;
public class TSDBDriver implements java.sql.Driver {
@Deprecated
private static final String URL_PREFIX1 = "jdbc:TSDB://";
private static final String URL_PREFIX = "jdbc:TAOS://";
private static final String URL_PREFIX1 = "jdbc:tsdb://";
private static final String URL_PREFIX = "jdbc:taos://";
/**
* Key used to retrieve the database value from the properties instance passed
@ -188,7 +185,7 @@ public class TSDBDriver implements java.sql.Driver {
}
public boolean acceptsURL(String url) throws SQLException {
return StringUtils.isNotBlank(url) && url.startsWith(URL_PREFIX);
return (url != null && url.length() > 0 && url.trim().length() > 0) && url.toLowerCase().startsWith(URL_PREFIX);
}
public DriverPropertyInfo[] getPropertyInfo(String url, Properties info) throws SQLException {
@ -238,7 +235,8 @@ public class TSDBDriver implements java.sql.Driver {
return null;
}
if (!StringUtils.startsWithIgnoreCase(url, URL_PREFIX) && !StringUtils.startsWithIgnoreCase(url, URL_PREFIX1)) {
String lowerUrl = url.toLowerCase();
if (!lowerUrl.startsWith(URL_PREFIX) && !lowerUrl.startsWith(URL_PREFIX1)) {
return null;
}

View File

@ -101,6 +101,7 @@ extern const int32_t TYPE_BYTES[11];
#define TSDB_TIME_PRECISION_MILLI 0
#define TSDB_TIME_PRECISION_MICRO 1
#define TSDB_TIME_PRECISION_NANO 2
#define TSDB_TICK_PER_SECOND(precision) ((precision)==TSDB_TIME_PRECISION_MILLI ? 1e3L : ((precision)==TSDB_TIME_PRECISION_MICRO ? 1e6L : 1e9L))
#define TSDB_TIME_PRECISION_MILLI_STR "ms"
#define TSDB_TIME_PRECISION_MICRO_STR "us"
@ -198,7 +199,7 @@ void setNullN(char *val, int32_t type, int32_t bytes, int32_t numOfElems);
void* getNullValue(int32_t type);
void assignVal(char *val, const char *src, int32_t len, int32_t type);
void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf);
// TODO: check if below is necessary
#define TSDB_RELATION_INVALID 0
@ -209,21 +210,24 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
#define TSDB_RELATION_GREATER_EQUAL 5
#define TSDB_RELATION_NOT_EQUAL 6
#define TSDB_RELATION_LIKE 7
#define TSDB_RELATION_IN 8
#define TSDB_RELATION_ISNULL 8
#define TSDB_RELATION_NOTNULL 9
#define TSDB_RELATION_IN 10
#define TSDB_RELATION_AND 9
#define TSDB_RELATION_OR 10
#define TSDB_RELATION_NOT 11
#define TSDB_RELATION_AND 11
#define TSDB_RELATION_OR 12
#define TSDB_RELATION_NOT 13
#define TSDB_BINARY_OP_ADD 12
#define TSDB_BINARY_OP_SUBTRACT 13
#define TSDB_BINARY_OP_MULTIPLY 14
#define TSDB_BINARY_OP_DIVIDE 15
#define TSDB_BINARY_OP_REMAINDER 16
#define TSDB_BINARY_OP_ADD 30
#define TSDB_BINARY_OP_SUBTRACT 31
#define TSDB_BINARY_OP_MULTIPLY 32
#define TSDB_BINARY_OP_DIVIDE 33
#define TSDB_BINARY_OP_REMAINDER 34
#define TS_PATH_DELIMITER_LEN 1
#define TSDB_UNI_LEN 24
#define TSDB_USER_LEN TSDB_UNI_LEN
// ACCOUNT is a 32 bit positive integer
// this is the length of its string representation
// including the terminator zero

View File

@ -97,8 +97,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_APP_ERROR, 0, 0x0211, "Applicatio
TAOS_DEFINE_ERROR(TSDB_CODE_TSC_ACTION_IN_PROGRESS, 0, 0x0212, "Action in progress")
TAOS_DEFINE_ERROR(TSDB_CODE_TSC_DISCONNECTED, 0, 0x0213, "Disconnected from service")
TAOS_DEFINE_ERROR(TSDB_CODE_TSC_NO_WRITE_AUTH, 0, 0x0214, "No write permission")
TAOS_DEFINE_ERROR(TSDB_CODE_TSC_CONN_KILLED, 0, 0x0215, "Connection killed")
TAOS_DEFINE_ERROR(TSDB_CODE_TSC_SQL_SYNTAX_ERROR, 0, 0x0216, "Syntax errr in SQL")
TAOS_DEFINE_ERROR(TSDB_CODE_TSC_CONN_KILLED, 0, 0x0215, "Connection killed")
TAOS_DEFINE_ERROR(TSDB_CODE_TSC_SQL_SYNTAX_ERROR, 0, 0x0216, "Syntax errr in SQL")
// mnode
TAOS_DEFINE_ERROR(TSDB_CODE_MND_MSG_NOT_PROCESSED, 0, 0x0300, "Message not processed")
@ -247,6 +247,106 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SYN_NOT_ENABLED, 0, 0x0901, "Sync modul
// wal
TAOS_DEFINE_ERROR(TSDB_CODE_WAL_APP_ERROR, 0, 0x1000, "Unexpected generic error in wal")
// http
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_SERVER_OFFLINE, 0, 0x1100, "http server is not onlin")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_UNSUPPORT_URL, 0, 0x1101, "url is not support")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVLALID_URL, 0, 0x1102, "invalid url format")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_NO_ENOUGH_MEMORY, 0, 0x1103, "no enough memory")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_REQUSET_TOO_BIG, 0, 0x1104, "request size is too big")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_NO_AUTH_INFO, 0, 0x1105, "no auth info input")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_NO_MSG_INPUT, 0, 0x1106, "request is empty")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_NO_SQL_INPUT, 0, 0x1107, "no sql input")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_NO_EXEC_USEDB, 0, 0x1108, "no need to execute use db cmd")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_SESSION_FULL, 0, 0x1109, "session list was full")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_GEN_TAOSD_TOKEN_ERR, 0, 0x110A, "generate taosd token error")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_MULTI_REQUEST, 0, 0x110B, "size of multi request is 0")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_CREATE_GZIP_FAILED, 0, 0x110C, "failed to create gzip")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_FINISH_GZIP_FAILED, 0, 0x110D, "failed to finish gzip")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_LOGIN_FAILED, 0, 0x110E, "failed to login")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_VERSION, 0, 0x1120, "invalid http version")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_CONTENT_LENGTH, 0, 0x1121, "invalid content length")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_AUTH_TYPE, 0, 0x1122, "invalid type of Authorization")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_AUTH_FORMAT, 0, 0x1123, "invalid format of Authorization")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_BASIC_AUTH, 0, 0x1124, "invalid basic Authorization")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_INVALID_TAOSD_AUTH, 0, 0x1125, "invalid taosd Authorization")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_METHOD_FAILED, 0, 0x1126, "failed to parse method")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_TARGET_FAILED, 0, 0x1127, "failed to parse target")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_VERSION_FAILED, 0, 0x1128, "failed to parse http version")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_SP_FAILED, 0, 0x1129, "failed to parse sp")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_STATUS_FAILED, 0, 0x112A, "failed to parse status")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_PHRASE_FAILED, 0, 0x112B, "failed to parse phrase")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_CRLF_FAILED, 0, 0x112C, "failed to parse crlf")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_HEADER_FAILED, 0, 0x112D, "failed to parse header")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_HEADER_KEY_FAILED, 0, 0x112E, "failed to parse header key")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_HEADER_VAL_FAILED, 0, 0x112F, "failed to parse header val")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_CHUNK_SIZE_FAILED, 0, 0x1130, "failed to parse chunk size")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_CHUNK_FAILED, 0, 0x1131, "failed to parse chunk")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_END_FAILED, 0, 0x1132, "failed to parse end section")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_INVALID_STATE, 0, 0x1134, "invalid parse state")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_PARSE_ERROR_STATE, 0, 0x1135, "failed to parse error section")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_GC_QUERY_NULL, 0, 0x1150, "query size is 0")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_GC_QUERY_SIZE, 0, 0x1151, "query size can not more than 100")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_GC_REQ_PARSE_ERROR, 0, 0x1152, "parse grafana json error")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_DB_NOT_INPUT, 0, 0x1160, "database name can not be null")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_DB_TOO_LONG, 0, 0x1161, "database name too long")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_INVALID_JSON, 0, 0x1162, "invalid telegraf json fromat")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_METRICS_NULL, 0, 0x1163, "metrics size is 0")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_METRICS_SIZE, 0, 0x1164, "metrics size can not more than 1K")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_METRIC_NULL, 0, 0x1165, "metric name not find")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_METRIC_TYPE, 0, 0x1166, "metric name type should be string")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_METRIC_NAME_NULL, 0, 0x1167, "metric name length is 0")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_METRIC_NAME_LONG, 0, 0x1168, "metric name length too long")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TIMESTAMP_NULL, 0, 0x1169, "timestamp not find")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TIMESTAMP_TYPE, 0, 0x116A, "timestamp type should be integer")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TIMESTAMP_VAL_NULL, 0, 0x116B, "timestamp value smaller than 0")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAGS_NULL, 0, 0x116C, "tags not find")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAGS_SIZE_0, 0, 0x116D, "tags size is 0")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAGS_SIZE_LONG, 0, 0x116E, "tags size too long")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAG_NULL, 0, 0x116F, "tag is null")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAG_NAME_NULL, 0, 0x1170, "tag name is null")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAG_NAME_SIZE, 0, 0x1171, "tag name length too long")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAG_VALUE_TYPE, 0, 0x1172, "tag value type should be number or string")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TAG_VALUE_NULL, 0, 0x1173, "tag value is null")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TABLE_NULL, 0, 0x1174, "table is null")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_TABLE_SIZE, 0, 0x1175, "table name length too long")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_FIELDS_NULL, 0, 0x1176, "fields not find")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_FIELDS_SIZE_0, 0, 0x1177, "fields size is 0")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_FIELDS_SIZE_LONG, 0, 0x1178, "fields size too long")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_FIELD_NULL, 0, 0x1179, "field is null")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_FIELD_NAME_NULL, 0, 0x117A, "field name is null")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_FIELD_NAME_SIZE, 0, 0x117B, "field name length too long")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_FIELD_VALUE_TYPE, 0, 0x117C, "field value type should be number or string")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_FIELD_VALUE_NULL, 0, 0x117D, "field value is null")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_HOST_NOT_STRING, 0, 0x117E, "host type should be string")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_TG_STABLE_NOT_EXIST, 0, 0x117F, "stable not exist")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_DB_NOT_INPUT, 0, 0x1190, "database name can not be null")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_DB_TOO_LONG, 0, 0x1191, "database name too long")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_INVALID_JSON, 0, 0x1192, "invalid opentsdb json fromat")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_METRICS_NULL, 0, 0x1193, "metrics size is 0")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_METRICS_SIZE, 0, 0x1194, "metrics size can not more than 10K")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_METRIC_NULL, 0, 0x1195, "metric name not find")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_METRIC_TYPE, 0, 0x1196, "metric name type should be string")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_METRIC_NAME_NULL, 0, 0x1197, "metric name length is 0")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_METRIC_NAME_LONG, 0, 0x1198, "metric name length can not more than 22")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TIMESTAMP_NULL, 0, 0x1199, "timestamp not find")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TIMESTAMP_TYPE, 0, 0x119A, "timestamp type should be integer")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TIMESTAMP_VAL_NULL, 0, 0x119B, "timestamp value smaller than 0")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAGS_NULL, 0, 0x119C, "tags not find")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAGS_SIZE_0, 0, 0x119D, "tags size is 0")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAGS_SIZE_LONG, 0, 0x119E, "tags size too long")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAG_NULL, 0, 0x119F, "tag is null")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAG_NAME_NULL, 0, 0x11A0, "tag name is null")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAG_NAME_SIZE, 0, 0x11A1, "tag name length too long")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAG_VALUE_TYPE, 0, 0x11A2, "tag value type should be boolean, number or string")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAG_VALUE_NULL, 0, 0x11A3, "tag value is null")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAG_VALUE_TOO_LONG, 0, 0x11A4, "tag value can not more than 64")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_VALUE_NULL, 0, 0x11A5, "value not find")
TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_VALUE_TYPE, 0, 0x11A6, "value type should be boolean, number or string")
#ifdef TAOS_ERROR_C
};
#endif

View File

@ -460,11 +460,7 @@ typedef struct {
int16_t order;
int16_t orderColId;
int16_t numOfCols; // the number of columns will be load from vnode
int64_t intervalTime; // time interval for aggregation, in million second
int64_t intervalOffset; // start offset for interval query
int64_t slidingTime; // value for sliding window
char intervalTimeUnit;
char slidingTimeUnit; // time interval type, for revisement of interval(1d)
SInterval interval;
uint16_t tagCondLen; // tag length in current query
int16_t numOfGroupCols; // num of group by columns
int16_t orderByIdx;

View File

@ -765,7 +765,9 @@ void read_history() {
FILE *f = fopen(f_history, "r");
if (f == NULL) {
#ifndef WINDOWS
fprintf(stderr, "Failed to open file %s\n", f_history);
if (errno != ENOENT) {
fprintf(stderr, "Failed to open file %s, reason:%s\n", f_history, strerror(errno));
}
#endif
return;
}
@ -792,7 +794,7 @@ void write_history() {
FILE *f = fopen(f_history, "w");
if (f == NULL) {
#ifndef WINDOWS
fprintf(stderr, "Failed to open file %s for write\n", f_history);
fprintf(stderr, "Failed to open file %s for write, reason:%s\n", f_history, strerror(errno));
#endif
return;
}

View File

@ -30,8 +30,6 @@ extern "C" {
#define MILLISECOND_PER_HOUR (MILLISECOND_PER_MINUTE * 60)
#define MILLISECOND_PER_DAY (MILLISECOND_PER_HOUR * 24)
#define MILLISECOND_PER_WEEK (MILLISECOND_PER_DAY * 7)
#define MILLISECOND_PER_MONTH (MILLISECOND_PER_DAY * 30)
#define MILLISECOND_PER_YEAR (MILLISECOND_PER_DAY * 365)
//@return timestamp in second
int32_t taosGetTimestampSec();
@ -63,8 +61,22 @@ static FORCE_INLINE int64_t taosGetTimestamp(int32_t precision) {
}
}
int32_t getTimestampInUsFromStr(char* token, int32_t tokenlen, int64_t* ts);
int32_t parseDuration(const char* token, int32_t tokenLen, int64_t* duration, char* unit);
typedef struct SInterval {
char intervalUnit;
char slidingUnit;
char offsetUnit;
int64_t interval;
int64_t sliding;
int64_t offset;
} SInterval;
int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision);
int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precision);
int32_t taosTimeCountInterval(int64_t skey, int64_t ekey, int64_t interval, char unit, int32_t precision);
int32_t parseAbsoluteDuration(char* token, int32_t tokenlen, int64_t* ts);
int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* duration, char* unit);
int32_t taosParseTime(char* timestr, int64_t* time, int32_t len, int32_t timePrec, int8_t dayligth);
void deltaToUtcInitOnce();

View File

@ -321,7 +321,7 @@ int32_t parseLocaltimeWithDst(char* timestr, int64_t* time, int32_t timePrec) {
}
static int32_t getTimestampInUsFromStrImpl(int64_t val, char unit, int64_t* result) {
static int32_t getDurationInUs(int64_t val, char unit, int64_t* result) {
*result = val;
int64_t factor = 1000L;
@ -342,19 +342,12 @@ static int32_t getTimestampInUsFromStrImpl(int64_t val, char unit, int64_t* resu
case 'w':
(*result) *= MILLISECOND_PER_WEEK*factor;
break;
case 'n':
(*result) *= MILLISECOND_PER_MONTH*factor;
break;
case 'y':
(*result) *= MILLISECOND_PER_YEAR*factor;
break;
case 'a':
(*result) *= factor;
break;
case 'u':
break;
default: {
;
return -1;
}
}
@ -373,7 +366,7 @@ static int32_t getTimestampInUsFromStrImpl(int64_t val, char unit, int64_t* resu
* n - Months (30 days)
* y - Years (365 days)
*/
int32_t getTimestampInUsFromStr(char* token, int32_t tokenlen, int64_t* ts) {
int32_t parseAbsoluteDuration(char* token, int32_t tokenlen, int64_t* duration) {
errno = 0;
char* endPtr = NULL;
@ -383,10 +376,16 @@ int32_t getTimestampInUsFromStr(char* token, int32_t tokenlen, int64_t* ts) {
return -1;
}
return getTimestampInUsFromStrImpl(timestamp, token[tokenlen - 1], ts);
/* natual month/year are not allowed in absolute duration */
char unit = token[tokenlen - 1];
if (unit == 'n' || unit == 'y') {
return -1;
}
return getDurationInUs(timestamp, unit, duration);
}
int32_t parseDuration(const char* token, int32_t tokenLen, int64_t* duration, char* unit) {
int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* duration, char* unit) {
errno = 0;
/* get the basic numeric value */
@ -400,7 +399,121 @@ int32_t parseDuration(const char* token, int32_t tokenLen, int64_t* duration, ch
return 0;
}
return getTimestampInUsFromStrImpl(*duration, *unit, duration);
return getDurationInUs(*duration, *unit, duration);
}
int64_t taosTimeAdd(int64_t t, int64_t duration, char unit, int32_t precision) {
if (duration == 0) {
return t;
}
if (unit == 'y') {
duration *= 12;
} else if (unit != 'n') {
return t + duration;
}
struct tm tm;
time_t tt = (time_t)(t / TSDB_TICK_PER_SECOND(precision));
localtime_r(&tt, &tm);
int mon = tm.tm_year * 12 + tm.tm_mon + (int)duration;
tm.tm_year = mon / 12;
tm.tm_mon = mon % 12;
return (int64_t)(mktime(&tm) * TSDB_TICK_PER_SECOND(precision));
}
int32_t taosTimeCountInterval(int64_t skey, int64_t ekey, int64_t interval, char unit, int32_t precision) {
if (ekey < skey) {
int64_t tmp = ekey;
ekey = skey;
skey = tmp;
}
if (unit != 'n' && unit != 'y') {
return (int32_t)((ekey - skey) / interval);
}
skey /= (int64_t)(TSDB_TICK_PER_SECOND(precision));
ekey /= (int64_t)(TSDB_TICK_PER_SECOND(precision));
struct tm tm;
time_t t = (time_t)skey;
localtime_r(&t, &tm);
int smon = tm.tm_year * 12 + tm.tm_mon;
t = (time_t)ekey;
localtime_r(&t, &tm);
int emon = tm.tm_year * 12 + tm.tm_mon;
if (unit == 'y') {
interval *= 12;
}
return (emon - smon) / (int32_t)interval;
}
int64_t taosTimeTruncate(int64_t t, const SInterval* pInterval, int32_t precision) {
if (pInterval->sliding == 0) {
assert(pInterval->interval == 0);
return t;
}
int64_t start = t;
if (pInterval->slidingUnit == 'n' || pInterval->slidingUnit == 'y') {
start /= (int64_t)(TSDB_TICK_PER_SECOND(precision));
struct tm tm;
time_t tt = (time_t)start;
localtime_r(&tt, &tm);
tm.tm_sec = 0;
tm.tm_min = 0;
tm.tm_hour = 0;
tm.tm_mday = 1;
if (pInterval->slidingUnit == 'y') {
tm.tm_mon = 0;
tm.tm_year = (int)(tm.tm_year / pInterval->sliding * pInterval->sliding);
} else {
int mon = tm.tm_year * 12 + tm.tm_mon;
mon = (int)(mon / pInterval->sliding * pInterval->sliding);
tm.tm_year = mon / 12;
tm.tm_mon = mon % 12;
}
start = (int64_t)(mktime(&tm) * TSDB_TICK_PER_SECOND(precision));
} else {
int64_t delta = t - pInterval->interval;
int32_t factor = delta > 0 ? 1 : -1;
start = (delta / pInterval->sliding + factor) * pInterval->sliding;
if (pInterval->intervalUnit == 'd' || pInterval->intervalUnit == 'w') {
/*
* here we revised the start time of day according to the local time zone,
* but in case of DST, the start time of one day need to be dynamically decided.
*/
// todo refactor to extract function that is available for Linux/Windows/Mac platform
#if defined(WINDOWS) && _MSC_VER >= 1900
// see https://docs.microsoft.com/en-us/cpp/c-runtime-library/daylight-dstbias-timezone-and-tzname?view=vs-2019
int64_t timezone = _timezone;
int32_t daylight = _daylight;
char** tzname = _tzname;
#endif
start += (int64_t)(timezone * TSDB_TICK_PER_SECOND(precision));
}
int64_t end = start + pInterval->interval - 1;
if (end < t) {
start += pInterval->sliding;
}
}
if (pInterval->offset > 0) {
start = taosTimeAdd(start, pInterval->offset, pInterval->offsetUnit, precision);
if (start > t) {
start = taosTimeAdd(start, -pInterval->interval, pInterval->intervalUnit, precision);
}
}
return start;
}
// internal function, when program is paused in debugger,
@ -411,24 +524,38 @@ int32_t parseDuration(const char* token, int32_t tokenLen, int64_t* duration, ch
// 2020-07-03 17:48:42
// and the parameter can also be a variable.
const char* fmtts(int64_t ts) {
static char buf[32];
static char buf[96];
size_t pos = 0;
struct tm tm;
time_t tt;
if (ts > -62135625943 && ts < 32503651200) {
tt = ts;
} else if (ts > -62135625943000 && ts < 32503651200000) {
tt = ts / 1000;
} else {
tt = ts / 1000000;
time_t t = (time_t)ts;
localtime_r(&t, &tm);
pos += strftime(buf + pos, sizeof(buf), "s=%Y-%m-%d %H:%M:%S", &tm);
}
struct tm* ptm = localtime(&tt);
size_t pos = strftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S", ptm);
if (ts > -62135625943000 && ts < 32503651200000) {
time_t t = (time_t)(ts / 1000);
localtime_r(&t, &tm);
if (pos > 0) {
buf[pos++] = ' ';
buf[pos++] = '|';
buf[pos++] = ' ';
}
pos += strftime(buf + pos, sizeof(buf), "ms=%Y-%m-%d %H:%M:%S", &tm);
pos += sprintf(buf + pos, ".%03d", (int)(ts % 1000));
}
if (ts <= -62135625943000 || ts >= 32503651200000) {
sprintf(buf + pos, ".%06d", (int)(ts % 1000000));
} else if (ts <= -62135625943 || ts >= 32503651200) {
sprintf(buf + pos, ".%03d", (int)(ts % 1000));
{
time_t t = (time_t)(ts / 1000000);
localtime_r(&t, &tm);
if (pos > 0) {
buf[pos++] = ' ';
buf[pos++] = '|';
buf[pos++] = ' ';
}
pos += strftime(buf + pos, sizeof(buf), "us=%Y-%m-%d %H:%M:%S", &tm);
pos += sprintf(buf + pos, ".%06d", (int)(ts % 1000000));
}
return buf;

View File

@ -16,8 +16,8 @@
#ifndef TDENGINE_HTTP_TOKEN_H
#define TDENGINE_HTTP_TOKEN_H
bool httpParseBasicAuthToken(HttpContext *pContext, char *token, int len);
bool httpParseTaosdAuthToken(HttpContext *pContext, char *token, int len);
bool httpGenTaosdAuthToken(HttpContext *pContext, char *token, int maxLen);
int32_t httpParseBasicAuthToken(HttpContext *pContext, char *token, int32_t len);
int32_t httpParseTaosdAuthToken(HttpContext *pContext, char *token, int32_t len);
int32_t httpGenTaosdAuthToken(HttpContext *pContext, char *token, int32_t maxLen);
#endif

View File

@ -1,112 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_HTTP_CODE_H
#define TDENGINE_HTTP_CODE_H
//for fixed msg info
#define HTTP_SUCCESS 0
#define HTTP_SERVER_OFFLINE 1
#define HTTP_UNSUPPORT_URL 2
#define HTTP_PARSE_HTTP_METHOD_ERROR 3
#define HTTP_PARSE_HTTP_VERSION_ERROR 4
#define HTTP_PARSE_HEAD_ERROR 5
#define HTTP_REQUSET_TOO_BIG 6
#define HTTP_PARSE_BODY_ERROR 7
#define HTTP_PARSE_CHUNKED_BODY_ERROR 8
#define HTTP_PARSE_URL_ERROR 9
#define HTTP_INVALID_AUTH_TOKEN 10
#define HTTP_PARSE_USR_ERROR 11
#define HTTP_NO_SQL_INPUT 12
#define HTTP_SESSION_FULL 13
#define HTTP_NO_ENOUGH_MEMORY 14
#define HTTP_GEN_TAOSD_TOKEN_ERR 15
#define HTTP_INVALID_DB_TABLE 16
#define HTTP_NO_EXEC_USEDB 17
#define HTTP_PARSE_GC_REQ_ERROR 18
#define HTTP_INVALID_MULTI_REQUEST 19
#define HTTP_NO_MSG_INPUT 20
#define HTTP_NO_ENOUGH_SESSIONS 21
//telegraf
#define HTTP_TG_DB_NOT_INPUT 22
#define HTTP_TG_DB_TOO_LONG 23
#define HTTP_TG_INVALID_JSON 24
#define HTTP_TG_METRICS_NULL 25
#define HTTP_TG_METRICS_SIZE 26
#define HTTP_TG_METRIC_NULL 27
#define HTTP_TG_METRIC_TYPE 28
#define HTTP_TG_METRIC_NAME_NULL 29
#define HTTP_TG_METRIC_NAME_LONG 30
#define HTTP_TG_TIMESTAMP_NULL 31
#define HTTP_TG_TIMESTAMP_TYPE 32
#define HTTP_TG_TIMESTAMP_VAL_NULL 33
#define HTTP_TG_TAGS_NULL 34
#define HTTP_TG_TAGS_SIZE_0 35
#define HTTP_TG_TAGS_SIZE_LONG 36
#define HTTP_TG_TAG_NULL 37
#define HTTP_TG_TAG_NAME_NULL 38
#define HTTP_TG_TAG_NAME_SIZE 39
#define HTTP_TG_TAG_VALUE_TYPE 40
#define HTTP_TG_TAG_VALUE_NULL 41
#define HTTP_TG_TABLE_NULL 42
#define HTTP_TG_TABLE_SIZE 43
#define HTTP_TG_FIELDS_NULL 44
#define HTTP_TG_FIELDS_SIZE_0 45
#define HTTP_TG_FIELDS_SIZE_LONG 46
#define HTTP_TG_FIELD_NULL 47
#define HTTP_TG_FIELD_NAME_NULL 48
#define HTTP_TG_FIELD_NAME_SIZE 49
#define HTTP_TG_FIELD_VALUE_TYPE 50
#define HTTP_TG_FIELD_VALUE_NULL 51
#define HTTP_INVALID_BASIC_AUTH_TOKEN 52
#define HTTP_INVALID_TAOSD_AUTH_TOKEN 53
#define HTTP_TG_HOST_NOT_STRING 54
//grafana
#define HTTP_GC_QUERY_NULL 55
#define HTTP_GC_QUERY_SIZE 56
//opentsdb
#define HTTP_OP_DB_NOT_INPUT 57
#define HTTP_OP_DB_TOO_LONG 58
#define HTTP_OP_INVALID_JSON 59
#define HTTP_OP_METRICS_NULL 60
#define HTTP_OP_METRICS_SIZE 61
#define HTTP_OP_METRIC_NULL 62
#define HTTP_OP_METRIC_TYPE 63
#define HTTP_OP_METRIC_NAME_NULL 64
#define HTTP_OP_METRIC_NAME_LONG 65
#define HTTP_OP_TIMESTAMP_NULL 66
#define HTTP_OP_TIMESTAMP_TYPE 67
#define HTTP_OP_TIMESTAMP_VAL_NULL 68
#define HTTP_OP_TAGS_NULL 69
#define HTTP_OP_TAGS_SIZE_0 70
#define HTTP_OP_TAGS_SIZE_LONG 71
#define HTTP_OP_TAG_NULL 72
#define HTTP_OP_TAG_NAME_NULL 73
#define HTTP_OP_TAG_NAME_SIZE 74
#define HTTP_OP_TAG_VALUE_TYPE 75
#define HTTP_OP_TAG_VALUE_NULL 76
#define HTTP_OP_TAG_VALUE_TOO_LONG 77
#define HTTP_OP_VALUE_NULL 78
#define HTTP_OP_VALUE_TYPE 79
//tgf
#define HTTP_TG_STABLE_NOT_EXIST 80
extern char *httpMsg[];
#endif

View File

@ -25,7 +25,7 @@ const char *httpContextStateStr(HttpContextState state);
HttpContext *httpCreateContext(int32_t fd);
bool httpInitContext(HttpContext *pContext);
HttpContext *httpGetContext(void * pContext);
void httpReleaseContext(HttpContext *pContext);
void httpReleaseContext(HttpContext *pContext, bool clearRes);
void httpCloseContextByServer(HttpContext *pContext);
void httpCloseContextByApp(HttpContext *pContext);
void httpNotifyContextClose(HttpContext *pContext);

View File

@ -24,7 +24,7 @@ void gcCleanQueryJson(HttpContext *pContext);
void gcStartQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result);
void gcStopQueryJson(HttpContext *pContext, HttpSqlCmd *cmd);
bool gcBuildQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int numOfRows);
bool gcBuildQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows);
void gcSendHeartBeatResp(HttpContext *pContext, HttpSqlCmd *cmd);

View File

@ -0,0 +1,43 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef HTTP_GZIP_H
#define HTTP_GZIP_H
#define EHTTP_GZIP_CHUNK_SIZE_DEFAULT (1024*16)
typedef struct ehttp_gzip_s ehttp_gzip_t;
typedef struct ehttp_gzip_callbacks_s ehttp_gzip_callbacks_t;
typedef struct ehttp_gzip_conf_s ehttp_gzip_conf_t;
struct ehttp_gzip_callbacks_s {
void (*on_data)(ehttp_gzip_t *gzip, void *arg, const char *buf, int32_t len);
};
struct ehttp_gzip_conf_s {
int32_t get_header:2; // 0: not fetching header info
int32_t chunk_size; // 0: fallback to default: EHTTP_GZIP_CHUNK_SIZE_DEFAULT
};
ehttp_gzip_t* ehttp_gzip_create_decompressor(ehttp_gzip_conf_t conf, ehttp_gzip_callbacks_t callbacks, void *arg);
ehttp_gzip_t* ehttp_gzip_create_compressor(ehttp_gzip_conf_t conf, ehttp_gzip_callbacks_t callbacks, void *arg);
void ehttp_gzip_destroy(ehttp_gzip_t *gzip);
int32_t ehttp_gzip_write(ehttp_gzip_t *gzip, const char *buf, int32_t len);
int32_t ehttp_gzip_finish(ehttp_gzip_t *gzip);
#endif // _ehttp_gzip_h_9196791b_ac2a_4d73_9979_f4b41abbc4c0_

View File

@ -24,56 +24,31 @@
#include "tutil.h"
#include "zlib.h"
#include "http.h"
#include "httpCode.h"
#include "httpLog.h"
#include "httpJson.h"
#include "httpParser.h"
#define HTTP_MAX_CMD_SIZE 1024
#define HTTP_MAX_BUFFER_SIZE 1024*1024
#define HTTP_MAX_BUFFER_SIZE 1024*1024*8
#define HTTP_LABEL_SIZE 8
#define HTTP_MAX_EVENTS 10
#define HTTP_BUFFER_SIZE 1024*65 //65k
#define HTTP_DECOMPRESS_BUF_SIZE 1024*64
#define HTTP_STEP_SIZE 1024 //http message get process step by step
#define HTTP_MAX_URL 5 //http url stack size
#define HTTP_BUFFER_INIT 4096
#define HTTP_BUFFER_SIZE 8388608
#define HTTP_STEP_SIZE 4096 //http message get process step by step
#define HTTP_METHOD_SCANNER_SIZE 7 //http method fp size
#define HTTP_GC_TARGET_SIZE 512
#define HTTP_VERSION_10 0
#define HTTP_VERSION_11 1
//#define HTTP_VERSION_12 2
#define HTTP_UNCUNKED 0
#define HTTP_CHUNKED 1
#define HTTP_KEEPALIVE_NO_INPUT 0
#define HTTP_KEEPALIVE_ENABLE 1
#define HTTP_KEEPALIVE_DISABLE 2
#define HTTP_REQTYPE_OTHERS 0
#define HTTP_REQTYPE_LOGIN 1
#define HTTP_REQTYPE_HEARTBEAT 2
#define HTTP_REQTYPE_SINGLE_SQL 3
#define HTTP_REQTYPE_MULTI_SQL 4
#define HTTP_CHECK_BODY_ERROR -1
#define HTTP_CHECK_BODY_CONTINUE 0
#define HTTP_CHECK_BODY_SUCCESS 1
#define HTTP_READ_DATA_SUCCESS 0
#define HTTP_READ_DATA_FAILED 1
#define HTTP_WRITE_RETRY_TIMES 500
#define HTTP_WRITE_WAIT_TIME_MS 5
#define HTTP_EXPIRED_TIME 60000
#define HTTP_DELAY_CLOSE_TIME_MS 500
#define HTTP_COMPRESS_IDENTITY 0
#define HTTP_COMPRESS_GZIP 2
#define HTTP_SESSION_ID_LEN (TSDB_USER_LEN + TSDB_PASSWORD_LEN)
typedef enum HttpReqType {
HTTP_REQTYPE_OTHERS = 0,
HTTP_REQTYPE_LOGIN = 1,
HTTP_REQTYPE_HEARTBEAT = 2,
HTTP_REQTYPE_SINGLE_SQL = 3,
HTTP_REQTYPE_MULTI_SQL = 4
} HttpReqType;
typedef enum {
HTTP_SERVER_INIT,
HTTP_SERVER_RUNNING,
@ -82,21 +57,12 @@ typedef enum {
} HttpServerStatus;
typedef enum {
HTTP_CONTEXT_STATE_READY,
HTTP_CONTEXT_STATE_HANDLING,
HTTP_CONTEXT_STATE_DROPPING,
HTTP_CONTEXT_STATE_CLOSED
HTTP_CONTEXT_STATE_READY,
HTTP_CONTEXT_STATE_HANDLING,
HTTP_CONTEXT_STATE_DROPPING,
HTTP_CONTEXT_STATE_CLOSED
} HttpContextState;
struct HttpContext;
struct HttpThread;
typedef struct {
char id[HTTP_SESSION_ID_LEN];
int refCount;
void *taos;
} HttpSession;
typedef enum {
HTTP_CMD_TYPE_UN_SPECIFIED,
HTTP_CMD_TYPE_CREATE_DB,
@ -108,6 +74,15 @@ typedef enum { HTTP_CMD_STATE_NOT_RUN_YET, HTTP_CMD_STATE_RUN_FINISHED } HttpSql
typedef enum { HTTP_CMD_RETURN_TYPE_WITH_RETURN, HTTP_CMD_RETURN_TYPE_NO_RETURN } HttpSqlCmdReturnType;
struct HttpContext;
struct HttpThread;
typedef struct {
char id[HTTP_SESSION_ID_LEN];
int32_t refCount;
void * taos;
} HttpSession;
typedef struct {
// used by single cmd
char *nativSql;
@ -157,34 +132,17 @@ typedef struct {
void (*setNextCmdFp)(struct HttpContext *pContext, HttpSqlCmd *cmd, int code);
} HttpEncodeMethod;
typedef struct {
char *pos;
int32_t len;
} HttpBuf;
typedef struct {
char buffer[HTTP_BUFFER_SIZE];
int bufsize;
char *pLast;
char *pCur;
HttpBuf method;
HttpBuf path[HTTP_MAX_URL]; // url: dbname/meter/query
HttpBuf data; // body content
HttpBuf token; // auth token
HttpDecodeMethod *pMethod;
} HttpParser;
typedef enum {
EHTTP_CONTEXT_PROCESS_FAILED = 0x01,
EHTTP_CONTEXT_PARSER_FAILED = 0x02
} EHTTP_CONTEXT_FAILED_CAUSE;
typedef struct HttpContext {
int32_t refCount;
int fd;
int32_t fd;
uint32_t accessTimes;
uint32_t lastAccessTime;
int32_t state;
uint8_t httpVersion;
uint8_t httpChunked;
uint8_t httpKeepAlive; // http1.0 and not keep-alive, close connection immediately
uint8_t acceptEncoding;
uint8_t contentEncoding;
uint8_t reqType;
uint8_t parsed;
char ipstr[22];
@ -194,12 +152,12 @@ typedef struct HttpContext {
void * ppContext;
HttpSession *session;
z_stream gzipStream;
HttpParser parser;
HttpParser *parser;
HttpSqlCmd singleCmd;
HttpSqlCmds *multiCmds;
JsonBuf * jsonBuf;
void * timer;
HttpEncodeMethod * encodeMethod;
HttpEncodeMethod *encodeMethod;
HttpDecodeMethod *decodeMethod;
struct HttpThread *pThread;
} HttpContext;
@ -208,9 +166,9 @@ typedef struct HttpThread {
HttpContext * pHead;
pthread_mutex_t threadMutex;
bool stop;
int pollFd;
int numOfContexts;
int threadId;
int32_t pollFd;
int32_t numOfContexts;
int32_t threadId;
char label[HTTP_LABEL_SIZE];
bool (*processData)(HttpContext *pContext);
} HttpThread;
@ -219,9 +177,9 @@ typedef struct HttpServer {
char label[HTTP_LABEL_SIZE];
uint32_t serverIp;
uint16_t serverPort;
int fd;
int numOfThreads;
int methodScannerLen;
int32_t fd;
int32_t numOfThreads;
int32_t methodScannerLen;
int32_t requestNum;
int32_t status;
pthread_t thread;

View File

@ -37,65 +37,65 @@ extern char JsonTrueTkn[];
extern char JsonFalseTkn[];
typedef struct {
int size;
int total;
char* lst;
char buf[JSON_BUFFER_SIZE];
struct HttpContext* pContext;
int32_t size;
int32_t total;
char* lst;
char buf[JSON_BUFFER_SIZE];
struct HttpContext* pContext;
} JsonBuf;
// http response
int httpWriteBuf(struct HttpContext* pContext, const char* buf, int sz);
int httpWriteBufNoTrace(struct HttpContext* pContext, const char* buf, int sz);
int httpWriteBufByFd(struct HttpContext* pContext, const char* buf, int sz);
int32_t httpWriteBuf(struct HttpContext* pContext, const char* buf, int32_t sz);
int32_t httpWriteBufNoTrace(struct HttpContext* pContext, const char* buf, int32_t sz);
int32_t httpWriteBufByFd(struct HttpContext* pContext, const char* buf, int32_t sz);
// builder callback
typedef void (*httpJsonBuilder)(JsonBuf* buf, void* jsnHandle);
// buffer
void httpInitJsonBuf(JsonBuf* buf, struct HttpContext* pContext);
void httpWriteJsonBufHead(JsonBuf* buf);
int httpWriteJsonBufBody(JsonBuf* buf, bool isTheLast);
void httpWriteJsonBufEnd(JsonBuf* buf);
void httpInitJsonBuf(JsonBuf* buf, struct HttpContext* pContext);
void httpWriteJsonBufHead(JsonBuf* buf);
int32_t httpWriteJsonBufBody(JsonBuf* buf, bool isTheLast);
void httpWriteJsonBufEnd(JsonBuf* buf);
// value
void httpJsonString(JsonBuf* buf, char* sVal, int len);
void httpJsonOriginString(JsonBuf* buf, char* sVal, int len);
void httpJsonStringForTransMean(JsonBuf* buf, char* SVal, int maxLen);
void httpJsonString(JsonBuf* buf, char* sVal, int32_t len);
void httpJsonOriginString(JsonBuf* buf, char* sVal, int32_t len);
void httpJsonStringForTransMean(JsonBuf* buf, char* SVal, int32_t maxLen);
void httpJsonInt64(JsonBuf* buf, int64_t num);
void httpJsonTimestamp(JsonBuf* buf, int64_t t, bool us);
void httpJsonUtcTimestamp(JsonBuf* buf, int64_t t, bool us);
void httpJsonInt(JsonBuf* buf, int num);
void httpJsonInt(JsonBuf* buf, int32_t num);
void httpJsonFloat(JsonBuf* buf, float num);
void httpJsonDouble(JsonBuf* buf, double num);
void httpJsonNull(JsonBuf* buf);
void httpJsonBool(JsonBuf* buf, int val);
void httpJsonBool(JsonBuf* buf, int32_t val);
// pair
void httpJsonPair(JsonBuf* buf, char* name, int nameLen, char* sVal, int valLen);
void httpJsonPairOriginString(JsonBuf* buf, char* name, int nameLen, char* sVal, int valLen);
void httpJsonPairHead(JsonBuf* buf, char* name, int len);
void httpJsonPairIntVal(JsonBuf* buf, char* name, int nNameLen, int num);
void httpJsonPairInt64Val(JsonBuf* buf, char* name, int nNameLen, int64_t num);
void httpJsonPairBoolVal(JsonBuf* buf, char* name, int nNameLen, int num);
void httpJsonPairFloatVal(JsonBuf* buf, char* name, int nNameLen, float num);
void httpJsonPairDoubleVal(JsonBuf* buf, char* name, int nNameLen, double num);
void httpJsonPairNullVal(JsonBuf* buf, char* name, int nNameLen);
void httpJsonPair(JsonBuf* buf, char* name, int32_t nameLen, char* sVal, int32_t valLen);
void httpJsonPairOriginString(JsonBuf* buf, char* name, int32_t nameLen, char* sVal, int32_t valLen);
void httpJsonPairHead(JsonBuf* buf, char* name, int32_t len);
void httpJsonPairIntVal(JsonBuf* buf, char* name, int32_t nNameLen, int32_t num);
void httpJsonPairInt64Val(JsonBuf* buf, char* name, int32_t nNameLen, int64_t num);
void httpJsonPairBoolVal(JsonBuf* buf, char* name, int32_t nNameLen, int32_t num);
void httpJsonPairFloatVal(JsonBuf* buf, char* name, int32_t nNameLen, float num);
void httpJsonPairDoubleVal(JsonBuf* buf, char* name, int32_t nNameLen, double num);
void httpJsonPairNullVal(JsonBuf* buf, char* name, int32_t nNameLen);
// object
void httpJsonPairArray(JsonBuf* buf, char* name, int nLen, httpJsonBuilder builder, void* dsHandle);
void httpJsonPairObject(JsonBuf* buf, char* name, int nLen, httpJsonBuilder builder, void* dsHandle);
void httpJsonPairArray(JsonBuf* buf, char* name, int32_t nLen, httpJsonBuilder builder, void* dsHandle);
void httpJsonPairObject(JsonBuf* buf, char* name, int32_t nLen, httpJsonBuilder builder, void* dsHandle);
void httpJsonObject(JsonBuf* buf, httpJsonBuilder fnBuilder, void* dsHandle);
void httpJsonArray(JsonBuf* buf, httpJsonBuilder fnBuidler, void* jsonHandle);
// print
void httpJsonTestBuf(JsonBuf* buf, int safety);
void httpJsonTestBuf(JsonBuf* buf, int32_t safety);
void httpJsonToken(JsonBuf* buf, char c);
void httpJsonItemToken(JsonBuf* buf);
void httpJsonPrint(JsonBuf* buf, const char* json, int len);
void httpJsonPrint(JsonBuf* buf, const char* json, int32_t len);
// quick
void httpJsonPairStatus(JsonBuf* buf, int code);
void httpJsonPairStatus(JsonBuf* buf, int32_t code);
// http json printer
JsonBuf* httpMallocJsonBuf(struct HttpContext* pContext);

View File

@ -0,0 +1,119 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef HTTP_PARSER_H
#define HTTP_PARSER_H
#include "httpGzip.h"
#define HTTP_MAX_URL 5 // http url stack size
typedef enum HTTP_PARSER_STATE {
HTTP_PARSER_BEGIN,
HTTP_PARSER_REQUEST_OR_RESPONSE,
HTTP_PARSER_METHOD,
HTTP_PARSER_TARGET,
HTTP_PARSER_HTTP_VERSION,
HTTP_PARSER_SP,
HTTP_PARSER_STATUS_CODE,
HTTP_PARSER_REASON_PHRASE,
HTTP_PARSER_CRLF,
HTTP_PARSER_HEADER,
HTTP_PARSER_HEADER_KEY,
HTTP_PARSER_HEADER_VAL,
HTTP_PARSER_CHUNK_SIZE,
HTTP_PARSER_CHUNK,
HTTP_PARSER_END,
HTTP_PARSER_ERROR,
} HTTP_PARSER_STATE;
typedef enum HTTP_AUTH_TYPE {
HTTP_INVALID_AUTH,
HTTP_BASIC_AUTH,
HTTP_TAOSD_AUTH
} HTTP_AUTH_TYPE;
typedef enum HTTP_VERSION {
HTTP_VERSION_10 = 0,
HTTP_VERSION_11 = 1,
HTTP_VERSION_12 = 2,
HTTP_INVALID_VERSION
} HTTP_VERSION;
typedef enum HTTP_KEEPALIVE {
HTTP_KEEPALIVE_NO_INPUT = 0,
HTTP_KEEPALIVE_ENABLE = 1,
HTTP_KEEPALIVE_DISABLE = 2
} HTTP_KEEPALIVE;
typedef struct HttpString {
char * str;
int32_t pos;
int32_t size;
} HttpString;
typedef struct HttpStatus {
int32_t code;
char * desc;
} HttpStatus;
typedef struct HttpStack{
int8_t *stacks;
int32_t pos;
int32_t size;
} HttpStack;
struct HttpContext;
typedef struct HttpParser {
struct HttpContext *pContext;
ehttp_gzip_t *gzip;
HttpStack stacks;
HttpString str;
HttpString body;
HttpString path[HTTP_MAX_URL];
char * method;
char * target;
char * version;
char * reasonPhrase;
char * key;
char * val;
char * authContent;
int8_t httpVersion;
int8_t acceptEncodingGzip;
int8_t acceptEncodingChunked;
int8_t contentLengthSpecified;
int8_t contentChunked;
int8_t transferGzip;
int8_t transferChunked;
int8_t keepAlive;
int8_t authType;
int32_t contentLength;
int32_t chunkSize;
int32_t receivedChunkSize;
int32_t receivedSize;
int32_t statusCode;
int8_t inited;
int8_t parsed;
int16_t httpCode;
int32_t parseCode;
} HttpParser;
void httpInitParser(HttpParser *parser);
HttpParser *httpCreateParser(struct HttpContext *pContext);
void httpClearParser(HttpParser *parser);
void httpDestroyParser(HttpParser *parser);
int32_t httpParseBuf(HttpParser *parser, const char *buf, int32_t len);
char * httpGetStatusDesc(int32_t statusCode);
#endif

View File

@ -32,9 +32,7 @@ enum _httpRespTempl {
extern const char *httpRespTemplate[];
void httpSendErrorResp(HttpContext *pContext, int errNo);
void httpSendErrorRespWithDesc(HttpContext *pContext, int errNo, char *desc);
void httpSendTaosdErrorResp(HttpContext *pContext, int errCode);
void httpSendErrorResp(HttpContext *pContext, int32_t errNo);
void httpSendTaosdInvalidSqlErrorResp(HttpContext *pContext, char* errMsg);
void httpSendSuccResp(HttpContext *pContext, char *desc);
void httpSendOptionResp(HttpContext *pContext, char *desc);

View File

@ -43,12 +43,12 @@
#define REST_TIMESTAMP_FMT_TIMESTAMP 1
#define REST_TIMESTAMP_FMT_UTC_STRING 2
void restBuildSqlAffectRowsJson(HttpContext *pContext, HttpSqlCmd *cmd, int affect_rows);
void restBuildSqlAffectRowsJson(HttpContext *pContext, HttpSqlCmd *cmd, int32_t affect_rows);
void restStartSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result);
bool restBuildSqlTimestampJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int numOfRows);
bool restBuildSqlLocalTimeStringJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int numOfRows);
bool restBuildSqlUtcTimeStringJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int numOfRows);
bool restBuildSqlTimestampJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows);
bool restBuildSqlLocalTimeStringJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows);
bool restBuildSqlUtcTimeStringJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows);
void restStopSqlJson(HttpContext *pContext, HttpSqlCmd *cmd);
#endif

View File

@ -21,8 +21,7 @@
bool httpInitConnect();
void httpCleanUpConnect();
void *httpInitServer(char *ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle);
void httpCleanUpServer(HttpServer *pServer);
int httpReadDataImp(HttpContext *pContext);
void *httpInitServer(char *ip, uint16_t port, char *label, int32_t numOfThreads, void *fp, void *shandle);
void httpCleanUpServer(HttpServer *pServer);
#endif

View File

@ -19,20 +19,20 @@
int32_t httpAddToSqlCmdBuffer(HttpContext *pContext, const char *const format, ...);
int32_t httpAddToSqlCmdBufferNoTerminal(HttpContext *pContext, const char *const format, ...);
int32_t httpAddToSqlCmdBufferWithSize(HttpContext *pContext, int mallocSize);
int32_t httpAddToSqlCmdBufferWithSize(HttpContext *pContext, int32_t mallocSize);
int32_t httpAddToSqlCmdBufferTerminal(HttpContext *pContext);
bool httpMallocMultiCmds(HttpContext *pContext, int cmdSize, int bufferSize);
bool httpReMallocMultiCmdsSize(HttpContext *pContext, int cmdSize);
bool httpReMallocMultiCmdsBuffer(HttpContext *pContext, int bufferSize);
bool httpMallocMultiCmds(HttpContext *pContext, int32_t cmdSize, int32_t bufferSize);
bool httpReMallocMultiCmdsSize(HttpContext *pContext, int32_t cmdSize);
bool httpReMallocMultiCmdsBuffer(HttpContext *pContext, int32_t bufferSize);
void httpFreeMultiCmds(HttpContext *pContext);
HttpSqlCmd *httpNewSqlCmd(HttpContext *pContext);
HttpSqlCmd *httpCurrSqlCmd(HttpContext *pContext);
int httpCurSqlCmdPos(HttpContext *pContext);
int32_t httpCurSqlCmdPos(HttpContext *pContext);
void httpTrimTableName(char *name);
int httpShrinkTableName(HttpContext *pContext, int pos, char *name);
char *httpGetCmdsString(HttpContext *pContext, int pos);
void httpTrimTableName(char *name);
int32_t httpShrinkTableName(HttpContext *pContext, int32_t pos, char *name);
char * httpGetCmdsString(HttpContext *pContext, int32_t pos);
#endif

View File

@ -24,8 +24,8 @@ void tgInitQueryJson(HttpContext *pContext);
void tgCleanQueryJson(HttpContext *pContext);
void tgStartQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result);
void tgStopQueryJson(HttpContext *pContext, HttpSqlCmd *cmd);
void tgBuildSqlAffectRowsJson(HttpContext *pContext, HttpSqlCmd *cmd, int affect_rows);
bool tgCheckFinished(struct HttpContext *pContext, HttpSqlCmd *cmd, int code);
void tgSetNextCmd(struct HttpContext *pContext, HttpSqlCmd *cmd, int code);
void tgBuildSqlAffectRowsJson(HttpContext *pContext, HttpSqlCmd *cmd, int32_t affect_rows);
bool tgCheckFinished(struct HttpContext *pContext, HttpSqlCmd *cmd, int32_t code);
void tgSetNextCmd(struct HttpContext *pContext, HttpSqlCmd *cmd, int32_t code);
#endif

View File

@ -17,21 +17,19 @@
#define TDENGINE_HTTP_UTIL_H
bool httpCheckUsedbSql(char *sql);
void httpTimeToString(time_t t, char *buf, int buflen);
void httpTimeToString(time_t t, char *buf, int32_t buflen);
bool httpUrlMatch(HttpContext *pContext, int pos, char *cmp);
bool httpParseRequest(HttpContext *pContext);
int httpCheckReadCompleted(HttpContext *pContext);
void httpReadDirtyData(HttpContext *pContext);
bool httpUrlMatch(HttpContext *pContext, int32_t pos, char *cmp);
bool httpParseRequest(HttpContext *pContext);
int32_t httpCheckReadCompleted(HttpContext *pContext);
void httpReadDirtyData(HttpContext *pContext);
int httpGzipDeCompress(char *srcData, int32_t nSrcData, char *destData, int32_t *nDestData);
int httpGzipCompressInit(HttpContext *pContext);
int httpGzipCompress(HttpContext *pContext, char *inSrcData, int32_t inSrcDataLen,
int32_t httpGzipDeCompress(char *srcData, int32_t nSrcData, char *destData, int32_t *nDestData);
int32_t httpGzipCompressInit(HttpContext *pContext);
int32_t httpGzipCompress(HttpContext *pContext, char *inSrcData, int32_t inSrcDataLen,
char *outDestData, int32_t *outDestDataLen, bool isTheLast);
// http request parser
void httpAddMethod(HttpServer *pServer, HttpDecodeMethod *pMethod);
#endif

View File

@ -23,85 +23,80 @@
#define KEY_DES_4 4971256377704625728L
bool httpParseBasicAuthToken(HttpContext *pContext, char *token, int len) {
int32_t httpParseBasicAuthToken(HttpContext *pContext, char *token, int32_t len) {
token[len] = '\0';
int outlen = 0;
int32_t outlen = 0;
char *base64 = (char *)base64_decode(token, len, &outlen);
if (base64 == NULL || outlen == 0) {
httpError("context:%p, fd:%d, ip:%s, basic token:%s parsed error", pContext, pContext->fd, pContext->ipstr, token);
httpError("context:%p, fd:%d, basic token:%s parsed error", pContext, pContext->fd, token);
free(base64);
return false;
return -1;
}
char *user = strstr(base64, ":");
if (user == NULL) {
httpError("context:%p, fd:%d, ip:%s, basic token:%s invalid format", pContext, pContext->fd, pContext->ipstr,
token);
httpError("context:%p, fd:%d, basic token:%s invalid format", pContext, pContext->fd, token);
free(base64);
return false;
return -1;
}
int user_len = (int)(user - base64);
int32_t user_len = (int32_t)(user - base64);
if (user_len < 1 || user_len >= TSDB_USER_LEN) {
httpError("context:%p, fd:%d, ip:%s, basic token:%s parse user error", pContext, pContext->fd, pContext->ipstr,
token);
httpError("context:%p, fd:%d, basic token:%s parse user error", pContext, pContext->fd, token);
free(base64);
return false;
return -1;
}
strncpy(pContext->user, base64, (size_t)user_len);
pContext->user[user_len] = 0;
char *password = user + 1;
int pass_len = (int)((base64 + outlen) - password);
int32_t pass_len = (int32_t)((base64 + outlen) - password);
if (pass_len < 1 || pass_len >= TSDB_PASSWORD_LEN) {
httpError("context:%p, fd:%d, ip:%s, basic token:%s parse password error", pContext, pContext->fd, pContext->ipstr,
token);
httpError("context:%p, fd:%d, basic token:%s parse password error", pContext, pContext->fd, token);
free(base64);
return false;
return -1;
}
strncpy(pContext->pass, password, (size_t)pass_len);
pContext->pass[pass_len] = 0;
free(base64);
httpDebug("context:%p, fd:%d, ip:%s, basic token parsed success, user:%s", pContext, pContext->fd, pContext->ipstr,
pContext->user);
return true;
httpDebug("context:%p, fd:%d, basic token parsed success, user:%s", pContext, pContext->fd, pContext->user);
return 0;
}
bool httpParseTaosdAuthToken(HttpContext *pContext, char *token, int len) {
int32_t httpParseTaosdAuthToken(HttpContext *pContext, char *token, int32_t len) {
token[len] = '\0';
int outlen = 0;
int32_t outlen = 0;
unsigned char *base64 = base64_decode(token, len, &outlen);
if (base64 == NULL || outlen == 0) {
httpError("context:%p, fd:%d, ip:%s, taosd token:%s parsed error", pContext, pContext->fd, pContext->ipstr, token);
httpError("context:%p, fd:%d, taosd token:%s parsed error", pContext, pContext->fd, token);
if (base64) free(base64);
return false;
return 01;
}
if (outlen != (TSDB_USER_LEN + TSDB_PASSWORD_LEN)) {
httpError("context:%p, fd:%d, ip:%s, taosd token:%s length error", pContext, pContext->fd, pContext->ipstr, token);
httpError("context:%p, fd:%d, taosd token:%s length error", pContext, pContext->fd, token);
free(base64);
return false;
return -1;
}
char *descrypt = taosDesDecode(KEY_DES_4, (char *)base64, outlen);
if (descrypt == NULL) {
httpError("context:%p, fd:%d, ip:%s, taosd token:%s descrypt error", pContext, pContext->fd, pContext->ipstr,
token);
httpError("context:%p, fd:%d, taosd token:%s descrypt error", pContext, pContext->fd, token);
free(base64);
return false;
return -1;
} else {
tstrncpy(pContext->user, descrypt, sizeof(pContext->user));
tstrncpy(pContext->pass, descrypt + TSDB_USER_LEN, sizeof(pContext->pass));
httpDebug("context:%p, fd:%d, ip:%s, taosd token:%s parsed success, user:%s", pContext, pContext->fd,
pContext->ipstr, token, pContext->user);
httpDebug("context:%p, fd:%d, taosd token:%s parsed success, user:%s", pContext, pContext->fd, token,
pContext->user);
free(base64);
free(descrypt);
return true;
return 0;
}
}
bool httpGenTaosdAuthToken(HttpContext *pContext, char *token, int maxLen) {
int32_t httpGenTaosdAuthToken(HttpContext *pContext, char *token, int32_t maxLen) {
char buffer[sizeof(pContext->user) + sizeof(pContext->pass)] = {0};
size_t size = sizeof(pContext->user);
tstrncpy(buffer, pContext->user, size);
@ -116,7 +111,7 @@ bool httpGenTaosdAuthToken(HttpContext *pContext, char *token, int maxLen) {
free(encrypt);
free(base64);
httpDebug("context:%p, fd:%d, ip:%s, gen taosd token:%s", pContext, pContext->fd, pContext->ipstr, token);
httpDebug("context:%p, fd:%d, generate taosd token:%s", pContext, pContext->fd, token);
return true;
return 0;
}

View File

@ -1,108 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
char* httpMsg[] = {
"success", // 0
"http server is not online", // 1
"http url is not support", // 2
"http method parse error", // 3
"http version should be 1.0, 1.1 or 1.2", // 4
"http head parse error", // 5
"request size is too big",
"http body size invalid",
"http chunked body parse error", // 8
"http url parse error", // 9
"invalid type of Authorization",
"no auth info input",
"no sql input",
"session list was full",
"no enough memory to alloc sqls",
"generate taosd token error",
"db and table can not be null",
"no need to execute use db cmd",
"parse grafana json error",
"size of multi request is 0", // 19
"request is empty", // 20
"no enough connections for http", // 21
// telegraf
"database name can not be null", // 22
"database name too long",
"invalid telegraf json fromat",
"metrics size is 0",
"metrics size can not more than 1K", // 26
"metric name not find",
"metric name type should be string",
"metric name length is 0",
"metric name length too long",
"timestamp not find", // 31
"timestamp type should be integer",
"timestamp value smaller than 0",
"tags not find",
"tags size is 0",
"tags size too long", // 36
"tag is null",
"tag name is null",
"tag name length too long", // 39
"tag value type should be number or string",
"tag value is null",
"table is null", // 42
"table name length too long",
"fields not find", // 44
"fields size is 0",
"fields size too long",
"field is null", // 47
"field name is null",
"field name length too long", // 49
"field value type should be number or string",
"field value is null", // 51
"parse basic auth token error",
"parse http auth token error",
"host type should be string",
// grafana
"query size is 0", // 55
"query size can not more than 100",
// opentsdb
"database name can not be null", // 57
"database name too long",
"invalid opentsdb json fromat", // 59
"metrics size is 0",
"metrics size can not more than 10K", // 61
"metric name not find",
"metric name type should be string",
"metric name length is 0",
"metric name length can not more than 22",
"timestamp not find",
"timestamp type should be integer",
"timestamp value smaller than 0",
"tags not find",
"tags size is 0",
"tags size too long", // 71
"tag is null",
"tag name is null",
"tag name length too long", // 74
"tag value type should be boolean, number or string",
"tag value is null",
"tag value can not more than 64", // 77
"value not find",
"value type should be boolean, number or string",
"stable not exist",
};

View File

@ -26,6 +26,10 @@
#include "httpResp.h"
#include "httpSql.h"
#include "httpSession.h"
#include "httpContext.h"
#include "httpParser.h"
static void httpDestroyContext(void *data);
static void httpRemoveContextFromEpoll(HttpContext *pContext) {
HttpThread *pThread = pContext->pThread;
@ -54,6 +58,11 @@ static void httpDestroyContext(void *data) {
httpFreeJsonBuf(pContext);
httpFreeMultiCmds(pContext);
if (pContext->parser) {
httpDestroyParser(pContext->parser);
pContext->parser = NULL;
}
taosTFree(pContext);
}
@ -104,9 +113,9 @@ HttpContext *httpCreateContext(int32_t fd) {
if (pContext == NULL) return NULL;
pContext->fd = fd;
pContext->httpVersion = HTTP_VERSION_10;
pContext->lastAccessTime = taosGetTimestampSec();
pContext->state = HTTP_CONTEXT_STATE_READY;
pContext->parser = httpCreateParser(pContext);
uint64_t handleVal = (uint64_t)pContext;
HttpContext **ppContext = taosCachePut(tsHttpServer.contextCache, &handleVal, sizeof(int64_t), &pContext, sizeof(int64_t), 3000);
@ -127,22 +136,26 @@ HttpContext *httpGetContext(void *ptr) {
HttpContext *pContext = *ppContext;
if (pContext) {
int32_t refCount = atomic_add_fetch_32(&pContext->refCount, 1);
httpDebug("context:%p, fd:%d, is accquired, data:%p refCount:%d", pContext, pContext->fd, ppContext, refCount);
httpTrace("context:%p, fd:%d, is accquired, data:%p refCount:%d", pContext, pContext->fd, ppContext, refCount);
return pContext;
}
}
return NULL;
}
void httpReleaseContext(HttpContext *pContext) {
void httpReleaseContext(HttpContext *pContext, bool clearRes) {
int32_t refCount = atomic_sub_fetch_32(&pContext->refCount, 1);
if (refCount < 0) {
httpError("context:%p, is already released, refCount:%d", pContext, refCount);
return;
}
if (clearRes) {
httpClearParser(pContext->parser);
}
HttpContext **ppContext = pContext->ppContext;
httpDebug("context:%p, is released, data:%p refCount:%d", pContext, ppContext, refCount);
httpTrace("context:%p, is released, data:%p refCount:%d", pContext, ppContext, refCount);
if (tsHttpServer.contextCache != NULL) {
taosCacheRelease(tsHttpServer.contextCache, (void **)(&ppContext), false);
@ -155,79 +168,66 @@ void httpReleaseContext(HttpContext *pContext) {
bool httpInitContext(HttpContext *pContext) {
pContext->accessTimes++;
pContext->lastAccessTime = taosGetTimestampSec();
pContext->httpVersion = HTTP_VERSION_10;
pContext->httpKeepAlive = HTTP_KEEPALIVE_NO_INPUT;
pContext->httpChunked = HTTP_UNCUNKED;
pContext->acceptEncoding = HTTP_COMPRESS_IDENTITY;
pContext->contentEncoding = HTTP_COMPRESS_IDENTITY;
pContext->reqType = HTTP_REQTYPE_OTHERS;
pContext->encodeMethod = NULL;
pContext->timer = NULL;
memset(&pContext->singleCmd, 0, sizeof(HttpSqlCmd));
HttpParser *pParser = &pContext->parser;
memset(pParser, 0, sizeof(HttpParser));
pParser->pCur = pParser->pLast = pParser->buffer;
httpDebug("context:%p, fd:%d, ip:%s, accessTimes:%d, parsed:%d", pContext, pContext->fd, pContext->ipstr,
pContext->accessTimes, pContext->parsed);
httpTrace("context:%p, fd:%d, parsed:%d", pContext, pContext->fd, pContext->parsed);
return true;
}
void httpCloseContextByApp(HttpContext *pContext) {
HttpParser *parser = pContext->parser;
pContext->parsed = false;
bool keepAlive = true;
if (pContext->httpVersion == HTTP_VERSION_10 && pContext->httpKeepAlive != HTTP_KEEPALIVE_ENABLE) {
if (parser->httpVersion == HTTP_VERSION_10 && parser->keepAlive != HTTP_KEEPALIVE_ENABLE) {
keepAlive = false;
} else if (pContext->httpVersion != HTTP_VERSION_10 && pContext->httpKeepAlive == HTTP_KEEPALIVE_DISABLE) {
} else if (parser->httpVersion != HTTP_VERSION_10 && parser->keepAlive == HTTP_KEEPALIVE_DISABLE) {
keepAlive = false;
} else {
}
if (keepAlive) {
if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_HANDLING, HTTP_CONTEXT_STATE_READY)) {
httpDebug("context:%p, fd:%d, ip:%s, last state:handling, keepAlive:true, reuse context", pContext, pContext->fd,
pContext->ipstr);
httpTrace("context:%p, fd:%d, last state:handling, keepAlive:true, reuse context", pContext, pContext->fd);
} else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_DROPPING, HTTP_CONTEXT_STATE_CLOSED)) {
httpRemoveContextFromEpoll(pContext);
httpDebug("context:%p, fd:%d, ip:%s, last state:dropping, keepAlive:true, close connect", pContext, pContext->fd,
pContext->ipstr);
httpTrace("context:%p, fd:%d, ast state:dropping, keepAlive:true, close connect", pContext, pContext->fd);
} else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_READY, HTTP_CONTEXT_STATE_READY)) {
httpDebug("context:%p, fd:%d, ip:%s, last state:ready, keepAlive:true, reuse context", pContext, pContext->fd,
pContext->ipstr);
httpTrace("context:%p, fd:%d, last state:ready, keepAlive:true, reuse context", pContext, pContext->fd);
} else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_CLOSED, HTTP_CONTEXT_STATE_CLOSED)) {
httpRemoveContextFromEpoll(pContext);
httpDebug("context:%p, fd:%d, ip:%s, last state:ready, keepAlive:true, close connect", pContext, pContext->fd,
pContext->ipstr);
httpTrace("context:%p, fd:%d, last state:ready, keepAlive:true, close connect", pContext, pContext->fd);
} else {
httpRemoveContextFromEpoll(pContext);
httpError("context:%p, fd:%d, ip:%s, last state:%s:%d, keepAlive:true, close connect", pContext, pContext->fd,
pContext->ipstr, httpContextStateStr(pContext->state), pContext->state);
httpError("context:%p, fd:%d, last state:%s:%d, keepAlive:true, close connect", pContext, pContext->fd,
httpContextStateStr(pContext->state), pContext->state);
}
} else {
httpRemoveContextFromEpoll(pContext);
httpDebug("context:%p, fd:%d, ip:%s, last state:%s:%d, keepAlive:false, close context", pContext, pContext->fd,
pContext->ipstr, httpContextStateStr(pContext->state), pContext->state);
httpTrace("context:%p, fd:%d, ilast state:%s:%d, keepAlive:false, close context", pContext, pContext->fd,
httpContextStateStr(pContext->state), pContext->state);
}
httpReleaseContext(pContext);
httpReleaseContext(pContext, true);
}
void httpCloseContextByServer(HttpContext *pContext) {
if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_HANDLING, HTTP_CONTEXT_STATE_DROPPING)) {
httpDebug("context:%p, fd:%d, ip:%s, epoll finished, still used by app", pContext, pContext->fd, pContext->ipstr);
httpTrace("context:%p, fd:%d, epoll finished, still used by app", pContext, pContext->fd);
} else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_DROPPING, HTTP_CONTEXT_STATE_DROPPING)) {
httpDebug("context:%p, fd:%d, ip:%s, epoll already finished, wait app finished", pContext, pContext->fd, pContext->ipstr);
httpTrace("context:%p, fd:%d, epoll already finished, wait app finished", pContext, pContext->fd);
} else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_READY, HTTP_CONTEXT_STATE_CLOSED)) {
httpDebug("context:%p, fd:%d, ip:%s, epoll finished, close connect", pContext, pContext->fd, pContext->ipstr);
httpTrace("context:%p, fd:%d, epoll finished, close connect", pContext, pContext->fd);
} else if (httpAlterContextState(pContext, HTTP_CONTEXT_STATE_CLOSED, HTTP_CONTEXT_STATE_CLOSED)) {
httpDebug("context:%p, fd:%d, ip:%s, epoll finished, will be closed soon", pContext, pContext->fd, pContext->ipstr);
httpTrace("context:%p, fd:%d, epoll finished, will be closed soon", pContext, pContext->fd);
} else {
httpError("context:%p, fd:%d, ip:%s, unknown state:%d", pContext, pContext->fd, pContext->ipstr, pContext->state);
httpError("context:%p, fd:%d, unknown state:%d", pContext, pContext->fd, pContext->state);
}
pContext->parsed = false;
httpRemoveContextFromEpoll(pContext);
httpReleaseContext(pContext);
}

View File

@ -15,11 +15,12 @@
#define _DEFAULT_SOURCE
#include "os.h"
#include "taosdef.h"
#include "taoserror.h"
#include "cJSON.h"
#include "httpLog.h"
#include "gcHandle.h"
#include "gcJson.h"
#include "taosdef.h"
#include "httpGcHandle.h"
#include "httpGcJson.h"
static HttpDecodeMethod gcDecodeMethod = {"grafana", gcProcessRequest};
static HttpEncodeMethod gcHeartBeatMethod = {
@ -47,28 +48,27 @@ static HttpEncodeMethod gcQueryMethod = {
void gcInitHandle(HttpServer* pServer) { httpAddMethod(pServer, &gcDecodeMethod); }
bool gcGetUserFromUrl(HttpContext* pContext) {
HttpParser* pParser = &pContext->parser;
if (pParser->path[GC_USER_URL_POS].len >= TSDB_USER_LEN || pParser->path[GC_USER_URL_POS].len <= 0) {
HttpParser* pParser = pContext->parser;
if (pParser->path[GC_USER_URL_POS].pos >= TSDB_USER_LEN || pParser->path[GC_USER_URL_POS].pos <= 0) {
return false;
}
tstrncpy(pContext->user, pParser->path[GC_USER_URL_POS].pos, TSDB_USER_LEN);
tstrncpy(pContext->user, pParser->path[GC_USER_URL_POS].str, TSDB_USER_LEN);
return true;
}
bool gcGetPassFromUrl(HttpContext* pContext) {
HttpParser* pParser = &pContext->parser;
if (pParser->path[GC_PASS_URL_POS].len >= TSDB_PASSWORD_LEN || pParser->path[GC_PASS_URL_POS].len <= 0) {
HttpParser* pParser = pContext->parser;
if (pParser->path[GC_PASS_URL_POS].pos >= TSDB_PASSWORD_LEN || pParser->path[GC_PASS_URL_POS].pos <= 0) {
return false;
}
tstrncpy(pContext->pass, pParser->path[GC_PASS_URL_POS].pos, TSDB_PASSWORD_LEN);
tstrncpy(pContext->pass, pParser->path[GC_PASS_URL_POS].str, TSDB_PASSWORD_LEN);
return true;
}
bool gcProcessLoginRequest(HttpContext* pContext) {
httpDebug("context:%p, fd:%d, ip:%s, user:%s, process grafana login msg", pContext, pContext->fd, pContext->ipstr,
pContext->user);
httpDebug("context:%p, fd:%d, user:%s, process grafana login msg", pContext, pContext->fd, pContext->user);
pContext->reqType = HTTP_REQTYPE_LOGIN;
return true;
}
@ -143,65 +143,61 @@ bool gcProcessLoginRequest(HttpContext* pContext) {
//}]
bool gcProcessQueryRequest(HttpContext* pContext) {
httpDebug("context:%p, fd:%d, ip:%s, process grafana query msg", pContext, pContext->fd, pContext->ipstr);
httpDebug("context:%p, fd:%d, process grafana query msg", pContext, pContext->fd);
HttpParser* pParser = &pContext->parser;
char* filter = pParser->data.pos;
char* filter = pContext->parser->body.str;
if (filter == NULL) {
httpSendErrorResp(pContext, HTTP_NO_MSG_INPUT);
httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_MSG_INPUT);
return false;
}
cJSON* root = cJSON_Parse(filter);
if (root == NULL) {
httpSendErrorResp(pContext, HTTP_PARSE_GC_REQ_ERROR);
httpSendErrorResp(pContext, TSDB_CODE_HTTP_GC_REQ_PARSE_ERROR);
return false;
}
int size = cJSON_GetArraySize(root);
int32_t size = cJSON_GetArraySize(root);
if (size <= 0) {
httpSendErrorResp(pContext, HTTP_GC_QUERY_NULL);
httpSendErrorResp(pContext, TSDB_CODE_HTTP_GC_QUERY_NULL);
cJSON_Delete(root);
return false;
}
if (size > 100) {
httpSendErrorResp(pContext, HTTP_GC_QUERY_SIZE);
httpSendErrorResp(pContext, TSDB_CODE_HTTP_GC_QUERY_SIZE);
cJSON_Delete(root);
return false;
}
if (!httpMallocMultiCmds(pContext, size, HTTP_BUFFER_SIZE)) {
httpSendErrorResp(pContext, HTTP_NO_ENOUGH_MEMORY);
httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_ENOUGH_MEMORY);
cJSON_Delete(root);
return false;
}
for (int i = 0; i < size; ++i) {
for (int32_t i = 0; i < size; ++i) {
cJSON* query = cJSON_GetArrayItem(root, i);
if (query == NULL) continue;
cJSON* refId = cJSON_GetObjectItem(query, "refId");
if (refId == NULL || refId->valuestring == NULL || strlen(refId->valuestring) == 0) {
httpDebug("context:%p, fd:%d, ip:%s, user:%s, refId is null", pContext, pContext->fd, pContext->ipstr,
pContext->user);
httpDebug("context:%p, fd:%d, user:%s, refId is null", pContext, pContext->fd, pContext->user);
continue;
}
int refIdBuffer = httpAddToSqlCmdBuffer(pContext, refId->valuestring);
int32_t refIdBuffer = httpAddToSqlCmdBuffer(pContext, refId->valuestring);
if (refIdBuffer == -1) {
httpWarn("context:%p, fd:%d, ip:%s, user:%s, refId buffer is full", pContext, pContext->fd, pContext->ipstr,
pContext->user);
httpWarn("context:%p, fd:%d, user:%s, refId buffer is full", pContext, pContext->fd, pContext->user);
break;
}
cJSON* alias = cJSON_GetObjectItem(query, "alias");
int aliasBuffer = -1;
int32_t aliasBuffer = -1;
if (!(alias == NULL || alias->valuestring == NULL || strlen(alias->valuestring) == 0)) {
aliasBuffer = httpAddToSqlCmdBuffer(pContext, alias->valuestring);
if (aliasBuffer == -1) {
httpWarn("context:%p, fd:%d, ip:%s, user:%s, alias buffer is full", pContext, pContext->fd, pContext->ipstr,
pContext->user);
httpWarn("context:%p, fd:%d, user:%s, alias buffer is full", pContext, pContext->fd, pContext->user);
break;
}
}
@ -211,21 +207,19 @@ bool gcProcessQueryRequest(HttpContext* pContext) {
cJSON* sql = cJSON_GetObjectItem(query, "sql");
if (sql == NULL || sql->valuestring == NULL || strlen(sql->valuestring) == 0) {
httpDebug("context:%p, fd:%d, ip:%s, user:%s, sql is null", pContext, pContext->fd, pContext->ipstr,
pContext->user);
httpDebug("context:%p, fd:%d, user:%s, sql is null", pContext, pContext->fd, pContext->user);
continue;
}
int sqlBuffer = httpAddToSqlCmdBuffer(pContext, sql->valuestring);
int32_t sqlBuffer = httpAddToSqlCmdBuffer(pContext, sql->valuestring);
if (sqlBuffer == -1) {
httpWarn("context:%p, fd:%d, ip:%s, user:%s, sql buffer is full", pContext, pContext->fd, pContext->ipstr,
pContext->user);
httpWarn("context:%p, fd:%d, user:%s, sql buffer is full", pContext, pContext->fd, pContext->user);
break;
}
HttpSqlCmd* cmd = httpNewSqlCmd(pContext);
if (cmd == NULL) {
httpSendErrorResp(pContext, HTTP_NO_ENOUGH_MEMORY);
httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_ENOUGH_MEMORY);
cJSON_Delete(root);
return false;
}
@ -237,8 +231,8 @@ bool gcProcessQueryRequest(HttpContext* pContext) {
cmd->timestamp = httpAddToSqlCmdBufferWithSize(pContext, HTTP_GC_TARGET_SIZE + 1); // hack way
if (cmd->timestamp == -1) {
httpWarn("context:%p, fd:%d, ip:%s, user:%s, cant't malloc target size, sql buffer is full",
pContext, pContext->fd, pContext->ipstr, pContext->user);
httpWarn("context:%p, fd:%d, user:%s, cant't malloc target size, sql buffer is full", pContext, pContext->fd,
pContext->user);
break;
}
}
@ -251,7 +245,7 @@ bool gcProcessQueryRequest(HttpContext* pContext) {
}
bool gcProcessHeartbeatRequest(HttpContext* pContext) {
httpDebug("context:%p, fd:%d, ip:%s, process grafana heartbeat msg", pContext, pContext->fd, pContext->ipstr);
httpDebug("context:%p, fd:%d, process grafana heartbeat msg", pContext, pContext->fd);
pContext->reqType = HTTP_REQTYPE_HEARTBEAT;
pContext->encodeMethod = &gcHeartBeatMethod;
return true;
@ -267,7 +261,7 @@ bool gcProcessRequest(struct HttpContext* pContext) {
}
if (strlen(pContext->user) == 0 || strlen(pContext->pass) == 0) {
httpSendErrorResp(pContext, HTTP_PARSE_USR_ERROR);
httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_AUTH_INFO);
return false;
}

View File

@ -15,8 +15,8 @@
#define _DEFAULT_SOURCE
#include "os.h"
#include "gcHandle.h"
#include "gcJson.h"
#include "httpGcHandle.h"
#include "httpGcJson.h"
#include "httpJson.h"
#include "httpResp.h"
@ -54,8 +54,8 @@ void gcWriteTargetStartJson(JsonBuf *jsonBuf, char *refId, char *target) {
httpJsonToken(jsonBuf, JsonObjStt);
// target section
httpJsonPair(jsonBuf, "refId", 5, refId, (int)strlen(refId));
httpJsonPair(jsonBuf, "target", 6, target, (int)strlen(target));
httpJsonPair(jsonBuf, "refId", 5, refId, (int32_t)strlen(refId));
httpJsonPair(jsonBuf, "target", 6, target, (int32_t)strlen(target));
// data begin
httpJsonPairHead(jsonBuf, "datapoints", 10);
@ -82,25 +82,25 @@ void gcStopQueryJson(HttpContext *pContext, HttpSqlCmd *cmd) {
}
}
bool gcBuildQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int numOfRows) {
bool gcBuildQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows) {
JsonBuf *jsonBuf = httpMallocJsonBuf(pContext);
if (jsonBuf == NULL) return false;
int num_fields = taos_num_fields(result);
int32_t num_fields = taos_num_fields(result);
TAOS_FIELD *fields = taos_fetch_fields(result);
if (num_fields == 0) {
return false;
}
int precision = taos_result_precision(result);
int32_t precision = taos_result_precision(result);
// such as select count(*) from sys.cpu
// such as select count(*) from sys.cpu group by ipaddr
// such as select count(*) from sys.cpu interval(1d)
// such as select count(*) from sys.cpu interval(1d) group by ipaddr
// such as select count(*) count(*) from sys.cpu group by ipaddr interval(1d)
int dataFields = -1;
int groupFields = -1;
int32_t dataFields = -1;
int32_t groupFields = -1;
bool hasTimestamp = fields[0].type == TSDB_DATA_TYPE_TIMESTAMP;
if (hasTimestamp) {
dataFields = 1;
@ -119,7 +119,7 @@ bool gcBuildQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result,
}
cmd->numOfRows += numOfRows;
for (int k = 0; k < numOfRows; ++k) {
for (int32_t k = 0; k < numOfRows; ++k) {
TAOS_ROW row = taos_fetch_row(result);
if (row == NULL) {
cmd->numOfRows--;
@ -130,9 +130,9 @@ bool gcBuildQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result,
// for group by
if (groupFields != -1) {
char target[HTTP_GC_TARGET_SIZE] = {0};
int len;
int32_t len;
len = snprintf(target,HTTP_GC_TARGET_SIZE,"%s{",aliasBuffer);
for (int i = dataFields + 1; i<num_fields; i++){
for (int32_t i = dataFields + 1; i<num_fields; i++){
switch (fields[i].type) {
case TSDB_DATA_TYPE_BOOL:
case TSDB_DATA_TYPE_TINYINT:
@ -188,7 +188,7 @@ bool gcBuildQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result,
httpJsonItemToken(jsonBuf);
httpJsonToken(jsonBuf, JsonArrStt);
for (int i = dataFields; i >= 0; i--) {
for (int32_t i = dataFields; i >= 0; i--) {
httpJsonItemToken(jsonBuf);
if (row[i] == NULL) {
httpJsonOriginString(jsonBuf, "null", 4);
@ -253,13 +253,13 @@ void gcSendHeartBeatResp(HttpContext *pContext, HttpSqlCmd *cmd) {
httpInitJsonBuf(jsonBuf, pContext);
httpJsonToken(jsonBuf, JsonObjStt);
httpJsonPair(jsonBuf, "message", (int)strlen("message"), desc, (int)strlen(desc));
httpJsonPair(jsonBuf, "message", (int32_t)strlen("message"), desc, (int32_t)strlen(desc));
httpJsonToken(jsonBuf, JsonObjEnd);
char head[1024];
int hLen = sprintf(head, httpRespTemplate[HTTP_RESPONSE_GRAFANA], httpVersionStr[pContext->httpVersion],
httpKeepAliveStr[pContext->httpKeepAlive], (jsonBuf->lst - jsonBuf->buf));
int32_t hLen = sprintf(head, httpRespTemplate[HTTP_RESPONSE_GRAFANA], httpVersionStr[pContext->parser->httpVersion],
httpKeepAliveStr[pContext->parser->keepAlive], (jsonBuf->lst - jsonBuf->buf));
httpWriteBuf(pContext, head, hLen);
httpWriteBuf(pContext, jsonBuf->buf, (int)(jsonBuf->lst - jsonBuf->buf));
httpWriteBuf(pContext, jsonBuf->buf, (int32_t)(jsonBuf->lst - jsonBuf->buf));
}

View File

@ -0,0 +1,167 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "zlib.h"
#include "httpGzip.h"
typedef enum {
EHTTP_GZIP_INITING,
EHTTP_GZIP_READY,
EHTTP_GZIP_CLOSED,
} EHTTP_GZIP_STATE;
struct ehttp_gzip_s {
ehttp_gzip_conf_t conf;
ehttp_gzip_callbacks_t callbacks;
void *arg;
z_stream *gzip;
gz_header *header;
char *chunk;
int32_t state;
};
static void dummy_on_data(ehttp_gzip_t *gzip, void *arg, const char *buf, int32_t len) {
}
static void ehttp_gzip_cleanup(ehttp_gzip_t *gzip) {
switch(gzip->state) {
case EHTTP_GZIP_READY: {
inflateEnd(gzip->gzip);
} break;
default: break;
}
if (gzip->gzip) {
free(gzip->gzip);
gzip->gzip = NULL;
}
if (gzip->header) {
free(gzip->header);
gzip->header = NULL;
}
if (gzip->chunk) {
free(gzip->chunk);
gzip->chunk = NULL;
}
gzip->state = EHTTP_GZIP_CLOSED;
}
ehttp_gzip_t* ehttp_gzip_create_decompressor(ehttp_gzip_conf_t conf, ehttp_gzip_callbacks_t callbacks, void *arg) {
ehttp_gzip_t *gzip = (ehttp_gzip_t*)calloc(1, sizeof(*gzip));
if (!gzip) return NULL;
do {
gzip->conf = conf;
gzip->callbacks = callbacks;
gzip->arg = arg;
if (gzip->callbacks.on_data == NULL) gzip->callbacks.on_data = dummy_on_data;
gzip->gzip = (z_stream*)calloc(1, sizeof(*gzip->gzip));
if (gzip->conf.get_header) {
gzip->header = (gz_header*)calloc(1, sizeof(*gzip->header));
}
if (gzip->conf.chunk_size<=0) gzip->conf.chunk_size = EHTTP_GZIP_CHUNK_SIZE_DEFAULT;
gzip->chunk = (char*)malloc(gzip->conf.chunk_size);
if (!gzip->gzip || (gzip->conf.get_header && !gzip->header) || !gzip->chunk) break;
gzip->gzip->zalloc = Z_NULL;
gzip->gzip->zfree = Z_NULL;
gzip->gzip->opaque = Z_NULL;
// 863 windowBits can also be greater than 15 for optional gzip decoding. Add
// 864 32 to windowBits to enable zlib and gzip decoding with automatic header
// 865 detection, or add 16 to decode only the gzip format (the zlib format will
// 866 return a Z_DATA_ERROR). If a gzip stream is being decoded, strm->adler is a
// 867 CRC-32 instead of an Adler-32. Unlike the gunzip utility and gzread() (see
// 868 below), inflate() will not automatically decode concatenated gzip streams.
// 869 inflate() will return Z_STREAM_END at the end of the gzip stream. The state
// 870 would need to be reset to continue decoding a subsequent gzip stream.
int32_t ret = inflateInit2(gzip->gzip, 32); // 32/16? 32/16 + MAX_WBITS
if (ret != Z_OK) break;
if (gzip->header) {
ret = inflateGetHeader(gzip->gzip, gzip->header);
}
if (ret != Z_OK) break;
gzip->gzip->next_out = (z_const Bytef*)gzip->chunk;
gzip->gzip->avail_out = gzip->conf.chunk_size;
gzip->state = EHTTP_GZIP_READY;
return gzip;
} while (0);
ehttp_gzip_destroy(gzip);
return NULL;
}
ehttp_gzip_t* ehttp_gzip_create_compressor(ehttp_gzip_conf_t conf, ehttp_gzip_callbacks_t callbacks, void *arg);
void ehttp_gzip_destroy(ehttp_gzip_t *gzip) {
ehttp_gzip_cleanup(gzip);
free(gzip);
}
int32_t ehttp_gzip_write(ehttp_gzip_t *gzip, const char *buf, int32_t len) {
if (gzip->state != EHTTP_GZIP_READY) return -1;
if (len <= 0) return 0;
gzip->gzip->next_in = (z_const Bytef*)buf;
gzip->gzip->avail_in = len;
while (gzip->gzip->avail_in) {
int32_t ret;
if (gzip->header) {
ret = inflate(gzip->gzip, Z_BLOCK);
} else {
ret = inflate(gzip->gzip, Z_SYNC_FLUSH);
}
if (ret != Z_OK && ret != Z_STREAM_END) return -1;
if (gzip->gzip->avail_out>0) {
if (ret!=Z_STREAM_END) continue;
}
int32_t len = gzip->gzip->next_out - (z_const Bytef*)gzip->chunk;
gzip->gzip->next_out[0] = '\0';
gzip->callbacks.on_data(gzip, gzip->arg, gzip->chunk, len);
gzip->gzip->next_out = (z_const Bytef*)gzip->chunk;
gzip->gzip->avail_out = gzip->conf.chunk_size;
}
return 0;
}
int32_t ehttp_gzip_finish(ehttp_gzip_t *gzip) {
if (gzip->state != EHTTP_GZIP_READY) return -1;
gzip->gzip->next_in = NULL;
gzip->gzip->avail_in = 0;
int32_t ret;
ret = inflate(gzip->gzip, Z_FINISH);
if (ret != Z_STREAM_END) return -1;
int32_t len = gzip->gzip->next_out - (z_const Bytef*)gzip->chunk;
gzip->gzip->next_out[0] = '\0';
gzip->callbacks.on_data(gzip, gzip->arg, gzip->chunk, len);
gzip->gzip->next_out = NULL;
gzip->gzip->avail_out = 0;
return 0;
}

View File

@ -15,382 +15,33 @@
#define _DEFAULT_SOURCE
#include "os.h"
#include "taos.h"
#include "tglobal.h"
#include "tsocket.h"
#include "ttimer.h"
#include "httpInt.h"
#include "httpResp.h"
#include "httpAuth.h"
#include "httpServer.h"
#include "httpContext.h"
#include "httpHandle.h"
void httpToLowerUrl(char* url) {
/*ignore case */
while (*url) {
if (*url >= 'A' && *url <= 'Z') {
*url = *url | 0x20;
}
url++;
}
}
bool httpUrlMatch(HttpContext* pContext, int pos, char* cmp) {
HttpParser* pParser = &pContext->parser;
if (pos < 0 || pos >= HTTP_MAX_URL) {
return false;
}
if (pParser->path[pos].len <= 0) {
return false;
}
if (strcmp(pParser->path[pos].pos, cmp) != 0) {
return false;
}
return true;
}
// /account/db/meter HTTP/1.1\r\nHost
bool httpParseURL(HttpContext* pContext) {
HttpParser* pParser = &pContext->parser;
char* pSeek;
char* pEnd = strchr(pParser->pLast, ' ');
if (pEnd == NULL) {
httpSendErrorResp(pContext, HTTP_UNSUPPORT_URL);
return false;
}
if (*pParser->pLast != '/') {
httpSendErrorResp(pContext, HTTP_UNSUPPORT_URL);
return false;
}
pParser->pLast++;
for (int i = 0; i < HTTP_MAX_URL; i++) {
pSeek = strchr(pParser->pLast, '/');
if (pSeek == NULL) {
break;
}
pParser->path[i].pos = pParser->pLast;
if (pSeek <= pEnd) {
pParser->path[i].len = (int16_t)(pSeek - pParser->pLast);
pParser->path[i].pos[pParser->path[i].len] = 0;
httpToLowerUrl(pParser->path[i].pos);
pParser->pLast = pSeek + 1;
} else {
pParser->path[i].len = (int16_t)(pEnd - pParser->pLast);
pParser->path[i].pos[pParser->path[i].len] = 0;
httpToLowerUrl(pParser->path[i].pos);
pParser->pLast = pEnd + 1;
break;
}
}
pParser->pLast = pEnd + 1;
if (pParser->path[0].len == 0) {
httpSendErrorResp(pContext, HTTP_UNSUPPORT_URL);
return false;
}
return true;
}
bool httpParseHttpVersion(HttpContext* pContext) {
HttpParser* pParser = &pContext->parser;
char* pEnd = strchr(pParser->pLast, '1');
if (pEnd == NULL) {
httpError("context:%p, fd:%d, ip:%s, can't find http version at position:%s", pContext, pContext->fd,
pContext->ipstr, pParser->pLast);
httpSendErrorResp(pContext, HTTP_PARSE_HTTP_VERSION_ERROR);
return false;
}
if (*(pEnd + 1) != '.') {
httpError("context:%p, fd:%d, ip:%s, can't find http version at position:%s", pContext, pContext->fd,
pContext->ipstr, pParser->pLast);
httpSendErrorResp(pContext, HTTP_PARSE_HTTP_VERSION_ERROR);
return false;
}
if (*(pEnd + 2) == '0')
pContext->httpVersion = HTTP_VERSION_10;
else if (*(pEnd + 2) == '1')
pContext->httpVersion = HTTP_VERSION_11;
else if (*(pEnd + 2) == '2')
pContext->httpVersion = HTTP_VERSION_11;
else
pContext->httpVersion = HTTP_VERSION_10;
httpDebug("context:%p, fd:%d, ip:%s, httpVersion:1.%d", pContext, pContext->fd, pContext->ipstr,
pContext->httpVersion);
return true;
}
bool httpGetNextLine(HttpContext* pContext) {
HttpParser* pParser = &pContext->parser;
while (pParser->buffer + pParser->bufsize - pParser->pCur++ > 0) {
if (*(pParser->pCur) == '\n' && *(pParser->pCur - 1) == '\r') {
// cut the string
*pParser->pCur = 0;
return true;
}
}
httpSendErrorResp(pContext, HTTP_PARSE_HEAD_ERROR);
return false;
}
bool httpGetHttpMethod(HttpContext* pContext) {
HttpParser* pParser = &pContext->parser;
char* pSeek = strchr(pParser->pLast, ' ');
if (pSeek == NULL) {
httpSendErrorResp(pContext, HTTP_PARSE_HTTP_METHOD_ERROR);
return false;
}
pParser->method.pos = pParser->pLast;
pParser->method.len = (int16_t)(pSeek - pParser->pLast);
pParser->method.pos[pParser->method.len] = 0;
pParser->pLast = pSeek + 1;
httpTrace("context:%p, fd:%d, ip:%s, httpMethod:%s", pContext, pContext->fd, pContext->ipstr, pParser->method.pos);
return true;
}
bool httpGetDecodeMethod(HttpContext* pContext) {
HttpParser* pParser = &pContext->parser;
HttpServer* pServer = &tsHttpServer;
int methodLen = pServer->methodScannerLen;
for (int i = 0; i < methodLen; i++) {
HttpDecodeMethod* method = pServer->methodScanner[i];
if (strcmp(method->module, pParser->path[0].pos) != 0) {
continue;
}
pParser->pMethod = method;
return true;
}
httpError("context:%p, fd:%d, ip:%s, error:the url is not support, method:%s, path:%s",
pContext, pContext->fd, pContext->ipstr, pParser->method.pos, pParser->path[0].pos);
httpSendErrorResp(pContext, HTTP_UNSUPPORT_URL);
return false;
}
bool httpParseHead(HttpContext* pContext) {
HttpParser* pParser = &pContext->parser;
if (strncasecmp(pParser->pLast, "Content-Length: ", 16) == 0) {
pParser->data.len = (int32_t)atoi(pParser->pLast + 16);
httpTrace("context:%p, fd:%d, ip:%s, Content-Length:%d", pContext, pContext->fd, pContext->ipstr,
pParser->data.len);
} else if (strncasecmp(pParser->pLast, "Accept-Encoding: ", 17) == 0) {
if (tsHttpEnableCompress && strstr(pParser->pLast + 17, "gzip") != NULL) {
pContext->acceptEncoding = HTTP_COMPRESS_GZIP;
httpTrace("context:%p, fd:%d, ip:%s, Accept-Encoding:gzip", pContext, pContext->fd, pContext->ipstr);
} else {
pContext->acceptEncoding = HTTP_COMPRESS_IDENTITY;
httpTrace("context:%p, fd:%d, ip:%s, Accept-Encoding:identity", pContext, pContext->fd, pContext->ipstr);
}
} else if (strncasecmp(pParser->pLast, "Content-Encoding: ", 18) == 0) {
if (strstr(pParser->pLast + 18, "gzip") != NULL) {
pContext->contentEncoding = HTTP_COMPRESS_GZIP;
httpTrace("context:%p, fd:%d, ip:%s, Content-Encoding:gzip", pContext, pContext->fd, pContext->ipstr);
} else {
pContext->contentEncoding = HTTP_COMPRESS_IDENTITY;
httpTrace("context:%p, fd:%d, ip:%s, Content-Encoding:identity", pContext, pContext->fd, pContext->ipstr);
}
} else if (strncasecmp(pParser->pLast, "Connection: ", 12) == 0) {
if (strncasecmp(pParser->pLast + 12, "Keep-Alive", 10) == 0) {
pContext->httpKeepAlive = HTTP_KEEPALIVE_ENABLE;
} else {
pContext->httpKeepAlive = HTTP_KEEPALIVE_DISABLE;
}
httpTrace("context:%p, fd:%d, ip:%s, keepAlive:%d", pContext, pContext->fd, pContext->ipstr,
pContext->httpKeepAlive);
} else if (strncasecmp(pParser->pLast, "Transfer-Encoding: ", 19) == 0) {
if (strncasecmp(pParser->pLast + 19, "chunked", 7) == 0) {
pContext->httpChunked = HTTP_CHUNKED;
}
} else if (strncasecmp(pParser->pLast, "Authorization: ", 15) == 0) {
if (strncasecmp(pParser->pLast + 15, "Basic ", 6) == 0) {
pParser->token.pos = pParser->pLast + 21;
pParser->token.len = (int16_t)(pParser->pCur - pParser->token.pos - 1);
bool parsed = httpParseBasicAuthToken(pContext, pParser->token.pos, pParser->token.len);
if (!parsed) {
httpSendErrorResp(pContext, HTTP_INVALID_BASIC_AUTH_TOKEN);
return false;
}
} else if (strncasecmp(pParser->pLast + 15, "Taosd ", 6) == 0) {
pParser->token.pos = pParser->pLast + 21;
pParser->token.len = (int16_t)(pParser->pCur - pParser->token.pos - 1);
bool parsed = httpParseTaosdAuthToken(pContext, pParser->token.pos, pParser->token.len);
if (!parsed) {
httpSendErrorResp(pContext, HTTP_INVALID_TAOSD_AUTH_TOKEN);
return false;
}
} else {
httpSendErrorResp(pContext, HTTP_INVALID_AUTH_TOKEN);
return false;
}
} else {
}
return true;
}
bool httpParseChunkedBody(HttpContext* pContext, HttpParser* pParser, bool test) {
char* pEnd = pParser->buffer + pParser->bufsize;
char* pRet = pParser->data.pos;
char* pSize = pParser->data.pos;
size_t size = strtoul(pSize, NULL, 16);
if (size <= 0) return false;
while (size > 0) {
char* pData = strstr(pSize, "\r\n");
if (pData == NULL || pData >= pEnd) return false;
pData += 2;
pSize = strstr(pData, "\r\n");
if (pSize == NULL || pSize >= pEnd) return false;
if ((size_t)(pSize - pData) != size) return false;
pSize += 2;
if (!test) {
memmove(pRet, pData, size);
pRet += size;
}
size = strtoul(pSize, NULL, 16);
}
if (!test) {
*pRet = '\0';
}
return true;
}
int httpReadChunkedBody(HttpContext* pContext, HttpParser* pParser) {
bool parsedOk = httpParseChunkedBody(pContext, pParser, true);
if (parsedOk) {
httpParseChunkedBody(pContext, pParser, false);
return HTTP_CHECK_BODY_SUCCESS;
} else {
httpTrace("context:%p, fd:%d, ip:%s, chunked body not finished, continue read", pContext, pContext->fd, pContext->ipstr);
if (httpReadDataImp(pContext) != HTTP_READ_DATA_SUCCESS) {
httpError("context:%p, fd:%d, ip:%s, read chunked request error", pContext, pContext->fd, pContext->ipstr);
return HTTP_CHECK_BODY_ERROR;
} else {
return HTTP_CHECK_BODY_CONTINUE;
}
}
}
int httpReadUnChunkedBody(HttpContext* pContext, HttpParser* pParser) {
int dataReadLen = pParser->bufsize - (int)(pParser->data.pos - pParser->buffer);
if (dataReadLen > pParser->data.len) {
httpError("context:%p, fd:%d, ip:%s, un-chunked body length invalid, read size:%d dataReadLen:%d > pContext->data.len:%d",
pContext, pContext->fd, pContext->ipstr, pContext->parser.bufsize, dataReadLen, pParser->data.len);
return HTTP_CHECK_BODY_ERROR;
} else if (dataReadLen < pParser->data.len) {
httpTrace("context:%p, fd:%d, ip:%s, un-chunked body not finished, read size:%d dataReadLen:%d < pContext->data.len:%d, continue read",
pContext, pContext->fd, pContext->ipstr, pContext->parser.bufsize, dataReadLen, pParser->data.len);
return HTTP_CHECK_BODY_CONTINUE;
} else {
return HTTP_CHECK_BODY_SUCCESS;
}
}
bool httpParseRequest(HttpContext* pContext) {
HttpParser *pParser = &pContext->parser;
if (pContext->parsed) {
return true;
}
httpTraceL("context:%p, fd:%d, ip:%s, thread:%s, numOfContexts:%d, read size:%d, raw data:\n%s", pContext,
pContext->fd, pContext->ipstr, pContext->pThread->label, pContext->pThread->numOfContexts,
pContext->parser.bufsize, pContext->parser.buffer);
if (!httpGetHttpMethod(pContext)) {
return false;
}
if (!httpParseURL(pContext)) {
return false;
}
if (!httpParseHttpVersion(pContext)) {
return false;
}
if (!httpGetDecodeMethod(pContext)) {
return false;
}
do {
if (!httpGetNextLine(pContext)) {
return false;
}
// Empty line, end of the HTTP HEAD
if (pParser->pCur - pParser->pLast == 1) {
pParser->data.pos = ++pParser->pCur;
break;
}
if (!httpParseHead(pContext)) {
return false;
}
pParser->pLast = ++pParser->pCur;
} while (1);
httpDebug("context:%p, fd:%d, ip:%s, parse http head ok", pContext, pContext->fd, pContext->ipstr);
pContext->parsed = true;
return true;
}
int httpCheckReadCompleted(HttpContext* pContext) {
HttpParser* pParser = &pContext->parser;
if (pContext->httpChunked == HTTP_UNCUNKED) {
return httpReadUnChunkedBody(pContext, pParser);
} else {
return httpReadChunkedBody(pContext, pParser);
}
}
bool httpDecodeRequest(HttpContext* pContext) {
HttpParser* pParser = &pContext->parser;
if (pParser->pMethod->decodeFp == NULL) {
if (pContext->decodeMethod->decodeFp == NULL) {
return false;
}
return (*pParser->pMethod->decodeFp)(pContext);
return (*pContext->decodeMethod->decodeFp)(pContext);
}
/**
* Process the request from http pServer
*/
bool httpProcessData(HttpContext* pContext) {
if (!httpAlterContextState(pContext, HTTP_CONTEXT_STATE_READY, HTTP_CONTEXT_STATE_HANDLING)) {
httpDebug("context:%p, fd:%d, ip:%s, state:%s not in ready state, stop process request",
pContext, pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state));
httpTrace("context:%p, fd:%d, state:%s not in ready state, stop process request", pContext, pContext->fd,
httpContextStateStr(pContext->state));
httpCloseContextByApp(pContext);
return false;
}
// handle Cross-domain request
if (strcmp(pContext->parser.method.pos, "OPTIONS") == 0) {
httpDebug("context:%p, fd:%d, ip:%s, process options request", pContext, pContext->fd, pContext->ipstr);
if (strcmp(pContext->parser->method, "OPTIONS") == 0) {
httpTrace("context:%p, fd:%d, process options request", pContext, pContext->fd);
httpSendOptionResp(pContext, "process options request success");
} else {
if (!httpDecodeRequest(pContext)) {

View File

@ -17,9 +17,9 @@
#include "os.h"
#include "taosmsg.h"
#include "taoserror.h"
#include "tglobal.h"
#include "http.h"
#include "httpLog.h"
#include "httpCode.h"
#include "httpJson.h"
#include "httpResp.h"
#include "httpUtil.h"
@ -38,28 +38,26 @@ char JsonNulTkn[] = "null";
char JsonTrueTkn[] = "true";
char JsonFalseTkn[] = "false";
int httpWriteBufByFd(struct HttpContext* pContext, const char* buf, int sz) {
int len;
int countWait = 0;
int writeLen = 0;
int32_t httpWriteBufByFd(struct HttpContext* pContext, const char* buf, int32_t sz) {
int32_t len;
int32_t countWait = 0;
int32_t writeLen = 0;
do {
if (pContext->fd > 2){
len = (int)taosSend(pContext->fd, buf + writeLen, (size_t)(sz - writeLen), MSG_NOSIGNAL);
len = (int32_t)taosSend(pContext->fd, buf + writeLen, (size_t)(sz - writeLen), MSG_NOSIGNAL);
}
else {
return sz;
}
if (len < 0) {
httpDebug("context:%p, fd:%d, ip:%s, socket write errno:%d, times:%d",
pContext, pContext->fd, pContext->ipstr, errno, countWait);
httpDebug("context:%p, fd:%d, socket write errno:%d, times:%d", pContext, pContext->fd, errno, countWait);
if (++countWait > HTTP_WRITE_RETRY_TIMES) break;
taosMsleep(HTTP_WRITE_WAIT_TIME_MS);
continue;
} else if (len == 0) {
httpDebug("context:%p, fd:%d, ip:%s, socket write errno:%d, connect already closed",
pContext, pContext->fd, pContext->ipstr, errno);
httpDebug("context:%p, fd:%d, socket write errno:%d, connect already closed", pContext, pContext->fd, errno);
break;
} else {
countWait = 0;
@ -70,36 +68,35 @@ int httpWriteBufByFd(struct HttpContext* pContext, const char* buf, int sz) {
return writeLen;
}
int httpWriteBuf(struct HttpContext *pContext, const char *buf, int sz) {
int writeSz = httpWriteBufByFd(pContext, buf, sz);
int32_t httpWriteBuf(struct HttpContext* pContext, const char* buf, int32_t sz) {
int32_t writeSz = httpWriteBufByFd(pContext, buf, sz);
if (writeSz != sz) {
httpError("context:%p, fd:%d, ip:%s, dataSize:%d, writeSize:%d, failed to send response:\n%s",
pContext, pContext->fd, pContext->ipstr, sz, writeSz, buf);
httpError("context:%p, fd:%d, dataSize:%d, writeSize:%d, failed to send response:\n%s", pContext, pContext->fd, sz,
writeSz, buf);
} else {
httpTrace("context:%p, fd:%d, ip:%s, dataSize:%d, writeSize:%d, response:\n%s", pContext, pContext->fd,
pContext->ipstr, sz, writeSz, buf);
httpTrace("context:%p, fd:%d, dataSize:%d, writeSize:%d, response:\n%s", pContext, pContext->fd, sz, writeSz, buf);
}
return writeSz;
}
int httpWriteBufNoTrace(struct HttpContext *pContext, const char *buf, int sz) {
int writeSz = httpWriteBufByFd(pContext, buf, sz);
int32_t httpWriteBufNoTrace(struct HttpContext *pContext, const char *buf, int32_t sz) {
int32_t writeSz = httpWriteBufByFd(pContext, buf, sz);
if (writeSz != sz) {
httpError("context:%p, fd:%d, ip:%s, dataSize:%d, writeSize:%d, failed to send response",
pContext, pContext->fd, pContext->ipstr, sz, writeSz);
httpError("context:%p, fd:%d, dataSize:%d, writeSize:%d, failed to send response", pContext, pContext->fd, sz,
writeSz);
}
return writeSz;
}
int httpWriteJsonBufBody(JsonBuf* buf, bool isTheLast) {
int remain = 0;
int32_t httpWriteJsonBufBody(JsonBuf* buf, bool isTheLast) {
int32_t remain = 0;
char sLen[24];
uint64_t srcLen = (uint64_t) (buf->lst - buf->buf);
if (buf->pContext->fd <= 0) {
httpTrace("context:%p, fd:%d, ip:%s, write json body error", buf->pContext, buf->pContext->fd, buf->pContext->ipstr);
httpTrace("context:%p, fd:%d, write json body error", buf->pContext, buf->pContext->fd);
buf->pContext->fd = -1;
}
@ -111,44 +108,44 @@ int httpWriteJsonBufBody(JsonBuf* buf, bool isTheLast) {
* The remote endpoint then decodes the stream by concatenating the chunks and uncompressing the result.
*/
if (buf->pContext->acceptEncoding == HTTP_COMPRESS_IDENTITY) {
if (buf->pContext->parser->acceptEncodingGzip == 0 || !tsHttpEnableCompress) {
if (buf->lst == buf->buf) {
httpTrace("context:%p, fd:%d, ip:%s, no data need dump", buf->pContext, buf->pContext->fd, buf->pContext->ipstr);
httpTrace("context:%p, fd:%d, no data need dump", buf->pContext, buf->pContext->fd);
return 0; // there is no data to dump.
} else {
int len = sprintf(sLen, "%lx\r\n", srcLen);
httpTrace("context:%p, fd:%d, ip:%s, write body, chunkSize:%" PRIu64 ", response:\n%s",
buf->pContext, buf->pContext->fd, buf->pContext->ipstr, srcLen, buf->buf);
int32_t len = sprintf(sLen, "%lx\r\n", srcLen);
httpTrace("context:%p, fd:%d, write body, chunkSize:%" PRIu64 ", response:\n%s", buf->pContext, buf->pContext->fd,
srcLen, buf->buf);
httpWriteBufNoTrace(buf->pContext, sLen, len);
remain = httpWriteBufNoTrace(buf->pContext, buf->buf, (int) srcLen);
remain = httpWriteBufNoTrace(buf->pContext, buf->buf, (int32_t)srcLen);
}
} else {
char compressBuf[JSON_BUFFER_SIZE] = {0};
int32_t compressBufLen = JSON_BUFFER_SIZE;
int ret = httpGzipCompress(buf->pContext, buf->buf, srcLen, compressBuf, &compressBufLen, isTheLast);
int32_t ret = httpGzipCompress(buf->pContext, buf->buf, srcLen, compressBuf, &compressBufLen, isTheLast);
if (ret == 0) {
if (compressBufLen > 0) {
int len = sprintf(sLen, "%x\r\n", compressBufLen);
httpTrace("context:%p, fd:%d, ip:%s, write body, chunkSize:%" PRIu64 ", compressSize:%d, last:%d, response:\n%s",
buf->pContext, buf->pContext->fd, buf->pContext->ipstr, srcLen, compressBufLen, isTheLast, buf->buf);
int32_t len = sprintf(sLen, "%x\r\n", compressBufLen);
httpTrace("context:%p, fd:%d, write body, chunkSize:%" PRIu64 ", compressSize:%d, last:%d, response:\n%s",
buf->pContext, buf->pContext->fd, srcLen, compressBufLen, isTheLast, buf->buf);
httpWriteBufNoTrace(buf->pContext, sLen, len);
remain = httpWriteBufNoTrace(buf->pContext, (const char *) compressBuf, (int) compressBufLen);
remain = httpWriteBufNoTrace(buf->pContext, (const char*)compressBuf, compressBufLen);
} else {
httpTrace("context:%p, fd:%d, ip:%s, last:%d, compress already dumped, response:\n%s",
buf->pContext, buf->pContext->fd, buf->pContext->ipstr, isTheLast, buf->buf);
httpTrace("context:%p, fd:%d, last:%d, compress already dumped, response:\n%s", buf->pContext,
buf->pContext->fd, isTheLast, buf->buf);
return 0; // there is no data to dump.
}
} else {
httpError("context:%p, fd:%d, ip:%s, failed to compress data, chunkSize:%" PRIu64 ", last:%d, error:%d, response:\n%s",
buf->pContext, buf->pContext->fd, buf->pContext->ipstr, srcLen, isTheLast, ret, buf->buf);
httpError("context:%p, fd:%d, failed to compress data, chunkSize:%" PRIu64 ", last:%d, error:%d, response:\n%s",
buf->pContext, buf->pContext->fd, srcLen, isTheLast, ret, buf->buf);
return 0;
}
}
httpWriteBufNoTrace(buf->pContext, "\r\n", 2);
buf->total += (int) (buf->lst - buf->buf);
buf->total += (int32_t)(buf->lst - buf->buf);
buf->lst = buf->buf;
memset(buf->buf, 0, (size_t) buf->size);
memset(buf->buf, 0, (size_t)buf->size);
return remain;
}
@ -158,14 +155,14 @@ void httpWriteJsonBufHead(JsonBuf* buf) {
}
char msg[1024] = {0};
int len = -1;
int32_t len = -1;
if (buf->pContext->acceptEncoding == HTTP_COMPRESS_IDENTITY) {
len = sprintf(msg, httpRespTemplate[HTTP_RESPONSE_CHUNKED_UN_COMPRESS], httpVersionStr[buf->pContext->httpVersion],
httpKeepAliveStr[buf->pContext->httpKeepAlive]);
if (buf->pContext->parser->acceptEncodingGzip == 0 || !tsHttpEnableCompress) {
len = sprintf(msg, httpRespTemplate[HTTP_RESPONSE_CHUNKED_UN_COMPRESS], httpVersionStr[buf->pContext->parser->httpVersion],
httpKeepAliveStr[buf->pContext->parser->keepAlive]);
} else {
len = sprintf(msg, httpRespTemplate[HTTP_RESPONSE_CHUNKED_COMPRESS], httpVersionStr[buf->pContext->httpVersion],
httpKeepAliveStr[buf->pContext->httpKeepAlive]);
len = sprintf(msg, httpRespTemplate[HTTP_RESPONSE_CHUNKED_COMPRESS], httpVersionStr[buf->pContext->parser->httpVersion],
httpKeepAliveStr[buf->pContext->parser->keepAlive]);
}
httpWriteBuf(buf->pContext, (const char*)msg, len);
@ -173,7 +170,7 @@ void httpWriteJsonBufHead(JsonBuf* buf) {
void httpWriteJsonBufEnd(JsonBuf* buf) {
if (buf->pContext->fd <= 0) {
httpTrace("context:%p, fd:%d, ip:%s, json buf fd is 0", buf->pContext, buf->pContext->fd, buf->pContext->ipstr);
httpTrace("context:%p, fd:%d, json buf fd is 0", buf->pContext, buf->pContext->fd);
buf->pContext->fd = -1;
}
@ -188,11 +185,11 @@ void httpInitJsonBuf(JsonBuf* buf, struct HttpContext* pContext) {
buf->pContext = pContext;
memset(buf->lst, 0, JSON_BUFFER_SIZE);
if (pContext->acceptEncoding == HTTP_COMPRESS_GZIP) {
if (pContext->parser->acceptEncodingGzip == 1 && tsHttpEnableCompress) {
httpGzipCompressInit(buf->pContext);
}
httpDebug("context:%p, fd:%d, ip:%s, json buffer initialized", buf->pContext, buf->pContext->fd, buf->pContext->ipstr);
httpTrace("context:%p, fd:%d, json buffer initialized", buf->pContext, buf->pContext->fd);
}
void httpJsonItemToken(JsonBuf* buf) {
@ -203,19 +200,19 @@ void httpJsonItemToken(JsonBuf* buf) {
if (buf->lst > buf->buf) httpJsonToken(buf, JsonItmTkn);
}
void httpJsonString(JsonBuf* buf, char* sVal, int len) {
void httpJsonString(JsonBuf* buf, char* sVal, int32_t len) {
httpJsonItemToken(buf);
httpJsonToken(buf, JsonStrStt);
httpJsonPrint(buf, sVal, len);
httpJsonToken(buf, JsonStrEnd);
}
void httpJsonOriginString(JsonBuf* buf, char* sVal, int len) {
void httpJsonOriginString(JsonBuf* buf, char* sVal, int32_t len) {
httpJsonItemToken(buf);
httpJsonPrint(buf, sVal, len);
}
void httpJsonStringForTransMean(JsonBuf* buf, char* sVal, int maxLen) {
void httpJsonStringForTransMean(JsonBuf* buf, char* sVal, int32_t maxLen) {
httpJsonItemToken(buf);
httpJsonToken(buf, JsonStrStt);
@ -224,18 +221,18 @@ void httpJsonStringForTransMean(JsonBuf* buf, char* sVal, int maxLen) {
char* lastPos = sVal;
char* curPos = sVal;
for (int i = 0; i < maxLen; ++i) {
for (int32_t i = 0; i < maxLen; ++i) {
if (*curPos == 0) {
break;
}
if (*curPos == '\"') {
httpJsonPrint(buf, lastPos, (int)(curPos - lastPos));
httpJsonPrint(buf, lastPos, (int32_t)(curPos - lastPos));
curPos++;
lastPos = curPos;
httpJsonPrint(buf, "\\\"", 2);
} else if (*curPos == '\\') {
httpJsonPrint(buf, lastPos, (int)(curPos - lastPos));
httpJsonPrint(buf, lastPos, (int32_t)(curPos - lastPos));
curPos++;
lastPos = curPos;
httpJsonPrint(buf, "\\\\", 2);
@ -245,7 +242,7 @@ void httpJsonStringForTransMean(JsonBuf* buf, char* sVal, int maxLen) {
}
if (*lastPos) {
httpJsonPrint(buf, lastPos, (int)(curPos - lastPos));
httpJsonPrint(buf, lastPos, (int32_t)(curPos - lastPos));
}
}
@ -261,14 +258,14 @@ void httpJsonInt64(JsonBuf* buf, int64_t num) {
void httpJsonTimestamp(JsonBuf* buf, int64_t t, bool us) {
char ts[35] = {0};
struct tm *ptm;
int precision = 1000;
int32_t precision = 1000;
if (us) {
precision = 1000000;
}
time_t tt = t / precision;
ptm = localtime(&tt);
int length = (int) strftime(ts, 35, "%Y-%m-%d %H:%M:%S", ptm);
int32_t length = (int32_t) strftime(ts, 35, "%Y-%m-%d %H:%M:%S", ptm);
if (us) {
length += snprintf(ts + length, 8, ".%06ld", t % precision);
} else {
@ -281,25 +278,25 @@ void httpJsonTimestamp(JsonBuf* buf, int64_t t, bool us) {
void httpJsonUtcTimestamp(JsonBuf* buf, int64_t t, bool us) {
char ts[40] = {0};
struct tm *ptm;
int precision = 1000;
int32_t precision = 1000;
if (us) {
precision = 1000000;
}
time_t tt = t / precision;
ptm = localtime(&tt);
int length = (int) strftime(ts, 40, "%Y-%m-%dT%H:%M:%S", ptm);
int32_t length = (int32_t)strftime(ts, 40, "%Y-%m-%dT%H:%M:%S", ptm);
if (us) {
length += snprintf(ts + length, 8, ".%06ld", t % precision);
} else {
length += snprintf(ts + length, 5, ".%03ld", t % precision);
}
length += (int) strftime(ts + length, 40 - length, "%z", ptm);
length += (int32_t)strftime(ts + length, 40 - length, "%z", ptm);
httpJsonString(buf, ts, length);
}
void httpJsonInt(JsonBuf* buf, int num) {
void httpJsonInt(JsonBuf* buf, int32_t num) {
httpJsonItemToken(buf);
httpJsonTestBuf(buf, MAX_NUM_STR_SZ);
buf->lst += snprintf(buf->lst, MAX_NUM_STR_SZ, "%d", num);
@ -331,65 +328,65 @@ void httpJsonDouble(JsonBuf* buf, double num) {
void httpJsonNull(JsonBuf* buf) { httpJsonString(buf, "null", 4); }
void httpJsonBool(JsonBuf* buf, int val) {
void httpJsonBool(JsonBuf* buf, int32_t val) {
if (val == 0)
httpJsonPrint(buf, JsonFalseTkn, sizeof(JsonFalseTkn));
else
httpJsonPrint(buf, JsonTrueTkn, sizeof(JsonTrueTkn));
}
void httpJsonPairHead(JsonBuf* buf, char* name, int len) {
void httpJsonPairHead(JsonBuf* buf, char* name, int32_t len) {
httpJsonItemToken(buf);
httpJsonString(buf, name, len);
httpJsonToken(buf, JsonPairTkn);
}
void httpJsonPair(JsonBuf* buf, char* name, int nameLen, char* sVal, int valLen) {
void httpJsonPair(JsonBuf* buf, char* name, int32_t nameLen, char* sVal, int32_t valLen) {
httpJsonPairHead(buf, name, nameLen);
httpJsonString(buf, sVal, valLen);
}
void httpJsonPairOriginString(JsonBuf* buf, char* name, int nameLen, char* sVal, int valLen) {
void httpJsonPairOriginString(JsonBuf* buf, char* name, int32_t nameLen, char* sVal, int32_t valLen) {
httpJsonPairHead(buf, name, nameLen);
httpJsonOriginString(buf, sVal, valLen);
}
void httpJsonPairIntVal(JsonBuf* buf, char* name, int nNameLen, int num) {
void httpJsonPairIntVal(JsonBuf* buf, char* name, int32_t nNameLen, int32_t num) {
httpJsonPairHead(buf, name, nNameLen);
httpJsonInt(buf, num);
}
void httpJsonPairInt64Val(JsonBuf* buf, char* name, int nNameLen, int64_t num) {
void httpJsonPairInt64Val(JsonBuf* buf, char* name, int32_t nNameLen, int64_t num) {
httpJsonPairHead(buf, name, nNameLen);
httpJsonInt64(buf, num);
}
void httpJsonPairBoolVal(JsonBuf* buf, char* name, int nNameLen, int num) {
void httpJsonPairBoolVal(JsonBuf* buf, char* name, int32_t nNameLen, int32_t num) {
httpJsonPairHead(buf, name, nNameLen);
httpJsonBool(buf, num);
}
void httpJsonPairFloatVal(JsonBuf* buf, char* name, int nNameLen, float num) {
void httpJsonPairFloatVal(JsonBuf* buf, char* name, int32_t nNameLen, float num) {
httpJsonPairHead(buf, name, nNameLen);
httpJsonFloat(buf, num);
}
void httpJsonPairDoubleVal(JsonBuf* buf, char* name, int nNameLen, double num) {
void httpJsonPairDoubleVal(JsonBuf* buf, char* name, int32_t nNameLen, double num) {
httpJsonPairHead(buf, name, nNameLen);
httpJsonDouble(buf, num);
}
void httpJsonPairNullVal(JsonBuf* buf, char* name, int nNameLen) {
void httpJsonPairNullVal(JsonBuf* buf, char* name, int32_t nNameLen) {
httpJsonPairHead(buf, name, nNameLen);
httpJsonNull(buf);
}
void httpJsonPairArray(JsonBuf* buf, char* name, int len, httpJsonBuilder fnBuilder, void* dsHandle) {
void httpJsonPairArray(JsonBuf* buf, char* name, int32_t len, httpJsonBuilder fnBuilder, void* dsHandle) {
httpJsonPairHead(buf, name, len);
httpJsonArray(buf, fnBuilder, dsHandle);
}
void httpJsonPairObject(JsonBuf* buf, char* name, int len, httpJsonBuilder fnBuilder, void* dsHandle) {
void httpJsonPairObject(JsonBuf* buf, char* name, int32_t len, httpJsonBuilder fnBuilder, void* dsHandle) {
httpJsonPairHead(buf, name, len);
httpJsonObject(buf, fnBuilder, dsHandle);
}
@ -408,7 +405,7 @@ void httpJsonArray(JsonBuf* buf, httpJsonBuilder fnBuilder, void* jsonHandle) {
httpJsonToken(buf, JsonArrEnd);
}
void httpJsonTestBuf(JsonBuf* buf, int safety) {
void httpJsonTestBuf(JsonBuf* buf, int32_t safety) {
if ((buf->lst - buf->buf + safety) < buf->size) return;
// buf->slot = *buf->lst;
httpWriteJsonBufBody(buf, false);
@ -419,7 +416,7 @@ void httpJsonToken(JsonBuf* buf, char c) {
*buf->lst++ = c;
}
void httpJsonPrint(JsonBuf* buf, const char* json, int len) {
void httpJsonPrint(JsonBuf* buf, const char* json, int32_t len) {
if (len == 0 || len >= JSON_BUFFER_SIZE) {
return;
}
@ -435,7 +432,7 @@ void httpJsonPrint(JsonBuf* buf, const char* json, int len) {
buf->lst += len;
}
void httpJsonPairStatus(JsonBuf* buf, int code) {
void httpJsonPairStatus(JsonBuf* buf, int32_t code) {
if (code == 0) {
httpJsonPair(buf, "status", 6, "succ", 4);
} else {
@ -448,7 +445,7 @@ void httpJsonPairStatus(JsonBuf* buf, int code) {
} else if (code == TSDB_CODE_MND_INVALID_TABLE_NAME) {
httpJsonPair(buf, "desc", 4, "failed to create table", 22);
} else {
httpJsonPair(buf, "desc", 4, (char*)tstrerror(code), (int)strlen(tstrerror(code)));
httpJsonPair(buf, "desc", 4, (char*)tstrerror(code), (int32_t)strlen(tstrerror(code)));
}
}
}

File diff suppressed because it is too large Load Diff

View File

@ -39,15 +39,15 @@ typedef struct {
typedef struct {
void *param;
void *result;
int numOfRows;
void (*fp)(void *param, void *result, int numOfRows);
int32_t numOfRows;
void (*fp)(void *param, void *result, int32_t numOfRows);
} SHttpResult;
static SHttpWorkerPool tsHttpPool;
static taos_qset tsHttpQset;
static taos_queue tsHttpQueue;
void httpDispatchToResultQueue(void *param, TAOS_RES *result, int numOfRows, void (*fp)(void *param, void *result, int numOfRows)) {
void httpDispatchToResultQueue(void *param, TAOS_RES *result, int32_t numOfRows, void (*fp)(void *param, void *result, int32_t numOfRows)) {
if (tsHttpQueue != NULL) {
SHttpResult *pMsg = (SHttpResult *)taosAllocateQitem(sizeof(SHttpResult));
pMsg->param = param;
@ -71,7 +71,7 @@ static void *httpProcessResultQueue(void *param) {
break;
}
httpDebug("context:%p, res:%p will be processed in result queue", pMsg->param, pMsg->result);
httpTrace("context:%p, res:%p will be processed in result queue", pMsg->param, pMsg->result);
(*pMsg->fp)(pMsg->param, pMsg->result, pMsg->numOfRows);
taosFreeQitem(pMsg);
}

View File

@ -19,7 +19,6 @@
#include "taosmsg.h"
#include "httpLog.h"
#include "httpResp.h"
#include "httpCode.h"
#include "httpJson.h"
#include "httpContext.h"
@ -45,155 +44,112 @@ const char *httpRespTemplate[] = {
"%s 200 OK\r\nAccess-Control-Allow-Origin:*\r\n%sAccess-Control-Allow-Methods:POST, GET, OPTIONS, DELETE, PUT\r\nAccess-Control-Allow-Headers:Accept, Content-Type\r\nContent-Type: application/json;charset=utf-8\r\nContent-Length: %d\r\n\r\n"
};
static void httpSendErrorRespImp(HttpContext *pContext, int httpCode, char *httpCodeStr, int errNo, char *desc) {
httpError("context:%p, fd:%d, ip:%s, code:%d, error:%s", pContext, pContext->fd, pContext->ipstr, httpCode, desc);
static void httpSendErrorRespImp(HttpContext *pContext, int32_t httpCode, char *httpCodeStr, int32_t errNo, const char *desc) {
httpError("context:%p, fd:%d, code:%d, error:%s", pContext, pContext->fd, httpCode, desc);
char head[512] = {0};
char body[512] = {0};
int bodyLen = sprintf(body, httpRespTemplate[HTTP_RESPONSE_JSON_ERROR], errNo, desc);
int headLen = sprintf(head, httpRespTemplate[HTTP_RESPONSE_ERROR], httpVersionStr[pContext->httpVersion], httpCode,
httpCodeStr, httpKeepAliveStr[pContext->httpKeepAlive], bodyLen);
int32_t bodyLen = sprintf(body, httpRespTemplate[HTTP_RESPONSE_JSON_ERROR], errNo, desc);
int32_t headLen = sprintf(head, httpRespTemplate[HTTP_RESPONSE_ERROR], httpVersionStr[pContext->parser->httpVersion],
httpCode, httpCodeStr, httpKeepAliveStr[pContext->parser->keepAlive], bodyLen);
httpWriteBuf(pContext, head, headLen);
httpWriteBuf(pContext, body, bodyLen);
httpCloseContextByApp(pContext);
}
void httpSendErrorRespWithDesc(HttpContext *pContext, int errNo, char *desc) {
int httpCode = 500;
char *httpCodeStr = "Internal Server Error";
switch (errNo) {
case HTTP_SUCCESS:
httpCode = 200;
httpCodeStr = "OK";
break;
case HTTP_SERVER_OFFLINE:
case HTTP_UNSUPPORT_URL:
httpCode = 404;
httpCodeStr = "Not Found";
break;
case HTTP_PARSE_HTTP_METHOD_ERROR:
httpCode = 405;
httpCodeStr = "Method Not Allowed";
break;
case HTTP_PARSE_HTTP_VERSION_ERROR:
httpCode = 505;
httpCodeStr = "HTTP Version Not Supported";
break;
case HTTP_PARSE_HEAD_ERROR:
httpCode = 406;
httpCodeStr = "Not Acceptable";
break;
case HTTP_REQUSET_TOO_BIG:
httpCode = 413;
httpCodeStr = "Request Entity Too Large";
break;
case HTTP_PARSE_BODY_ERROR:
case HTTP_PARSE_CHUNKED_BODY_ERROR:
httpCode = 409;
httpCodeStr = "Conflict";
break;
case HTTP_PARSE_URL_ERROR:
httpCode = 414;
httpCodeStr = "Request-URI Invalid";
break;
case HTTP_INVALID_AUTH_TOKEN:
case HTTP_PARSE_USR_ERROR:
httpCode = 401;
httpCodeStr = "Unauthorized";
break;
case HTTP_NO_SQL_INPUT:
httpCode = 400;
httpCodeStr = "Bad Request";
break;
case HTTP_SESSION_FULL:
httpCode = 421;
httpCodeStr = "Too many connections";
break;
case HTTP_NO_ENOUGH_MEMORY:
case HTTP_GEN_TAOSD_TOKEN_ERR:
httpCode = 507;
httpCodeStr = "Insufficient Storage";
break;
case HTTP_INVALID_DB_TABLE:
case HTTP_NO_EXEC_USEDB:
case HTTP_PARSE_GC_REQ_ERROR:
case HTTP_INVALID_MULTI_REQUEST:
case HTTP_NO_MSG_INPUT:
httpCode = 400;
httpCodeStr = "Bad Request";
break;
case HTTP_NO_ENOUGH_SESSIONS:
httpCode = 421;
httpCodeStr = "Too many connections";
break;
// telegraf
case HTTP_TG_DB_NOT_INPUT:
case HTTP_TG_DB_TOO_LONG:
case HTTP_TG_INVALID_JSON:
case HTTP_TG_METRICS_NULL:
case HTTP_TG_METRICS_SIZE:
case HTTP_TG_METRIC_NULL:
case HTTP_TG_METRIC_TYPE:
case HTTP_TG_METRIC_NAME_NULL:
case HTTP_TG_METRIC_NAME_LONG:
case HTTP_TG_TIMESTAMP_NULL:
case HTTP_TG_TIMESTAMP_TYPE:
case HTTP_TG_TIMESTAMP_VAL_NULL:
case HTTP_TG_TAGS_NULL:
case HTTP_TG_TAGS_SIZE_0:
case HTTP_TG_TAGS_SIZE_LONG:
case HTTP_TG_TAG_NULL:
case HTTP_TG_TAG_NAME_NULL:
case HTTP_TG_TAG_NAME_SIZE:
case HTTP_TG_TAG_VALUE_TYPE:
case HTTP_TG_TAG_VALUE_NULL:
case HTTP_TG_TABLE_NULL:
case HTTP_TG_TABLE_SIZE:
case HTTP_TG_FIELDS_NULL:
case HTTP_TG_FIELDS_SIZE_0:
case HTTP_TG_FIELDS_SIZE_LONG:
case HTTP_TG_FIELD_NULL:
case HTTP_TG_FIELD_NAME_NULL:
case HTTP_TG_FIELD_NAME_SIZE:
case HTTP_TG_FIELD_VALUE_TYPE:
case HTTP_TG_FIELD_VALUE_NULL:
case HTTP_INVALID_BASIC_AUTH_TOKEN:
case HTTP_INVALID_TAOSD_AUTH_TOKEN:
case HTTP_TG_HOST_NOT_STRING:
// grafana
case HTTP_GC_QUERY_NULL:
case HTTP_GC_QUERY_SIZE:
httpCode = 400;
httpCodeStr = "Bad Request";
break;
default:
httpError("context:%p, fd:%d, ip:%s, error:%d not recognized", pContext, pContext->fd, pContext->ipstr, errNo);
break;
void httpSendErrorResp(HttpContext *pContext, int32_t errNo) {
int32_t httpCode = 500;
if (errNo == TSDB_CODE_SUCCESS)
httpCode = 200;
else if (errNo == TSDB_CODE_HTTP_SERVER_OFFLINE)
httpCode = 404;
else if (errNo == TSDB_CODE_HTTP_UNSUPPORT_URL)
httpCode = 404;
else if (errNo == TSDB_CODE_HTTP_INVLALID_URL)
httpCode = 404;
else if (errNo == TSDB_CODE_HTTP_NO_ENOUGH_MEMORY)
httpCode = 507;
else if (errNo == TSDB_CODE_HTTP_REQUSET_TOO_BIG)
httpCode = 413;
else if (errNo == TSDB_CODE_HTTP_NO_AUTH_INFO)
httpCode = 401;
else if (errNo == TSDB_CODE_HTTP_NO_MSG_INPUT)
httpCode = 400;
else if (errNo == TSDB_CODE_HTTP_NO_SQL_INPUT)
httpCode = 400;
else if (errNo == TSDB_CODE_HTTP_NO_EXEC_USEDB)
httpCode = 400;
else if (errNo == TSDB_CODE_HTTP_SESSION_FULL)
httpCode = 421;
else if (errNo == TSDB_CODE_HTTP_GEN_TAOSD_TOKEN_ERR)
httpCode = 507;
else if (errNo == TSDB_CODE_HTTP_INVALID_MULTI_REQUEST)
httpCode = 400;
else if (errNo == TSDB_CODE_HTTP_CREATE_GZIP_FAILED)
httpCode = 507;
else if (errNo == TSDB_CODE_HTTP_FINISH_GZIP_FAILED)
httpCode = 507;
else if (errNo == TSDB_CODE_HTTP_INVALID_VERSION)
httpCode = 406;
else if (errNo == TSDB_CODE_HTTP_INVALID_CONTENT_LENGTH)
httpCode = 406;
else if (errNo == TSDB_CODE_HTTP_INVALID_AUTH_TYPE)
httpCode = 406;
else if (errNo == TSDB_CODE_HTTP_INVALID_AUTH_FORMAT)
httpCode = 406;
else if (errNo == TSDB_CODE_HTTP_INVALID_BASIC_AUTH)
httpCode = 406;
else if (errNo == TSDB_CODE_HTTP_INVALID_TAOSD_AUTH)
httpCode = 406;
else if (errNo == TSDB_CODE_HTTP_PARSE_METHOD_FAILED)
httpCode = 406;
else if (errNo == TSDB_CODE_HTTP_PARSE_TARGET_FAILED)
httpCode = 406;
else if (errNo == TSDB_CODE_HTTP_PARSE_VERSION_FAILED)
httpCode = 406;
else if (errNo == TSDB_CODE_HTTP_PARSE_SP_FAILED)
httpCode = 406;
else if (errNo == TSDB_CODE_HTTP_PARSE_STATUS_FAILED)
httpCode = 406;
else if (errNo == TSDB_CODE_HTTP_PARSE_PHRASE_FAILED)
httpCode = 406;
else if (errNo == TSDB_CODE_HTTP_PARSE_CRLF_FAILED)
httpCode = 406;
else if (errNo == TSDB_CODE_HTTP_PARSE_HEADER_FAILED)
httpCode = 406;
else if (errNo == TSDB_CODE_HTTP_PARSE_HEADER_KEY_FAILED)
httpCode = 406;
else if (errNo == TSDB_CODE_HTTP_PARSE_HEADER_VAL_FAILED)
httpCode = 406;
else if (errNo == TSDB_CODE_HTTP_PARSE_CHUNK_SIZE_FAILED)
httpCode = 406;
else if (errNo == TSDB_CODE_HTTP_PARSE_CHUNK_FAILED)
httpCode = 406;
else if (errNo == TSDB_CODE_HTTP_PARSE_END_FAILED)
httpCode = 406;
else if (errNo == TSDB_CODE_HTTP_PARSE_INVALID_STATE)
httpCode = 406;
else if (errNo == TSDB_CODE_HTTP_PARSE_ERROR_STATE)
httpCode = 406;
else
httpCode = 400;
if (pContext->parser->httpCode != 0) {
httpCode = pContext->parser->httpCode;
}
if (desc == NULL) {
httpSendErrorRespImp(pContext, httpCode, httpCodeStr, errNo + 5000, httpMsg[errNo]);
} else {
httpSendErrorRespImp(pContext, httpCode, httpCodeStr, errNo + 5000, desc);
}
char *httpCodeStr = httpGetStatusDesc(httpCode);
httpSendErrorRespImp(pContext, httpCode, httpCodeStr, errNo & 0XFFFF, tstrerror(errNo));
}
void httpSendErrorResp(HttpContext *pContext, int errNo) { httpSendErrorRespWithDesc(pContext, errNo, NULL); }
void httpSendTaosdInvalidSqlErrorResp(HttpContext *pContext, char *errMsg) {
int32_t httpCode = 400;
char temp[512] = {0};
int32_t len = sprintf(temp, "invalid SQL: %s", errMsg);
void httpSendTaosdErrorResp(HttpContext *pContext, int errCode) {
int httpCode = 400;
httpSendErrorRespImp(pContext, httpCode, "Bad Request", errCode & 0XFFFF, (char*)tstrerror(errCode));
}
void httpSendTaosdInvalidSqlErrorResp(HttpContext *pContext, char* errMsg) {
int httpCode = 400;
char temp[512] = {0};
int len = sprintf(temp, "invalid SQL: %s", errMsg);
for (int i = 0; i < len; ++i) {
for (int32_t i = 0; i < len; ++i) {
if (temp[i] == '\"') {
temp[i] = '\'';
} else if (temp[i] == '\n') {
@ -208,9 +164,9 @@ void httpSendSuccResp(HttpContext *pContext, char *desc) {
char head[1024] = {0};
char body[1024] = {0};
int bodyLen = sprintf(body, httpRespTemplate[HTTP_RESPONSE_JSON_OK], HTTP_SUCCESS, desc);
int headLen = sprintf(head, httpRespTemplate[HTTP_RESPONSE_OK], httpVersionStr[pContext->httpVersion],
httpKeepAliveStr[pContext->httpKeepAlive], bodyLen);
int32_t bodyLen = sprintf(body, httpRespTemplate[HTTP_RESPONSE_JSON_OK], TSDB_CODE_SUCCESS, desc);
int32_t headLen = sprintf(head, httpRespTemplate[HTTP_RESPONSE_OK], httpVersionStr[pContext->parser->httpVersion],
httpKeepAliveStr[pContext->parser->keepAlive], bodyLen);
httpWriteBuf(pContext, head, headLen);
httpWriteBuf(pContext, body, bodyLen);
@ -221,9 +177,9 @@ void httpSendOptionResp(HttpContext *pContext, char *desc) {
char head[1024] = {0};
char body[1024] = {0};
int bodyLen = sprintf(body, httpRespTemplate[HTTP_RESPONSE_JSON_OK], HTTP_SUCCESS, desc);
int headLen = sprintf(head, httpRespTemplate[HTTP_RESPONSE_OPTIONS], httpVersionStr[pContext->httpVersion],
httpKeepAliveStr[pContext->httpKeepAlive], bodyLen);
int32_t bodyLen = sprintf(body, httpRespTemplate[HTTP_RESPONSE_JSON_OK], TSDB_CODE_SUCCESS, desc);
int32_t headLen = sprintf(head, httpRespTemplate[HTTP_RESPONSE_OPTIONS], httpVersionStr[pContext->parser->httpVersion],
httpKeepAliveStr[pContext->parser->keepAlive], bodyLen);
httpWriteBuf(pContext, head, headLen);
httpWriteBuf(pContext, body, bodyLen);

View File

@ -15,9 +15,10 @@
#define _DEFAULT_SOURCE
#include "os.h"
#include "taoserror.h"
#include "httpLog.h"
#include "restHandle.h"
#include "restJson.h"
#include "httpRestHandle.h"
#include "httpRestJson.h"
static HttpDecodeMethod restDecodeMethod = {"rest", restProcessRequest};
static HttpDecodeMethod restDecodeMethod2 = {"restful", restProcessRequest};
@ -60,39 +61,37 @@ void restInitHandle(HttpServer* pServer) {
}
bool restGetUserFromUrl(HttpContext* pContext) {
HttpParser* pParser = &pContext->parser;
if (pParser->path[REST_USER_URL_POS].len >= TSDB_USER_LEN || pParser->path[REST_USER_URL_POS].len <= 0) {
HttpParser* pParser = pContext->parser;
if (pParser->path[REST_USER_URL_POS].pos >= TSDB_USER_LEN || pParser->path[REST_USER_URL_POS].pos <= 0) {
return false;
}
tstrncpy(pContext->user, pParser->path[REST_USER_URL_POS].pos, TSDB_USER_LEN);
tstrncpy(pContext->user, pParser->path[REST_USER_URL_POS].str, TSDB_USER_LEN);
return true;
}
bool restGetPassFromUrl(HttpContext* pContext) {
HttpParser* pParser = &pContext->parser;
if (pParser->path[REST_PASS_URL_POS].len >= TSDB_PASSWORD_LEN || pParser->path[REST_PASS_URL_POS].len <= 0) {
HttpParser* pParser = pContext->parser;
if (pParser->path[REST_PASS_URL_POS].pos >= TSDB_PASSWORD_LEN || pParser->path[REST_PASS_URL_POS].pos <= 0) {
return false;
}
tstrncpy(pContext->pass, pParser->path[REST_PASS_URL_POS].pos, TSDB_PASSWORD_LEN);
tstrncpy(pContext->pass, pParser->path[REST_PASS_URL_POS].str, TSDB_PASSWORD_LEN);
return true;
}
bool restProcessLoginRequest(HttpContext* pContext) {
httpDebug("context:%p, fd:%d, ip:%s, user:%s, process restful login msg", pContext, pContext->fd, pContext->ipstr,
pContext->user);
httpDebug("context:%p, fd:%d, user:%s, process restful login msg", pContext, pContext->fd, pContext->user);
pContext->reqType = HTTP_REQTYPE_LOGIN;
return true;
}
bool restProcessSqlRequest(HttpContext* pContext, int timestampFmt) {
httpDebug("context:%p, fd:%d, ip:%s, user:%s, process restful sql msg", pContext, pContext->fd, pContext->ipstr,
pContext->user);
bool restProcessSqlRequest(HttpContext* pContext, int32_t timestampFmt) {
httpDebug("context:%p, fd:%d, user:%s, process restful sql msg", pContext, pContext->fd, pContext->user);
char* sql = pContext->parser.data.pos;
char* sql = pContext->parser->body.str;
if (sql == NULL) {
httpSendErrorResp(pContext, HTTP_NO_SQL_INPUT);
httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_SQL_INPUT);
return false;
}
@ -101,7 +100,7 @@ bool restProcessSqlRequest(HttpContext* pContext, int timestampFmt) {
* for async test
*
if (httpCheckUsedbSql(sql)) {
httpSendErrorResp(pContext, HTTP_NO_EXEC_USEDB);
httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_EXEC_USEDB);
return false;
}
*/
@ -128,7 +127,7 @@ bool restProcessRequest(struct HttpContext* pContext) {
}
if (strlen(pContext->user) == 0 || strlen(pContext->pass) == 0) {
httpSendErrorResp(pContext, HTTP_PARSE_USR_ERROR);
httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_AUTH_INFO);
return false;
}
@ -143,6 +142,6 @@ bool restProcessRequest(struct HttpContext* pContext) {
} else {
}
httpSendErrorResp(pContext, HTTP_PARSE_URL_ERROR);
httpSendErrorResp(pContext, TSDB_CODE_HTTP_INVLALID_URL);
return false;
}

View File

@ -18,10 +18,10 @@
#include "tglobal.h"
#include "httpLog.h"
#include "httpJson.h"
#include "restHandle.h"
#include "restJson.h"
#include "httpRestHandle.h"
#include "httpRestJson.h"
void restBuildSqlAffectRowsJson(HttpContext *pContext, HttpSqlCmd *cmd, int affect_rows) {
void restBuildSqlAffectRowsJson(HttpContext *pContext, HttpSqlCmd *cmd, int32_t affect_rows) {
JsonBuf *jsonBuf = httpMallocJsonBuf(pContext);
if (jsonBuf == NULL) return;
@ -43,7 +43,7 @@ void restStartSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result)
if (jsonBuf == NULL) return;
TAOS_FIELD *fields = taos_fetch_fields(result);
int num_fields = taos_num_fields(result);
int32_t num_fields = taos_num_fields(result);
httpInitJsonBuf(jsonBuf, pContext);
httpWriteJsonBufHead(jsonBuf);
@ -66,9 +66,9 @@ void restStartSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result)
httpJsonItemToken(jsonBuf);
httpJsonString(jsonBuf, REST_JSON_AFFECT_ROWS, REST_JSON_AFFECT_ROWS_LEN);
} else {
for (int i = 0; i < num_fields; ++i) {
for (int32_t i = 0; i < num_fields; ++i) {
httpJsonItemToken(jsonBuf);
httpJsonString(jsonBuf, fields[i].name, (int)strlen(fields[i].name));
httpJsonString(jsonBuf, fields[i].name, (int32_t)strlen(fields[i].name));
}
}
@ -83,19 +83,16 @@ void restStartSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result)
httpJsonToken(jsonBuf, JsonArrStt);
}
bool restBuildSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int numOfRows, int timestampFormat) {
bool restBuildSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows, int32_t timestampFormat) {
JsonBuf *jsonBuf = httpMallocJsonBuf(pContext);
if (jsonBuf == NULL) return false;
cmd->numOfRows += numOfRows;
int num_fields = taos_num_fields(result);
int32_t num_fields = taos_num_fields(result);
TAOS_FIELD *fields = taos_fetch_fields(result);
for (int k = 0; k < numOfRows; ++k) {
for (int32_t k = 0; k < numOfRows; ++k) {
TAOS_ROW row = taos_fetch_row(result);
if (row == NULL) {
cmd->numOfRows--;
continue;
}
int32_t* length = taos_fetch_lengths(result);
@ -104,7 +101,7 @@ bool restBuildSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result,
httpJsonItemToken(jsonBuf);
httpJsonToken(jsonBuf, JsonArrStt);
for (int i = 0; i < num_fields; i++) {
for (int32_t i = 0; i < num_fields; i++) {
httpJsonItemToken(jsonBuf);
if (row[i] == NULL) {
@ -151,37 +148,34 @@ bool restBuildSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result,
}
// data row array end
httpJsonToken(jsonBuf, JsonArrEnd);
}
httpJsonToken(jsonBuf, JsonArrEnd);
cmd->numOfRows ++;
if (cmd->numOfRows >= tsRestRowLimit) {
httpDebug("context:%p, fd:%d, ip:%s, user:%s, retrieve rows:%d larger than limit:%d, abort retrieve", pContext,
pContext->fd, pContext->ipstr, pContext->user, cmd->numOfRows, tsRestRowLimit);
return false;
}
else {
if (pContext->fd <= 0) {
httpError("context:%p, fd:%d, ip:%s, user:%s, connection is closed, abort retrieve", pContext, pContext->fd,
pContext->ipstr, pContext->user);
httpError("context:%p, fd:%d, user:%s, conn closed, abort retrieve", pContext, pContext->fd, pContext->user);
return false;
}
else {
httpDebug("context:%p, fd:%d, ip:%s, user:%s, total rows:%d retrieved", pContext, pContext->fd, pContext->ipstr,
pContext->user, cmd->numOfRows);
return true;
if (cmd->numOfRows >= tsRestRowLimit) {
httpDebug("context:%p, fd:%d, user:%s, retrieve rows:%d larger than limit:%d, abort retrieve", pContext,
pContext->fd, pContext->user, cmd->numOfRows, tsRestRowLimit);
return false;
}
}
httpDebug("context:%p, fd:%d, user:%s, retrieved row:%d", pContext, pContext->fd, pContext->user, cmd->numOfRows);
return true;
}
bool restBuildSqlTimestampJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int numOfRows) {
bool restBuildSqlTimestampJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows) {
return restBuildSqlJson(pContext,cmd, result, numOfRows, REST_TIMESTAMP_FMT_TIMESTAMP);
}
bool restBuildSqlLocalTimeStringJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int numOfRows) {
bool restBuildSqlLocalTimeStringJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows) {
return restBuildSqlJson(pContext,cmd, result, numOfRows, REST_TIMESTAMP_FMT_LOCAL_STRING);
}
bool restBuildSqlUtcTimeStringJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int numOfRows) {
bool restBuildSqlUtcTimeStringJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, int32_t numOfRows) {
return restBuildSqlJson(pContext,cmd, result, numOfRows, REST_TIMESTAMP_FMT_UTC_STRING);
}

View File

@ -29,6 +29,8 @@
#define EPOLLWAKEUP (1u << 29)
#endif
static bool httpReadData(HttpContext *pContext);
static void httpStopThread(HttpThread* pThread) {
pThread->stop = true;
@ -59,7 +61,7 @@ void httpCleanUpConnect() {
if (pServer->pThreads == NULL) return;
pthread_join(pServer->thread, NULL);
for (int i = 0; i < pServer->numOfThreads; ++i) {
for (int32_t i = 0; i < pServer->numOfThreads; ++i) {
HttpThread* pThread = pServer->pThreads + i;
if (pThread != NULL) {
httpStopThread(pThread);
@ -69,119 +71,11 @@ void httpCleanUpConnect() {
httpDebug("http server:%s is cleaned up", pServer->label);
}
int httpReadDataImp(HttpContext *pContext) {
HttpParser *pParser = &pContext->parser;
while (pParser->bufsize <= (HTTP_BUFFER_SIZE - HTTP_STEP_SIZE)) {
int nread = (int)taosReadSocket(pContext->fd, pParser->buffer + pParser->bufsize, HTTP_STEP_SIZE);
if (nread >= 0 && nread < HTTP_STEP_SIZE) {
pParser->bufsize += nread;
break;
} else if (nread < 0) {
if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) {
httpDebug("context:%p, fd:%d, ip:%s, read from socket error:%d, wait another event",
pContext, pContext->fd, pContext->ipstr, errno);
break;
} else {
httpError("context:%p, fd:%d, ip:%s, read from socket error:%d, close connect",
pContext, pContext->fd, pContext->ipstr, errno);
return HTTP_READ_DATA_FAILED;
}
} else {
pParser->bufsize += nread;
}
if (pParser->bufsize >= (HTTP_BUFFER_SIZE - HTTP_STEP_SIZE)) {
httpError("context:%p, fd:%d, ip:%s, thread:%s, request big than:%d",
pContext, pContext->fd, pContext->ipstr, pContext->pThread->label, HTTP_BUFFER_SIZE);
return HTTP_REQUSET_TOO_BIG;
}
}
pParser->buffer[pParser->bufsize] = 0;
return HTTP_READ_DATA_SUCCESS;
}
static bool httpDecompressData(HttpContext *pContext) {
if (pContext->contentEncoding != HTTP_COMPRESS_GZIP) {
httpTraceL("context:%p, fd:%d, ip:%s, content:%s", pContext, pContext->fd, pContext->ipstr, pContext->parser.data.pos);
return true;
}
char *decompressBuf = calloc(HTTP_DECOMPRESS_BUF_SIZE, 1);
int32_t decompressBufLen = HTTP_DECOMPRESS_BUF_SIZE;
size_t bufsize = sizeof(pContext->parser.buffer) - (pContext->parser.data.pos - pContext->parser.buffer) - 1;
if (decompressBufLen > (int)bufsize) {
decompressBufLen = (int)bufsize;
}
int ret = httpGzipDeCompress(pContext->parser.data.pos, pContext->parser.data.len, decompressBuf, &decompressBufLen);
if (ret == 0) {
memcpy(pContext->parser.data.pos, decompressBuf, decompressBufLen);
pContext->parser.data.pos[decompressBufLen] = 0;
httpTraceL("context:%p, fd:%d, ip:%s, rawSize:%d, decompressSize:%d, content:%s", pContext, pContext->fd,
pContext->ipstr, pContext->parser.data.len, decompressBufLen, decompressBuf);
pContext->parser.data.len = decompressBufLen;
} else {
httpError("context:%p, fd:%d, ip:%s, failed to decompress data, rawSize:%d, error:%d",
pContext, pContext->fd, pContext->ipstr, pContext->parser.data.len, ret);
}
free(decompressBuf);
return ret == 0;
}
static bool httpReadData(HttpContext *pContext) {
if (!pContext->parsed) {
httpInitContext(pContext);
}
int32_t code = httpReadDataImp(pContext);
if (code != HTTP_READ_DATA_SUCCESS) {
if (code == HTTP_READ_DATA_FAILED) {
httpReleaseContext(pContext);
} else {
httpSendErrorResp(pContext, code);
httpNotifyContextClose(pContext);
}
return false;
}
if (!httpParseRequest(pContext)) {
httpNotifyContextClose(pContext);
return false;
}
int ret = httpCheckReadCompleted(pContext);
if (ret == HTTP_CHECK_BODY_CONTINUE) {
//httpDebug("context:%p, fd:%d, ip:%s, not finished yet, wait another event", pContext, pContext->fd, pContext->ipstr);
httpReleaseContext(pContext);
return false;
} else if (ret == HTTP_CHECK_BODY_SUCCESS){
httpDebug("context:%p, fd:%d, ip:%s, thread:%s, read size:%d, dataLen:%d",
pContext, pContext->fd, pContext->ipstr, pContext->pThread->label, pContext->parser.bufsize, pContext->parser.data.len);
if (httpDecompressData(pContext)) {
return true;
} else {
httpNotifyContextClose(pContext);
httpReleaseContext(pContext);
return false;
}
} else {
httpError("context:%p, fd:%d, ip:%s, failed to read http body, close connect", pContext, pContext->fd, pContext->ipstr);
httpNotifyContextClose(pContext);
httpReleaseContext(pContext);
return false;
}
}
static void httpProcessHttpData(void *param) {
HttpServer *pServer = &tsHttpServer;
HttpThread *pThread = (HttpThread *)param;
HttpContext *pContext;
int fdNum;
int32_t fdNum;
sigset_t set;
sigemptyset(&set);
@ -198,7 +92,7 @@ static void httpProcessHttpData(void *param) {
}
if (fdNum <= 0) continue;
for (int i = 0; i < fdNum; ++i) {
for (int32_t i = 0; i < fdNum; ++i) {
pContext = httpGetContext(events[i].data.ptr);
if (pContext == NULL) {
httpError("context:%p, is already released, close connect", events[i].data.ptr);
@ -208,49 +102,51 @@ static void httpProcessHttpData(void *param) {
}
if (events[i].events & EPOLLPRI) {
httpDebug("context:%p, fd:%d, ip:%s, state:%s, EPOLLPRI events occured, accessed:%d, close connect",
pContext, pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state), pContext->accessTimes);
httpDebug("context:%p, fd:%d, state:%s, EPOLLPRI events occured, accessed:%d, close connect", pContext,
pContext->fd, httpContextStateStr(pContext->state), pContext->accessTimes);
httpCloseContextByServer(pContext);
continue;
}
if (events[i].events & EPOLLRDHUP) {
httpDebug("context:%p, fd:%d, ip:%s, state:%s, EPOLLRDHUP events occured, accessed:%d, close connect",
pContext, pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state), pContext->accessTimes);
httpDebug("context:%p, fd:%d, state:%s, EPOLLRDHUP events occured, accessed:%d, close connect", pContext,
pContext->fd, httpContextStateStr(pContext->state), pContext->accessTimes);
httpCloseContextByServer(pContext);
continue;
}
if (events[i].events & EPOLLERR) {
httpDebug("context:%p, fd:%d, ip:%s, state:%s, EPOLLERR events occured, accessed:%d, close connect",
pContext, pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state), pContext->accessTimes);
httpDebug("context:%p, fd:%d, state:%s, EPOLLERR events occured, accessed:%d, close connect", pContext,
pContext->fd, httpContextStateStr(pContext->state), pContext->accessTimes);
httpCloseContextByServer(pContext);
continue;
}
if (events[i].events & EPOLLHUP) {
httpDebug("context:%p, fd:%d, ip:%s, state:%s, EPOLLHUP events occured, accessed:%d, close connect",
pContext, pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state), pContext->accessTimes);
httpDebug("context:%p, fd:%d, state:%s, EPOLLHUP events occured, accessed:%d, close connect", pContext,
pContext->fd, httpContextStateStr(pContext->state), pContext->accessTimes);
httpCloseContextByServer(pContext);
continue;
}
if (!httpAlterContextState(pContext, HTTP_CONTEXT_STATE_READY, HTTP_CONTEXT_STATE_READY)) {
httpDebug("context:%p, fd:%d, ip:%s, state:%s, not in ready state, ignore read events",
pContext, pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state));
httpReleaseContext(pContext);
httpDebug("context:%p, fd:%d, state:%s, not in ready state, ignore read events", pContext, pContext->fd,
httpContextStateStr(pContext->state));
httpReleaseContext(pContext, true);
continue;
}
if (pServer->status != HTTP_SERVER_RUNNING) {
httpDebug("context:%p, fd:%d, ip:%s, state:%s, server is not running, accessed:%d, close connect", pContext,
pContext->fd, pContext->ipstr, httpContextStateStr(pContext->state), pContext->accessTimes);
httpSendErrorResp(pContext, HTTP_SERVER_OFFLINE);
httpDebug("context:%p, fd:%d, state:%s, server is not running, accessed:%d, close connect", pContext,
pContext->fd, httpContextStateStr(pContext->state), pContext->accessTimes);
httpSendErrorResp(pContext, TSDB_CODE_HTTP_SERVER_OFFLINE);
httpNotifyContextClose(pContext);
} else {
if (httpReadData(pContext)) {
(*(pThread->processData))(pContext);
atomic_fetch_add_32(&pServer->requestNum, 1);
} else {
httpReleaseContext(pContext, false);
}
}
}
@ -258,13 +154,13 @@ static void httpProcessHttpData(void *param) {
}
static void *httpAcceptHttpConnection(void *arg) {
int connFd = -1;
int32_t connFd = -1;
struct sockaddr_in clientAddr;
int threadId = 0;
int32_t threadId = 0;
HttpServer * pServer = &tsHttpServer;
HttpThread * pThread = NULL;
HttpContext * pContext = NULL;
int totalFds = 0;
int32_t totalFds = 0;
sigset_t set;
sigemptyset(&set);
@ -284,7 +180,7 @@ static void *httpAcceptHttpConnection(void *arg) {
while (1) {
socklen_t addrlen = sizeof(clientAddr);
connFd = (int)accept(pServer->fd, (struct sockaddr *)&clientAddr, &addrlen);
connFd = (int32_t)accept(pServer->fd, (struct sockaddr *)&clientAddr, &addrlen);
if (connFd == -1) {
if (errno == EINVAL) {
httpDebug("http server:%s socket was shutdown, exiting...", pServer->label);
@ -295,7 +191,7 @@ static void *httpAcceptHttpConnection(void *arg) {
}
totalFds = 1;
for (int i = 0; i < pServer->numOfThreads; ++i) {
for (int32_t i = 0; i < pServer->numOfThreads; ++i) {
totalFds += pServer->pThreads[i].numOfContexts;
}
@ -332,7 +228,7 @@ static void *httpAcceptHttpConnection(void *arg) {
httpError("context:%p, fd:%d, ip:%s, thread:%s, failed to add http fd for epoll, error:%s", pContext, connFd,
pContext->ipstr, pThread->label, strerror(errno));
taosClose(pContext->fd);
httpReleaseContext(pContext);
httpReleaseContext(pContext, true);
continue;
}
@ -359,7 +255,7 @@ bool httpInitConnect() {
}
HttpThread *pThread = pServer->pThreads;
for (int i = 0; i < pServer->numOfThreads; ++i) {
for (int32_t i = 0; i < pServer->numOfThreads; ++i) {
sprintf(pThread->label, "%s%d", pServer->label, i);
pThread->processData = pServer->processData;
pThread->threadId = i;
@ -405,3 +301,62 @@ bool httpInitConnect() {
pServer->serverPort, pServer->numOfThreads);
return true;
}
static bool httpReadData(HttpContext *pContext) {
HttpParser *pParser = pContext->parser;
if (!pParser->inited) {
httpInitParser(pParser);
}
if (pParser->parsed) {
httpDebug("context:%p, fd:%d, not in ready state, parsed:%d", pContext, pContext->fd, pParser->parsed);
return false;
}
pContext->accessTimes++;
pContext->lastAccessTime = taosGetTimestampSec();
char buf[HTTP_STEP_SIZE + 1] = {0};
while (1) {
int32_t nread = (int32_t)taosReadSocket(pContext->fd, buf, HTTP_STEP_SIZE);
if (nread > 0) {
buf[nread] = '\0';
httpTraceL("context:%p, fd:%d, nread:%d content:%s", pContext, pContext->fd, nread, buf);
int32_t ok = httpParseBuf(pParser, buf, nread);
if (ok) {
httpError("context:%p, fd:%d, parse failed, ret:%d code:%d close connect", pContext, pContext->fd, ok,
pParser->parseCode);
httpSendErrorResp(pContext, pParser->parseCode);
httpNotifyContextClose(pContext);
return false;
}
if (pParser->parseCode) {
httpError("context:%p, fd:%d, parse failed, code:%d close connect", pContext, pContext->fd, pParser->parseCode);
httpSendErrorResp(pContext, pParser->parseCode);
httpNotifyContextClose(pContext);
return false;
}
if (!pParser->parsed) {
httpTrace("context:%p, fd:%d, read not finished", pContext, pContext->fd);
continue;
} else {
httpDebug("context:%p, fd:%d, bodyLen:%d", pContext, pContext->fd, pParser->body.pos);
return true;
}
} else if (nread < 0) {
if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) {
httpDebug("context:%p, fd:%d, read from socket error:%d, wait another event", pContext, pContext->fd, errno);
return false; // later again
} else {
httpError("context:%p, fd:%d, read from socket error:%d, close connect", pContext, pContext->fd, errno);
return false;
}
} else {
httpError("context:%p, fd:%d, nread:%d, wait another event", pContext, pContext->fd, nread);
return false;
}
}
}

View File

@ -16,6 +16,7 @@
#define _DEFAULT_SOURCE
#include "os.h"
#include "taos.h"
#include "taoserror.h"
#include "tglobal.h"
#include "tcache.h"
#include "httpInt.h"
@ -39,15 +40,15 @@ void httpCreateSession(HttpContext *pContext, void *taos) {
// taosCacheRelease(server->sessionCache, (void **)&temp, false);
if (pContext->session == NULL) {
httpError("context:%p, fd:%d, ip:%s, user:%s, error:%s", pContext, pContext->fd, pContext->ipstr, pContext->user,
httpMsg[HTTP_SESSION_FULL]);
httpError("context:%p, fd:%d, user:%s, error:%s", pContext, pContext->fd, pContext->user,
tstrerror(TSDB_CODE_HTTP_SESSION_FULL));
taos_close(taos);
pthread_mutex_unlock(&server->serverMutex);
return;
}
httpDebug("context:%p, fd:%d, ip:%s, user:%s, create a new session:%p:%p sessionRef:%d", pContext, pContext->fd,
pContext->ipstr, pContext->user, pContext->session, pContext->session->taos, pContext->session->refCount);
httpDebug("context:%p, fd:%d, user:%s, create a new session:%p:%p sessionRef:%d", pContext, pContext->fd,
pContext->user, pContext->session, pContext->session->taos, pContext->session->refCount);
pthread_mutex_unlock(&server->serverMutex);
}
@ -61,11 +62,10 @@ static void httpFetchSessionImp(HttpContext *pContext) {
pContext->session = taosCacheAcquireByKey(server->sessionCache, sessionId, len);
if (pContext->session != NULL) {
atomic_add_fetch_32(&pContext->session->refCount, 1);
httpDebug("context:%p, fd:%d, ip:%s, user:%s, find an exist session:%p:%p, sessionRef:%d", pContext, pContext->fd,
pContext->ipstr, pContext->user, pContext->session, pContext->session->taos, pContext->session->refCount);
httpDebug("context:%p, fd:%d, user:%s, find an exist session:%p:%p, sessionRef:%d", pContext, pContext->fd,
pContext->user, pContext->session, pContext->session->taos, pContext->session->refCount);
} else {
httpDebug("context:%p, fd:%d, ip:%s, user:%s, session not found", pContext, pContext->fd, pContext->ipstr,
pContext->user);
httpDebug("context:%p, fd:%d, user:%s, session not found", pContext, pContext->fd, pContext->user);
}
pthread_mutex_unlock(&server->serverMutex);

View File

@ -17,6 +17,7 @@
#include "os.h"
#include "tnote.h"
#include "taos.h"
#include "taoserror.h"
#include "tsclient.h"
#include "httpInt.h"
#include "httpContext.h"
@ -50,24 +51,20 @@ void httpProcessMultiSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int n
}
}
// if (tscResultsetFetchCompleted(result)) {
// isContinue = false;
// }
if (isContinue) {
// retrieve next batch of rows
httpDebug("context:%p, fd:%d, ip:%s, user:%s, process pos:%d, continue retrieve, numOfRows:%d, sql:%s",
pContext, pContext->fd, pContext->ipstr, pContext->user, multiCmds->pos, numOfRows, sql);
httpDebug("context:%p, fd:%d, user:%s, process pos:%d, continue retrieve, numOfRows:%d, sql:%s", pContext,
pContext->fd, pContext->user, multiCmds->pos, numOfRows, sql);
taos_fetch_rows_a(result, httpProcessMultiSqlRetrieveCallBack, param);
} else {
httpDebug("context:%p, fd:%d, ip:%s, user:%s, process pos:%d, stop retrieve, numOfRows:%d, sql:%s",
pContext, pContext->fd, pContext->ipstr, pContext->user, multiCmds->pos, numOfRows, sql);
httpDebug("context:%p, fd:%d, user:%s, process pos:%d, stop retrieve, numOfRows:%d, sql:%s", pContext, pContext->fd,
pContext->user, multiCmds->pos, numOfRows, sql);
if (numOfRows < 0) {
httpError("context:%p, fd:%d, ip:%s, user:%s, process pos:%d, retrieve failed code:%s, sql:%s",
pContext, pContext->fd, pContext->ipstr, pContext->user, multiCmds->pos, tstrerror(numOfRows), sql);
}
httpError("context:%p, fd:%d, user:%s, process pos:%d, retrieve failed code:%s, sql:%s", pContext, pContext->fd,
pContext->user, multiCmds->pos, tstrerror(numOfRows), sql);
}
taos_free_result(result);
if (singleCmd->cmdReturnType == HTTP_CMD_RETURN_TYPE_WITH_RETURN && encode->stopJsonFp) {
@ -94,20 +91,20 @@ void httpProcessMultiSqlCallBackImp(void *param, TAOS_RES *result, int code) {
char * sql = httpGetCmdsString(pContext, singleCmd->sql);
if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
httpWarn("context:%p, fd:%d, ip:%s, user:%s, process pos:%d, code:%s:inprogress, sql:%s",
pContext, pContext->fd, pContext->ipstr, pContext->user, multiCmds->pos, tstrerror(code), sql);
httpWarn("context:%p, fd:%d, user:%s, process pos:%d, code:%s:inprogress, sql:%s", pContext, pContext->fd,
pContext->user, multiCmds->pos, tstrerror(code), sql);
return;
}
if (code < 0) {
if (encode->checkFinishedFp != NULL && !encode->checkFinishedFp(pContext, singleCmd, code)) {
singleCmd->code = code;
httpDebug("context:%p, fd:%d, ip:%s, user:%s, process pos jump to:%d, last code:%s, last sql:%s",
pContext, pContext->fd, pContext->ipstr, pContext->user, multiCmds->pos + 1, tstrerror(code), sql);
httpDebug("context:%p, fd:%d, user:%s, process pos jump to:%d, last code:%s, last sql:%s", pContext, pContext->fd,
pContext->user, multiCmds->pos + 1, tstrerror(code), sql);
} else {
singleCmd->code = code;
httpError("context:%p, fd:%d, ip:%s, user:%s, process pos:%d, error code:%s, sql:%s",
pContext, pContext->fd, pContext->ipstr, pContext->user, multiCmds->pos, tstrerror(code), sql);
httpError("context:%p, fd:%d, user:%s, process pos:%d, error code:%s, sql:%s", pContext, pContext->fd,
pContext->user, multiCmds->pos, tstrerror(code), sql);
if (singleCmd->cmdReturnType == HTTP_CMD_RETURN_TYPE_WITH_RETURN) {
if (encode->startJsonFp) (encode->startJsonFp)(pContext, singleCmd, result);
@ -125,8 +122,8 @@ void httpProcessMultiSqlCallBackImp(void *param, TAOS_RES *result, int code) {
if (isUpdate) {
// not select or show commands
int affectRows = taos_affected_rows(result);
httpDebug("context:%p, fd:%d, ip:%s, user:%s, process pos:%d, affect rows:%d, sql:%s",
pContext, pContext->fd, pContext->ipstr, pContext->user, multiCmds->pos, affectRows, sql);
httpDebug("context:%p, fd:%d, user:%s, process pos:%d, affect rows:%d, sql:%s", pContext, pContext->fd,
pContext->user, multiCmds->pos, affectRows, sql);
singleCmd->code = 0;
@ -151,8 +148,8 @@ void httpProcessMultiSqlCallBackImp(void *param, TAOS_RES *result, int code) {
taos_free_result(result);
httpProcessMultiSql(pContext);
} else {
httpDebug("context:%p, fd:%d, ip:%s, user:%s, process pos:%d, start retrieve, sql:%s",
pContext, pContext->fd, pContext->ipstr, pContext->user, multiCmds->pos, sql);
httpDebug("context:%p, fd:%d, user:%s, process pos:%d, start retrieve, sql:%s", pContext, pContext->fd,
pContext->user, multiCmds->pos, sql);
if (singleCmd->cmdReturnType == HTTP_CMD_RETURN_TYPE_WITH_RETURN && encode->startJsonFp) {
(encode->startJsonFp)(pContext, singleCmd, result);
@ -170,8 +167,8 @@ void httpProcessMultiSql(HttpContext *pContext) {
HttpEncodeMethod *encode = pContext->encodeMethod;
if (multiCmds->pos >= multiCmds->size) {
httpDebug("context:%p, fd:%d, ip:%s, user:%s, process pos:%d, size:%d, stop mulit-querys",
pContext, pContext->fd, pContext->ipstr, pContext->user, multiCmds->pos, multiCmds->size);
httpDebug("context:%p, fd:%d, user:%s, process pos:%d, size:%d, stop mulit-querys", pContext, pContext->fd,
pContext->user, multiCmds->pos, multiCmds->size);
if (encode->cleanJsonFp) {
(encode->cleanJsonFp)(pContext);
}
@ -182,8 +179,8 @@ void httpProcessMultiSql(HttpContext *pContext) {
HttpSqlCmd *cmd = multiCmds->cmds + multiCmds->pos;
char *sql = httpGetCmdsString(pContext, cmd->sql);
httpTraceL("context:%p, fd:%d, ip:%s, user:%s, process pos:%d, start query, sql:%s", pContext, pContext->fd,
pContext->ipstr, pContext->user, multiCmds->pos, sql);
httpTraceL("context:%p, fd:%d, user:%s, process pos:%d, start query, sql:%s", pContext, pContext->fd, pContext->user,
multiCmds->pos, sql);
taosNotePrintHttp(sql);
taos_query_a(pContext->session->taos, sql, httpProcessMultiSqlCallBack, (void *)pContext);
}
@ -193,12 +190,12 @@ void httpProcessMultiSqlCmd(HttpContext *pContext) {
HttpSqlCmds *multiCmds = pContext->multiCmds;
if (multiCmds == NULL || multiCmds->size <= 0 || multiCmds->pos >= multiCmds->size || multiCmds->pos < 0) {
httpSendErrorResp(pContext, HTTP_INVALID_MULTI_REQUEST);
httpSendErrorResp(pContext, TSDB_CODE_HTTP_INVALID_MULTI_REQUEST);
return;
}
httpDebug("context:%p, fd:%d, ip:%s, user:%s, start multi-querys pos:%d, size:%d", pContext, pContext->fd,
pContext->ipstr, pContext->user, multiCmds->pos, multiCmds->size);
httpDebug("context:%p, fd:%d, user:%s, start multi-querys pos:%d, size:%d", pContext, pContext->fd, pContext->user,
multiCmds->pos, multiCmds->size);
HttpEncodeMethod *encode = pContext->encodeMethod;
if (encode->initJsonFp) {
(encode->initJsonFp)(pContext);
@ -223,27 +220,18 @@ void httpProcessSingleSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int
}
}
#if 0
// todo refactor
if (tscResultsetFetchCompleted(result)) {
httpDebug("context:%p, fd:%d, ip:%s, user:%s, resultset fetch completed", pContext, pContext->fd, pContext->ipstr,
pContext->user);
isContinue = false;
}
#endif
if (isContinue) {
// retrieve next batch of rows
httpDebug("context:%p, fd:%d, ip:%s, user:%s, continue retrieve, numOfRows:%d", pContext, pContext->fd,
pContext->ipstr, pContext->user, numOfRows);
httpDebug("context:%p, fd:%d, user:%s, continue retrieve, numOfRows:%d", pContext, pContext->fd, pContext->user,
numOfRows);
taos_fetch_rows_a(result, httpProcessSingleSqlRetrieveCallBack, param);
} else {
httpDebug("context:%p, fd:%d, ip:%s, user:%s, stop retrieve, numOfRows:%d", pContext, pContext->fd, pContext->ipstr,
pContext->user, numOfRows);
httpDebug("context:%p, fd:%d, user:%s, stop retrieve, numOfRows:%d", pContext, pContext->fd, pContext->user,
numOfRows);
if (numOfRows < 0) {
httpError("context:%p, fd:%d, ip:%s, user:%s, retrieve failed, code:%s", pContext, pContext->fd, pContext->ipstr,
pContext->user, tstrerror(numOfRows));
httpError("context:%p, fd:%d, user:%s, retrieve failed, code:%s", pContext, pContext->fd, pContext->user,
tstrerror(numOfRows));
}
taos_free_result(result);
@ -269,21 +257,21 @@ void httpProcessSingleSqlCallBackImp(void *param, TAOS_RES *result, int unUsedCo
HttpEncodeMethod *encode = pContext->encodeMethod;
if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
httpError("context:%p, fd:%d, ip:%s, user:%s, query error, taos:%p, code:%s:inprogress, sqlObj:%p",
pContext, pContext->fd, pContext->ipstr, pContext->user, pContext->session->taos, tstrerror(code), (SSqlObj *)result);
httpError("context:%p, fd:%d, user:%s, query error, taos:%p, code:%s:inprogress, sqlObj:%p", pContext, pContext->fd,
pContext->user, pContext->session->taos, tstrerror(code), (SSqlObj *)result);
return;
}
if (code < 0) {
SSqlObj *pObj = (SSqlObj *)result;
if (code == TSDB_CODE_TSC_INVALID_SQL) {
httpError("context:%p, fd:%d, ip:%s, user:%s, query error, taos:%p, code:%s, sqlObj:%p, error:%s",
pContext, pContext->fd, pContext->ipstr, pContext->user, pContext->session->taos, tstrerror(code), pObj, pObj->cmd.payload);
httpError("context:%p, fd:%d, user:%s, query error, taos:%p, code:%s, sqlObj:%p, error:%s", pContext,
pContext->fd, pContext->user, pContext->session->taos, tstrerror(code), pObj, pObj->cmd.payload);
httpSendTaosdInvalidSqlErrorResp(pContext, pObj->cmd.payload);
} else {
httpError("context:%p, fd:%d, ip:%s, user:%s, query error, taos:%p, code:%s, sqlObj:%p",
pContext, pContext->fd, pContext->ipstr, pContext->user, pContext->session->taos, tstrerror(code), pObj);
httpSendTaosdErrorResp(pContext, code);
httpError("context:%p, fd:%d, user:%s, query error, taos:%p, code:%s, sqlObj:%p", pContext, pContext->fd,
pContext->user, pContext->session->taos, tstrerror(code), pObj);
httpSendErrorResp(pContext, code);
}
taos_free_result(result);
return;
@ -294,8 +282,8 @@ void httpProcessSingleSqlCallBackImp(void *param, TAOS_RES *result, int unUsedCo
// not select or show commands
int affectRows = taos_affected_rows(result);
httpDebug("context:%p, fd:%d, ip:%s, user:%s, affect rows:%d, stop query, sqlObj:%p",
pContext, pContext->fd, pContext->ipstr, pContext->user, affectRows, result);
httpDebug("context:%p, fd:%d, user:%s, affect rows:%d, stop query, sqlObj:%p", pContext, pContext->fd,
pContext->user, affectRows, result);
if (encode->startJsonFp) {
(encode->startJsonFp)(pContext, &pContext->singleCmd, result);
@ -312,8 +300,7 @@ void httpProcessSingleSqlCallBackImp(void *param, TAOS_RES *result, int unUsedCo
taos_free_result(result);
httpCloseContextByApp(pContext);
} else {
httpDebug("context:%p, fd:%d, ip:%s, user:%s, start retrieve", pContext, pContext->fd, pContext->ipstr,
pContext->user);
httpDebug("context:%p, fd:%d, user:%s, start retrieve", pContext, pContext->fd, pContext->user);
if (encode->startJsonFp) {
(encode->startJsonFp)(pContext, &pContext->singleCmd, result);
@ -333,25 +320,23 @@ void httpProcessSingleSqlCmd(HttpContext *pContext) {
HttpSession *pSession = pContext->session;
if (sql == NULL || sql[0] == 0) {
httpError("context:%p, fd:%d, ip:%s, user:%s, error:no sql input", pContext, pContext->fd, pContext->ipstr,
pContext->user);
httpSendErrorResp(pContext, HTTP_NO_SQL_INPUT);
httpError("context:%p, fd:%d, user:%s, error:no sql input", pContext, pContext->fd, pContext->user);
httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_SQL_INPUT);
return;
}
httpTraceL("context:%p, fd:%d, ip:%s, user:%s, start query, sql:%s", pContext, pContext->fd, pContext->ipstr,
pContext->user, sql);
httpTraceL("context:%p, fd:%d, user:%s, start query, sql:%s", pContext, pContext->fd, pContext->user, sql);
taosNotePrintHttp(sql);
taos_query_a(pSession->taos, sql, httpProcessSingleSqlCallBack, (void *)pContext);
}
void httpProcessLoginCmd(HttpContext *pContext) {
char token[128] = {0};
if (!httpGenTaosdAuthToken(pContext, token, 128)) {
httpSendErrorResp(pContext, HTTP_GEN_TAOSD_TOKEN_ERR);
if (httpGenTaosdAuthToken(pContext, token, 128) != 0) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_GEN_TAOSD_TOKEN_ERR);
} else {
httpDebug("context:%p, fd:%d, ip:%s, user:%s, login via http, return token:%s",
pContext, pContext->fd, pContext->ipstr, pContext->user, token);
httpDebug("context:%p, fd:%d, user:%s, login via http, return token:%s", pContext, pContext->fd, pContext->user,
token);
httpSendSuccResp(pContext, token);
}
}
@ -397,25 +382,24 @@ void httpProcessRequestCb(void *param, TAOS_RES *result, int code) {
if (pContext == NULL) return;
if (code < 0) {
httpError("context:%p, fd:%d, ip:%s, user:%s, login error, code:%s", pContext, pContext->fd, pContext->ipstr,
pContext->user, tstrerror(code));
httpSendTaosdErrorResp(pContext, code);
httpError("context:%p, fd:%d, user:%s, login error, code:%s", pContext, pContext->fd, pContext->user,
tstrerror(code));
httpSendErrorResp(pContext, code);
return;
}
httpDebug("context:%p, fd:%d, ip:%s, user:%s, connect tdengine success, taos:%p", pContext, pContext->fd,
pContext->ipstr, pContext->user, pContext->taos);
httpDebug("context:%p, fd:%d, user:%s, connect tdengine success, taos:%p", pContext, pContext->fd, pContext->user,
pContext->taos);
if (pContext->taos == NULL) {
httpError("context:%p, fd:%d, ip:%s, user:%s, login error, taos is empty", pContext, pContext->fd, pContext->ipstr,
pContext->user);
httpSendErrorResp(pContext, HTTP_NO_ENOUGH_SESSIONS);
httpError("context:%p, fd:%d, user:%s, login error, taos is empty", pContext, pContext->fd, pContext->user);
httpSendErrorResp(pContext, TSDB_CODE_HTTP_LOGIN_FAILED);
return;
}
httpCreateSession(pContext, pContext->taos);
if (pContext->session == NULL) {
httpSendErrorResp(pContext, HTTP_SESSION_FULL);
httpSendErrorResp(pContext, TSDB_CODE_HTTP_SESSION_FULL);
httpCloseContextByApp(pContext);
} else {
httpExecCmd(pContext);
@ -428,8 +412,8 @@ void httpProcessRequest(HttpContext *pContext) {
if (pContext->session == NULL || pContext->reqType == HTTP_REQTYPE_LOGIN) {
taos_connect_a(NULL, pContext->user, pContext->pass, "", 0, httpProcessRequestCb, (void *)pContext,
&(pContext->taos));
httpDebug("context:%p, fd:%d, ip:%s, user:%s, try connect tdengine, taos:%p", pContext, pContext->fd,
pContext->ipstr, pContext->user, pContext->taos);
httpDebug("context:%p, fd:%d, user:%s, try connect tdengine, taos:%p", pContext, pContext->fd, pContext->user,
pContext->taos);
} else {
httpExecCmd(pContext);
}

View File

@ -27,9 +27,9 @@
#include "httpResp.h"
#include "httpHandle.h"
#include "httpQueue.h"
#include "gcHandle.h"
#include "restHandle.h"
#include "tgHandle.h"
#include "httpGcHandle.h"
#include "httpRestHandle.h"
#include "httpTgHandle.h"
#ifndef _ADMIN
void adminInitHandle(HttpServer* pServer) {}
@ -37,9 +37,9 @@ void opInitHandle(HttpServer* pServer) {}
#endif
HttpServer tsHttpServer;
void taosInitNote(int numOfNoteLines, int maxNotes, char* lable);
void taosInitNote(int32_t numOfNoteLines, int32_t maxNotes, char* lable);
int httpInitSystem() {
int32_t httpInitSystem() {
strcpy(tsHttpServer.label, "rest");
tsHttpServer.serverIp = 0;
tsHttpServer.serverPort = tsHttpPort;
@ -60,7 +60,7 @@ int httpInitSystem() {
return 0;
}
int httpStartSystem() {
int32_t httpStartSystem() {
httpInfo("start http server ...");
if (tsHttpServer.status != HTTP_SERVER_INIT) {

View File

@ -19,8 +19,8 @@
#include "taosdef.h"
#include "taosmsg.h"
#include "httpInt.h"
#include "tgHandle.h"
#include "tgJson.h"
#include "httpTgHandle.h"
#include "httpTgJson.h"
#include "cJSON.h"
/*
@ -83,16 +83,16 @@ static const char DEFAULT_TELEGRAF_CFG[] =
"]}";
typedef struct {
char *name;
char *tbName;
char **fields;
int fieldNum;
char * name;
char * tbName;
char ** fields;
int32_t fieldNum;
} STgSchema;
typedef struct {
STgSchema *schemas;
int size;
int pos;
int32_t size;
int32_t pos;
} STgSchemas;
static STgSchemas tgSchemas = {0};
@ -107,7 +107,7 @@ void tgFreeSchema(STgSchema *schema) {
schema->tbName = NULL;
}
if (schema->fields != NULL) {
for (int f = 0; f < schema->fieldNum; ++f) {
for (int32_t f = 0; f < schema->fieldNum; ++f) {
if (schema->fields[f] != NULL) {
free(schema->fields[f]);
schema->fields[f] = NULL;
@ -121,7 +121,7 @@ void tgFreeSchema(STgSchema *schema) {
void tgFreeSchemas() {
if (tgSchemas.schemas != NULL) {
for (int s = 0; s < tgSchemas.size; ++s) {
for (int32_t s = 0; s < tgSchemas.size; ++s) {
tgFreeSchema(&tgSchemas.schemas[s]);
}
free(tgSchemas.schemas);
@ -130,7 +130,7 @@ void tgFreeSchemas() {
}
}
void tgInitSchemas(int size) {
void tgInitSchemas(int32_t size) {
tgFreeSchemas();
tgSchemas.schemas = calloc(sizeof(STgSchema), size);
tgSchemas.size = 0;
@ -154,7 +154,7 @@ void tgParseSchemaMetric(cJSON *metric) {
parsedOk = false;
goto ParseEnd;
}
int nameLen = (int)strlen(name->valuestring);
int32_t nameLen = (int32_t)strlen(name->valuestring);
if (nameLen == 0) {
parsedOk = false;
goto ParseEnd;
@ -177,7 +177,7 @@ void tgParseSchemaMetric(cJSON *metric) {
parsedOk = false;
goto ParseEnd;
}
int tbnameLen = (int)strlen(tbname->valuestring);
int32_t tbnameLen = (int32_t)strlen(tbname->valuestring);
if (tbnameLen == 0) {
parsedOk = false;
goto ParseEnd;
@ -191,7 +191,7 @@ void tgParseSchemaMetric(cJSON *metric) {
if (fields == NULL) {
goto ParseEnd;
}
int fieldSize = cJSON_GetArraySize(fields);
int32_t fieldSize = cJSON_GetArraySize(fields);
if (fieldSize <= 0 || fieldSize > TSDB_MAX_COLUMNS) {
goto ParseEnd;
}
@ -199,7 +199,7 @@ void tgParseSchemaMetric(cJSON *metric) {
if (fieldSize > 0) {
schema.fields = calloc(sizeof(STgSchema), (size_t)fieldSize);
schema.fieldNum = fieldSize;
for (int i = 0; i < fieldSize; i++) {
for (int32_t i = 0; i < fieldSize; i++) {
cJSON *field = cJSON_GetArrayItem(fields, i);
if (field == NULL) {
parsedOk = false;
@ -209,7 +209,7 @@ void tgParseSchemaMetric(cJSON *metric) {
parsedOk = false;
goto ParseEnd;
}
int nameLen = (int)strlen(field->valuestring);
int32_t nameLen = (int32_t)strlen(field->valuestring);
if (nameLen == 0 || nameLen >= TSDB_TABLE_NAME_LEN) {
parsedOk = false;
goto ParseEnd;
@ -227,13 +227,13 @@ ParseEnd:
}
}
int tgParseSchema(const char *content, char*fileName) {
int32_t tgParseSchema(const char *content, char*fileName) {
cJSON *root = cJSON_Parse(content);
if (root == NULL) {
httpError("failed to parse telegraf schema file:%s, invalid json format, content:%s", fileName, content);
return -1;
}
int size = 0;
int32_t size = 0;
cJSON *metrics = cJSON_GetObjectItem(root, "metrics");
if (metrics != NULL) {
size = cJSON_GetArraySize(metrics);
@ -244,7 +244,7 @@ int tgParseSchema(const char *content, char*fileName) {
}
tgInitSchemas(size);
for (int i = 0; i < size; i++) {
for (int32_t i = 0; i < size; i++) {
cJSON *metric = cJSON_GetArrayItem(metrics, i);
if (metric != NULL) {
tgParseSchemaMetric(metric);
@ -260,7 +260,7 @@ int tgParseSchema(const char *content, char*fileName) {
return size;
}
int tgReadSchema(char *fileName) {
int32_t tgReadSchema(char *fileName) {
FILE *fp = fopen(fileName, "r");
if (fp == NULL) {
return -1;
@ -286,7 +286,7 @@ int tgReadSchema(char *fileName) {
}
content[contentSize] = 0;
int schemaNum = tgParseSchema(content, fileName);
int32_t schemaNum = tgParseSchema(content, fileName);
free(content);
fclose(fp);
@ -313,53 +313,53 @@ void tgCleanupHandle() {
}
bool tgGetUserFromUrl(HttpContext *pContext) {
HttpParser *pParser = &pContext->parser;
if (pParser->path[TG_USER_URL_POS].len >= TSDB_USER_LEN || pParser->path[TG_USER_URL_POS].len <= 0) {
HttpParser *pParser = pContext->parser;
if (pParser->path[TG_USER_URL_POS].pos >= TSDB_USER_LEN || pParser->path[TG_USER_URL_POS].pos <= 0) {
return false;
}
tstrncpy(pContext->user, pParser->path[TG_USER_URL_POS].pos, sizeof(pContext->user));
tstrncpy(pContext->user, pParser->path[TG_USER_URL_POS].str, sizeof(pContext->user));
return true;
}
bool tgGetPassFromUrl(HttpContext *pContext) {
HttpParser *pParser = &pContext->parser;
if (pParser->path[TG_PASS_URL_POS].len >= TSDB_PASSWORD_LEN || pParser->path[TG_PASS_URL_POS].len <= 0) {
HttpParser *pParser = pContext->parser;
if (pParser->path[TG_PASS_URL_POS].pos >= TSDB_PASSWORD_LEN || pParser->path[TG_PASS_URL_POS].pos <= 0) {
return false;
}
tstrncpy(pContext->pass, pParser->path[TG_PASS_URL_POS].pos, sizeof(pContext->pass));
tstrncpy(pContext->pass, pParser->path[TG_PASS_URL_POS].str, sizeof(pContext->pass));
return true;
}
char *tgGetDbFromUrl(HttpContext *pContext) {
HttpParser *pParser = &pContext->parser;
if (pParser->path[TG_DB_URL_POS].len <= 0) {
httpSendErrorResp(pContext, HTTP_TG_DB_NOT_INPUT);
HttpParser *pParser = pContext->parser;
if (pParser->path[TG_DB_URL_POS].pos <= 0) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_DB_NOT_INPUT);
return NULL;
}
if (pParser->path[TG_DB_URL_POS].len >= TSDB_DB_NAME_LEN) {
httpSendErrorResp(pContext, HTTP_TG_DB_TOO_LONG);
if (pParser->path[TG_DB_URL_POS].pos >= TSDB_DB_NAME_LEN) {
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_DB_TOO_LONG);
return NULL;
}
return pParser->path[TG_DB_URL_POS].pos;
return pParser->path[TG_DB_URL_POS].str;
}
char *tgGetStableName(char *stname, cJSON *fields, int fieldsSize) {
for (int s = 0; s < tgSchemas.size; ++s) {
char *tgGetStableName(char *stname, cJSON *fields, int32_t fieldsSize) {
for (int32_t s = 0; s < tgSchemas.size; ++s) {
STgSchema *schema = &tgSchemas.schemas[s];
if (strcasecmp(schema->name, stname) != 0) {
continue;
}
bool schemaMatched = true;
for (int f = 0; f < schema->fieldNum; ++f) {
for (int32_t f = 0; f < schema->fieldNum; ++f) {
char *fieldName = schema->fields[f];
bool fieldMatched = false;
for (int i = 0; i < fieldsSize; i++) {
for (int32_t i = 0; i < fieldsSize; i++) {
cJSON *field = cJSON_GetArrayItem(fields, i);
if (strcasecmp(field->string, fieldName) == 0) {
fieldMatched = true;
@ -401,70 +401,70 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) {
// metric name
cJSON *name = cJSON_GetObjectItem(metric, "name");
if (name == NULL) {
httpSendErrorResp(pContext, HTTP_TG_METRIC_NULL);
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_METRIC_NULL);
return false;
}
if (name->type != cJSON_String) {
httpSendErrorResp(pContext, HTTP_TG_METRIC_TYPE);
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_METRIC_TYPE);
return false;
}
if (name->valuestring == NULL) {
httpSendErrorResp(pContext, HTTP_TG_METRIC_NAME_NULL);
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_METRIC_NAME_NULL);
return false;
}
int nameLen = (int)strlen(name->valuestring);
int32_t nameLen = (int32_t)strlen(name->valuestring);
if (nameLen == 0) {
httpSendErrorResp(pContext, HTTP_TG_METRIC_NAME_NULL);
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_METRIC_NAME_NULL);
return false;
}
if (nameLen >= TSDB_TABLE_NAME_LEN - 8) {
httpSendErrorResp(pContext, HTTP_TG_METRIC_NAME_LONG);
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_METRIC_NAME_LONG);
return false;
}
// timestamp
cJSON *timestamp = cJSON_GetObjectItem(metric, "timestamp");
if (timestamp == NULL) {
httpSendErrorResp(pContext, HTTP_TG_TIMESTAMP_NULL);
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_TIMESTAMP_NULL);
return false;
}
if (timestamp->type != cJSON_Number) {
httpSendErrorResp(pContext, HTTP_TG_TIMESTAMP_TYPE);
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_TIMESTAMP_TYPE);
return false;
}
if (timestamp->valueint <= 0) {
httpSendErrorResp(pContext, HTTP_TG_TIMESTAMP_VAL_NULL);
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_TIMESTAMP_VAL_NULL);
return false;
}
// tags
cJSON *tags = cJSON_GetObjectItem(metric, "tags");
if (tags == NULL) {
httpSendErrorResp(pContext, HTTP_TG_TAGS_NULL);
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_TAGS_NULL);
return false;
}
int tagsSize = cJSON_GetArraySize(tags);
int32_t tagsSize = cJSON_GetArraySize(tags);
if (tagsSize <= 0) {
httpSendErrorResp(pContext, HTTP_TG_TAGS_SIZE_0);
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_TAGS_SIZE_0);
return false;
}
if (tagsSize > TG_MAX_SORT_TAG_SIZE) {
httpSendErrorResp(pContext, HTTP_TG_TAGS_SIZE_LONG);
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_TAGS_SIZE_LONG);
return false;
}
cJSON *host = NULL;
for (int i = 0; i < tagsSize; i++) {
for (int32_t i = 0; i < tagsSize; i++) {
cJSON *tag = cJSON_GetArrayItem(tags, i);
if (tag == NULL) {
httpSendErrorResp(pContext, HTTP_TG_TAG_NULL);
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_TAG_NULL);
return false;
}
if (tag->string == NULL || strlen(tag->string) == 0) {
httpSendErrorResp(pContext, HTTP_TG_TAG_NAME_NULL);
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_TAG_NAME_NULL);
return false;
}
@ -474,19 +474,19 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) {
*/
if (0) {
if (strlen(tag->string) >= TSDB_COL_NAME_LEN) {
httpSendErrorResp(pContext, HTTP_TG_TAG_NAME_SIZE);
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_TAG_NAME_SIZE);
return false;
}
}
if (tag->type != cJSON_Number && tag->type != cJSON_String) {
httpSendErrorResp(pContext, HTTP_TG_TAG_VALUE_TYPE);
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_TAG_VALUE_TYPE);
return false;
}
if (tag->type == cJSON_String) {
if (tag->valuestring == NULL || strlen(tag->valuestring) == 0) {
httpSendErrorResp(pContext, HTTP_TG_TAG_VALUE_NULL);
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_TAG_VALUE_NULL);
return false;
}
}
@ -497,46 +497,46 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) {
}
if (host == NULL) {
httpSendErrorResp(pContext, HTTP_TG_TABLE_NULL);
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_TABLE_NULL);
return false;
}
if (host->type != cJSON_String) {
httpSendErrorResp(pContext, HTTP_TG_HOST_NOT_STRING);
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_HOST_NOT_STRING);
return false;
}
if (strlen(host->valuestring) >= TSDB_TABLE_NAME_LEN - 1) {
httpSendErrorResp(pContext, HTTP_TG_TABLE_SIZE);
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_TABLE_SIZE);
return false;
}
// fields
cJSON *fields = cJSON_GetObjectItem(metric, "fields");
if (fields == NULL) {
httpSendErrorResp(pContext, HTTP_TG_FIELDS_NULL);
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_FIELDS_NULL);
return false;
}
int fieldsSize = cJSON_GetArraySize(fields);
int32_t fieldsSize = cJSON_GetArraySize(fields);
if (fieldsSize <= 0) {
httpSendErrorResp(pContext, HTTP_TG_FIELDS_SIZE_0);
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_FIELDS_SIZE_0);
return false;
}
if (fieldsSize > (TSDB_MAX_COLUMNS - TSDB_MAX_TAGS - 1)) {
httpSendErrorResp(pContext, HTTP_TG_FIELDS_SIZE_LONG);
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_FIELDS_SIZE_LONG);
return false;
}
for (int i = 0; i < fieldsSize; i++) {
for (int32_t i = 0; i < fieldsSize; i++) {
cJSON *field = cJSON_GetArrayItem(fields, i);
if (field == NULL) {
httpSendErrorResp(pContext, HTTP_TG_FIELD_NULL);
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_FIELD_NULL);
return false;
}
if (field->string == NULL || strlen(field->string) == 0) {
httpSendErrorResp(pContext, HTTP_TG_FIELD_NAME_NULL);
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_FIELD_NAME_NULL);
return false;
}
/*
@ -545,17 +545,17 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) {
*/
if (0) {
if (strlen(field->string) >= TSDB_COL_NAME_LEN) {
httpSendErrorResp(pContext, HTTP_TG_FIELD_NAME_SIZE);
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_FIELD_NAME_SIZE);
return false;
}
}
if (field->type != cJSON_Number && field->type != cJSON_String) {
httpSendErrorResp(pContext, HTTP_TG_FIELD_VALUE_TYPE);
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_FIELD_VALUE_TYPE);
return false;
}
if (field->type == cJSON_String) {
if (field->valuestring == NULL || strlen(field->valuestring) == 0) {
httpSendErrorResp(pContext, HTTP_TG_FIELD_VALUE_NULL);
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_FIELD_VALUE_NULL);
return false;
}
}
@ -564,7 +564,7 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) {
// assembling cmds
HttpSqlCmd *stable_cmd = httpNewSqlCmd(pContext);
if (stable_cmd == NULL) {
httpSendErrorResp(pContext, HTTP_NO_ENOUGH_MEMORY);
httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_ENOUGH_MEMORY);
return false;
}
stable_cmd->cmdType = HTTP_CMD_TYPE_CREATE_STBALE;
@ -572,18 +572,18 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) {
HttpSqlCmd *table_cmd = httpNewSqlCmd(pContext);
if (table_cmd == NULL) {
httpSendErrorResp(pContext, HTTP_NO_ENOUGH_MEMORY);
httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_ENOUGH_MEMORY);
return false;
}
table_cmd->cmdType = HTTP_CMD_TYPE_INSERT;
// order by tag name
cJSON *orderedTags[TG_MAX_SORT_TAG_SIZE] = {0};
int orderTagsLen = 0;
for (int i = 0; i < tagsSize; ++i) {
int32_t orderTagsLen = 0;
for (int32_t i = 0; i < tagsSize; ++i) {
cJSON *tag = cJSON_GetArrayItem(tags, i);
orderedTags[orderTagsLen++] = tag;
for (int j = orderTagsLen - 1; j >= 1; --j) {
for (int32_t j = orderTagsLen - 1; j >= 1; --j) {
cJSON *tag1 = orderedTags[j];
cJSON *tag2 = orderedTags[j - 1];
if (strcasecmp(tag1->string, "host") == 0 || strcmp(tag1->string, tag2->string) < 0) {
@ -609,7 +609,7 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) {
httpShrinkTableName(pContext, table_cmd->stable, httpGetCmdsString(pContext, table_cmd->stable));
// stable tag for detail
for (int i = 0; i < orderTagsLen; ++i) {
for (int32_t i = 0; i < orderTagsLen; ++i) {
cJSON *tag = orderedTags[i];
stable_cmd->tagNames[i] = table_cmd->tagNames[i] = httpAddToSqlCmdBuffer(pContext, tag->string);
@ -631,7 +631,7 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) {
} else {
table_cmd->table = stable_cmd->table = httpAddToSqlCmdBufferNoTerminal(pContext, "%s_%d_%d_%s", stname, fieldsSize, orderTagsLen, host->valuestring);
}
for (int i = 0; i < orderTagsLen; ++i) {
for (int32_t i = 0; i < orderTagsLen; ++i) {
cJSON *tag = orderedTags[i];
if (tag == host) continue;
if (tag->type == cJSON_String)
@ -653,7 +653,7 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) {
// assembling create stable sql
stable_cmd->sql = httpAddToSqlCmdBufferNoTerminal(pContext, "create table if not exists %s.%s(ts timestamp", db,
httpGetCmdsString(pContext, table_cmd->stable));
for (int i = 0; i < fieldsSize; ++i) {
for (int32_t i = 0; i < fieldsSize; ++i) {
cJSON *field = cJSON_GetArrayItem(fields, i);
char * field_type = "double";
if (field->type == cJSON_String)
@ -668,7 +668,7 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) {
}
httpAddToSqlCmdBufferNoTerminal(pContext, ") tags(");
for (int i = 0; i < orderTagsLen; ++i) {
for (int32_t i = 0; i < orderTagsLen; ++i) {
cJSON *tag = orderedTags[i];
char * tag_type = "bigint";
if (tag->type == cJSON_String)
@ -689,7 +689,7 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) {
table_cmd->sql = httpAddToSqlCmdBufferNoTerminal(pContext, "import into %s.%s using %s.%s tags(", db,
httpGetCmdsString(pContext, table_cmd->table), db,
httpGetCmdsString(pContext, table_cmd->stable));
for (int i = 0; i < orderTagsLen; ++i) {
for (int32_t i = 0; i < orderTagsLen; ++i) {
cJSON *tag = orderedTags[i];
if (i != orderTagsLen - 1) {
if (tag->type == cJSON_Number)
@ -719,7 +719,7 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) {
}
httpAddToSqlCmdBufferNoTerminal(pContext, " values(%" PRId64 ",", timestamp->valueint);
for (int i = 0; i < fieldsSize; ++i) {
for (int32_t i = 0; i < fieldsSize; ++i) {
cJSON *field = cJSON_GetArrayItem(fields, i);
if (i != fieldsSize - 1) {
if (field->type == cJSON_Number)
@ -800,48 +800,46 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) {
}
*/
bool tgProcessQueryRequest(HttpContext *pContext, char *db) {
httpDebug("context:%p, fd:%d, ip:%s, process telegraf query msg", pContext, pContext->fd, pContext->ipstr);
httpDebug("context:%p, fd:%d, process telegraf query msg", pContext, pContext->fd);
HttpParser *pParser = &pContext->parser;
char * filter = pParser->data.pos;
char *filter = pContext->parser->body.str;
if (filter == NULL) {
httpSendErrorResp(pContext, HTTP_NO_MSG_INPUT);
httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_MSG_INPUT);
return false;
}
cJSON *root = cJSON_Parse(filter);
if (root == NULL) {
httpSendErrorResp(pContext, HTTP_TG_INVALID_JSON);
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_INVALID_JSON);
return false;
}
cJSON *metrics = cJSON_GetObjectItem(root, "metrics");
if (metrics != NULL) {
int size = cJSON_GetArraySize(metrics);
httpDebug("context:%p, fd:%d, ip:%s, multiple metrics:%d at one time", pContext, pContext->fd, pContext->ipstr,
size);
int32_t size = cJSON_GetArraySize(metrics);
httpDebug("context:%p, fd:%d, multiple metrics:%d at one time", pContext, pContext->fd, size);
if (size <= 0) {
httpSendErrorResp(pContext, HTTP_TG_METRICS_NULL);
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_METRICS_NULL);
cJSON_Delete(root);
return false;
}
int cmdSize = size * 2 + 1;
int32_t cmdSize = size * 2 + 1;
if (cmdSize > HTTP_MAX_CMD_SIZE) {
httpSendErrorResp(pContext, HTTP_TG_METRICS_SIZE);
httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_METRICS_SIZE);
cJSON_Delete(root);
return false;
}
if (!httpMallocMultiCmds(pContext, cmdSize, HTTP_BUFFER_SIZE)) {
httpSendErrorResp(pContext, HTTP_NO_ENOUGH_MEMORY);
httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_ENOUGH_MEMORY);
cJSON_Delete(root);
return false;
}
HttpSqlCmd *cmd = httpNewSqlCmd(pContext);
if (cmd == NULL) {
httpSendErrorResp(pContext, HTTP_NO_ENOUGH_MEMORY);
httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_ENOUGH_MEMORY);
cJSON_Delete(root);
return false;
}
@ -849,7 +847,7 @@ bool tgProcessQueryRequest(HttpContext *pContext, char *db) {
cmd->cmdReturnType = HTTP_CMD_RETURN_TYPE_NO_RETURN;
cmd->sql = httpAddToSqlCmdBuffer(pContext, "create database if not exists %s", db);
for (int i = 0; i < size; i++) {
for (int32_t i = 0; i < size; i++) {
cJSON *metric = cJSON_GetArrayItem(metrics, i);
if (metric != NULL) {
if (!tgProcessSingleMetric(pContext, metric, db)) {
@ -859,17 +857,17 @@ bool tgProcessQueryRequest(HttpContext *pContext, char *db) {
}
}
} else {
httpDebug("context:%p, fd:%d, ip:%s, single metric", pContext, pContext->fd, pContext->ipstr);
httpDebug("context:%p, fd:%d, single metric", pContext, pContext->fd);
if (!httpMallocMultiCmds(pContext, 3, HTTP_BUFFER_SIZE)) {
httpSendErrorResp(pContext, HTTP_NO_ENOUGH_MEMORY);
httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_ENOUGH_MEMORY);
cJSON_Delete(root);
return false;
}
HttpSqlCmd *cmd = httpNewSqlCmd(pContext);
if (cmd == NULL) {
httpSendErrorResp(pContext, HTTP_NO_ENOUGH_MEMORY);
httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_ENOUGH_MEMORY);
cJSON_Delete(root);
return false;
}
@ -894,7 +892,7 @@ bool tgProcessQueryRequest(HttpContext *pContext, char *db) {
bool tgProcessRquest(struct HttpContext *pContext) {
if (strlen(pContext->user) == 0 || strlen(pContext->pass) == 0) {
httpSendErrorResp(pContext, HTTP_PARSE_USR_ERROR);
httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_AUTH_INFO);
return false;
}

View File

@ -19,8 +19,8 @@
#include "httpLog.h"
#include "httpJson.h"
#include "httpResp.h"
#include "tgHandle.h"
#include "tgJson.h"
#include "httpTgHandle.h"
#include "httpTgJson.h"
void tgInitQueryJson(HttpContext *pContext) {
JsonBuf *jsonBuf = httpMallocJsonBuf(pContext);
@ -61,19 +61,19 @@ void tgStartQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result)
// data
httpJsonItemToken(jsonBuf);
httpJsonPair(jsonBuf, "metric", 6, httpGetCmdsString(pContext, cmd->stable),
(int)strlen(httpGetCmdsString(pContext, cmd->metric)));
(int32_t)strlen(httpGetCmdsString(pContext, cmd->metric)));
httpJsonItemToken(jsonBuf);
httpJsonPair(jsonBuf, "stable", 6, httpGetCmdsString(pContext, cmd->stable),
(int)strlen(httpGetCmdsString(pContext, cmd->stable)));
(int32_t)strlen(httpGetCmdsString(pContext, cmd->stable)));
httpJsonItemToken(jsonBuf);
httpJsonPair(jsonBuf, "table", 5, httpGetCmdsString(pContext, cmd->table),
(int)strlen(httpGetCmdsString(pContext, cmd->table)));
(int32_t)strlen(httpGetCmdsString(pContext, cmd->table)));
httpJsonItemToken(jsonBuf);
httpJsonPair(jsonBuf, "timestamp", 9, httpGetCmdsString(pContext, cmd->timestamp),
(int)strlen(httpGetCmdsString(pContext, cmd->timestamp))); // hack way
(int32_t)strlen(httpGetCmdsString(pContext, cmd->timestamp))); // hack way
}
void tgStopQueryJson(HttpContext *pContext, HttpSqlCmd *cmd) {
@ -88,7 +88,7 @@ void tgStopQueryJson(HttpContext *pContext, HttpSqlCmd *cmd) {
httpJsonToken(jsonBuf, JsonObjEnd);
}
void tgBuildSqlAffectRowsJson(HttpContext *pContext, HttpSqlCmd *cmd, int affect_rows) {
void tgBuildSqlAffectRowsJson(HttpContext *pContext, HttpSqlCmd *cmd, int32_t affect_rows) {
JsonBuf *jsonBuf = httpMallocJsonBuf(pContext);
if (jsonBuf == NULL) return;
@ -96,10 +96,10 @@ void tgBuildSqlAffectRowsJson(HttpContext *pContext, HttpSqlCmd *cmd, int affect
httpJsonPairIntVal(jsonBuf, "affected_rows", 13, affect_rows);
}
bool tgCheckFinished(struct HttpContext *pContext, HttpSqlCmd *cmd, int code) {
bool tgCheckFinished(struct HttpContext *pContext, HttpSqlCmd *cmd, int32_t code) {
HttpSqlCmds *multiCmds = pContext->multiCmds;
httpDebug("context:%p, fd:%d, ip:%s, check telegraf command, code:%s, state:%d, type:%d, rettype:%d, tags:%d",
pContext, pContext->fd, pContext->ipstr, tstrerror(code), cmd->cmdState, cmd->cmdType, cmd->cmdReturnType, cmd->tagNum);
httpDebug("context:%p, fd:%d, check telegraf command, code:%s, state:%d, type:%d, rettype:%d, tags:%d", pContext,
pContext->fd, tstrerror(code), cmd->cmdState, cmd->cmdType, cmd->cmdReturnType, cmd->tagNum);
if (cmd->cmdType == HTTP_CMD_TYPE_INSERT) {
if (cmd->cmdState == HTTP_CMD_STATE_NOT_RUN_YET) {
@ -107,16 +107,14 @@ bool tgCheckFinished(struct HttpContext *pContext, HttpSqlCmd *cmd, int code) {
cmd->cmdState = HTTP_CMD_STATE_RUN_FINISHED;
if (multiCmds->cmds[0].cmdState == HTTP_CMD_STATE_NOT_RUN_YET) {
multiCmds->pos = (int16_t)-1;
httpDebug("context:%p, fd:%d, ip:%s, import failed, try create database", pContext, pContext->fd,
pContext->ipstr);
httpDebug("context:%p, fd:%d, import failed, try create database", pContext, pContext->fd);
return false;
}
} else if (code == TSDB_CODE_MND_INVALID_TABLE_NAME) {
cmd->cmdState = HTTP_CMD_STATE_RUN_FINISHED;
if (multiCmds->cmds[multiCmds->pos - 1].cmdState == HTTP_CMD_STATE_NOT_RUN_YET) {
multiCmds->pos = (int16_t)(multiCmds->pos - 2);
httpDebug("context:%p, fd:%d, ip:%s, import failed, try create stable", pContext, pContext->fd,
pContext->ipstr);
httpDebug("context:%p, fd:%d, import failed, try create stable", pContext, pContext->fd);
return false;
}
} else {
@ -125,22 +123,21 @@ bool tgCheckFinished(struct HttpContext *pContext, HttpSqlCmd *cmd, int code) {
}
} else if (cmd->cmdType == HTTP_CMD_TYPE_CREATE_DB) {
cmd->cmdState = HTTP_CMD_STATE_RUN_FINISHED;
httpDebug("context:%p, fd:%d, ip:%s, code:%s, create database failed", pContext, pContext->fd, pContext->ipstr,
tstrerror(code));
httpDebug("context:%p, fd:%d, code:%s, create database failed", pContext, pContext->fd, tstrerror(code));
} else if (cmd->cmdType == HTTP_CMD_TYPE_CREATE_STBALE) {
cmd->cmdState = HTTP_CMD_STATE_RUN_FINISHED;
httpDebug("context:%p, fd:%d, ip:%s, code:%s, create stable failed", pContext, pContext->fd, pContext->ipstr, tstrerror(code));
httpDebug("context:%p, fd:%d, code:%s, create stable failed", pContext, pContext->fd, tstrerror(code));
} else {
}
return true;
}
void tgSetNextCmd(struct HttpContext *pContext, HttpSqlCmd *cmd, int code) {
void tgSetNextCmd(struct HttpContext *pContext, HttpSqlCmd *cmd, int32_t code) {
HttpSqlCmds *multiCmds = pContext->multiCmds;
httpDebug("context:%p, fd:%d, ip:%s, get telegraf next command, pos:%d, code:%s, state:%d, type:%d, rettype:%d, tags:%d",
pContext, pContext->fd, pContext->ipstr, multiCmds->pos, tstrerror(code), cmd->cmdState, cmd->cmdType,
cmd->cmdReturnType, cmd->tagNum);
httpDebug("context:%p, fd:%d, get telegraf next command, pos:%d, code:%s, state:%d, type:%d, rettype:%d, tags:%d",
pContext, pContext->fd, multiCmds->pos, tstrerror(code), cmd->cmdState, cmd->cmdType, cmd->cmdReturnType,
cmd->tagNum);
if (cmd->cmdType == HTTP_CMD_TYPE_INSERT) {
multiCmds->pos = (int16_t)(multiCmds->pos + 2);

View File

@ -29,7 +29,7 @@ bool httpCheckUsedbSql(char *sql) {
return false;
}
void httpTimeToString(time_t t, char *buf, int buflen) {
void httpTimeToString(time_t t, char *buf, int32_t buflen) {
memset(buf, 0, (size_t)buflen);
char ts[32] = {0};
@ -44,13 +44,13 @@ int32_t httpAddToSqlCmdBuffer(HttpContext *pContext, const char *const format, .
HttpSqlCmds *cmd = pContext->multiCmds;
if (cmd->buffer == NULL) return -1;
int remainLength = cmd->bufferSize - cmd->bufferPos;
int32_t remainLength = cmd->bufferSize - cmd->bufferPos;
if (remainLength < 4096) {
if (!httpReMallocMultiCmdsBuffer(pContext, cmd->bufferSize * 2)) return -1;
}
char *buffer = cmd->buffer + cmd->bufferPos;
int len = 0;
char * buffer = cmd->buffer + cmd->bufferPos;
int32_t len = 0;
va_list argpointer;
va_start(argpointer, format);
@ -76,13 +76,13 @@ int32_t httpAddToSqlCmdBufferNoTerminal(HttpContext *pContext, const char *const
HttpSqlCmds *cmd = pContext->multiCmds;
if (cmd->buffer == NULL) return -1;
int remainLength = cmd->bufferSize - cmd->bufferPos;
int32_t remainLength = cmd->bufferSize - cmd->bufferPos;
if (remainLength < 4096) {
if (!httpReMallocMultiCmdsBuffer(pContext, cmd->bufferSize * 2)) return -1;
}
char *buffer = cmd->buffer + cmd->bufferPos;
int len = 0;
char * buffer = cmd->buffer + cmd->bufferPos;
int32_t len = 0;
va_list argpointer;
va_start(argpointer, format);
@ -107,7 +107,7 @@ int32_t httpAddToSqlCmdBufferTerminal(HttpContext *pContext) {
HttpSqlCmds *cmd = pContext->multiCmds;
if (cmd->buffer == NULL) return -1;
int remainLength = cmd->bufferSize - cmd->bufferPos;
int32_t remainLength = cmd->bufferSize - cmd->bufferPos;
if (remainLength < 4096) {
if (!httpReMallocMultiCmdsBuffer(pContext, cmd->bufferSize * 2)) return -1;
}
@ -124,7 +124,7 @@ int32_t httpAddToSqlCmdBufferTerminal(HttpContext *pContext) {
return (int32_t)(buffer - cmd->buffer);
}
int32_t httpAddToSqlCmdBufferWithSize(HttpContext *pContext, int mallocSize) {
int32_t httpAddToSqlCmdBufferWithSize(HttpContext *pContext, int32_t mallocSize) {
HttpSqlCmds *cmd = pContext->multiCmds;
if (cmd->buffer == NULL) return -1;
@ -139,18 +139,17 @@ int32_t httpAddToSqlCmdBufferWithSize(HttpContext *pContext, int mallocSize) {
return (int32_t)(buffer - cmd->buffer);
}
bool httpMallocMultiCmds(HttpContext *pContext, int cmdSize, int bufferSize) {
bool httpMallocMultiCmds(HttpContext *pContext, int32_t cmdSize, int32_t bufferSize) {
if (cmdSize > HTTP_MAX_CMD_SIZE) {
httpError("context:%p, fd:%d, ip:%s, user:%s, mulitcmd size:%d large then %d", pContext, pContext->fd,
pContext->ipstr, pContext->user, cmdSize, HTTP_MAX_CMD_SIZE);
httpError("context:%p, fd:%d, user:%s, mulitcmd size:%d large then %d", pContext, pContext->fd, pContext->user,
cmdSize, HTTP_MAX_CMD_SIZE);
return false;
}
if (pContext->multiCmds == NULL) {
pContext->multiCmds = (HttpSqlCmds *)malloc(sizeof(HttpSqlCmds));
if (pContext->multiCmds == NULL) {
httpError("context:%p, fd:%d, ip:%s, user:%s, malloc multiCmds error", pContext, pContext->fd, pContext->ipstr,
pContext->user);
httpError("context:%p, fd:%d, user:%s, malloc multiCmds error", pContext, pContext->fd, pContext->user);
return false;
}
memset(pContext->multiCmds, 0, sizeof(HttpSqlCmds));
@ -161,7 +160,7 @@ bool httpMallocMultiCmds(HttpContext *pContext, int cmdSize, int bufferSize) {
free(multiCmds->cmds);
multiCmds->cmds = (HttpSqlCmd *)malloc((size_t)cmdSize * sizeof(HttpSqlCmd));
if (multiCmds->cmds == NULL) {
httpError("context:%p, fd:%d, ip:%s, user:%s, malloc cmds:%d error", pContext, pContext->fd, pContext->ipstr,
httpError("context:%p, fd:%d, user:%s, malloc cmds:%d error", pContext, pContext->fd,
pContext->user, cmdSize);
return false;
}
@ -172,8 +171,8 @@ bool httpMallocMultiCmds(HttpContext *pContext, int cmdSize, int bufferSize) {
free(multiCmds->buffer);
multiCmds->buffer = (char *)malloc((size_t)bufferSize);
if (multiCmds->buffer == NULL) {
httpError("context:%p, fd:%d, ip:%s, user:%s, malloc buffer:%d error", pContext, pContext->fd, pContext->ipstr,
pContext->user, bufferSize);
httpError("context:%p, fd:%d, user:%s, malloc buffer:%d error", pContext, pContext->fd, pContext->user,
bufferSize);
return false;
}
multiCmds->bufferSize = bufferSize;
@ -187,19 +186,18 @@ bool httpMallocMultiCmds(HttpContext *pContext, int cmdSize, int bufferSize) {
return true;
}
bool httpReMallocMultiCmdsSize(HttpContext *pContext, int cmdSize) {
bool httpReMallocMultiCmdsSize(HttpContext *pContext, int32_t cmdSize) {
HttpSqlCmds *multiCmds = pContext->multiCmds;
if (cmdSize > HTTP_MAX_CMD_SIZE) {
httpError("context:%p, fd:%d, ip:%s, user:%s, mulitcmd size:%d large then %d", pContext, pContext->fd,
pContext->ipstr, pContext->user, cmdSize, HTTP_MAX_CMD_SIZE);
httpError("context:%p, fd:%d, user:%s, mulitcmd size:%d large then %d", pContext, pContext->fd, pContext->user,
cmdSize, HTTP_MAX_CMD_SIZE);
return false;
}
multiCmds->cmds = (HttpSqlCmd *)realloc(multiCmds->cmds, (size_t)cmdSize * sizeof(HttpSqlCmd));
if (multiCmds->cmds == NULL) {
httpError("context:%p, fd:%d, ip:%s, user:%s, malloc cmds:%d error", pContext, pContext->fd, pContext->ipstr,
pContext->user, cmdSize);
httpError("context:%p, fd:%d, user:%s, malloc cmds:%d error", pContext, pContext->fd, pContext->user, cmdSize);
return false;
}
memset(multiCmds->cmds + multiCmds->maxSize, 0, (size_t)(cmdSize - multiCmds->maxSize) * sizeof(HttpSqlCmd));
@ -208,19 +206,18 @@ bool httpReMallocMultiCmdsSize(HttpContext *pContext, int cmdSize) {
return true;
}
bool httpReMallocMultiCmdsBuffer(HttpContext *pContext, int bufferSize) {
bool httpReMallocMultiCmdsBuffer(HttpContext *pContext, int32_t bufferSize) {
HttpSqlCmds *multiCmds = pContext->multiCmds;
if (bufferSize > HTTP_MAX_BUFFER_SIZE) {
httpError("context:%p, fd:%d, ip:%s, user:%s, mulitcmd buffer size:%d large then %d",
pContext, pContext->fd, pContext->ipstr, pContext->user, bufferSize, HTTP_MAX_BUFFER_SIZE);
httpError("context:%p, fd:%d, user:%s, mulitcmd buffer size:%d large then %d", pContext, pContext->fd,
pContext->user, bufferSize, HTTP_MAX_BUFFER_SIZE);
return false;
}
multiCmds->buffer = (char *)realloc(multiCmds->buffer, (size_t)bufferSize);
if (multiCmds->buffer == NULL) {
httpError("context:%p, fd:%d, ip:%s, user:%s, malloc buffer:%d error", pContext, pContext->fd, pContext->ipstr,
pContext->user, bufferSize);
httpError("context:%p, fd:%d, user:%s, malloc buffer:%d error", pContext, pContext->fd, pContext->user, bufferSize);
return false;
}
memset(multiCmds->buffer + multiCmds->bufferSize, 0, (size_t)(bufferSize - multiCmds->bufferSize));
@ -261,7 +258,7 @@ bool httpCompareMethod(HttpDecodeMethod *pSrc, HttpDecodeMethod *pCmp) {
}
void httpAddMethod(HttpServer *pServer, HttpDecodeMethod *pMethod) {
int pos = 0;
int32_t pos = 0;
for (pos = 0; pos < pServer->methodScannerLen; ++pos) {
if (httpCompareMethod(pServer->methodScanner[pos], pMethod)) {
break;
@ -296,13 +293,13 @@ HttpSqlCmd *httpCurrSqlCmd(HttpContext *pContext) {
return multiCmds->cmds + multiCmds->size - 1;
}
int httpNextSqlCmdPos(HttpContext *pContext) {
int32_t httpNextSqlCmdPos(HttpContext *pContext) {
HttpSqlCmds *multiCmds = pContext->multiCmds;
return multiCmds->size;
}
void httpTrimTableName(char *name) {
for (int i = 0; name[i] != 0; i++) {
for (int32_t i = 0; name[i] != 0; i++) {
if (name[i] == ' ' || name[i] == ':' || name[i] == '.' || name[i] == '-' || name[i] == '/' || name[i] == '\'')
name[i] = '_';
if (i == TSDB_TABLE_NAME_LEN) {
@ -312,9 +309,9 @@ void httpTrimTableName(char *name) {
}
}
int httpShrinkTableName(HttpContext *pContext, int pos, char *name) {
int len = 0;
for (int i = 0; name[i] != 0; i++) {
int32_t httpShrinkTableName(HttpContext *pContext, int32_t pos, char *name) {
int32_t len = 0;
for (int32_t i = 0; name[i] != 0; i++) {
if (name[i] == ' ' || name[i] == ':' || name[i] == '.' || name[i] == '-' || name[i] == '/' || name[i] == '\'' ||
name[i] == '\"')
name[i] = '_';
@ -330,7 +327,7 @@ int httpShrinkTableName(HttpContext *pContext, int pos, char *name) {
MD5Update(&context, (uint8_t *)name, (uint32_t)len);
MD5Final(&context);
int table_name = httpAddToSqlCmdBuffer(
int32_t table_name = httpAddToSqlCmdBuffer(
pContext, "%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x", context.digest[0],
context.digest[1], context.digest[2], context.digest[3], context.digest[4], context.digest[5], context.digest[6],
context.digest[7], context.digest[8], context.digest[9], context.digest[10], context.digest[11],
@ -343,7 +340,7 @@ int httpShrinkTableName(HttpContext *pContext, int pos, char *name) {
return table_name;
}
char *httpGetCmdsString(HttpContext *pContext, int pos) {
char *httpGetCmdsString(HttpContext *pContext, int32_t pos) {
HttpSqlCmds *multiCmds = pContext->multiCmds;
if (pos < 0 || pos >= multiCmds->bufferSize) {
return "";
@ -352,8 +349,8 @@ char *httpGetCmdsString(HttpContext *pContext, int pos) {
return multiCmds->buffer + pos;
}
int httpGzipDeCompress(char *srcData, int32_t nSrcData, char *destData, int32_t *nDestData) {
int err = 0;
int32_t httpGzipDeCompress(char *srcData, int32_t nSrcData, char *destData, int32_t *nDestData) {
int32_t err = 0;
z_stream gzipStream = {0};
static char dummyHead[2] = {
@ -396,7 +393,7 @@ int httpGzipDeCompress(char *srcData, int32_t nSrcData, char *destData, int32_t
return 0;
}
int httpGzipCompressInit(HttpContext *pContext) {
int32_t httpGzipCompressInit(HttpContext *pContext) {
pContext->gzipStream.zalloc = (alloc_func) 0;
pContext->gzipStream.zfree = (free_func) 0;
pContext->gzipStream.opaque = (voidpf) 0;
@ -407,8 +404,8 @@ int httpGzipCompressInit(HttpContext *pContext) {
return 0;
}
int httpGzipCompress(HttpContext *pContext, char *srcData, int32_t nSrcData, char *destData, int32_t *nDestData, bool isTheLast) {
int err = 0;
int32_t httpGzipCompress(HttpContext *pContext, char *srcData, int32_t nSrcData, char *destData, int32_t *nDestData, bool isTheLast) {
int32_t err = 0;
pContext->gzipStream.next_in = (Bytef *) srcData;
pContext->gzipStream.avail_in = (uLong) nSrcData;
pContext->gzipStream.next_out = (Bytef *) destData;
@ -442,3 +439,21 @@ int httpGzipCompress(HttpContext *pContext, char *srcData, int32_t nSrcData, cha
*nDestData = (int32_t) (pContext->gzipStream.total_out);
return 0;
}
bool httpUrlMatch(HttpContext* pContext, int32_t pos, char* cmp) {
HttpParser* pParser = pContext->parser;
if (pos < 0 || pos >= HTTP_MAX_URL) {
return false;
}
if (pParser->path[pos].pos <= 0) {
return false;
}
if (strcmp(pParser->path[pos].str, cmp) != 0) {
return false;
}
return true;
}

View File

@ -32,6 +32,7 @@ struct tExprNode;
struct SSchema;
enum {
TSQL_NODE_DUMMY = 0x0,
TSQL_NODE_EXPR = 0x1,
TSQL_NODE_COL = 0x2,
TSQL_NODE_VALUE = 0x4,

View File

@ -57,7 +57,7 @@ typedef struct SWindowResult {
uint16_t numOfRows; // number of rows of current time window
bool closed; // this result status: closed or opened
SResultInfo* resultInfo; // For each result column, there is a resultInfo
TSKEY skey; // start key of current time window
union {STimeWindow win; char* key;}; // start key of current time window
} SWindowResult;
/**
@ -132,12 +132,9 @@ typedef struct SQueryCostInfo {
typedef struct SQuery {
int16_t numOfCols;
int16_t numOfTags;
char intervalTimeUnit;
char slidingTimeUnit; // interval data type, used for daytime revise
SOrderVal order;
STimeWindow window;
int64_t intervalTime;
int64_t slidingTime; // sliding time for sliding window query
SInterval interval;
int16_t precision;
int16_t numOfOutput;
int16_t fillType;

View File

@ -51,12 +51,11 @@ typedef struct SFillInfo {
int32_t rowSize; // size of each row
// char ** pTags; // tags value for current interpolation
SFillTagColInfo* pTags; // tags value for filling gap
int64_t slidingTime; // sliding value to determine the number of result for a given time window
SInterval interval;
char * prevValues; // previous row of data, to generate the interpolation results
char * nextValues; // next row of data
char** pData; // original result data block involved in filling data
int32_t capacityInRows; // data buffer size in rows
int8_t slidingUnit; // sliding time unit
int8_t precision; // time resoluation
SFillColInfo* pFillCol; // column info for fill operations
} SFillInfo;

View File

@ -64,11 +64,11 @@ typedef struct tMemBucket {
__perc_hash_func_t hashFunc;
} tMemBucket;
tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType);
tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval, double maxval);
void tMemBucketDestroy(tMemBucket *pBucket);
void tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size);
int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size);
double getPercentile(tMemBucket *pMemBucket, double percent);

View File

@ -65,6 +65,11 @@ typedef struct tVariantList {
tVariantListItem *a; /* One entry for each expression */
} tVariantList;
typedef struct SIntervalVal {
SStrToken interval;
SStrToken offset;
} SIntervalVal;
typedef struct SQuerySQL {
struct tSQLExprList *pSelection; // select clause
tVariantList * from; // from clause
@ -72,6 +77,7 @@ typedef struct SQuerySQL {
tVariantList * pGroupby; // groupby clause, only for tags[optional]
tVariantList * pSortOrder; // orderby [optional]
SStrToken interval; // interval [optional]
SStrToken offset; // offset window [optional]
SStrToken sliding; // sliding window [optional]
SLimitVal limit; // limit offset [optional]
SLimitVal slimit; // group limit offset [optional]
@ -259,7 +265,7 @@ tSQLExprList *tSQLExprListAppend(tSQLExprList *pList, tSQLExpr *pNode, SStrToken
void tSQLExprListDestroy(tSQLExprList *pList);
SQuerySQL *tSetQuerySQLElems(SStrToken *pSelectToken, tSQLExprList *pSelection, tVariantList *pFrom, tSQLExpr *pWhere,
tVariantList *pGroupby, tVariantList *pSortOrder, SStrToken *pInterval,
tVariantList *pGroupby, tVariantList *pSortOrder, SIntervalVal *pInterval,
SStrToken *pSliding, tVariantList *pFill, SLimitVal *pLimit, SLimitVal *pGLimit);
SCreateTableSQL *tSetCreateSQLElems(tFieldList *pCols, tFieldList *pTags, SStrToken *pMetricName,

View File

@ -50,14 +50,16 @@ static FORCE_INLINE char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int3
tFilePage* page) {
assert(pResult != NULL && pRuntimeEnv != NULL);
SQuery *pQuery = pRuntimeEnv->pQuery;
// tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pos.pageId);
SQuery *pQuery = pRuntimeEnv->pQuery;
int32_t realRowId = (int32_t)(pResult->pos.rowId * GET_ROW_PARAM_FOR_MULTIOUTPUT(pQuery, pRuntimeEnv->topBotQuery, pRuntimeEnv->stableQuery));
return ((char *)page->data) + pRuntimeEnv->offset[columnIndex] * pRuntimeEnv->numOfRowsPerPage +
pQuery->pSelectExpr[columnIndex].bytes * realRowId;
}
bool isNull_filter(SColumnFilterElem *pFilter, char* minval, char* maxval);
bool notNull_filter(SColumnFilterElem *pFilter, char* minval, char* maxval);
__filter_func_t *getRangeFilterFuncArray(int32_t type);
__filter_func_t *getValueFilterFuncArray(int32_t type);

View File

@ -458,9 +458,10 @@ tablelist(A) ::= tablelist(Y) COMMA ids(X) cpxName(Z) ids(F). {
%type tmvar {SStrToken}
tmvar(A) ::= VARIABLE(X). {A = X;}
%type interval_opt {SStrToken}
interval_opt(N) ::= INTERVAL LP tmvar(E) RP. {N = E; }
interval_opt(N) ::= . {N.n = 0; N.z = NULL; N.type = 0; }
%type interval_opt {SIntervalVal}
interval_opt(N) ::= INTERVAL LP tmvar(E) RP. {N.interval = E; N.offset.n = 0; N.offset.z = NULL; N.offset.type = 0;}
interval_opt(N) ::= INTERVAL LP tmvar(E) COMMA tmvar(O) RP. {N.interval = E; N.offset = O;}
interval_opt(N) ::= . {memset(&N, 0, sizeof(N));}
%type fill_opt {tVariantList*}
%destructor fill_opt {tVariantListDestroy($$);}
@ -567,53 +568,53 @@ where_opt(A) ::= WHERE expr(X). {A = X;}
%type expr {tSQLExpr*}
%destructor expr {tSQLExprDestroy($$);}
expr(A) ::= LP expr(X) RP. {A = X; }
expr(A) ::= LP expr(X) RP. {A = X; }
expr(A) ::= ID(X). {A = tSQLExprIdValueCreate(&X, TK_ID);}
expr(A) ::= ID(X) DOT ID(Y). {X.n += (1+Y.n); A = tSQLExprIdValueCreate(&X, TK_ID);}
expr(A) ::= ID(X) DOT STAR(Y). {X.n += (1+Y.n); A = tSQLExprIdValueCreate(&X, TK_ALL);}
expr(A) ::= ID(X). {A = tSQLExprIdValueCreate(&X, TK_ID);}
expr(A) ::= ID(X) DOT ID(Y). {X.n += (1+Y.n); A = tSQLExprIdValueCreate(&X, TK_ID);}
expr(A) ::= ID(X) DOT STAR(Y). {X.n += (1+Y.n); A = tSQLExprIdValueCreate(&X, TK_ALL);}
expr(A) ::= INTEGER(X). {A = tSQLExprIdValueCreate(&X, TK_INTEGER);}
expr(A) ::= INTEGER(X). {A = tSQLExprIdValueCreate(&X, TK_INTEGER);}
expr(A) ::= MINUS(X) INTEGER(Y). {X.n += Y.n; X.type = TK_INTEGER; A = tSQLExprIdValueCreate(&X, TK_INTEGER);}
expr(A) ::= PLUS(X) INTEGER(Y). {X.n += Y.n; X.type = TK_INTEGER; A = tSQLExprIdValueCreate(&X, TK_INTEGER);}
expr(A) ::= FLOAT(X). {A = tSQLExprIdValueCreate(&X, TK_FLOAT);}
expr(A) ::= MINUS(X) FLOAT(Y). {X.n += Y.n; X.type = TK_FLOAT; A = tSQLExprIdValueCreate(&X, TK_FLOAT);}
expr(A) ::= PLUS(X) FLOAT(Y). {X.n += Y.n; X.type = TK_FLOAT; A = tSQLExprIdValueCreate(&X, TK_FLOAT);}
expr(A) ::= STRING(X). {A = tSQLExprIdValueCreate(&X, TK_STRING);}
expr(A) ::= NOW(X). {A = tSQLExprIdValueCreate(&X, TK_NOW); }
expr(A) ::= VARIABLE(X). {A = tSQLExprIdValueCreate(&X, TK_VARIABLE);}
expr(A) ::= BOOL(X). {A = tSQLExprIdValueCreate(&X, TK_BOOL);}
// normal functions: min(x)
expr(A) ::= ID(X) LP exprlist(Y) RP(E). {
A = tSQLExprCreateFunction(Y, &X, &E, X.type);
}
expr(A) ::= FLOAT(X). {A = tSQLExprIdValueCreate(&X, TK_FLOAT);}
expr(A) ::= MINUS(X) FLOAT(Y). {X.n += Y.n; X.type = TK_FLOAT; A = tSQLExprIdValueCreate(&X, TK_FLOAT);}
expr(A) ::= PLUS(X) FLOAT(Y). {X.n += Y.n; X.type = TK_FLOAT; A = tSQLExprIdValueCreate(&X, TK_FLOAT);}
expr(A) ::= STRING(X). {A = tSQLExprIdValueCreate(&X, TK_STRING);}
expr(A) ::= NOW(X). {A = tSQLExprIdValueCreate(&X, TK_NOW); }
expr(A) ::= VARIABLE(X). {A = tSQLExprIdValueCreate(&X, TK_VARIABLE);}
expr(A) ::= BOOL(X). {A = tSQLExprIdValueCreate(&X, TK_BOOL);}
// this is for: count(*)/first(*)/last(*) operation
expr(A) ::= ID(X) LP STAR RP(Y). {
A = tSQLExprCreateFunction(NULL, &X, &Y, X.type);
}
// ordinary functions: min(x), max(x), top(k, 20)
expr(A) ::= ID(X) LP exprlist(Y) RP(E). { A = tSQLExprCreateFunction(Y, &X, &E, X.type); }
//binary expression: a+2, b+3
expr(A) ::= expr(X) AND expr(Y). {A = tSQLExprCreate(X, Y, TK_AND);}
expr(A) ::= expr(X) OR expr(Y). {A = tSQLExprCreate(X, Y, TK_OR); }
// for parsing sql functions with wildcard for parameters. e.g., count(*)/first(*)/last(*) operation
expr(A) ::= ID(X) LP STAR RP(Y). { A = tSQLExprCreateFunction(NULL, &X, &Y, X.type); }
//binary relational expression
expr(A) ::= expr(X) LT expr(Y). {A = tSQLExprCreate(X, Y, TK_LT);}
expr(A) ::= expr(X) GT expr(Y). {A = tSQLExprCreate(X, Y, TK_GT);}
expr(A) ::= expr(X) LE expr(Y). {A = tSQLExprCreate(X, Y, TK_LE);}
expr(A) ::= expr(X) GE expr(Y). {A = tSQLExprCreate(X, Y, TK_GE);}
expr(A) ::= expr(X) NE expr(Y). {A = tSQLExprCreate(X, Y, TK_NE);}
expr(A) ::= expr(X) EQ expr(Y). {A = tSQLExprCreate(X, Y, TK_EQ);}
// is (not) null expression
expr(A) ::= expr(X) IS NULL. {A = tSQLExprCreate(X, NULL, TK_ISNULL);}
expr(A) ::= expr(X) IS NOT NULL. {A = tSQLExprCreate(X, NULL, TK_NOTNULL);}
//binary arithmetic expression
// relational expression
expr(A) ::= expr(X) LT expr(Y). {A = tSQLExprCreate(X, Y, TK_LT);}
expr(A) ::= expr(X) GT expr(Y). {A = tSQLExprCreate(X, Y, TK_GT);}
expr(A) ::= expr(X) LE expr(Y). {A = tSQLExprCreate(X, Y, TK_LE);}
expr(A) ::= expr(X) GE expr(Y). {A = tSQLExprCreate(X, Y, TK_GE);}
expr(A) ::= expr(X) NE expr(Y). {A = tSQLExprCreate(X, Y, TK_NE);}
expr(A) ::= expr(X) EQ expr(Y). {A = tSQLExprCreate(X, Y, TK_EQ);}
expr(A) ::= expr(X) AND expr(Y). {A = tSQLExprCreate(X, Y, TK_AND);}
expr(A) ::= expr(X) OR expr(Y). {A = tSQLExprCreate(X, Y, TK_OR); }
// binary arithmetic expression
expr(A) ::= expr(X) PLUS expr(Y). {A = tSQLExprCreate(X, Y, TK_PLUS); }
expr(A) ::= expr(X) MINUS expr(Y). {A = tSQLExprCreate(X, Y, TK_MINUS); }
expr(A) ::= expr(X) STAR expr(Y). {A = tSQLExprCreate(X, Y, TK_STAR); }
expr(A) ::= expr(X) SLASH expr(Y). {A = tSQLExprCreate(X, Y, TK_DIVIDE);}
expr(A) ::= expr(X) REM expr(Y). {A = tSQLExprCreate(X, Y, TK_REM); }
//like expression
expr(A) ::= expr(X) LIKE expr(Y). {A = tSQLExprCreate(X, Y, TK_LIKE); }
// like expression
expr(A) ::= expr(X) LIKE expr(Y). {A = tSQLExprCreate(X, Y, TK_LIKE); }
//in expression
expr(A) ::= expr(X) IN LP exprlist(Y) RP. {A = tSQLExprCreate(X, (tSQLExpr*)Y, TK_IN); }
@ -625,9 +626,9 @@ expr(A) ::= expr(X) IN LP exprlist(Y) RP. {A = tSQLExprCreate(X, (tSQLExpr*)Y,
%destructor expritem {tSQLExprDestroy($$);}
exprlist(A) ::= exprlist(X) COMMA expritem(Y). {A = tSQLExprListAppend(X,Y,0);}
exprlist(A) ::= expritem(X). {A = tSQLExprListAppend(0,X,0);}
expritem(A) ::= expr(X). {A = X;}
expritem(A) ::= . {A = 0;}
exprlist(A) ::= expritem(X). {A = tSQLExprListAppend(0,X,0);}
expritem(A) ::= expr(X). {A = X;}
expritem(A) ::= . {A = 0;}
///////////////////////////////////reset query cache//////////////////////////////////////
cmd ::= RESET QUERY CACHE. { setDCLSQLElems(pInfo, TSDB_SQL_RESET_CACHE, 0);}

View File

@ -177,7 +177,7 @@ typedef struct SQLFunctionCtx {
int16_t outputType;
int16_t outputBytes; // size of results, determined by function and input column data type
bool hasNull; // null value exist in current block
bool requireNull; // require null in some function
bool requireNull; // require null in some function
int16_t functionId; // function id
void * aInputElemBuf;
char * aOutputBuf; // final result output buffer, point to sdata->data

View File

@ -188,6 +188,10 @@ uint8_t getBinaryExprOptr(SStrToken *pToken) {
return TSDB_BINARY_OP_REMAINDER;
case TK_LIKE:
return TSDB_RELATION_LIKE;
case TK_ISNULL:
return TSDB_RELATION_ISNULL;
case TK_NOTNULL:
return TSDB_RELATION_NOTNULL;
default: { return 0; }
}
}
@ -486,29 +490,42 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr
} else {
int32_t optr = cond.end ? cond.end->optr : TSDB_RELATION_INVALID;
if (optr == TSDB_RELATION_LESS || optr == TSDB_RELATION_LESS_EQUAL) {
bool comp = true;
bool comp = true;
int32_t ret = 0;
while(tSkipListIterNext(iter)) {
SSkipListNode* pNode = tSkipListIterGet(iter);
while (tSkipListIterNext(iter)) {
SSkipListNode *pNode = tSkipListIterGet(iter);
if (comp) {
ret = pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.end->v);
assert(ret <= 0);
}
if (ret == 0 && optr == TSDB_RELATION_LESS) {
continue;
} else {
STableKeyInfo info = {.pTable = *(void**)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
STableKeyInfo info = {.pTable = *(void **)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
taosArrayPush(result, &info);
comp = false; // no need to compare anymore
}
}
} else {
assert(pQueryInfo->optr == TSDB_RELATION_ISNULL || pQueryInfo->optr == TSDB_RELATION_NOTNULL);
while (tSkipListIterNext(iter)) {
SSkipListNode *pNode = tSkipListIterGet(iter);
bool isnull = isNull(SL_GET_NODE_KEY(pSkipList, pNode), pQueryInfo->sch.type);
if ((pQueryInfo->optr == TSDB_RELATION_ISNULL && isnull) ||
(pQueryInfo->optr == TSDB_RELATION_NOTNULL && (!isnull))) {
STableKeyInfo info = {.pTable = *(void **)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL};
taosArrayPush(result, &info);
}
}
}
}
free(cond.start);
free(cond.start);
free(cond.end);
tSkipListDestroyIter(iter);
}
@ -683,6 +700,7 @@ static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo,
char * pData = SL_GET_NODE_DATA(pNode);
tstr *name = (tstr*) tsdbGetTableName(*(void**) pData);
// todo speed up by using hash
if (pQueryInfo->sch.colId == TSDB_TBNAME_COLUMN_INDEX) {
if (pQueryInfo->optr == TSDB_RELATION_IN) {
@ -714,7 +732,7 @@ void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, S
// column project
if (pLeft->nodeType != TSQL_NODE_EXPR && pRight->nodeType != TSQL_NODE_EXPR) {
assert(pLeft->nodeType == TSQL_NODE_COL && pRight->nodeType == TSQL_NODE_VALUE);
assert(pLeft->nodeType == TSQL_NODE_COL && (pRight->nodeType == TSQL_NODE_VALUE || pRight->nodeType == TSQL_NODE_DUMMY));
param->setupInfoFn(pExpr, param->pExtInfo);
if (pSkipList == NULL) {

View File

@ -131,21 +131,21 @@ static UNUSED_FUNC void* u_realloc(void* p, size_t __size) {
static void setQueryStatus(SQuery *pQuery, int8_t status);
static void finalizeQueryResult(SQueryRuntimeEnv *pRuntimeEnv);
#define QUERY_IS_INTERVAL_QUERY(_q) ((_q)->intervalTime > 0)
#define QUERY_IS_INTERVAL_QUERY(_q) ((_q)->interval.interval > 0)
static void getNextTimeWindow(SQuery* pQuery, STimeWindow* tw) {
int32_t factor = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order);
if (pQuery->intervalTimeUnit != 'n' && pQuery->intervalTimeUnit != 'y') {
tw->skey += pQuery->slidingTime * factor;
tw->ekey = tw->skey + pQuery->intervalTime - 1;
if (pQuery->interval.intervalUnit != 'n' && pQuery->interval.intervalUnit != 'y') {
tw->skey += pQuery->interval.sliding * factor;
tw->ekey = tw->skey + pQuery->interval.interval - 1;
return;
}
int64_t key = tw->skey / 1000, interval = pQuery->intervalTime;
int64_t key = tw->skey / 1000, interval = pQuery->interval.interval;
if (pQuery->precision == TSDB_TIME_PRECISION_MICRO) {
key /= 1000;
}
if (pQuery->intervalTimeUnit == 'y') {
if (pQuery->interval.intervalUnit == 'y') {
interval *= 12;
}
@ -200,14 +200,28 @@ bool doFilterData(SQuery *pQuery, int32_t elemPos) {
SSingleColumnFilterInfo *pFilterInfo = &pQuery->pFilterInfo[k];
char *pElem = (char*)pFilterInfo->pData + pFilterInfo->info.bytes * elemPos;
if (isNull(pElem, pFilterInfo->info.type)) {
return false;
}
bool qualified = false;
for (int32_t j = 0; j < pFilterInfo->numOfFilters; ++j) {
SColumnFilterElem *pFilterElem = &pFilterInfo->pFilters[j];
bool isnull = isNull(pElem, pFilterInfo->info.type);
if (isnull) {
if (pFilterElem->fp == isNull_filter) {
qualified = true;
break;
} else {
continue;
}
} else {
if (pFilterElem->fp == notNull_filter) {
qualified = true;
break;
} else if (pFilterElem->fp == isNull_filter) {
continue;
}
}
if (pFilterElem->fp(pFilterElem, pElem, pElem)) {
qualified = true;
break;
@ -496,35 +510,35 @@ static STimeWindow getActiveTimeWindow(SWindowResInfo *pWindowResInfo, int64_t t
if (pWindowResInfo->curIndex == -1) { // the first window, from the previous stored value
w.skey = pWindowResInfo->prevSKey;
if (pQuery->intervalTimeUnit == 'n' || pQuery->intervalTimeUnit == 'y') {
w.ekey = taosAddNatualInterval(w.skey, pQuery->intervalTime, pQuery->intervalTimeUnit, pQuery->precision) - 1;
if (pQuery->interval.intervalUnit == 'n' || pQuery->interval.intervalUnit == 'y') {
w.ekey = taosTimeAdd(w.skey, pQuery->interval.interval, pQuery->interval.intervalUnit, pQuery->precision);
} else {
w.ekey = w.skey + pQuery->intervalTime - 1;
w.ekey = w.skey + pQuery->interval.interval - 1;
}
} else {
int32_t slot = curTimeWindowIndex(pWindowResInfo);
SWindowResult* pWindowRes = getWindowResult(pWindowResInfo, slot);
w = GET_TIMEWINDOW(pWindowResInfo, pWindowRes);
w = pWindowRes->win;
}
if (w.skey > ts || w.ekey < ts) {
if (pQuery->intervalTimeUnit == 'n' || pQuery->intervalTimeUnit == 'y') {
w.skey = taosGetIntervalStartTimestamp(ts, pQuery->slidingTime, pQuery->intervalTime, pQuery->intervalTimeUnit, pQuery->precision);
w.ekey = taosAddNatualInterval(w.skey, pQuery->intervalTime, pQuery->intervalTimeUnit, pQuery->precision) - 1;
if (pQuery->interval.intervalUnit == 'n' || pQuery->interval.intervalUnit == 'y') {
w.skey = taosTimeTruncate(ts, &pQuery->interval, pQuery->precision);
w.ekey = taosTimeAdd(w.skey, pQuery->interval.interval, pQuery->interval.intervalUnit, pQuery->precision) - 1;
} else {
int64_t st = w.skey;
if (st > ts) {
st -= ((st - ts + pQuery->slidingTime - 1) / pQuery->slidingTime) * pQuery->slidingTime;
st -= ((st - ts + pQuery->interval.sliding - 1) / pQuery->interval.sliding) * pQuery->interval.sliding;
}
int64_t et = st + pQuery->intervalTime - 1;
int64_t et = st + pQuery->interval.interval - 1;
if (et < ts) {
st += ((ts - et + pQuery->slidingTime - 1) / pQuery->slidingTime) * pQuery->slidingTime;
st += ((ts - et + pQuery->interval.sliding - 1) / pQuery->interval.sliding) * pQuery->interval.sliding;
}
w.skey = st;
w.ekey = w.skey + pQuery->intervalTime - 1;
w.ekey = w.skey + pQuery->interval.interval - 1;
}
}
@ -536,8 +550,6 @@ static STimeWindow getActiveTimeWindow(SWindowResInfo *pWindowResInfo, int64_t t
w.ekey = pQuery->window.ekey;
}
assert(ts >= w.skey && ts <= w.ekey);
return w;
}
@ -610,7 +622,7 @@ static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowRes
}
// set time window for current result
pWindowRes->skey = win->skey;
pWindowRes->win = (*win);
setWindowResOutputBufInitCtx(pRuntimeEnv, pWindowRes);
return TSDB_CODE_SUCCESS;
@ -683,12 +695,12 @@ static int32_t doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKe
continue;
}
TSKEY ekey = pResult->skey + pWindowResInfo->interval;
TSKEY ekey = pResult->win.ekey;
if ((ekey <= lastKey && QUERY_IS_ASC_QUERY(pQuery)) ||
(pResult->skey >= lastKey && !QUERY_IS_ASC_QUERY(pQuery))) {
(pResult->win.skey >= lastKey && !QUERY_IS_ASC_QUERY(pQuery))) {
closeTimeWindow(pWindowResInfo, i);
} else {
skey = pResult->skey;
skey = pResult->win.skey;
break;
}
}
@ -701,7 +713,7 @@ static int32_t doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKe
pWindowResInfo->curIndex = i;
}
pWindowResInfo->prevSKey = pWindowResInfo->pResult[pWindowResInfo->curIndex].skey;
pWindowResInfo->prevSKey = pWindowResInfo->pResult[pWindowResInfo->curIndex].win.skey;
// the number of completed slots are larger than the threshold, return current generated results to client.
if (numOfClosed > pWindowResInfo->threshold) {
@ -729,9 +741,9 @@ static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SDataBlockInfo *pDataBlo
int32_t startPos, TSKEY ekey, __block_search_fn_t searchFn, bool updateLastKey) {
assert(startPos >= 0 && startPos < pDataBlockInfo->rows);
int32_t num = -1;
int32_t num = -1;
int32_t order = pQuery->order.order;
int32_t step = GET_FORWARD_DIRECTION_FACTOR(order);
int32_t step = GET_FORWARD_DIRECTION_FACTOR(order);
STableQueryInfo* item = pQuery->current;
@ -765,27 +777,24 @@ static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SDataBlockInfo *pDataBlo
return num;
}
static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, bool closed, STimeWindow *pWin,
int32_t offset, int32_t forwardStep, TSKEY *tsBuf, int32_t numOfTotal) {
static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, bool closed, STimeWindow *pWin, int32_t offset,
int32_t forwardStep, TSKEY *tsCol, int32_t numOfTotal) {
SQuery * pQuery = pRuntimeEnv->pQuery;
SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx;
if (IS_MASTER_SCAN(pRuntimeEnv) || closed) {
for (int32_t k = 0; k < pQuery->numOfOutput; ++k) {
int32_t functionId = pQuery->pSelectExpr[k].base.functionId;
pCtx[k].nStartQueryTimestamp = pWin->skey;
pCtx[k].size = forwardStep;
pCtx[k].startOffset = (QUERY_IS_ASC_QUERY(pQuery)) ? offset : offset - (forwardStep - 1);
int32_t functionId = pQuery->pSelectExpr[k].base.functionId;
if ((aAggs[functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) != 0) {
pCtx[k].ptsList = &tsBuf[offset];
pCtx[k].ptsList = &tsCol[pCtx[k].startOffset];
}
// not a whole block involved in query processing, statistics data can not be used
if (forwardStep != numOfTotal) {
pCtx[k].preAggVals.isSet = false;
}
pCtx[k].preAggVals.isSet = (forwardStep == numOfTotal);
if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) {
aAggs[functionId].xFunction(&pCtx[k]);
@ -837,7 +846,7 @@ static int32_t getNextQualifiedWindow(SQueryRuntimeEnv *pRuntimeEnv, STimeWindow
int32_t startPos = 0;
// tumbling time window query, a special case of sliding time window query
if (pQuery->slidingTime == pQuery->intervalTime && prevPosition != -1) {
if (pQuery->interval.sliding == pQuery->interval.interval && prevPosition != -1) {
int32_t factor = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order);
startPos = prevPosition + factor;
} else {
@ -850,21 +859,21 @@ static int32_t getNextQualifiedWindow(SQueryRuntimeEnv *pRuntimeEnv, STimeWindow
*/
if (QUERY_IS_ASC_QUERY(pQuery) && primaryKeys[startPos] > pNext->ekey) {
TSKEY next = primaryKeys[startPos];
if (pQuery->intervalTimeUnit == 'n' || pQuery->intervalTimeUnit == 'y') {
pNext->skey = taosGetIntervalStartTimestamp(next, pQuery->slidingTime, pQuery->intervalTime, pQuery->intervalTimeUnit, pQuery->precision);
pNext->ekey = taosAddNatualInterval(pNext->skey, pQuery->intervalTime, pQuery->intervalTimeUnit, pQuery->precision) - 1;
if (pQuery->interval.intervalUnit == 'n' || pQuery->interval.intervalUnit == 'y') {
pNext->skey = taosTimeTruncate(next, &pQuery->interval, pQuery->precision);
pNext->ekey = taosTimeAdd(pNext->skey, pQuery->interval.interval, pQuery->interval.intervalUnit, pQuery->precision) - 1;
} else {
pNext->ekey += ((next - pNext->ekey + pQuery->slidingTime - 1)/pQuery->slidingTime) * pQuery->slidingTime;
pNext->skey = pNext->ekey - pQuery->intervalTime + 1;
pNext->ekey += ((next - pNext->ekey + pQuery->interval.sliding - 1)/pQuery->interval.sliding) * pQuery->interval.sliding;
pNext->skey = pNext->ekey - pQuery->interval.interval + 1;
}
} else if ((!QUERY_IS_ASC_QUERY(pQuery)) && primaryKeys[startPos] < pNext->skey) {
TSKEY next = primaryKeys[startPos];
if (pQuery->intervalTimeUnit == 'n' || pQuery->intervalTimeUnit == 'y') {
pNext->skey = taosGetIntervalStartTimestamp(next, pQuery->slidingTime, pQuery->intervalTime, pQuery->intervalTimeUnit, pQuery->precision);
pNext->ekey = taosAddNatualInterval(pNext->skey, pQuery->intervalTime, pQuery->intervalTimeUnit, pQuery->precision) - 1;
if (pQuery->interval.intervalUnit == 'n' || pQuery->interval.intervalUnit == 'y') {
pNext->skey = taosTimeTruncate(next, &pQuery->interval, pQuery->precision);
pNext->ekey = taosTimeAdd(pNext->skey, pQuery->interval.interval, pQuery->interval.intervalUnit, pQuery->precision) - 1;
} else {
pNext->skey -= ((pNext->skey - next + pQuery->slidingTime - 1) / pQuery->slidingTime) * pQuery->slidingTime;
pNext->ekey = pNext->skey + pQuery->intervalTime - 1;
pNext->skey -= ((pNext->skey - next + pQuery->interval.sliding - 1) / pQuery->interval.sliding) * pQuery->interval.sliding;
pNext->ekey = pNext->skey + pQuery->interval.interval - 1;
}
}
@ -910,19 +919,11 @@ static char *getDataBlock(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas
char *dataBlock = NULL;
SQuery *pQuery = pRuntimeEnv->pQuery;
SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx;
int32_t functionId = pQuery->pSelectExpr[col].base.functionId;
if (functionId == TSDB_FUNC_ARITHM) {
sas->pArithExpr = &pQuery->pSelectExpr[col];
// set the start offset to be the lowest start position, no matter asc/desc query order
if (QUERY_IS_ASC_QUERY(pQuery)) {
pCtx->startOffset = pQuery->pos;
} else {
pCtx->startOffset = pQuery->pos - (size - 1);
}
sas->offset = 0;
sas->colList = pQuery->colList;
sas->numOfCols = pQuery->numOfCols;
@ -1002,7 +1003,7 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *
}
int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order);
if (QUERY_IS_INTERVAL_QUERY(pQuery)/* && tsCols != NULL*/) {
if (QUERY_IS_INTERVAL_QUERY(pQuery)) {
TSKEY ts = TSKEY_INITIAL_VAL;
if (tsCols == NULL) {
@ -1094,8 +1095,25 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat
SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf;
int64_t v = -1;
// not assign result buffer yet, add new result buffer
char* d = pData;
int16_t len = bytes;
if (type == TSDB_DATA_TYPE_BINARY||type == TSDB_DATA_TYPE_NCHAR) {
d = varDataVal(pData);
len = varDataLen(pData);
} else if (type == TSDB_DATA_TYPE_FLOAT || type == TSDB_DATA_TYPE_DOUBLE) {
SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv);
qError("QInfo:%p group by not supported on double/float/binary/nchar columns, abort", pQInfo);
longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR);
}
SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, d, len, true);
if (pWindowRes == NULL) {
return -1;
}
int64_t v = -1;
switch(type) {
case TSDB_DATA_TYPE_BOOL:
case TSDB_DATA_TYPE_TINYINT: v = GET_INT8_VAL(pData); break;
@ -1104,12 +1122,14 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat
case TSDB_DATA_TYPE_BIGINT: v = GET_INT64_VAL(pData); break;
}
SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, pData, bytes, true);
if (pWindowRes == NULL) {
return -1;
if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
pWindowRes->key = malloc(varDataTLen(pData));
varDataCopy(pWindowRes->key, pData);
} else {
pWindowRes->win.skey = v;
pWindowRes->win.ekey = v;
}
pWindowRes->skey = v;
assert(pRuntimeEnv->windowResInfo.interval == 0);
if (pWindowRes->pos.pageId == -1) {
@ -1180,7 +1200,7 @@ static int32_t doTSJoinFilter(SQueryRuntimeEnv *pRuntimeEnv, int32_t offset) {
#if defined(_DEBUG_VIEW)
printf("elem in comp ts file:%" PRId64 ", key:%" PRId64 ", tag:%"PRIu64", query order:%d, ts order:%d, traverse:%d, index:%d\n",
elem.ts, key, elem.tag, pQuery->order.order, pRuntimeEnv->pTSBuf->tsOrder,
elem.ts, key, elem.tag.i64Key, pQuery->order.order, pRuntimeEnv->pTSBuf->tsOrder,
pRuntimeEnv->pTSBuf->cur.order, pRuntimeEnv->pTSBuf->cur.tsIndex);
#endif
@ -1461,12 +1481,15 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY
pCtx->preAggVals.dataBlockLoaded = (inputData != NULL);
// limit/offset query will affect this value
pCtx->startOffset = QUERY_IS_ASC_QUERY(pQuery) ? pQuery->pos:0;
pCtx->size = QUERY_IS_ASC_QUERY(pQuery) ? pBlockInfo->rows - pQuery->pos : pQuery->pos + 1;
// minimum value no matter ascending/descending order query
pCtx->startOffset = QUERY_IS_ASC_QUERY(pQuery) ? pQuery->pos: (pQuery->pos - pCtx->size + 1);
assert(pCtx->startOffset >= 0);
uint32_t status = aAggs[functionId].nStatus;
if (((status & (TSDB_FUNCSTATE_SELECTIVITY | TSDB_FUNCSTATE_NEED_TS)) != 0) && (tsCol != NULL)) {
pCtx->ptsList = tsCol;
pCtx->ptsList = &tsCol[pCtx->startOffset];
}
if (functionId >= TSDB_FUNC_FIRST_DST && functionId <= TSDB_FUNC_LAST_DST) {
@ -1846,20 +1869,20 @@ static bool onlyQueryTags(SQuery* pQuery) {
/////////////////////////////////////////////////////////////////////////////////////////////
void getAlignQueryTimeWindow(SQuery *pQuery, int64_t key, int64_t keyFirst, int64_t keyLast, STimeWindow *win) {
assert(key >= keyFirst && key <= keyLast && pQuery->slidingTime <= pQuery->intervalTime);
win->skey = taosGetIntervalStartTimestamp(key, pQuery->slidingTime, pQuery->intervalTime, pQuery->slidingTimeUnit, pQuery->precision);
assert(key >= keyFirst && key <= keyLast && pQuery->interval.sliding <= pQuery->interval.interval);
win->skey = taosTimeTruncate(key, &pQuery->interval, pQuery->precision);
/*
* if the realSkey > INT64_MAX - pQuery->intervalTime, the query duration between
* if the realSkey > INT64_MAX - pQuery->interval.interval, the query duration between
* realSkey and realEkey must be less than one interval.Therefore, no need to adjust the query ranges.
*/
if (keyFirst > (INT64_MAX - pQuery->intervalTime)) {
assert(keyLast - keyFirst < pQuery->intervalTime);
if (keyFirst > (INT64_MAX - pQuery->interval.interval)) {
assert(keyLast - keyFirst < pQuery->interval.interval);
win->ekey = INT64_MAX;
} else if (pQuery->intervalTimeUnit == 'n' || pQuery->intervalTimeUnit == 'y') {
win->ekey = taosAddNatualInterval(win->skey, pQuery->intervalTime, pQuery->intervalTimeUnit, pQuery->precision) - 1;
} else if (pQuery->interval.intervalUnit == 'n' || pQuery->interval.intervalUnit == 'y') {
win->ekey = taosTimeAdd(win->skey, pQuery->interval.interval, pQuery->interval.intervalUnit, pQuery->precision) - 1;
} else {
win->ekey = win->skey + pQuery->intervalTime - 1;
win->ekey = win->skey + pQuery->interval.interval - 1;
}
}
@ -1951,40 +1974,40 @@ static void changeExecuteScanOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bo
// todo handle the case the the order irrelevant query type mixed up with order critical query type
// descending order query for last_row query
if (isFirstLastRowQuery(pQuery) && !QUERY_IS_ASC_QUERY(pQuery)) {
if (isFirstLastRowQuery(pQuery)) {
qDebug("QInfo:%p scan order changed for last_row query, old:%d, new:%d", GET_QINFO_ADDR(pQuery),
pQuery->order.order, TSDB_ORDER_ASC);
SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY);
pQuery->order.order = TSDB_ORDER_ASC;
assert (pQuery->window.skey <= pQuery->window.ekey);
if (pQuery->window.skey > pQuery->window.ekey) {
SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY);
}
return;
}
if (isGroupbyNormalCol(pQuery->pGroupbyExpr) && pQuery->order.order == TSDB_ORDER_DESC) {
pQuery->order.order = TSDB_ORDER_ASC;
if (pQuery->window.skey > pQuery->window.ekey) {
SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY);
}
doExchangeTimeWindow(pQInfo, &pQuery->window);
return;
}
if (isGroupbyNormalCol(pQuery->pGroupbyExpr) && !QUERY_IS_ASC_QUERY(pQuery)) {
pQuery->order.order = TSDB_ORDER_ASC;
SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY);
assert (pQuery->window.skey <= pQuery->window.ekey);
if (isPointInterpoQuery(pQuery) && pQuery->interval.interval == 0) {
if (!QUERY_IS_ASC_QUERY(pQuery)) {
qDebug(msg, GET_QINFO_ADDR(pQuery), "interp", pQuery->order.order, TSDB_ORDER_ASC, pQuery->window.skey,
pQuery->window.ekey, pQuery->window.ekey, pQuery->window.skey);
SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY);
}
doExchangeTimeWindow(pQInfo, &pQuery->window);
pQuery->order.order = TSDB_ORDER_ASC;
return;
}
if (isPointInterpoQuery(pQuery) && (pQuery->intervalTime == 0) && !QUERY_IS_ASC_QUERY(pQuery)) {
qDebug(msg, GET_QINFO_ADDR(pQuery), "interp", pQuery->order.order, TSDB_ORDER_ASC, pQuery->window.skey,
pQuery->window.ekey, pQuery->window.ekey, pQuery->window.skey);
SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY);
pQuery->order.order = TSDB_ORDER_ASC;
assert (pQuery->window.skey <= pQuery->window.ekey);
doExchangeTimeWindow(pQInfo, &pQuery->window);
return;
}
if (pQuery->intervalTime == 0) {
if (pQuery->interval.interval == 0) {
if (onlyFirstQuery(pQuery)) {
if (!QUERY_IS_ASC_QUERY(pQuery)) {
qDebug(msg, GET_QINFO_ADDR(pQuery), "only-first", pQuery->order.order, TSDB_ORDER_ASC, pQuery->window.skey,
@ -2185,43 +2208,43 @@ static bool overlapWithTimeWindow(SQuery* pQuery, SDataBlockInfo* pBlockInfo) {
return false;
}
int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, void* pQueryHandle, SDataBlockInfo* pBlockInfo, SDataStatis **pStatis, SArray** pDataBlock) {
int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, void* pQueryHandle, SDataBlockInfo* pBlockInfo, SDataStatis **pStatis, SArray** pDataBlock, uint32_t* status) {
SQuery *pQuery = pRuntimeEnv->pQuery;
uint32_t status = 0;
*status = 0;
if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf > 0) {
status = BLK_DATA_ALL_NEEDED;
*status = BLK_DATA_ALL_NEEDED;
} else { // check if this data block is required to load
// Calculate all time windows that are overlapping or contain current data block.
// If current data block is contained by all possible time window, do not load current data block.
if (QUERY_IS_INTERVAL_QUERY(pQuery) && overlapWithTimeWindow(pQuery, pBlockInfo)) {
status = BLK_DATA_ALL_NEEDED;
*status = BLK_DATA_ALL_NEEDED;
}
if (status != BLK_DATA_ALL_NEEDED) {
if ((*status) != BLK_DATA_ALL_NEEDED) {
for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
SSqlFuncMsg* pSqlFunc = &pQuery->pSelectExpr[i].base;
int32_t functionId = pSqlFunc->functionId;
int32_t colId = pSqlFunc->colInfo.colId;
status |= aAggs[functionId].dataReqFunc(&pRuntimeEnv->pCtx[i], pBlockInfo->window.skey, pBlockInfo->window.ekey, colId);
if ((status & BLK_DATA_ALL_NEEDED) == BLK_DATA_ALL_NEEDED) {
(*status) |= aAggs[functionId].dataReqFunc(&pRuntimeEnv->pCtx[i], pBlockInfo->window.skey, pBlockInfo->window.ekey, colId);
if (((*status) & BLK_DATA_ALL_NEEDED) == BLK_DATA_ALL_NEEDED) {
break;
}
}
}
}
if (status == BLK_DATA_NO_NEEDED) {
if ((*status) == BLK_DATA_NO_NEEDED) {
qDebug("QInfo:%p data block discard, brange:%"PRId64 "-%"PRId64", rows:%d", GET_QINFO_ADDR(pRuntimeEnv),
pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows);
pRuntimeEnv->summary.discardBlocks += 1;
} else if (status == BLK_DATA_STATIS_NEEDED) {
if (tsdbRetrieveDataBlockStatisInfo(pQueryHandle, pStatis) != TSDB_CODE_SUCCESS) {
// return DISK_DATA_LOAD_FAILED;
}
} else if ((*status) == BLK_DATA_STATIS_NEEDED) {
// this function never returns error?
tsdbRetrieveDataBlockStatisInfo(pQueryHandle, pStatis);
pRuntimeEnv->summary.loadBlockStatis += 1;
@ -2230,24 +2253,26 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, void* pQueryHandle,
pRuntimeEnv->summary.totalCheckedRows += pBlockInfo->rows;
}
} else {
assert(status == BLK_DATA_ALL_NEEDED);
assert((*status) == BLK_DATA_ALL_NEEDED);
// load the data block statistics to perform further filter
pRuntimeEnv->summary.loadBlockStatis += 1;
if (tsdbRetrieveDataBlockStatisInfo(pQueryHandle, pStatis) != TSDB_CODE_SUCCESS) {
}
tsdbRetrieveDataBlockStatisInfo(pQueryHandle, pStatis);
if (!needToLoadDataBlock(pRuntimeEnv, *pStatis, pRuntimeEnv->pCtx, pBlockInfo->rows)) {
// current block has been discard due to filter applied
pRuntimeEnv->summary.discardBlocks += 1;
qDebug("QInfo:%p data block discard, brange:%"PRId64 "-%"PRId64", rows:%d", GET_QINFO_ADDR(pRuntimeEnv),
pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows);
return BLK_DATA_DISCARD;
(*status) = BLK_DATA_DISCARD;
}
pRuntimeEnv->summary.totalCheckedRows += pBlockInfo->rows;
pRuntimeEnv->summary.loadBlocks += 1;
*pDataBlock = tsdbRetrieveDataBlock(pQueryHandle, NULL);
if (*pDataBlock == NULL) {
return terrno;
}
}
return TSDB_CODE_SUCCESS;
@ -2365,16 +2390,16 @@ static void ensureOutputBuffer(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pB
memset(tmp + sizeof(tFilePage) + bytes * pRec->rows, 0, (size_t)((newSize - pRec->rows) * bytes));
pQuery->sdata[i] = (tFilePage *)tmp;
}
// set the pCtx output buffer position
pRuntimeEnv->pCtx[i].aOutputBuf = pQuery->sdata[i]->data + pRec->rows * bytes;
int32_t functionId = pQuery->pSelectExpr[i].base.functionId;
if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM || functionId == TSDB_FUNC_DIFF) {
pRuntimeEnv->pCtx[i].ptsOutputBuf = pRuntimeEnv->pCtx[0].aOutputBuf;
}
}
qDebug("QInfo:%p realloc output buffer, new size: %d rows, old:%" PRId64 ", remain:%" PRId64, GET_QINFO_ADDR(pRuntimeEnv),
newSize, pRec->capacity, newSize - pRec->rows);
@ -2431,15 +2456,18 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) {
ensureOutputBuffer(pRuntimeEnv, &blockInfo);
SDataStatis *pStatis = NULL;
SArray *pDataBlock = NULL;
if (loadDataBlockOnDemand(pRuntimeEnv, pQueryHandle, &blockInfo, &pStatis, &pDataBlock) == BLK_DATA_DISCARD) {
pQuery->current->lastKey = QUERY_IS_ASC_QUERY(pQuery)? blockInfo.window.ekey + step:blockInfo.window.skey + step;
continue;
SArray * pDataBlock = NULL;
uint32_t status = 0;
int32_t ret = loadDataBlockOnDemand(pRuntimeEnv, pQueryHandle, &blockInfo, &pStatis, &pDataBlock, &status);
if (ret != TSDB_CODE_SUCCESS) {
break;
}
if (terrno != TSDB_CODE_SUCCESS) { // load data block failed, abort query
longjmp(pRuntimeEnv->env, terrno);
break;
if (status == BLK_DATA_DISCARD) {
pQuery->current->lastKey =
QUERY_IS_ASC_QUERY(pQuery) ? blockInfo.window.ekey + step : blockInfo.window.skey + step;
continue;
}
// query start position can not move into tableApplyFunctionsOnBlock due to limit/offset condition
@ -2806,6 +2834,7 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) {
// all results have been return to client, try next group
if (pGroupResInfo->pos.pageId == pGroupResInfo->numOfDataPages) {
pGroupResInfo->numOfDataPages = 0;
pGroupResInfo->pos.pageId = 0;
pGroupResInfo->pos.rowId = 0;
// current results of group has been sent to client, try next group
@ -2920,11 +2949,11 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) {
STableQueryInfo *item = taosArrayGetP(pGroup, i);
SIDList list = getDataBufPagesIdList(pRuntimeEnv->pResultBuf, TSDB_TABLEID(item->pTable)->tid);
pageList = list;
tid = TSDB_TABLEID(item->pTable)->tid;
if (taosArrayGetSize(list) > 0 && item->windowResInfo.size > 0) {
pTableList[numOfTables++] = item;
tid = TSDB_TABLEID(item->pTable)->tid;
pageList = list;
}
}
@ -2993,7 +3022,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) {
char *b = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes, page);
TSKEY ts = GET_INT64_VAL(b);
assert(ts == pWindowRes->skey);
assert(ts == pWindowRes->win.skey);
int64_t num = getNumOfResultWindowRes(pQuery, pWindowRes);
if (num <= 0) {
cs.position[pos] += 1;
@ -3357,7 +3386,7 @@ void skipResults(SQueryRuntimeEnv *pRuntimeEnv) {
for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
int32_t functionId = pQuery->pSelectExpr[i].base.functionId;
int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes;
memmove(pQuery->sdata[i]->data, (char*)pQuery->sdata[i]->data + bytes * numOfSkip, (size_t)(pQuery->rec.rows * bytes));
pRuntimeEnv->pCtx[i].aOutputBuf = ((char*) pQuery->sdata[i]->data) + pQuery->rec.rows * bytes;
@ -4238,8 +4267,8 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) {
}
/*
* 1. for interval without interpolation query we forward pQuery->intervalTime at a time for
* pQuery->limit.offset times. Since hole exists, pQuery->intervalTime*pQuery->limit.offset value is
* 1. for interval without interpolation query we forward pQuery->interval.interval at a time for
* pQuery->limit.offset times. Since hole exists, pQuery->interval.interval*pQuery->limit.offset value is
* not valid. otherwise, we only forward pQuery->limit.offset number of points
*/
assert(pRuntimeEnv->windowResInfo.prevSKey == TSKEY_INITIAL_VAL);
@ -4354,32 +4383,6 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) {
return true;
}
static void freeTableQueryInfo(STableGroupInfo* pTableGroupInfo) {
if (pTableGroupInfo->pGroupList == NULL) {
assert(pTableGroupInfo->numOfTables == 0);
} else {
size_t numOfGroups = taosArrayGetSize(pTableGroupInfo->pGroupList);
for (int32_t i = 0; i < numOfGroups; ++i) {
SArray *p = taosArrayGetP(pTableGroupInfo->pGroupList, i);
size_t num = taosArrayGetSize(p);
for(int32_t j = 0; j < num; ++j) {
STableQueryInfo* item = taosArrayGetP(p, j);
destroyTableQueryInfo(item);
}
taosArrayDestroy(p);
}
taosArrayDestroy(pTableGroupInfo->pGroupList);
pTableGroupInfo->pGroupList = NULL;
pTableGroupInfo->numOfTables = 0;
}
taosHashCleanup(pTableGroupInfo->map);
pTableGroupInfo->map = NULL;
}
static int32_t setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery) {
SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv;
SQuery *pQuery = pQInfo->runtimeEnv.pQuery;
@ -4415,22 +4418,20 @@ static int32_t setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery)
terrno = TSDB_CODE_SUCCESS;
if (isFirstLastRowQuery(pQuery)) {
pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo);
if (pRuntimeEnv->pQueryHandle == NULL) { // no data in current stable, clear all
freeTableQueryInfo(&pQInfo->tableqinfoGroupInfo);
} else { // update the query time window
pQuery->window = cond.twindow;
size_t numOfGroups = GET_NUM_OF_TABLEGROUP(pQInfo);
for (int32_t i = 0; i < numOfGroups; ++i) {
SArray *group = GET_TABLEGROUP(pQInfo, i);
// update the query time window
pQuery->window = cond.twindow;
size_t t = taosArrayGetSize(group);
for (int32_t j = 0; j < t; ++j) {
STableQueryInfo *pCheckInfo = taosArrayGetP(group, j);
size_t numOfGroups = GET_NUM_OF_TABLEGROUP(pQInfo);
for(int32_t i = 0; i < numOfGroups; ++i) {
SArray *group = GET_TABLEGROUP(pQInfo, i);
pCheckInfo->win = pQuery->window;
pCheckInfo->lastKey = pCheckInfo->win.skey;
}
size_t t = taosArrayGetSize(group);
for (int32_t j = 0; j < t; ++j) {
STableQueryInfo *pCheckInfo = taosArrayGetP(group, j);
pCheckInfo->win = pQuery->window;
pCheckInfo->lastKey = pCheckInfo->win.skey;
}
}
} else if (isPointInterpoQuery(pQuery)) {
@ -4484,12 +4485,6 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo
return code;
}
if (pQInfo->tableqinfoGroupInfo.numOfTables == 0) {
qDebug("QInfo:%p no table qualified for tag filter, abort query", pQInfo);
setQueryStatus(pQuery, QUERY_COMPLETED);
return TSDB_CODE_SUCCESS;
}
pQInfo->tsdb = tsdb;
pQInfo->vgId = vgId;
@ -4572,7 +4567,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo
getAlignQueryTimeWindow(pQuery, pQuery->window.skey, sk, ek, &w);
pRuntimeEnv->pFillInfo = taosInitFillInfo(pQuery->order.order, w.skey, 0, (int32_t)pQuery->rec.capacity, pQuery->numOfOutput,
pQuery->slidingTime, pQuery->slidingTimeUnit, (int8_t)pQuery->precision,
pQuery->interval.sliding, pQuery->interval.slidingUnit, (int8_t)pQuery->precision,
pQuery->fillType, pColInfo);
}
@ -4612,7 +4607,7 @@ static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) {
SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv;
SQuery* pQuery = pRuntimeEnv->pQuery;
SQueryCostInfo* summary = &pRuntimeEnv->summary;
int64_t st = taosGetTimestampMs();
TsdbQueryHandleT pQueryHandle = IS_MASTER_SCAN(pRuntimeEnv)? pRuntimeEnv->pQueryHandle : pRuntimeEnv->pSecQueryHandle;
@ -4622,7 +4617,7 @@ static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) {
while (tsdbNextDataBlock(pQueryHandle)) {
summary->totalBlocks += 1;
if (IS_QUERY_KILLED(pQInfo)) {
longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED);
}
@ -4651,15 +4646,23 @@ static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) {
}
SDataStatis *pStatis = NULL;
SArray *pDataBlock = NULL;
if (loadDataBlockOnDemand(pRuntimeEnv, pQueryHandle, &blockInfo, &pStatis, &pDataBlock) == BLK_DATA_DISCARD) {
pQuery->current->lastKey = QUERY_IS_ASC_QUERY(pQuery)? blockInfo.window.ekey + step:blockInfo.window.skey + step;
SArray * pDataBlock = NULL;
uint32_t status = 0;
int32_t ret = loadDataBlockOnDemand(pRuntimeEnv, pQueryHandle, &blockInfo, &pStatis, &pDataBlock, &status);
if (ret != TSDB_CODE_SUCCESS) {
break;
}
if (status == BLK_DATA_DISCARD) {
pQuery->current->lastKey =
QUERY_IS_ASC_QUERY(pQuery) ? blockInfo.window.ekey + step : blockInfo.window.skey + step;
continue;
}
summary->totalRows += blockInfo.rows;
stableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, pStatis, pDataBlock, binarySearchForKey);
qDebug("QInfo:%p check data block completed, uid:%"PRId64", tid:%d, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, "
"lastKey:%" PRId64,
pQInfo, blockInfo.uid, blockInfo.tid, blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows,
@ -5425,7 +5428,7 @@ static void stableQueryImpl(SQInfo *pQInfo) {
(isFixedOutputQuery(pRuntimeEnv) && (!isPointInterpoQuery(pQuery)) && (!pRuntimeEnv->groupbyNormalCol))) {
multiTableQueryProcess(pQInfo);
} else {
assert((pQuery->checkBuffer == 1 && pQuery->intervalTime == 0) || isPointInterpoQuery(pQuery) ||
assert((pQuery->checkBuffer == 1 && pQuery->interval.interval == 0) || isPointInterpoQuery(pQuery) ||
isFirstLastRowQuery(pQuery) || pRuntimeEnv->groupbyNormalCol);
sequentialTableProcess(pQInfo);
@ -5463,6 +5466,7 @@ static int32_t getColumnIndexInSource(SQueryTableMsg *pQueryMsg, SSqlFuncMsg *pE
}
}
assert(0);
return -1;
}
bool validateExprColumnInfo(SQueryTableMsg *pQueryMsg, SSqlFuncMsg *pExprMsg, SColumnInfo* pTagCols) {
@ -5471,8 +5475,8 @@ bool validateExprColumnInfo(SQueryTableMsg *pQueryMsg, SSqlFuncMsg *pExprMsg, SC
}
static bool validateQueryMsg(SQueryTableMsg *pQueryMsg) {
if (pQueryMsg->intervalTime < 0) {
qError("qmsg:%p illegal value of interval time %" PRId64, pQueryMsg, pQueryMsg->intervalTime);
if (pQueryMsg->interval.interval < 0) {
qError("qmsg:%p illegal value of interval time %" PRId64, pQueryMsg, pQueryMsg->interval.interval);
return false;
}
@ -5551,8 +5555,12 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList,
pQueryMsg->window.skey = htobe64(pQueryMsg->window.skey);
pQueryMsg->window.ekey = htobe64(pQueryMsg->window.ekey);
pQueryMsg->intervalTime = htobe64(pQueryMsg->intervalTime);
pQueryMsg->slidingTime = htobe64(pQueryMsg->slidingTime);
pQueryMsg->interval.interval = htobe64(pQueryMsg->interval.interval);
pQueryMsg->interval.sliding = htobe64(pQueryMsg->interval.sliding);
pQueryMsg->interval.offset = htobe64(pQueryMsg->interval.offset);
pQueryMsg->interval.intervalUnit = pQueryMsg->interval.intervalUnit;
pQueryMsg->interval.slidingUnit = pQueryMsg->interval.slidingUnit;
pQueryMsg->interval.offsetUnit = pQueryMsg->interval.offsetUnit;
pQueryMsg->limit = htobe64(pQueryMsg->limit);
pQueryMsg->offset = htobe64(pQueryMsg->offset);
@ -5765,7 +5773,7 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList,
qDebug("qmsg:%p query %d tables, type:%d, qrange:%" PRId64 "-%" PRId64 ", numOfGroupbyTagCols:%d, order:%d, "
"outputCols:%d, numOfCols:%d, interval:%" PRId64 ", fillType:%d, comptsLen:%d, compNumOfBlocks:%d, limit:%" PRId64 ", offset:%" PRId64,
pQueryMsg, pQueryMsg->numOfTables, pQueryMsg->queryType, pQueryMsg->window.skey, pQueryMsg->window.ekey, pQueryMsg->numOfGroupCols,
pQueryMsg->order, pQueryMsg->numOfOutput, pQueryMsg->numOfCols, pQueryMsg->intervalTime,
pQueryMsg->order, pQueryMsg->numOfOutput, pQueryMsg->numOfCols, pQueryMsg->interval.interval,
pQueryMsg->fillType, pQueryMsg->tsLen, pQueryMsg->tsNumOfBlocks, pQueryMsg->limit, pQueryMsg->offset);
return TSDB_CODE_SUCCESS;
@ -6104,10 +6112,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou
pQuery->order.orderColId = pQueryMsg->orderColId;
pQuery->pSelectExpr = pExprs;
pQuery->pGroupbyExpr = pGroupbyExpr;
pQuery->intervalTime = pQueryMsg->intervalTime;
pQuery->slidingTime = pQueryMsg->slidingTime;
pQuery->intervalTimeUnit = pQueryMsg->intervalTimeUnit;
pQuery->slidingTimeUnit = pQueryMsg->slidingTimeUnit;
memcpy(&pQuery->interval, &pQueryMsg->interval, sizeof(pQuery->interval));
pQuery->fillType = pQueryMsg->fillType;
pQuery->numOfTags = pQueryMsg->numOfTags;
pQuery->tagColList = pTagCols;
@ -6383,10 +6388,25 @@ static void freeQInfo(SQInfo *pQInfo) {
taosTFree(pQuery);
}
freeTableQueryInfo(&pQInfo->tableqinfoGroupInfo);
// todo refactor, extract method to destroytableDataInfo
if (pQInfo->tableqinfoGroupInfo.pGroupList != NULL) {
int32_t numOfGroups = (int32_t)(GET_NUM_OF_TABLEGROUP(pQInfo));
for (int32_t i = 0; i < numOfGroups; ++i) {
SArray *p = GET_TABLEGROUP(pQInfo, i);
size_t num = taosArrayGetSize(p);
for(int32_t j = 0; j < num; ++j) {
STableQueryInfo* item = taosArrayGetP(p, j);
destroyTableQueryInfo(item);
}
taosArrayDestroy(p);
}
}
taosTFree(pQInfo->pBuf);
taosArrayDestroy(pQInfo->tableqinfoGroupInfo.pGroupList);
taosHashCleanup(pQInfo->tableqinfoGroupInfo.map);
tsdbDestroyTableGroup(&pQInfo->tableGroupInfo);
taosArrayDestroy(pQInfo->arrTableIdInfo);

View File

@ -502,22 +502,22 @@ FORCE_INLINE int32_t compare_sd(tOrderDescriptor *pDescriptor, int32_t numOfRows
return compare_d(pDescriptor, numOfRows, idx1, data, numOfRows, idx2, data);
}
static void swap(SColumnModel *pColumnModel, int32_t count, int32_t s1, char *data1, int32_t s2) {
static void swap(SColumnModel *pColumnModel, int32_t count, int32_t s1, char *data1, int32_t s2, void* buf) {
for (int32_t i = 0; i < pColumnModel->numOfCols; ++i) {
void *first = COLMODEL_GET_VAL(data1, pColumnModel, count, s1, i);
void *second = COLMODEL_GET_VAL(data1, pColumnModel, count, s2, i);
SSchema* pSchema = &pColumnModel->pFields[i].field;
tsDataSwap(first, second, pSchema->type, pSchema->bytes);
tsDataSwap(first, second, pSchema->type, pSchema->bytes, buf);
}
}
static void tColDataInsertSort(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t start, int32_t end, char *data,
__col_compar_fn_t compareFn) {
__col_compar_fn_t compareFn, void* buf) {
for (int32_t i = start + 1; i <= end; ++i) {
for (int32_t j = i; j > start; --j) {
if (compareFn(pDescriptor, numOfRows, j, j - 1, data) == -1) {
swap(pDescriptor->pColumnModel, numOfRows, j - 1, data, j);
swap(pDescriptor->pColumnModel, numOfRows, j - 1, data, j, buf);
} else {
break;
}
@ -553,7 +553,7 @@ static void UNUSED_FUNC tSortDataPrint(int32_t type, char *prefix, char *startx,
}
static void median(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t start, int32_t end, char *data,
__col_compar_fn_t compareFn) {
__col_compar_fn_t compareFn, void* buf) {
int32_t midIdx = ((end - start) >> 1) + start;
#if defined(_DEBUG_VIEW)
@ -567,15 +567,16 @@ static void median(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t sta
tSortDataPrint(pDescriptor->pColumnModel->pFields[colIdx].field.type, "before", startx, midx, endx);
#endif
SColumnModel* pModel = pDescriptor->pColumnModel;
if (compareFn(pDescriptor, numOfRows, midIdx, start, data) == 1) {
swap(pDescriptor->pColumnModel, numOfRows, start, data, midIdx);
swap(pModel, numOfRows, start, data, midIdx, buf);
}
if (compareFn(pDescriptor, numOfRows, midIdx, end, data) == 1) {
swap(pDescriptor->pColumnModel, numOfRows, midIdx, data, start);
swap(pDescriptor->pColumnModel, numOfRows, midIdx, data, end);
swap(pModel, numOfRows, midIdx, data, start, buf);
swap(pModel, numOfRows, midIdx, data, end, buf);
} else if (compareFn(pDescriptor, numOfRows, start, end, data) == 1) {
swap(pDescriptor->pColumnModel, numOfRows, start, data, end);
swap(pModel, numOfRows, start, data, end, buf);
}
assert(compareFn(pDescriptor, numOfRows, midIdx, start, data) <= 0 &&
@ -626,32 +627,20 @@ static UNUSED_FUNC void tRowModelDisplay(tOrderDescriptor *pDescriptor, int32_t
printf("\n");
}
static int32_t qsort_call = 0;
void tColDataQSort(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t start, int32_t end, char *data,
int32_t orderType) {
// short array sort, incur another sort procedure instead of quick sort process
__col_compar_fn_t compareFn = (orderType == TSDB_ORDER_ASC) ? compare_sa : compare_sd;
if (end - start + 1 <= 8) {
tColDataInsertSort(pDescriptor, numOfRows, start, end, data, compareFn);
return;
}
static void columnwiseQSortImpl(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t start, int32_t end, char *data,
int32_t orderType, __col_compar_fn_t compareFn, void* buf) {
#ifdef _DEBUG_VIEW
// printf("before sort:\n");
// tRowModelDisplay(pDescriptor, numOfRows, data, end - start + 1);
printf("before sort:\n");
tRowModelDisplay(pDescriptor, numOfRows, data, end - start + 1);
#endif
int32_t s = start, e = end;
median(pDescriptor, numOfRows, start, end, data, compareFn);
median(pDescriptor, numOfRows, start, end, data, compareFn, buf);
#ifdef _DEBUG_VIEW
// printf("%s called: %d\n", __FUNCTION__, qsort_call++);
// printf("%s called: %d\n", __FUNCTION__, qsort_call++);
#endif
UNUSED(qsort_call);
int32_t end_same = end;
int32_t start_same = start;
@ -663,17 +652,17 @@ void tColDataQSort(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t sta
}
if (ret == 0 && e != end_same) {
swap(pDescriptor->pColumnModel, numOfRows, e, data, end_same--);
swap(pDescriptor->pColumnModel, numOfRows, e, data, end_same--, buf);
}
e--;
}
if (e != s) {
swap(pDescriptor->pColumnModel, numOfRows, s, data, e);
swap(pDescriptor->pColumnModel, numOfRows, s, data, e, buf);
}
#ifdef _DEBUG_VIEW
// tRowModelDisplay(pDescriptor, numOfRows, data, end - start + 1);
// tRowModelDisplay(pDescriptor, numOfRows, data, end - start + 1);
#endif
while (s < e) {
@ -683,16 +672,16 @@ void tColDataQSort(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t sta
}
if (ret == 0 && s != start_same) {
swap(pDescriptor->pColumnModel, numOfRows, s, data, start_same++);
swap(pDescriptor->pColumnModel, numOfRows, s, data, start_same++, buf);
}
s++;
}
if (s != e) {
swap(pDescriptor->pColumnModel, numOfRows, s, data, e);
swap(pDescriptor->pColumnModel, numOfRows, s, data, e, buf);
}
#ifdef _DEBUG_VIEW
// tRowModelDisplay(pDescriptor, numOfRows, data, end - start + 1);
// tRowModelDisplay(pDescriptor, numOfRows, data, end - start + 1);
#endif
}
@ -702,14 +691,14 @@ void tColDataQSort(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t sta
int32_t right = end;
while (right > end_same && left <= end_same) {
swap(pDescriptor->pColumnModel, numOfRows, left++, data, right--);
swap(pDescriptor->pColumnModel, numOfRows, left++, data, right--, buf);
}
// (pivotal+1) + steps of number that are identical pivotal
rightx += (end - end_same);
#ifdef _DEBUG_VIEW
// tRowModelDisplay(pDescriptor, numOfRows, data, end - start + 1);
// tRowModelDisplay(pDescriptor, numOfRows, data, end - start + 1);
#endif
}
@ -719,26 +708,52 @@ void tColDataQSort(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t sta
int32_t right = e - 1;
while (left < start_same && right >= start_same) {
swap(pDescriptor->pColumnModel, numOfRows, left++, data, right--);
swap(pDescriptor->pColumnModel, numOfRows, left++, data, right--, buf);
}
// (pivotal-1) - steps of number that are identical pivotal
leftx -= (start_same - start);
#ifdef _DEBUG_VIEW
// tRowModelDisplay(pDescriptor, numOfRows, data, end - start + 1);
// tRowModelDisplay(pDescriptor, numOfRows, data, end - start + 1);
#endif
}
if (leftx > start) {
tColDataQSort(pDescriptor, numOfRows, start, leftx, data, orderType);
columnwiseQSortImpl(pDescriptor, numOfRows, start, leftx, data, orderType, compareFn, buf);
}
if (rightx < end) {
tColDataQSort(pDescriptor, numOfRows, rightx, end, data, orderType);
columnwiseQSortImpl(pDescriptor, numOfRows, rightx, end, data, orderType, compareFn, buf);
}
}
void tColDataQSort(tOrderDescriptor *pDescriptor, int32_t numOfRows, int32_t start, int32_t end, char *data, int32_t order) {
// short array sort, incur another sort procedure instead of quick sort process
__col_compar_fn_t compareFn = (order == TSDB_ORDER_ASC) ? compare_sa : compare_sd;
SColumnModel* pModel = pDescriptor->pColumnModel;
size_t width = 0;
for(int32_t i = 0; i < pModel->numOfCols; ++i) {
SSchema* pSchema = &pModel->pFields[i].field;
if (width < pSchema->bytes) {
width = pSchema->bytes;
}
}
char* buf = malloc(width);
assert(width > 0 && buf != NULL);
if (end - start + 1 <= 8) {
tColDataInsertSort(pDescriptor, numOfRows, start, end, data, compareFn, buf);
} else {
columnwiseQSortImpl(pDescriptor, numOfRows, start, end, data, order, compareFn, buf);
}
free(buf);
}
/*
* deep copy of sschema
*/

View File

@ -38,8 +38,8 @@ SFillInfo* taosInitFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_
pFillInfo->numOfTags = numOfTags;
pFillInfo->numOfCols = numOfCols;
pFillInfo->precision = precision;
pFillInfo->slidingTime = slidingTime;
pFillInfo->slidingUnit = slidingUnit;
pFillInfo->interval.sliding = slidingTime;
pFillInfo->interval.slidingUnit = slidingUnit;
pFillInfo->pData = malloc(POINTER_BYTES * numOfCols);
if (numOfTags > 0) {
@ -108,21 +108,15 @@ void* taosDestoryFillInfo(SFillInfo* pFillInfo) {
return NULL;
}
static TSKEY taosGetRevisedEndKey(TSKEY ekey, int32_t order, int64_t timeInterval, int8_t slidingTimeUnit, int8_t precision) {
if (order == TSDB_ORDER_ASC) {
return ekey;
} else {
return taosGetIntervalStartTimestamp(ekey, timeInterval, timeInterval, slidingTimeUnit, precision);
}
}
void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey) {
if (pFillInfo->fillType == TSDB_FILL_NONE) {
return;
}
pFillInfo->endKey = taosGetRevisedEndKey(endKey, pFillInfo->order, pFillInfo->slidingTime, pFillInfo->slidingUnit,
pFillInfo->precision);
pFillInfo->endKey = endKey;
if (pFillInfo->order != TSDB_ORDER_ASC) {
pFillInfo->endKey = taosTimeTruncate(endKey, &pFillInfo->interval, pFillInfo->precision);
}
pFillInfo->rowIdx = 0;
pFillInfo->numOfRows = numOfRows;
@ -172,30 +166,34 @@ int64_t getFilledNumOfRes(SFillInfo* pFillInfo, TSKEY ekey, int32_t maxNumOfRows
int32_t numOfRows = taosNumOfRemainRows(pFillInfo);
TSKEY ekey1 = taosGetRevisedEndKey(ekey, pFillInfo->order, pFillInfo->slidingTime, pFillInfo->slidingUnit,
pFillInfo->precision);
TSKEY ekey1 = ekey;
if (pFillInfo->order != TSDB_ORDER_ASC) {
pFillInfo->endKey = taosTimeTruncate(ekey, &pFillInfo->interval, pFillInfo->precision);
}
int64_t numOfRes = -1;
if (numOfRows > 0) { // still fill gap within current data block, not generating data after the result set.
TSKEY lastKey = tsList[pFillInfo->numOfRows - 1];
if (pFillInfo->slidingUnit != 'y' && pFillInfo->slidingUnit != 'n') {
numOfRes = (int64_t)(ABS(lastKey - pFillInfo->start) / pFillInfo->slidingTime) + 1;
} else {
numOfRes = taosCountNatualInterval(lastKey, pFillInfo->start, pFillInfo->slidingTime, pFillInfo->slidingUnit, pFillInfo->precision) + 1;
}
numOfRes = taosTimeCountInterval(
lastKey,
pFillInfo->start,
pFillInfo->interval.sliding,
pFillInfo->interval.slidingUnit,
pFillInfo->precision);
numOfRes += 1;
assert(numOfRes >= numOfRows);
} else { // reach the end of data
if ((ekey1 < pFillInfo->start && FILL_IS_ASC_FILL(pFillInfo)) ||
(ekey1 > pFillInfo->start && !FILL_IS_ASC_FILL(pFillInfo))) {
return 0;
}
// the numOfRes rows are all filled with specified policy
if (pFillInfo->slidingUnit != 'y' && pFillInfo->slidingUnit != 'n') {
numOfRes = (ABS(ekey1 - pFillInfo->start) / pFillInfo->slidingTime) + 1;
} else {
numOfRes = taosCountNatualInterval(ekey1, pFillInfo->start, pFillInfo->slidingTime, pFillInfo->slidingUnit, pFillInfo->precision) + 1;
}
numOfRes = taosTimeCountInterval(
ekey1,
pFillInfo->start,
pFillInfo->interval.sliding,
pFillInfo->interval.slidingUnit,
pFillInfo->precision);
numOfRes += 1;
}
return (numOfRes > maxNumOfRows) ? maxNumOfRows : numOfRes;
@ -374,12 +372,7 @@ static void doFillResultImpl(SFillInfo* pFillInfo, tFilePage** data, int32_t* nu
setTagsValue(pFillInfo, data, *num);
}
// TODO natual sliding time
if (pFillInfo->slidingUnit != 'n' && pFillInfo->slidingUnit != 'y') {
pFillInfo->start += (pFillInfo->slidingTime * step);
} else {
pFillInfo->start = taosAddNatualInterval(pFillInfo->start, pFillInfo->slidingTime*step, pFillInfo->slidingUnit, pFillInfo->precision);
}
pFillInfo->start = taosTimeAdd(pFillInfo->start, pFillInfo->interval.sliding * step, pFillInfo->interval.slidingUnit, pFillInfo->precision);
pFillInfo->numOfCurrent++;
(*num) += 1;
@ -486,12 +479,7 @@ int32_t generateDataBlockImpl(SFillInfo* pFillInfo, tFilePage** data, int32_t nu
// set the tag value for final result
setTagsValue(pFillInfo, data, num);
// TODO natual sliding time
if (pFillInfo->slidingUnit != 'n' && pFillInfo->slidingUnit != 'y') {
pFillInfo->start += (pFillInfo->slidingTime * step);
} else {
pFillInfo->start = taosAddNatualInterval(pFillInfo->start, pFillInfo->slidingTime*step, pFillInfo->slidingUnit, pFillInfo->precision);
}
pFillInfo->start = taosTimeAdd(pFillInfo->start, pFillInfo->interval.sliding*step, pFillInfo->interval.slidingUnit, pFillInfo->precision);
pFillInfo->rowIdx += 1;
pFillInfo->numOfCurrent +=1;

View File

@ -284,6 +284,14 @@ bool nequal_nchar(SColumnFilterElem *pFilter, char* minval, char *maxval) {
return wcsncmp((wchar_t *)pFilter->filterInfo.pz, varDataVal(minval), varDataLen(minval)/TSDB_NCHAR_SIZE) != 0;
}
////////////////////////////////////////////////////////////////
bool isNull_filter(SColumnFilterElem *pFilter, char* minval, char* maxval) {
return true;
}
bool notNull_filter(SColumnFilterElem *pFilter, char* minval, char* maxval) {
return true;
}
////////////////////////////////////////////////////////////////
@ -398,6 +406,8 @@ bool (*filterFunc_i8[])(SColumnFilterElem *pFilter, char *minval, char *maxval)
largeEqual_i8,
nequal_i8,
NULL,
isNull_filter,
notNull_filter,
};
bool (*filterFunc_i16[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = {
@ -409,6 +419,8 @@ bool (*filterFunc_i16[])(SColumnFilterElem *pFilter, char *minval, char *maxval)
largeEqual_i16,
nequal_i16,
NULL,
isNull_filter,
notNull_filter,
};
bool (*filterFunc_i32[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = {
@ -420,6 +432,8 @@ bool (*filterFunc_i32[])(SColumnFilterElem *pFilter, char *minval, char *maxval)
largeEqual_i32,
nequal_i32,
NULL,
isNull_filter,
notNull_filter,
};
bool (*filterFunc_i64[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = {
@ -431,6 +445,8 @@ bool (*filterFunc_i64[])(SColumnFilterElem *pFilter, char *minval, char *maxval)
largeEqual_i64,
nequal_i64,
NULL,
isNull_filter,
notNull_filter,
};
bool (*filterFunc_ds[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = {
@ -442,6 +458,8 @@ bool (*filterFunc_ds[])(SColumnFilterElem *pFilter, char *minval, char *maxval)
largeEqual_ds,
nequal_ds,
NULL,
isNull_filter,
notNull_filter,
};
bool (*filterFunc_dd[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = {
@ -453,6 +471,8 @@ bool (*filterFunc_dd[])(SColumnFilterElem *pFilter, char *minval, char *maxval)
largeEqual_dd,
nequal_dd,
NULL,
isNull_filter,
notNull_filter,
};
bool (*filterFunc_str[])(SColumnFilterElem* pFilter, char* minval, char *maxval) = {
@ -464,6 +484,8 @@ bool (*filterFunc_str[])(SColumnFilterElem* pFilter, char* minval, char *maxval)
NULL,
nequal_str,
like_str,
isNull_filter,
notNull_filter,
};
bool (*filterFunc_nchar[])(SColumnFilterElem* pFitler, char* minval, char* maxval) = {
@ -475,6 +497,8 @@ bool (*filterFunc_nchar[])(SColumnFilterElem* pFitler, char* minval, char* maxva
NULL,
nequal_nchar,
like_nchar,
isNull_filter,
notNull_filter,
};
bool (*rangeFilterFunc_i8[])(SColumnFilterElem *pFilter, char *minval, char *maxval) = {

View File

@ -135,7 +135,7 @@ tSQLExpr *tSQLExprIdValueCreate(SStrToken *pToken, int32_t optrType) {
pSQLExpr->val.nType = TSDB_DATA_TYPE_BIGINT;
pSQLExpr->nSQLOptr = TK_TIMESTAMP; // TK_TIMESTAMP used to denote the time value is in microsecond
} else if (optrType == TK_VARIABLE) {
int32_t ret = getTimestampInUsFromStr(pToken->z, pToken->n, &pSQLExpr->val.i64Key);
int32_t ret = parseAbsoluteDuration(pToken->z, pToken->n, &pSQLExpr->val.i64Key);
UNUSED(ret);
pSQLExpr->val.nType = TSDB_DATA_TYPE_BIGINT;
@ -179,7 +179,7 @@ tSQLExpr *tSQLExprCreateFunction(tSQLExprList *pList, SStrToken *pFuncToken, SSt
tSQLExpr *tSQLExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optrType) {
tSQLExpr *pExpr = calloc(1, sizeof(tSQLExpr));
if (pRight != NULL && pLeft != NULL) {
if (pLeft != NULL && pRight != NULL && (optrType != TK_IN)) {
char* endPos = pRight->token.z + pRight->token.n;
pExpr->token.z = pLeft->token.z;
pExpr->token.n = (uint32_t)(endPos - pExpr->token.z);
@ -275,6 +275,11 @@ tSQLExpr *tSQLExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optrType) {
} else {
pExpr->nSQLOptr = optrType;
pExpr->pLeft = pLeft;
if (pRight == NULL) {
pRight = calloc(1, sizeof(tSQLExpr));
}
pExpr->pRight = pRight;
}
@ -438,44 +443,6 @@ void setDBName(SStrToken *pCpxName, SStrToken *pDB) {
pCpxName->n = pDB->n;
}
int32_t getTimestampInUsFromStrImpl(int64_t val, char unit, int64_t *result) {
*result = val;
switch (unit) {
case 's':
(*result) *= MILLISECOND_PER_SECOND;
break;
case 'm':
(*result) *= MILLISECOND_PER_MINUTE;
break;
case 'h':
(*result) *= MILLISECOND_PER_HOUR;
break;
case 'd':
(*result) *= MILLISECOND_PER_DAY;
break;
case 'w':
(*result) *= MILLISECOND_PER_WEEK;
break;
case 'n':
(*result) *= MILLISECOND_PER_MONTH;
break;
case 'y':
(*result) *= MILLISECOND_PER_YEAR;
break;
case 'a':
break;
default: {
;
return -1;
}
}
/* get the value in microsecond */
(*result) *= 1000L;
return 0;
}
void tSQLSetColumnInfo(TAOS_FIELD *pField, SStrToken *pName, TAOS_FIELD *pType) {
int32_t maxLen = sizeof(pField->name) / sizeof(pField->name[0]);
@ -530,7 +497,7 @@ void tSQLSetColumnType(TAOS_FIELD *pField, SStrToken *type) {
* extract the select info out of sql string
*/
SQuerySQL *tSetQuerySQLElems(SStrToken *pSelectToken, tSQLExprList *pSelection, tVariantList *pFrom, tSQLExpr *pWhere,
tVariantList *pGroupby, tVariantList *pSortOrder, SStrToken *pInterval,
tVariantList *pGroupby, tVariantList *pSortOrder, SIntervalVal *pInterval,
SStrToken *pSliding, tVariantList *pFill, SLimitVal *pLimit, SLimitVal *pGLimit) {
assert(pSelection != NULL);
@ -553,7 +520,8 @@ SQuerySQL *tSetQuerySQLElems(SStrToken *pSelectToken, tSQLExprList *pSelection,
}
if (pInterval != NULL) {
pQuery->interval = *pInterval;
pQuery->interval = pInterval->interval;
pQuery->offset = pInterval->offset;
}
if (pSliding != NULL) {

View File

@ -70,6 +70,33 @@ static void resetBoundingBox(MinMaxEntry* range, int32_t type) {
}
}
static int32_t setBoundingBox(MinMaxEntry* range, int16_t type, double minval, double maxval) {
if (minval > maxval) {
return -1;
}
switch(type) {
case TSDB_DATA_TYPE_TINYINT:
case TSDB_DATA_TYPE_SMALLINT:
case TSDB_DATA_TYPE_INT:
range->iMinVal = (int32_t) minval;
range->iMaxVal = (int32_t) maxval;
break;
case TSDB_DATA_TYPE_BIGINT:
range->i64MinVal = (int64_t) minval;
range->i64MaxVal = (int64_t) maxval;
break;
case TSDB_DATA_TYPE_FLOAT:
case TSDB_DATA_TYPE_DOUBLE:
range->dMinVal = minval;
range->dMaxVal = maxval;
break;
}
return 0;
}
static void resetPosInfo(SSlotInfo* pInfo) {
pInfo->size = 0;
pInfo->pageId = -1;
@ -135,6 +162,11 @@ int32_t tBucketBigIntHash(tMemBucket *pBucket, const void *value) {
return index;
} else {
// out of range
if (v < pBucket->range.i64MinVal || v > pBucket->range.i64MaxVal) {
return -1;
}
// todo hash for bigint and float and double
int64_t span = pBucket->range.i64MaxVal - pBucket->range.i64MinVal;
if (span < pBucket->numOfSlots) {
@ -179,6 +211,11 @@ int32_t tBucketIntHash(tMemBucket *pBucket, const void *value) {
return index;
} else {
// out of range
if (v < pBucket->range.iMinVal || v > pBucket->range.iMaxVal) {
return -1;
}
// divide a range of [iMinVal, iMaxVal] into 1024 buckets
int32_t span = pBucket->range.iMaxVal - pBucket->range.iMinVal;
if (span < pBucket->numOfSlots) {
@ -209,6 +246,12 @@ int32_t tBucketDoubleHash(tMemBucket *pBucket, const void *value) {
double posx = (v + DBL_MAX) / x;
return ((int32_t)posx) % pBucket->numOfSlots;
} else {
// out of range
if (v < pBucket->range.dMinVal || v > pBucket->range.dMaxVal) {
return -1;
}
// divide a range of [dMinVal, dMaxVal] into 1024 buckets
double span = pBucket->range.dMaxVal - pBucket->range.dMinVal;
if (span < pBucket->numOfSlots) {
@ -262,7 +305,7 @@ static void resetSlotInfo(tMemBucket* pBucket) {
}
}
tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType) {
tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval, double maxval) {
tMemBucket *pBucket = (tMemBucket *)calloc(1, sizeof(tMemBucket));
if (pBucket == NULL) {
return NULL;
@ -278,9 +321,14 @@ tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType) {
pBucket->maxCapacity = 200000;
if (setBoundingBox(&pBucket->range, pBucket->type, minval, maxval) != 0) {
uError("MemBucket:%p, invalid value range: %f-%f", pBucket, minval, maxval);
free(pBucket);
return NULL;
}
pBucket->elemPerPage = (pBucket->bufPageSize - sizeof(tFilePage))/pBucket->bytes;
pBucket->comparFn = getKeyComparFunc(pBucket->type);
resetBoundingBox(&pBucket->range, pBucket->type);
pBucket->hashFunc = getHashFunc(pBucket->type);
if (pBucket->hashFunc == NULL) {
@ -395,23 +443,25 @@ void tMemBucketUpdateBoundingBox(MinMaxEntry *r, char *data, int32_t dataType) {
/*
* in memory bucket, we only accept data array list
*/
void tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size) {
int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size) {
assert(pBucket != NULL && data != NULL && size > 0);
pBucket->total += (int32_t)size;
int32_t bytes = pBucket->bytes;
for (int32_t i = 0; i < size; ++i) {
char *d = (char *) data + i * bytes;
int32_t slotIdx = (pBucket->hashFunc)(pBucket, d);
assert(slotIdx >= 0);
int32_t index = (pBucket->hashFunc)(pBucket, d);
if (index == -1) { // the value is out of range, do not add it into bucket
return -1;
}
tMemBucketSlot *pSlot = &pBucket->pSlots[slotIdx];
tMemBucketSlot *pSlot = &pBucket->pSlots[index];
tMemBucketUpdateBoundingBox(&pSlot->range, d, pBucket->type);
// ensure available memory pages to allocate
int32_t groupId = getGroupId(pBucket->numOfSlots, slotIdx, pBucket->times);
int32_t groupId = getGroupId(pBucket->numOfSlots, index, pBucket->times);
int32_t pageId = -1;
if (pSlot->info.data == NULL || pSlot->info.data->num >= pBucket->elemPerPage) {
@ -432,10 +482,12 @@ void tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size) {
pSlot->info.data->num += 1;
pSlot->info.size += 1;
}
return 0;
}
////////////////////////////////////////////////////////////////////////////////////////////
static void findMaxMinValue(tMemBucket *pMemBucket, double *maxVal, double *minVal) {
static UNUSED_FUNC void findMaxMinValue(tMemBucket *pMemBucket, double *maxVal, double *minVal) {
*minVal = DBL_MAX;
*maxVal = -DBL_MAX;
@ -681,16 +733,29 @@ double getPercentile(tMemBucket *pMemBucket, double percent) {
// find the min/max value, no need to scan all data in bucket
if (fabs(percent - 100.0) < DBL_EPSILON || (percent < DBL_EPSILON)) {
double minx = 0, maxx = 0;
findMaxMinValue(pMemBucket, &maxx, &minx);
MinMaxEntry* pRange = &pMemBucket->range;
return fabs(percent - 100) < DBL_EPSILON ? maxx : minx;
switch(pMemBucket->type) {
case TSDB_DATA_TYPE_TINYINT:
case TSDB_DATA_TYPE_SMALLINT:
case TSDB_DATA_TYPE_INT:
return fabs(percent - 100) < DBL_EPSILON? pRange->iMaxVal:pRange->iMinVal;
case TSDB_DATA_TYPE_BIGINT: {
double v = (double)(fabs(percent - 100) < DBL_EPSILON ? pRange->i64MaxVal : pRange->i64MinVal);
return v;
}
case TSDB_DATA_TYPE_FLOAT:
case TSDB_DATA_TYPE_DOUBLE:
return fabs(percent - 100) < DBL_EPSILON? pRange->dMaxVal:pRange->dMinVal;
default:
return -1;
}
}
double percentVal = (percent * (pMemBucket->total - 1)) / ((double)100.0);
int32_t orderIdx = (int32_t)percentVal;
// do put data by using buckets
int32_t orderIdx = (int32_t)percentVal;
return getPercentileImpl(pMemBucket, orderIdx, percentVal - orderIdx);
}

View File

@ -54,7 +54,7 @@ int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, SQueryRuntimeEnv *pRun
return TSDB_CODE_QRY_OUT_OF_MEMORY;
}
pWindowResInfo->interval = pRuntimeEnv->pQuery->intervalTime;
pWindowResInfo->interval = pRuntimeEnv->pQuery->interval.interval;
pSummary->internalSupSize += sizeof(SWindowResult) * threshold;
pSummary->internalSupSize += (pRuntimeEnv->pQuery->numOfOutput * sizeof(SResultInfo) + pRuntimeEnv->interBufSize) * pWindowResInfo->capacity;
@ -126,11 +126,26 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) {
int32_t numOfClosed = numOfClosedTimeWindow(pWindowResInfo);
assert(num >= 0 && num <= numOfClosed);
int16_t type = pWindowResInfo->type;
char *key = NULL;
int16_t bytes = -1;
for (int32_t i = 0; i < num; ++i) {
SWindowResult *pResult = &pWindowResInfo->pResult[i];
if (pResult->closed) { // remove the window slot from hash table
taosHashRemove(pWindowResInfo->hashList, (const char *)&pResult->skey, pWindowResInfo->type);
// todo refactor
if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
key = varDataVal(pResult->key);
bytes = varDataLen(pResult->key);
} else {
key = (char*) &pResult->win.skey;
bytes = tDataTypeDesc[pWindowResInfo->type].nSize;
}
taosHashRemove(pWindowResInfo->hashList, (const char *)key, bytes);
} else {
break;
}
@ -150,15 +165,24 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) {
}
pWindowResInfo->size = remain;
for (int32_t k = 0; k < pWindowResInfo->size; ++k) {
SWindowResult *pResult = &pWindowResInfo->pResult[k];
int32_t *p = (int32_t *)taosHashGet(pWindowResInfo->hashList, (const char *)&pResult->skey,
tDataTypeDesc[pWindowResInfo->type].nSize);
if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
key = varDataVal(pResult->key);
bytes = varDataLen(pResult->key);
} else {
key = (char*) &pResult->win.skey;
bytes = tDataTypeDesc[pWindowResInfo->type].nSize;
}
int32_t *p = (int32_t *)taosHashGet(pWindowResInfo->hashList, (const char *)key, bytes);
assert(p != NULL);
int32_t v = (*p - num);
assert(v >= 0 && v <= pWindowResInfo->size);
taosHashPut(pWindowResInfo->hashList, (char *)&pResult->skey, tDataTypeDesc[pWindowResInfo->type].nSize,
(char *)&v, sizeof(int32_t));
taosHashPut(pWindowResInfo->hashList, (char *)key, bytes, (char *)&v, sizeof(int32_t));
}
pWindowResInfo->curIndex = -1;
@ -207,20 +231,19 @@ void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_
}
// get the result order
int32_t resultOrder = (pWindowResInfo->pResult[0].skey < pWindowResInfo->pResult[1].skey)? 1:-1;
int32_t resultOrder = (pWindowResInfo->pResult[0].win.skey < pWindowResInfo->pResult[1].win.skey)? 1:-1;
if (order != resultOrder) {
return;
}
int32_t i = 0;
if (order == QUERY_ASC_FORWARD_STEP) {
TSKEY ekey = pWindowResInfo->pResult[i].skey + pWindowResInfo->interval;
TSKEY ekey = pWindowResInfo->pResult[i].win.ekey;
while (i < pWindowResInfo->size && (ekey < lastKey)) {
++i;
}
} else if (order == QUERY_DESC_FORWARD_STEP) {
while (i < pWindowResInfo->size && (pWindowResInfo->pResult[i].skey > lastKey)) {
while (i < pWindowResInfo->size && (pWindowResInfo->pResult[i].win.skey > lastKey)) {
++i;
}
}
@ -258,7 +281,7 @@ void clearTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pWindow
pWindowRes->numOfRows = 0;
pWindowRes->pos = (SPosInfo){-1, -1};
pWindowRes->closed = false;
pWindowRes->skey = TSKEY_INITIAL_VAL;
pWindowRes->win = TSWINDOW_INITIALIZER;
}
/**
@ -268,7 +291,7 @@ void clearTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pWindow
*/
void copyTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *dst, const SWindowResult *src) {
dst->numOfRows = src->numOfRows;
dst->skey = src->skey;
dst->win = src->win;
dst->closed = src->closed;
int32_t nOutputCols = pRuntimeEnv->pQuery->numOfOutput;

File diff suppressed because it is too large Load Diff

View File

@ -245,6 +245,8 @@ static void *taosAcceptTcpConnection(void *arg) {
}
taosKeepTcpAlive(connFd);
struct timeval to={1, 0};
taosSetSockOpt(connFd, SOL_SOCKET, SO_RCVTIMEO, &to, sizeof(to));
// pick up the thread to handle this connection
pThreadObj = pServerObj->pThreadObj + threadId;

View File

@ -27,7 +27,8 @@ static int tsdbCompareSchemaVersion(const void *key1, const void *key2);
static int tsdbRestoreTable(void *pHandle, void *cont, int contLen);
static void tsdbOrgMeta(void *pHandle);
static char * getTagIndexKey(const void *pData);
static STable *tsdbNewTable(STableCfg *pCfg, bool isSuper);
static STable *tsdbNewTable();
static STable *tsdbCreateTableFromCfg(STableCfg *pCfg, bool isSuper);
static void tsdbFreeTable(STable *pTable);
static int tsdbAddTableToMeta(STsdbRepo *pRepo, STable *pTable, bool addIdx, bool lock);
static void tsdbRemoveTableFromMeta(STsdbRepo *pRepo, STable *pTable, bool rmFromIdx, bool lock);
@ -92,7 +93,7 @@ int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg) {
super = tsdbGetTableByUid(pMeta, pCfg->superUid);
if (super == NULL) { // super table not exists, try to create it
newSuper = 1;
super = tsdbNewTable(pCfg, true);
super = tsdbCreateTableFromCfg(pCfg, true);
if (super == NULL) goto _err;
} else {
if (TABLE_TYPE(super) != TSDB_SUPER_TABLE || TABLE_UID(super) != pCfg->superUid) {
@ -102,7 +103,7 @@ int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg) {
}
}
table = tsdbNewTable(pCfg, false);
table = tsdbCreateTableFromCfg(pCfg, false);
if (table == NULL) goto _err;
// Register to meta
@ -654,15 +655,24 @@ static char *getTagIndexKey(const void *pData) {
return res;
}
static STable *tsdbNewTable(STableCfg *pCfg, bool isSuper) {
static STable *tsdbNewTable() {
STable *pTable = (STable *)calloc(1, sizeof(*pTable));
if (pTable == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
return NULL;
}
pTable->lastKey = TSKEY_INITIAL_VAL;
return pTable;
}
static STable *tsdbCreateTableFromCfg(STableCfg *pCfg, bool isSuper) {
STable *pTable = NULL;
size_t tsize = 0;
pTable = (STable *)calloc(1, sizeof(STable));
if (pTable == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
goto _err;
}
pTable = tsdbNewTable();
if (pTable == NULL) goto _err;
if (isSuper) {
pTable->type = TSDB_SUPER_TABLE;
@ -731,8 +741,6 @@ static STable *tsdbNewTable(STableCfg *pCfg, bool isSuper) {
}
}
}
pTable->lastKey = TSKEY_INITIAL_VAL;
}
T_REF_INC(pTable);
@ -1124,11 +1132,9 @@ static int tsdbEncodeTable(void **buf, STable *pTable) {
}
static void *tsdbDecodeTable(void *buf, STable **pRTable) {
STable *pTable = (STable *)calloc(1, sizeof(STable));
if (pTable == NULL) {
terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
return NULL;
}
STable *pTable = tsdbNewTable();
if (pTable == NULL) return NULL;
uint8_t type = 0;
buf = taosDecodeFixedU8(buf, &type);

View File

@ -1348,7 +1348,7 @@ static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDa
int dcol = 0; // loop iter for SDataCols object
while (dcol < pDataCols->numOfCols) {
SDataCol *pDataCol = &(pDataCols->cols[dcol]);
if (ccol >= pCompData->numOfCols) {
if (dcol != 0 && ccol >= pCompData->numOfCols) {
// Set current column as NULL and forward
dataColSetNEleNull(pDataCol, pCompBlock->numOfRows, pDataCols->maxPoints);
dcol++;

View File

@ -248,6 +248,7 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab
STsdbMeta* pMeta = tsdbGetMeta(tsdb);
assert(pMeta != NULL && sizeOfGroup >= 1 && pCond != NULL && pCond->numOfCols > 0);
// todo apply the lastkey of table check to avoid to load header file
for (int32_t i = 0; i < sizeOfGroup; ++i) {
SArray* group = *(SArray**) taosArrayGet(groupList->pGroupList, i);
@ -388,9 +389,9 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh
SDataRow row = *(SDataRow *)SL_GET_NODE_DATA(node);
TSKEY key = dataRowKey(row); // first timestamp in buffer
tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in mem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64
"-%" PRId64 ", lastKey:%" PRId64 ", %p",
"-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", %p",
pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, key, order, pMem->keyFirst, pMem->keyLast,
pCheckInfo->lastKey, pHandle->qinfo);
pCheckInfo->lastKey, pMem->numOfRows, pHandle->qinfo);
if (ASCENDING_TRAVERSE(order)) {
assert(pCheckInfo->lastKey <= key);
@ -410,9 +411,9 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh
SDataRow row = *(SDataRow *)SL_GET_NODE_DATA(node);
TSKEY key = dataRowKey(row); // first timestamp in buffer
tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in imem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64
"-%" PRId64 ", lastKey:%" PRId64 ", %p",
"-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", %p",
pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, key, order, pIMem->keyFirst, pIMem->keyLast,
pCheckInfo->lastKey, pHandle->qinfo);
pCheckInfo->lastKey, pIMem->numOfRows, pHandle->qinfo);
if (ASCENDING_TRAVERSE(order)) {
assert(pCheckInfo->lastKey <= key);
@ -741,6 +742,7 @@ static int32_t doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* p
return TSDB_CODE_SUCCESS;
}
static int32_t getEndPosInDataBlock(STsdbQueryHandle* pQueryHandle, SDataBlockInfo* pBlockInfo);
static int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfRows, int32_t start, int32_t end);
static void moveDataToFront(STsdbQueryHandle* pQueryHandle, int32_t numOfRows, int32_t numOfCols);
static void doCheckGeneratedBlockRange(STsdbQueryHandle* pQueryHandle);
@ -798,9 +800,10 @@ static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBloc
* Here the buffer is not enough, so only part of file block can be loaded into memory buffer
*/
assert(pQueryHandle->outputCapacity >= binfo.rows);
int32_t endPos = getEndPosInDataBlock(pQueryHandle, &binfo);
if ((cur->pos == 0 && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
(cur->pos == (binfo.rows - 1) && (!ASCENDING_TRAVERSE(pQueryHandle->order)))) {
if ((cur->pos == 0 && endPos == binfo.rows -1 && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
(cur->pos == (binfo.rows - 1) && endPos == 0 && (!ASCENDING_TRAVERSE(pQueryHandle->order)))) {
pQueryHandle->realNumOfRows = binfo.rows;
cur->rows = binfo.rows;
@ -816,7 +819,6 @@ static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBloc
cur->pos = -1;
}
} else { // partially copy to dest buffer
int32_t endPos = ASCENDING_TRAVERSE(pQueryHandle->order)? (binfo.rows - 1): 0;
copyAllRemainRowsFromFileBlock(pQueryHandle, pCheckInfo, &binfo, endPos);
cur->mixBlock = true;
}
@ -1211,6 +1213,29 @@ static void copyAllRemainRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, STabl
cur->win.ekey, cur->rows, pQueryHandle->qinfo);
}
int32_t getEndPosInDataBlock(STsdbQueryHandle* pQueryHandle, SDataBlockInfo* pBlockInfo) {
// NOTE: reverse the order to find the end position in data block
int32_t endPos = -1;
int32_t order = ASCENDING_TRAVERSE(pQueryHandle->order)? TSDB_ORDER_DESC : TSDB_ORDER_ASC;
SQueryFilePos* cur = &pQueryHandle->cur;
SDataCols* pCols = pQueryHandle->rhelper.pDataCols[0];
if (ASCENDING_TRAVERSE(pQueryHandle->order) && pQueryHandle->window.ekey >= pBlockInfo->window.ekey) {
endPos = pBlockInfo->rows - 1;
cur->mixBlock = (cur->pos != 0);
} else if (!ASCENDING_TRAVERSE(pQueryHandle->order) && pQueryHandle->window.ekey <= pBlockInfo->window.skey) {
endPos = 0;
cur->mixBlock = (cur->pos != pBlockInfo->rows - 1);
} else {
assert(pCols->numOfRows > 0);
endPos = doBinarySearchKey(pCols->cols[0].pData, pCols->numOfRows, pQueryHandle->window.ekey, order);
cur->mixBlock = true;
}
return endPos;
}
// only return the qualified data to client in terms of query time window, data rows in the same block but do not
// be included in the query time window will be discarded
static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock) {
@ -1233,19 +1258,7 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pQueryHandle));
STable* pTable = pCheckInfo->pTableObj;
int32_t endPos = cur->pos;
if (ASCENDING_TRAVERSE(pQueryHandle->order) && pQueryHandle->window.ekey > blockInfo.window.ekey) {
endPos = blockInfo.rows - 1;
cur->mixBlock = (cur->pos != 0);
} else if (!ASCENDING_TRAVERSE(pQueryHandle->order) && pQueryHandle->window.ekey < blockInfo.window.skey) {
endPos = 0;
cur->mixBlock = (cur->pos != blockInfo.rows - 1);
} else {
assert(pCols->numOfRows > 0);
endPos = doBinarySearchKey(pCols->cols[0].pData, pCols->numOfRows, pQueryHandle->window.ekey, order);
cur->mixBlock = true;
}
int32_t endPos = getEndPosInDataBlock(pQueryHandle, &blockInfo);
tsdbDebug("%p uid:%" PRIu64",tid:%d start merge data block, file block range:%"PRIu64"-%"PRIu64" rows:%d, start:%d,"
"end:%d, %p",
@ -1366,8 +1379,8 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo*
}
cur->blockCompleted =
(((pos >= endPos || cur->lastKey > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
((pos <= endPos || cur->lastKey < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order)));
(((pos > endPos || cur->lastKey > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
((pos < endPos || cur->lastKey < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order)));
if (!ASCENDING_TRAVERSE(pQueryHandle->order)) {
SWAP(cur->win.skey, cur->win.ekey, TSKEY);
@ -2100,13 +2113,17 @@ STimeWindow changeTableGroupByLastrow(STableGroupInfo *groupList) {
if (keyInfo.pTable != NULL) {
totalNumOfTable++;
taosArrayPush(pGroup, &keyInfo);
} else {
taosArrayRemove(groupList->pGroupList, j);
numOfGroups -= 1;
j -= 1;
}
}
// window does not being updated, so set the original
if (window.skey == INT64_MAX && window.ekey == INT64_MIN) {
window = TSWINDOW_INITIALIZER;
assert(totalNumOfTable == 0);
assert(totalNumOfTable == 0 && taosArrayGetSize(groupList->pGroupList) == 0);
}
groupList->numOfTables = totalNumOfTable;
@ -2427,6 +2444,14 @@ static bool indexedNodeFilterFp(const void* pNode, void* param) {
val = tdGetKVRowValOfCol(pTable->tagVal, pInfo->sch.colId);
}
if (pInfo->optr == TSDB_RELATION_ISNULL || pInfo->optr == TSDB_RELATION_NOTNULL) {
if (pInfo->optr == TSDB_RELATION_ISNULL) {
return (val == NULL) || isNull(val, pInfo->sch.type);
} else if (pInfo->optr == TSDB_RELATION_NOTNULL) {
return (val != NULL) && (!isNull(val, pInfo->sch.type));
}
}
int32_t ret = 0;
if (val == NULL) { //the val is possible to be null, so check it out carefully
ret = -1; // val is missing in table tags value pairs

0
src/util/src/tarray.c Executable file → Normal file
View File

View File

@ -71,7 +71,7 @@ static SCacheDataNode *taosCreateCacheNode(const char *key, size_t keyLen, const
* @param pCacheObj Cache object
* @param pNode Cache slot object
*/
static void taosAddToTrash(SCacheObj *pCacheObj, SCacheDataNode *pNode);
static void taosAddToTrashcan(SCacheObj *pCacheObj, SCacheDataNode *pNode);
/**
* remove nodes in trash with refCount == 0 in cache
@ -80,7 +80,7 @@ static void taosAddToTrash(SCacheObj *pCacheObj, SCacheDataNode *pNode);
* @param force force model, if true, remove data in trash without check refcount.
* may cause corruption. So, forece model only applys before cache is closed
*/
static void taosTrashCanEmpty(SCacheObj *pCacheObj, bool force);
static void taosTrashcanEmpty(SCacheObj *pCacheObj, bool force);
/**
* release node
@ -165,7 +165,7 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool ext
return NULL;
}
// set free cache node callback function for hash table
// set free cache node callback function
pCacheObj->freeFp = fn;
pCacheObj->refreshTime = refreshTimeInSeconds * 1000;
pCacheObj->extendLifespan = extendLifespan;
@ -222,7 +222,7 @@ void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const v
taosTFree(p);
} else {
taosAddToTrash(pCacheObj, p);
taosAddToTrashcan(pCacheObj, p);
uDebug("cache:%s, key:%p, %p exist in cache, updated old:%p", pCacheObj->name, key, pNode1->data, p->data);
}
}
@ -322,7 +322,12 @@ void *taosCacheTransfer(SCacheObj *pCacheObj, void **data) {
}
void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) {
if (pCacheObj == NULL || (*data) == NULL || (taosHashGetSize(pCacheObj->pHashTable) + pCacheObj->numOfElemsInTrash == 0)) {
if (pCacheObj == NULL || taosHashGetSize(pCacheObj->pHashTable) + pCacheObj->numOfElemsInTrash == 0) {
return;
}
if ((*data) == NULL) {
uError("cache:%s, NULL data to release", pCacheObj->name);
return;
}
@ -394,19 +399,19 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) {
"others already", pCacheObj->name, pNode->key, p->data, T_REF_VAL_GET(p), pNode->data);
assert(p->pTNodeHeader == NULL);
taosAddToTrash(pCacheObj, p);
taosAddToTrashcan(pCacheObj, p);
} else {
uDebug("cache:%s, key:%p, %p successfully removed from hash table, refcnt:%d", pCacheObj->name, pNode->key,
pNode->data, ref);
if (ref > 0) {
assert(pNode->pTNodeHeader == NULL);
taosAddToTrash(pCacheObj, pNode);
taosAddToTrashcan(pCacheObj, pNode);
} else { // ref == 0
atomic_sub_fetch_64(&pCacheObj->totalSize, pNode->size);
int32_t size = (int32_t)taosHashGetSize(pCacheObj->pHashTable);
uDebug("cache:%s, key:%p, %p is destroyed from cache, size:%dbytes, num:%d size:%" PRId64 "bytes",
uDebug("cache:%s, key:%p, %p is destroyed from cache, size:%dbytes, totalNum:%d size:%" PRId64 "bytes",
pCacheObj->name, pNode->key, pNode->data, pNode->size, size, pCacheObj->totalSize);
if (pCacheObj->freeFp) {
@ -427,6 +432,26 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) {
char* key = pNode->key;
char* p = pNode->data;
// int32_t ref = T_REF_VAL_GET(pNode);
//
// if (ref == 1 && inTrashcan) {
// // If it is the last ref, remove it from trashcan linked-list first, and then destroy it.Otherwise, it may be
// // destroyed by refresh worker if decrease ref count before removing it from linked-list.
// assert(pNode->pTNodeHeader->pData == pNode);
//
// __cache_wr_lock(pCacheObj);
// doRemoveElemInTrashcan(pCacheObj, pNode->pTNodeHeader);
// __cache_unlock(pCacheObj);
//
// ref = T_REF_DEC(pNode);
// assert(ref == 0);
//
// doDestroyTrashcanElem(pCacheObj, pNode->pTNodeHeader);
// } else {
// ref = T_REF_DEC(pNode);
// assert(ref >= 0);
// }
int32_t ref = T_REF_DEC(pNode);
uDebug("cache:%s, key:%p, %p released, refcnt:%d, data in trashcan:%d", pCacheObj->name, key, p, ref, inTrashcan);
}
@ -447,7 +472,7 @@ static bool travHashTableEmptyFn(void* param, void* data) {
if (T_REF_VAL_GET(pNode) == 0) {
taosCacheReleaseNode(pCacheObj, pNode);
} else { // do add to trashcan
taosAddToTrash(pCacheObj, pNode);
taosAddToTrashcan(pCacheObj, pNode);
}
// this node should be remove from hash table
@ -458,7 +483,7 @@ void taosCacheEmpty(SCacheObj *pCacheObj) {
SHashTravSupp sup = {.pCacheObj = pCacheObj, .fp = NULL, .time = taosGetTimestampMs()};
taosHashCondTraverse(pCacheObj->pHashTable, travHashTableEmptyFn, &sup);
taosTrashCanEmpty(pCacheObj, false);
taosTrashcanEmpty(pCacheObj, false);
}
void taosCacheCleanup(SCacheObj *pCacheObj) {
@ -498,7 +523,7 @@ SCacheDataNode *taosCreateCacheNode(const char *key, size_t keyLen, const char *
return pNewNode;
}
void taosAddToTrash(SCacheObj *pCacheObj, SCacheDataNode *pNode) {
void taosAddToTrashcan(SCacheObj *pCacheObj, SCacheDataNode *pNode) {
if (pNode->inTrashcan) { /* node is already in trash */
assert(pNode->pTNodeHeader != NULL && pNode->pTNodeHeader->pData == pNode);
return;
@ -520,11 +545,11 @@ void taosAddToTrash(SCacheObj *pCacheObj, SCacheDataNode *pNode) {
pCacheObj->numOfElemsInTrash++;
__cache_unlock(pCacheObj);
uDebug("cache:%s key:%p, %p move to trash, numOfElem in trash:%d", pCacheObj->name, pNode->key, pNode->data,
uDebug("cache:%s key:%p, %p move to trashcan, numOfElem in trashcan:%d", pCacheObj->name, pNode->key, pNode->data,
pCacheObj->numOfElemsInTrash);
}
void taosTrashCanEmpty(SCacheObj *pCacheObj, bool force) {
void taosTrashcanEmpty(SCacheObj *pCacheObj, bool force) {
__cache_wr_lock(pCacheObj);
if (pCacheObj->numOfElemsInTrash == 0) {
@ -568,7 +593,7 @@ void doCleanupDataCache(SCacheObj *pCacheObj) {
// todo memory leak if there are object with refcount greater than 0 in hash table?
taosHashCleanup(pCacheObj->pHashTable);
taosTrashCanEmpty(pCacheObj, true);
taosTrashcanEmpty(pCacheObj, true);
__cache_lock_destroy(pCacheObj);
@ -643,7 +668,7 @@ void* taosCacheTimedRefresh(void *handle) {
doCacheRefresh(pCacheObj, now, NULL);
}
taosTrashCanEmpty(pCacheObj, false);
taosTrashcanEmpty(pCacheObj, false);
}
return NULL;

Some files were not shown because too many files have changed in this diff Show More