From e593c12cef6b7247d5d7ed24d372182fdbc31ee4 Mon Sep 17 00:00:00 2001 From: slguan Date: Mon, 2 Dec 2019 16:18:32 +0800 Subject: [PATCH 001/108] Instructions for using the RESTful interface --- .../webdocs/markdowndocs/connector-ch.md | 126 ++++++++++++++++-- 1 file changed, 116 insertions(+), 10 deletions(-) diff --git a/documentation/webdocs/markdowndocs/connector-ch.md b/documentation/webdocs/markdowndocs/connector-ch.md index 9261b9eef6..e629a44bf1 100644 --- a/documentation/webdocs/markdowndocs/connector-ch.md +++ b/documentation/webdocs/markdowndocs/connector-ch.md @@ -316,11 +316,23 @@ import taos 如：http://192.168.0.1:6020/rest/sql 是指向IP地址为192.168.0.1的URL. -HTTP请求的Header里需带有身份认证信息，TDengine单机版仅支持Basic认证机制。 +HTTP请求的Header里需带有身份认证信息，TDengine支持Basic认证与自定义认证两种机制，后续版本将提供标准安全的数字签名机制来做身份验证。 + +- 自定义身份认证信息如下所示（稍后介绍） + +``` +Authorization: Taosd +``` + +- Basic身份认证信息如下所示 + +``` +Authorization: Basic +``` HTTP请求的BODY里就是一个完整的SQL语句，SQL语句中的数据表应提供数据库前缀，例如\.\。如果表名不带数据库前缀，系统会返回错误。因为HTTP模块只是一个简单的转发，没有当前DB的概念。 -使用curl来发起一个HTTP Request, 语法如下： +使用curl通过自定义身份认证方式来发起一个HTTP Request, 语法如下： ``` curl -H 'Authorization: Basic ' -d '' :/rest/sql @@ -332,7 +344,7 @@ curl -H 'Authorization: Basic ' -d '' :/rest/sql curl -u username:password -d '' :/rest/sql ``` -其中，`TOKEN`为`{username}:{password}`经过Base64编码之后的字符串，例如`root:taosdata`编码后为`cm9vdDp0YW9zZGF0YQ==` +其中，`TOKEN`为`{username}:{password}`经过Base64编码之后的字符串, 例如`root:taosdata`编码后为`cm9vdDp0YW9zZGF0YQ==` ### HTTP返回格式 @@ -356,21 +368,55 @@ curl -u username:password -d '' :/rest/sql - 第三行是具体返回的数据，一排一排的呈现。如果不返回结果集，仅[[affected_rows]] - 第四行”rows”表明总共多少行数据 +### 自定义授权码 + +HTTP请求中需要带有授权码``, 用于身份识别。授权码通常由管理员提供, 可简单的通过发送`HTTP GET`请求来获取授权码, 操作如下： + +``` +curl http://:6020/rest/login// +``` + +其中, `ip`是TDengine数据库的IP地址, `username`为数据库用户名, `password`为数据库密码, 返回值为`JSON`格式, 各字段含义如下： + +- status：请求结果的标志位 + +- code：返回值代码 + +- desc: 授权码 + +获取授权码示例： + +``` +curl http://192.168.0.1:6020/rest/login/root/taosdata +``` + +返回值： + +``` +{ + "status": "succ", + "code": 0, + "desc": +"/KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04" +} +``` + ### 使用示例 - 在demo库里查询表t1的所有记录, curl如下： - `curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.t1' 192.168.0.1:6020/rest/sql` - - 返回值： +``` +curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.t1' 192.168.0.1:6020/rest/sql` +``` +返回值： ``` { "status": "succ", "head": ["column1","column2","column3"], "data": [ - ["2017-12-12 23:44:25.730", 1, 2.3], - ["2017-12-12 22:44:25.728", 4, 5.6] + ["2017-12-12 22:44:25.728",4,5.60000], + ["2017-12-12 23:44:25.730",1,2.30000] ], "rows": 2 } @@ -378,9 +424,11 @@ curl -u username:password -d '' :/rest/sql - 创建库demo： - `curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'create database demo' 192.168.0.1:6020/rest/sql` +``` +curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'create database demo' 192.168.0.1:6020/rest/sql` +``` - 返回值： +返回值： ``` { "status": "succ", @@ -390,6 +438,64 @@ curl -u username:password -d '' :/rest/sql } ``` +### 其他用法 + +#### 结果集采用Unix时间戳 + +HTTP请求URL采用`sqlt`时，返回结果集的时间戳将采用Unix时间戳格式表示，例如 + +``` +curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.t1' 192.168.0.1:6020/rest/sqlt` +``` + +返回值： + +``` +{ + "status": "succ", + "head": ["column1","column2","column3"], + "data": [ + [1513089865728,4,5.60000], + [1513093465730,1,2.30000] + ], + "rows": 2 +} +``` + +#### 结果集采用UTC时间字符串 + +HTTP请求URL采用`sqlutc`时，返回结果集的时间戳将采用UTC时间字符串表示，例如 +``` + curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.t1' 192.168.0.1:6020/rest/sqlutc` +``` + +返回值： + +``` +{ + "status": "succ", + "head": ["column1","column2","column3"], + "data": [ + ["2017-12-12T22:44:25.728+0800",4,5.60000], + ["2017-12-12T23:44:25.730+0800",1,2.30000] + ], + "rows": 2 +} +``` + +### 重要配置项 + +下面仅列出一些与RESTFul接口有关的配置参数，其他系统参数请看配置文件里的说明。注意：配置修改后，需要重启taosd服务才能生效 + +- httpIp: 对外提供RESTFul服务的IP地址，默认绑定到0.0.0.0 +- httpPort: 对外提供RESTFul服务的端口号，默认绑定到6020 +- httpMaxThreads: 启动的线程数量，默认为2 +- httpCacheSessions: 缓存连接的数量，并发请求数目需小于此数值的10倍，默认值为100 +- restfulRowLimit: 返回结果集（JSON格式）的最大条数，默认值为10240 +- httpEnableCompress: 是否支持压缩，默认不支持，目前TDengine仅支持gzip压缩格式 +- httpDebugFlag: 日志开关，131：仅错误和报警信息，135：所有，默认131 + + ## Go Connector #### 安装TDengine From 0517feb1fd1c52733ca8683b78c913a7eb594a43 Mon Sep 17 00:00:00 2001 From: slguan Date: Mon, 2 Dec 2019 16:31:38 +0800 Subject: [PATCH 002/108] Instructions for using the RESTful interface --- .../webdocs/markdowndocs/connector-ch.md | 30 ++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/documentation/webdocs/markdowndocs/connector-ch.md b/documentation/webdocs/markdowndocs/connector-ch.md index e629a44bf1..85596f2053 100644 --- a/documentation/webdocs/markdowndocs/connector-ch.md +++ b/documentation/webdocs/markdowndocs/connector-ch.md @@ -306,15 +306,16 @@ import taos ### HTTP请求格式 -​ `http://:/rest/sql` +``` +http://:/rest/sql +``` -​ 参数说明： +​参数说明： -​ IP: 集群中的任一台主机 +- IP: 集群中的任一台主机 +- PORT: 配置文件中httpPort配置项，缺省为6020 -​ PORT: 配置文件中httpPort配置项，缺省为6020 - -如：http://192.168.0.1:6020/rest/sql 是指向IP地址为192.168.0.1的URL. +例如：http://192.168.0.1:6020/rest/sql 是指向IP地址为192.168.0.1的URL. HTTP请求的Header里需带有身份认证信息，TDengine支持Basic认证与自定义认证两种机制，后续版本将提供标准安全的数字签名机制来做身份验证。 @@ -348,7 +349,8 @@ curl -u username:password -d '' :/rest/sql ### HTTP返回格式 -返回值为JSON格式，如下： +返回值为JSON格式，如下: + ``` { "status": "succ", @@ -363,10 +365,10 @@ curl -u username:password -d '' :/rest/sql 说明： -- 第一行”status”告知操作结果是成功还是失败; -- 第二行”head”是表的定义，如果不返回结果集，仅有一列“affected_rows”; -- 第三行是具体返回的数据，一排一排的呈现。如果不返回结果集，仅[[affected_rows]] -- 第四行”rows”表明总共多少行数据 +- status: 告知操作结果是成功还是失败 +- head: 表的定义，如果不返回结果集，仅有一列“affected_rows” +- data: 具体返回的数据，一排一排的呈现,如果不返回结果集，仅[[affected_rows]] +- rows: 表明总共多少行数据 ### 自定义授权码 @@ -403,7 +405,7 @@ curl http://192.168.0.1:6020/rest/login/root/taosdata ### 使用示例 -- 在demo库里查询表t1的所有记录, curl如下： +- 在demo库里查询表t1的所有记录： ``` curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.t1' 192.168.0.1:6020/rest/sql` @@ -445,7 +447,7 @@ curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'create database demo' 19 HTTP请求URL采用`sqlt`时，返回结果集的时间戳将采用Unix时间戳格式表示，例如 ``` -curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.t1' 192.168.0.1:6020/rest/sqlt` +curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.t1' 192.168.0.1:6020/rest/sqlt ``` 返回值： @@ -466,7 +468,7 @@ curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.t1' 1 HTTP请求URL采用`sqlutc`时，返回结果集的时间戳将采用UTC时间字符串表示，例如 ``` - curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.t1' 192.168.0.1:6020/rest/sqlutc` + curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.t1' 192.168.0.1:6020/rest/sqlutc ``` 返回值： From e5418dd3e13370be8a5d5b7518e5069472389e58 Mon Sep 17 00:00:00 2001 From: haojun Liao Date: Mon, 2 Dec 2019 18:37:51 +0800 Subject: [PATCH 003/108] Update TAOS SQL-ch.md Add some detailed description with respect to the time unit conversion for interval clause. --- documentation/webdocs/markdowndocs/TAOS SQL-ch.md | 1 + 1 file changed, 1 insertion(+) diff --git a/documentation/webdocs/markdowndocs/TAOS SQL-ch.md b/documentation/webdocs/markdowndocs/TAOS SQL-ch.md index 6a8549bbd2..347ac4f21f 100644 --- a/documentation/webdocs/markdowndocs/TAOS SQL-ch.md +++ b/documentation/webdocs/markdowndocs/TAOS SQL-ch.md @@ -18,6 +18,7 @@ TDengine提供类似SQL语法，用户可以在TDengine Shell中使用SQL语句 - 插入记录时，如果时间戳为0，插入数据时使用服务器当前时间 - Epoch Time: 时间戳也可以是一个长整数，表示从1970-01-01 08:00:00.000开始的毫秒数 - 时间可以加减，比如 now-2h，表明查询时刻向前推2个小时(最近2小时)。数字后面的时间单位：a(毫秒), s(秒), m(分), h(小时), d(天)，w(周), n(月), y(年)。比如select * from t1 where ts > now-2w and ts <= now-1w, 表示查询两周前整整一周的数据 +- TDengine暂不支持时间窗口按照自然年和自然月切分。Where条件中的时间窗口单位的换算关系如下：interval(1y) 等效于 interval(365d), interval(1n) 等效于 interval(30d), interval(1w) 等效于 interval(7d) TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMicrosecond就可支持微秒。 From 5d25f776a4bcb49b432e9809301eb623ec98cc2f Mon Sep 17 00:00:00 2001 From: fang Date: Mon, 2 Dec 2019 22:02:11 +0800 Subject: [PATCH 004/108] python connector user guide --- .../tdenginedocs-cn/connector/index.html | 90 +++++++++++++++---- .../tdenginedocs-en/connector/index.html | 66 +++++++++++++- 2 files changed, 138 insertions(+), 18 deletions(-) diff --git a/documentation/tdenginedocs-cn/connector/index.html b/documentation/tdenginedocs-cn/connector/index.html index fca343e977..d808e42dda 100644 --- a/documentation/tdenginedocs-cn/connector/index.html +++ b/documentation/tdenginedocs-cn/connector/index.html @@ -114,23 +114,83 @@ public Connection getConn() throws Exception{

对于TDengine操作的报错信息，用户可使用JDBCDriver包里提供的枚举类TSDBError.java来获取error message和error code的列表。对于更多的具体操作的相关代码，请参考TDengine提供的使用示范项目JDBCDemo。

Python Connector

-

Python客户端安装

-

用户可以在源代码的src/connector/python文件夹下找到python2和python3的安装包。用户可以通过pip命令安装：

-

​ pip install src/connector/python/python2/

-

或

-

​ pip install src/connector/python/python3/

-

如果机器上没有pip命令，用户可将src/connector/python/python3或src/connector/python/python2下的taos文件夹拷贝到应用程序的目录使用。

-

Python客户端接口

+

安装准备

+
  • 已安装TDengine, 如果客户端在Windows上，需要安装Windows 版本的TDengine客户端
    • +
  • 已安装python 2.7 or >= 3.4
    • +
  • 已安装pip
    • +

    安装

    +

    Linux

    +

    用户可以在源代码的src/connector/python文件夹下找到python2和python3的安装包， 然后通过pip命令安装

    +
    pip install src/connector/python/linux/python2/
    +

    或者

    +
    pip install src/connector/python/linux/python3/
    +

    Windows

    +

    在已安装Windows 客户端的情况下， 将文件"C:\TDengine\driver\taos.dll" 拷贝到 "C:\windows\system32" 目录下, 然后进入Windwos cmd 命令行界面

    +
    cd C:\TDengine\connector\python\windows
    +
    pip install python2\
    +

    或者

    +
    cd C:\TDengine\connector\python\windows
    +
    pip install python3\
    +

    * 如果机器上没有pip命令，用户可将src/connector/python/windows/python3或src/connector/python/windows/python2下的taos文件夹拷贝到应用程序的目录使用。

    +

    Python client interfaces

    在使用TDengine的python接口时，需导入TDengine客户端模块：

    -
    import taos
    -

    用户可通过python的帮助信息直接查看模块的使用信息，或者参考code/examples/python中的示例程序。以下为部分常用类和方法：

    +
    import taos
    +

    代码示例

    +
  • 获取连接
    • +
    
+conn = taos.connect(host="127.0.0.1", user="root", password="taosdata", config="/etc/taos")
+c1 = conn.cursor()
+
    +
  • 写入数据
    • +
    
+import datetime
+ 
+# create a database
+c1.execute('create database db')
+c1.execute('use db')
+# create a table
+c1.execute('create table tb (ts timestamp, temperature int, humidity float)')
+# insert a record
+start_time = datetime.datetime(2019, 11, 1)
+affected_rows = c1.execute('insert into tb values (\'%s\', 0, 0.0)' %start_time)
+# insert multiple records in a batch
+time_interval = datetime.timedelta(seconds=60)
+sqlcmd = ['insert into tb values']
+for irow in range(1,11):
+  start_time += time_interval
+  sqlcmd.append('(\'%s\', %d, %f)' %(start_time, irow, irow*1.2))
+affected_rows = c1.execute(' '.join(sqlcmd))
+
    +
  • 写入数据
    • +
    +c1.execute('select * from tb')
+# fetch all returned results
+data = c1.fetchall()
+# data is a list of returned rows with each row being a tuple
+numOfRows = c1.rowcount
+numOfCols = c1.descriptions
+for irow in range(numOfRows):
+  print("Row%d: ts=%s, temperature=%d, humidity=%f" %(irow, data[irow][0], data[irow][1],data[irow][2])
+  
+# use the cursor as an iterator to retrieve all returned results
+c1.execute('select * from tb')
+for data in c1:
+  print("ts=%s, temperature=%d, humidity=%f" %(data[0], data[1],data[2])
+
     +
  • 关闭连接
    • +
    +c1.close()
+conn.close()
+
     +

    帮助信息

    +

    用户可通过python的帮助信息直接查看模块的使用信息，或者参考code/examples/python中的示例程序。以下为部分常用类和方法:

      -

    • TaosConnection类

      -

      参考python中help(taos.TaosConnection)。

      • -

    • TaosCursor类

      -

      参考python中help(taos.TaosCursor)。

      • -

    • connect方法

      -

      用于生成taos.TaosConnection的实例。

      • +

    • TaosConnection 类

      +

      参考python中help(taos.TDengineConnection)

      • +

    • TaosCursor 类

      +

      参考python中help(taos.TDengineCursor)

      • +

    • connect 方法

      +

      用于生成taos.TDengineConnection的实例。

    RESTful Connector

    为支持各种不同类型平台的开发，TDengine提供符合REST设计标准的API，即RESTful API。为最大程度降低学习成本，不同于其他数据库RESTful API的设计方法，TDengine直接通过HTTP POST 请求BODY中包含的SQL语句来操作数据库，仅需要一个URL。

    diff --git a/documentation/tdenginedocs-en/connector/index.html b/documentation/tdenginedocs-en/connector/index.html index ce32c062ff..ba1cbcaf85 100644 --- a/documentation/tdenginedocs-en/connector/index.html +++ b/documentation/tdenginedocs-en/connector/index.html @@ -122,15 +122,75 @@ public Connection getConn() throws Exception{

    All the error codes and error messages can be found in TSDBError.java . For a more detailed coding example, please refer to the demo project JDBCDemo in TDengine's code examples.

    Python Connector

    +

    Pre-requirement

    +
  • TDengine installed, TDengine-client installed if on Windows
    • +
  • python 2.7 or >= 3.4
    • +
  • pip installed

    • Install TDengine Python client

    +

    Linux

    Users can find python client packages in our source code directory src/connector/python. There are two directories corresponding two python versions. Please choose the correct package to install. Users can use pip command to install:

    -
    pip install src/connector/python/python2/
    +
    pip install src/connector/python/linux/python2/

    or

    -
    pip install src/connector/python/python3/
    -

    If pip command is not installed on the system, users can choose to install pip or just copy the taos directory in the python client directory to the application directory to use.

    +
    pip install src/connector/python/linux/python3/
    +

    Windows

    +

    Assume you have installed the Windows client , copy the file "C:\TDengine\driver\taos.dll" to the folder "C:\windows\system32", and then enter the cmd command interface

    +
    cd C:\TDengine\connector\python\windows
    +
    pip install python2\
    +

    or

    +
    cd C:\TDengine\connector\python\windows
    +
    pip install python3\
    +

    * If pip command is not installed on the system, users can choose to install pip or just copy the taos directory in the python client directory to the application directory to use.

    Python client interfaces

    To use TDengine Python client, import TDengine module at first:

    import taos
    +

    Examples

    +
  • get the connection
    • +
    
+conn = taos.connect(host="127.0.0.1", user="root", password="taosdata", config="/etc/taos")
+c1 = conn.cursor()
+
    +
  • insert records into the database
    • +
    
+import datetime
+ 
+# create a database
+c1.execute('create database db')
+c1.execute('use db')
+# create a table
+c1.execute('create table tb (ts timestamp, temperature int, humidity float)')
+# insert a record
+start_time = datetime.datetime(2019, 11, 1)
+affected_rows = c1.execute('insert into tb values (\'%s\', 0, 0.0)' %start_time)
+# insert multiple records in a batch
+time_interval = datetime.timedelta(seconds=60)
+sqlcmd = ['insert into tb values']
+for irow in range(1,11):
+  start_time += time_interval
+  sqlcmd.append('(\'%s\', %d, %f)' %(start_time, irow, irow*1.2))
+affected_rows = c1.execute(' '.join(sqlcmd))
+
    +
  • query the database
    • +
    +c1.execute('select * from tb')
+# fetch all returned results
+data = c1.fetchall()
+# data is a list of returned rows with each row being a tuple
+numOfRows = c1.rowcount
+numOfCols = c1.descriptions
+for irow in range(numOfRows):
+  print("Row%d: ts=%s, temperature=%d, humidity=%f" %(irow, data[irow][0], data[irow][1],data[irow][2])
+  
+# use the cursor as an iterator to retrieve all returned results
+c1.execute('select * from tb')
+for data in c1:
+  print("ts=%s, temperature=%d, humidity=%f" %(data[0], data[1],data[2])
+
     +
  • close the connection
    • +
    +c1.close()
+conn.close()
+
     +

    Help information

    Users can get module information from Python help interface or refer to our [python code example](). We list the main classes and methods below:

    TDengine is specially designed and optimized for time-series data processing in IoT, connected cars, Industrial IoT, IT infrastructure and application monitoring, and other scenarios. Compared with other solutions, it is 10x faster on insert/query speed. With a single-core machine, over 20K requestes can be processed, millions data points can be ingested, and over 10 million data points can be retrieved in a second. Via column-based storage and tuned compression algorithm for different data types, less than 1/10 storage space is required.

    Explore More on TDengine

    -

    Please read through the whole documentation to learn more about TDengine.

    Back \ No newline at end of file +

    Please read through the whole documentation to learn more about TDengine.

    Back From 327eebb51ba0b76f343cb03356237f5871ba195f Mon Sep 17 00:00:00 2001 From: ATR Date: Mon, 9 Dec 2019 19:18:21 +0800 Subject: [PATCH 084/108] Update index.html create table t (ts timestamp, speed int); // fix 'cdata' to 'speed' --- documentation/tdenginedocs-cn/getting-started/index.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation/tdenginedocs-cn/getting-started/index.html b/documentation/tdenginedocs-cn/getting-started/index.html index 65ef667d00..d7d5d8540c 100644 --- a/documentation/tdenginedocs-cn/getting-started/index.html +++ b/documentation/tdenginedocs-cn/getting-started/index.html @@ -28,7 +28,7 @@

    在TDengine终端中，用户可以通过SQL命令来创建/删除数据库、表等，并进行插入查询操作。在终端中运行的SQL语句需要以分号结束来运行。示例：

    create database db;
 use db;
-create table t (ts timestamp, cdata int);
+create table t (ts timestamp, speed int);
 insert into t values ('2019-07-15 00:00:00', 10);
 insert into t values ('2019-07-15 01:00:00', 20);
 select * from t;
@@ -85,4 +85,4 @@ Query OK, 2 row(s) in set (0.001700s)

    TDengine是专为物联网、车联网、工业互联网、运维监测等场景优化设计的时序数据处理引擎。与其他方案相比，它的插入查询速度都快10倍以上。单核一秒钟就能插入100万数据点，读出1000万数据点。由于采用列式存储和优化的压缩算法，存储空间不及普通数据库的1/10.

    深入了解TDengine

    -

    请继续阅读文档来深入了解TDengine。

    回去 \ No newline at end of file +

    请继续阅读文档来深入了解TDengine。

    回去 From 4c7a815fdd29e1e8763231c924aeb7df667f6d72 Mon Sep 17 00:00:00 2001 From: lihui Date: Mon, 9 Dec 2019 19:19:13 +0800 Subject: [PATCH 085/108] [NONE] --- src/client/src/tscParseInsert.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 6317decbe0..0aa04f518c 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -444,7 +444,7 @@ int tsParseOneRowData(char **str, STableDataBlocks *pDataBlocks, SSchema schema[ } } - tmpTokenBuf[j] = sToken.z[i]; + tmpTokenBuf[j] = sToken.z[k]; j++; } tmpTokenBuf[j] = 0; From a5df5d938c991cb01cc7a0e968b30a1ae055f848 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Mon, 9 Dec 2019 22:19:35 +0800 Subject: [PATCH 086/108] suppress compile warnings --- src/client/src/tscAst.c | 7 +++---- src/client/src/tscServer.c | 9 ++++++--- src/client/src/tscSub.c | 4 ++-- src/client/src/tscUtil.c | 3 ++- 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/client/src/tscAst.c b/src/client/src/tscAst.c index a06a075248..d071358dbf 100644 --- a/src/client/src/tscAst.c +++ b/src/client/src/tscAst.c @@ -643,13 +643,12 @@ int32_t intersect(tQueryResultset *pLeft, tQueryResultset *pRight, tQueryResults } /* - * + * traverse the result and apply the function to each item to check if the item is qualified or not */ -void tSQLListTraverseOnResult(struct tSQLBinaryExpr *pExpr, bool (*fp)(tSkipListNode *, void *), - tQueryResultset * pResult) { +static void tSQLListTraverseOnResult(struct tSQLBinaryExpr *pExpr, __result_filter_fn_t fp, tQueryResultset *pResult) { assert(pExpr->pLeft->nodeType == TSQL_NODE_COL && pExpr->pRight->nodeType == TSQL_NODE_VALUE); - // brutal force search + // brutal force scan the result list and check for each item in the list int64_t num = pResult->num; for (int32_t i = 0, j = 0; i < pResult->num; ++i) { if (fp == NULL || (fp(pResult->pRes[i], pExpr->info) == true)) { diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 28cefeea73..0913cf956c 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -3350,7 +3350,7 @@ int tscProcessShowRsp(SSqlObj *pSql) { } int tscProcessConnectRsp(SSqlObj *pSql) { - char temp[TSDB_METER_ID_LEN]; + char temp[TSDB_METER_ID_LEN*2]; SConnectRsp *pConnect; STscObj *pObj = pSql->pTscObj; @@ -3358,8 +3358,11 @@ int tscProcessConnectRsp(SSqlObj *pSql) { pConnect = (SConnectRsp *)pRes->pRsp; strcpy(pObj->acctId, pConnect->acctId); // copy acctId from response - sprintf(temp, "%s%s%s", pObj->acctId, TS_PATH_DELIMITER, pObj->db); - strcpy(pObj->db, temp); + int32_t len =sprintf(temp, "%s%s%s", pObj->acctId, TS_PATH_DELIMITER, pObj->db); + + assert(len <= tListLen(pObj->db)); + strncpy(pObj->db, temp, tListLen(pObj->db)); + #ifdef CLUSTER SIpList * pIpList; char *rsp = pRes->pRsp + sizeof(SConnectRsp); diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c index d6bc1eebe9..f2e9395c68 100644 --- a/src/client/src/tscSub.c +++ b/src/client/src/tscSub.c @@ -56,7 +56,7 @@ TAOS_SUB *taos_subscribe(const char *host, const char *user, const char *pass, c if (pSub->taos == NULL) { tfree(pSub); } else { - char qstr[128]; + char qstr[256] = {0}; sprintf(qstr, "use %s", db); int res = taos_query(pSub->taos, qstr); if (res != 0) { @@ -64,7 +64,7 @@ TAOS_SUB *taos_subscribe(const char *host, const char *user, const char *pass, c taos_close(pSub->taos); tfree(pSub); } else { - sprintf(qstr, "select * from %s where _c0 > now+1000d", pSub->name); + snprintf(qstr, tListLen(qstr), "select * from %s where _c0 > now+1000d", pSub->name); if (taos_query(pSub->taos, qstr)) { tscTrace("failed to select, reason:%s", taos_errstr(pSub->taos)); taos_close(pSub->taos); diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 3ff262bf68..288e564cb0 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1138,7 +1138,8 @@ void tscColumnFilterInfoCopy(SColumnFilterInfo* dst, const SColumnFilterInfo* sr *dst = *src; if (dst->filterOnBinary) { size_t len = (size_t) dst->len + 1; - dst->pz = calloc(1, len); + char* pTmp = calloc(1, len); + dst->pz = (int64_t) pTmp; memcpy((char*) dst->pz, (char*) src->pz, (size_t) len); } } From 2ab2e2831e77ef51edfd5f9fff373db97ee603ab Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Mon, 9 Dec 2019 23:23:05 +0800 Subject: [PATCH 087/108] suppress some compile warnings --- src/inc/taosmsg.h | 2 +- src/system/detail/inc/mgmtUtil.h | 2 +- src/system/detail/src/mgmtSupertableQuery.c | 12 +++++++----- src/system/detail/src/vnodeShell.c | 2 +- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index f39786da2c..895cf23cf8 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -622,7 +622,7 @@ typedef struct { char repStrategy; char loadLatest; // load into mem or not - char precision; // time resoluation + uint8_t precision; // time resolution char reserved[16]; } SVnodeCfg, SCreateDbMsg, SDbCfg, SAlterDbMsg; diff --git a/src/system/detail/inc/mgmtUtil.h b/src/system/detail/inc/mgmtUtil.h index 04bacbe1db..1f70485894 100644 --- a/src/system/detail/inc/mgmtUtil.h +++ b/src/system/detail/inc/mgmtUtil.h @@ -37,6 +37,6 @@ int32_t mgmtRetrieveMetersFromMetric(SMetricMetaMsg* pInfo, int32_t tableIndex, int32_t mgmtDoJoin(SMetricMetaMsg* pMetricMetaMsg, tQueryResultset* pRes); void mgmtReorganizeMetersInMetricMeta(SMetricMetaMsg* pInfo, int32_t index, tQueryResultset* pRes); -bool tSkipListNodeFilterCallback(struct tSkipListNode *pNode, void *param); +bool tSkipListNodeFilterCallback(const void *pNode, void *param); #endif //TBASE_MGMTUTIL_H diff --git a/src/system/detail/src/mgmtSupertableQuery.c b/src/system/detail/src/mgmtSupertableQuery.c index 4dc7760d89..9b39532276 100644 --- a/src/system/detail/src/mgmtSupertableQuery.c +++ b/src/system/detail/src/mgmtSupertableQuery.c @@ -203,7 +203,7 @@ static bool mgmtTablenameFilterCallback(tSkipListNode* pNode, void* param) { static void mgmtRetrieveFromLikeOptr(tQueryResultset* pRes, const char* str, STabObj* pMetric) { SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER; - SMeterNameFilterSupporter supporter = {info, str}; + SMeterNameFilterSupporter supporter = {info, (char*) str}; pRes->num = tSkipListIterateList(pMetric->pSkipList, (tSkipListNode***)&pRes->pRes, mgmtTablenameFilterCallback, &supporter); @@ -230,7 +230,7 @@ static void mgmtFilterByTableNameCond(tQueryResultset* pRes, char* condStr, int3 free(str); } -static bool mgmtJoinFilterCallback(tSkipListNode* pNode, void* param) { +UNUSED_FUNC static bool mgmtJoinFilterCallback(tSkipListNode* pNode, void* param) { SJoinSupporter* pSupporter = (SJoinSupporter*)param; SSchema s = {0}; @@ -639,7 +639,8 @@ static void getTagColumnInfo(SSyntaxTreeFilterSupporter* pSupporter, SSchema* pS } } -void filterPrepare(tSQLBinaryExpr* pExpr, void* param) { +void filterPrepare(void* expr, void* param) { + tSQLBinaryExpr *pExpr = (tSQLBinaryExpr*) expr; if (pExpr->info != NULL) { return; } @@ -791,9 +792,10 @@ static char* getTagValueFromMeter(STabObj* pMeter, int32_t offset, void* param) } } -bool tSkipListNodeFilterCallback(tSkipListNode* pNode, void* param) { +bool tSkipListNodeFilterCallback(const void* pNode, void* param) { + tQueryInfo* pInfo = (tQueryInfo*)param; - STabObj* pMeter = (STabObj*)pNode->pData; + STabObj* pMeter = (STabObj*)(((tSkipListNode*)pNode)->pData); char name[TSDB_METER_NAME_LEN + 1] = {0}; char* val = getTagValueFromMeter(pMeter, pInfo->offset, name); diff --git a/src/system/detail/src/vnodeShell.c b/src/system/detail/src/vnodeShell.c index 5eea6632af..9ae33c43b7 100644 --- a/src/system/detail/src/vnodeShell.c +++ b/src/system/detail/src/vnodeShell.c @@ -473,7 +473,7 @@ void vnodeExecuteRetrieveReq(SSchedMsg *pSched) { if (numOfRows == 0 && (pRetrieve->qhandle == (uint64_t)pObj->qhandle) && (code != TSDB_CODE_ACTION_IN_PROGRESS)) { dTrace("QInfo:%p %s free qhandle code:%d", pObj->qhandle, __FUNCTION__, code); - vnodeFreeQInfo(pObj->qhandle); + vnodeFreeQInfo(pObj->qhandle, true); pObj->qhandle = NULL; } From c2a2c575a114378a2cc8f6f992b778dbef1215ef Mon Sep 17 00:00:00 2001 From: fang Date: Tue, 10 Dec 2019 08:46:01 +0800 Subject: [PATCH 088/108] solve some statich check questions --- src/client/src/tscPrepare.c | 1 - src/client/src/tscSQLParser.c | 1 - 2 files changed, 2 deletions(-) diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index 2f1db58117..ef8ddfd211 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -65,7 +65,6 @@ static int normalStmtAddPart(SNormalStmt* stmt, bool isParam, char* str, uint32_ } stmt->sizeParts = size; stmt->parts = (SNormalStmtPart*)tmp; - free(tmp); } stmt->parts[stmt->numParts].isParam = isParam; diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index cb73a6e9bd..ac2b958be7 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -2927,7 +2927,6 @@ static SColumnFilterInfo* addColumnFilterInfo(SColumnBase* pColumn) { char* tmp = realloc(pColumn->filterInfo, sizeof(SColumnFilterInfo) * (size)); if (tmp != NULL) { pColumn->filterInfo = (SColumnFilterInfo*)tmp; - free(tmp); } pColumn->numOfFilters++; From 959b6ec95ce9722d1698930d02f97453301f948e Mon Sep 17 00:00:00 2001 From: fang Date: Tue, 10 Dec 2019 09:41:10 +0800 Subject: [PATCH 089/108] solve some static check questions --- src/util/src/tutil.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/util/src/tutil.c b/src/util/src/tutil.c index e0dcb0aa3f..ee7795863a 100644 --- a/src/util/src/tutil.c +++ b/src/util/src/tutil.c @@ -106,6 +106,7 @@ char **strsplit(char *z, const char *delim, int32_t *num) { if ((*num) >= size) { size = (size << 1); split = realloc(split, POINTER_BYTES * size); + assert(NULL != split); } } From 103b8794da28359c9d19533a59625faa6bc3b968 Mon Sep 17 00:00:00 2001 From: lihui Date: Tue, 10 Dec 2019 10:36:12 +0800 Subject: [PATCH 090/108] [TBASE-1142/1319] --- packaging/release.sh | 1 + packaging/tools/install.sh | 4 ++-- packaging/tools/makeclient.sh | 5 ++--- src/inc/tglobalcfg.h | 1 + src/system/detail/src/dnodeService.c | 1 + src/util/src/tglobalcfg.c | 3 +++ src/util/src/version.c | 5 +++-- 7 files changed, 13 insertions(+), 7 deletions(-) diff --git a/packaging/release.sh b/packaging/release.sh index 58f69589d8..08edce249e 100755 --- a/packaging/release.sh +++ b/packaging/release.sh @@ -109,6 +109,7 @@ build_time=$(date +"%F %R") echo "char version[64] = \"${version}\";" > ${versioninfo} echo "char compatible_version[64] = \"${compatible_version}\";" >> ${versioninfo} echo "char gitinfo[128] = \"$(git rev-parse --verify HEAD)\";" >> ${versioninfo} +echo "char gitinfoOfInternal[128] = \"\";" >> ${versioninfo} echo "char buildinfo[512] = \"Built by ${USER} at ${build_time}\";" >> ${versioninfo} # 2. cmake executable file diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index 019e614e0b..e1bcce401d 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -313,9 +313,9 @@ vercomp () { function is_version_compatible() { - curr_version=$(${bin_dir}/taosd -V | head -1 | cut -d ' ' -f 2) + curr_version=$(${bin_dir}/taosd -V | head -1 | cut -d ' ' -f 3) - min_compatible_version=$(${script_dir}/bin/taosd -V | head -1 | cut -d ' ' -f 4) + min_compatible_version=$(${script_dir}/bin/taosd -V | head -1 | cut -d ' ' -f 5) vercomp $curr_version $min_compatible_version case $? in diff --git a/packaging/tools/makeclient.sh b/packaging/tools/makeclient.sh index b4948bb3a7..f8d21e202b 100755 --- a/packaging/tools/makeclient.sh +++ b/packaging/tools/makeclient.sh @@ -17,7 +17,6 @@ top_dir="$(readlink -m ${script_dir}/../..)" build_dir="${compile_dir}/build" code_dir="${top_dir}/src" release_dir="${top_dir}/release" -community_dir="${script_dir}/../../../community/src" #package_name='linux' install_dir="${release_dir}/TDengine-client-${version}" @@ -25,7 +24,7 @@ install_dir="${release_dir}/TDengine-client-${version}" # Directories and files. bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdump ${script_dir}/remove_client.sh" lib_files="${build_dir}/lib/libtaos.so.${version}" -header_files="${community_dir}/inc/taos.h ${community_dir}/inc/taoserror.h" +header_files="${code_dir}/inc/taos.h ${code_dir}/inc/taoserror.h" cfg_dir="${top_dir}/packaging/cfg" install_files="${script_dir}/install_client.sh" @@ -55,7 +54,7 @@ mkdir -p ${install_dir}/driver cp ${lib_files} ${install_dir}/driver # Copy connector -connector_dir="${community_dir}/connector" +connector_dir="${code_dir}/connector" mkdir -p ${install_dir}/connector cp ${build_dir}/lib/*.jar ${install_dir}/connector cp -r ${connector_dir}/grafana ${install_dir}/connector/ diff --git a/src/inc/tglobalcfg.h b/src/inc/tglobalcfg.h index ede3c97ce9..63cff06e83 100644 --- a/src/inc/tglobalcfg.h +++ b/src/inc/tglobalcfg.h @@ -179,6 +179,7 @@ extern int tsUdpDelay; extern char version[]; extern char compatible_version[]; extern char gitinfo[]; +extern char gitinfoOfInternal[]; extern char buildinfo[]; extern char tsTimezone[64]; diff --git a/src/system/detail/src/dnodeService.c b/src/system/detail/src/dnodeService.c index f03bd5f3bb..2de60bda91 100644 --- a/src/system/detail/src/dnodeService.c +++ b/src/system/detail/src/dnodeService.c @@ -62,6 +62,7 @@ int main(int argc, char *argv[]) { #endif printf("gitinfo: %s\n", gitinfo); + printf("gitinfoI: %s\n", gitinfoOfInternal); printf("buildinfo: %s\n", buildinfo); return 0; } else if (strcmp(argv[i], "-k") == 0) { diff --git a/src/util/src/tglobalcfg.c b/src/util/src/tglobalcfg.c index cef11d30cb..00da094d5f 100644 --- a/src/util/src/tglobalcfg.c +++ b/src/util/src/tglobalcfg.c @@ -783,6 +783,9 @@ static void doInitGlobalConfig() { tsInitConfigOption(cfg++, "gitinfo", gitinfo, TSDB_CFG_VTYPE_STRING, TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT, 0, 0, 0, TSDB_CFG_UTYPE_NONE); + tsInitConfigOption(cfg++, "gitinfoOfInternal", gitinfoOfInternal, TSDB_CFG_VTYPE_STRING, + TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT, + 0, 0, 0, TSDB_CFG_UTYPE_NONE); tsInitConfigOption(cfg++, "buildinfo", buildinfo, TSDB_CFG_VTYPE_STRING, TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT, 0, 0, 0, TSDB_CFG_UTYPE_NONE); diff --git a/src/util/src/version.c b/src/util/src/version.c index df490eb0aa..4f88d6170c 100644 --- a/src/util/src/version.c +++ b/src/util/src/version.c @@ -1,4 +1,5 @@ char version[64] = "1.6.4.2"; char compatible_version[64] = "1.6.1.0"; -char gitinfo[128] = "e6445addc77e8c96dcb57221fa6ab5dcde0458f7"; -char buildinfo[512] = "Built by ubuntu at 2019-12-06 13:59"; +char gitinfo[128] = "b9a62d60dc1d4a41452a9bc94e3a0924485c3a75"; +char gitinfoOfInternal[128] = "e6445addc77e8c96dcb57221fa6ab5dcde0458f7"; +char buildinfo[512] = "Built by root at 2019-12-10 10:31"; From d2ef1dad53744f8297b0b338cb390cbc8304e2c5 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Tue, 10 Dec 2019 12:16:10 +0800 Subject: [PATCH 091/108] [tbase-1287] --- src/client/inc/tscUtil.h | 6 +-- src/client/inc/tsclient.h | 25 ++++++----- src/client/src/tscParseInsert.c | 6 +-- src/client/src/tscUtil.c | 77 ++++++++++++++++++++++----------- 4 files changed, 72 insertions(+), 42 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 3deb4c463f..5d828d7cf0 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -67,7 +67,7 @@ typedef struct SJoinSubquerySupporter { } SJoinSubquerySupporter; void tscDestroyDataBlock(STableDataBlocks* pDataBlock); -STableDataBlocks* tscCreateDataBlock(int32_t size); +STableDataBlocks* tscCreateDataBlock(size_t initialBufSize, int32_t rowSize, int32_t startOffset, const char* name); void tscAppendDataBlock(SDataBlockList* pList, STableDataBlocks* pBlocks); SParamInfo* tscAddParamToDataBlock(STableDataBlocks* pDataBlock, char type, uint8_t timePrec, short bytes, uint32_t offset); @@ -78,9 +78,7 @@ int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDa void tscFreeUnusedDataBlocks(SDataBlockList* pList); int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SDataBlockList* pDataList); STableDataBlocks* tscGetDataBlockFromList(void* pHashList, SDataBlockList* pDataBlockList, int64_t id, int32_t size, - int32_t startOffset, int32_t rowSize, char* tableId); -STableDataBlocks* tscCreateDataBlockEx(size_t size, int32_t rowSize, int32_t startOffset, char* name); - + int32_t startOffset, int32_t rowSize, const char* tableId); SVnodeSidList* tscGetVnodeSidList(SMetricMeta* pMetricmeta, int32_t vnodeIdx); SMeterSidExtInfo* tscGetMeterSidInfo(SVnodeSidList* pSidList, int32_t idx); diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 4de33f5ac4..be9ba47f2a 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -231,17 +231,22 @@ typedef struct SParamInfo { typedef struct STableDataBlocks { char meterId[TSDB_METER_ID_LEN]; - int8_t tsSource; - bool ordered; + int8_t tsSource; // where does the UNIX timestamp come from, server or client + bool ordered; // if current rows are ordered or not + int64_t vgid; // virtual group id + int64_t prevTS; // previous timestamp, recorded to decide if the records array is ts ascending + int32_t numOfMeters; // number of tables in current submit block - int64_t vgid; - int64_t prevTS; - - int32_t numOfMeters; - - int32_t rowSize; + int32_t rowSize; // row size for current table uint32_t nAllocSize; uint32_t size; + + /* + * the metermeta for current table, the metermeta will be used during submit stage, keep a ref + * to avoid it to be removed from cache + */ + SMeterMeta* pMeterMeta; + union { char *filename; char *pData; @@ -255,8 +260,8 @@ typedef struct STableDataBlocks { typedef struct SDataBlockList { int32_t idx; - int32_t nSize; - int32_t nAlloc; + uint32_t nSize; + uint32_t nAlloc; char * userParam; /* user assigned parameters for async query */ void * udfp; /* user defined function pointer, used in async model */ STableDataBlocks **pData; diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 6317decbe0..a07ad6f211 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -985,7 +985,7 @@ int doParserInsertSql(SSqlObj *pSql, char *str) { strcpy(fname, full_path.we_wordv[0]); wordfree(&full_path); - STableDataBlocks *pDataBlock = tscCreateDataBlockEx(PATH_MAX, pMeterMetaInfo->pMeterMeta->rowSize, + STableDataBlocks *pDataBlock = tscCreateDataBlock(PATH_MAX, pMeterMetaInfo->pMeterMeta->rowSize, sizeof(SShellSubmitBlock), pMeterMetaInfo->name); tscAppendDataBlock(pCmd->pDataBlocks, pDataBlock); @@ -1222,8 +1222,8 @@ static int tscInsertDataFromFile(SSqlObj *pSql, FILE *fp, char *tmpTokenBuf) { int32_t rowSize = pMeterMeta->rowSize; pCmd->pDataBlocks = tscCreateBlockArrayList(); - STableDataBlocks *pTableDataBlock = - tscCreateDataBlockEx(TSDB_PAYLOAD_SIZE, pMeterMeta->rowSize, sizeof(SShellSubmitBlock), pMeterMetaInfo->name); + STableDataBlocks *pTableDataBlock = tscCreateDataBlock(TSDB_PAYLOAD_SIZE, pMeterMeta->rowSize, + sizeof(SShellSubmitBlock), pMeterMetaInfo->name); tscAppendDataBlock(pCmd->pDataBlocks, pTableDataBlock); diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 288e564cb0..7c7310a1c7 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -451,15 +451,6 @@ void tscFreeSqlObj(SSqlObj* pSql) { free(pSql); } -STableDataBlocks* tscCreateDataBlock(int32_t size) { - STableDataBlocks* dataBuf = (STableDataBlocks*)calloc(1, sizeof(STableDataBlocks)); - dataBuf->nAllocSize = (uint32_t)size; - dataBuf->pData = calloc(1, dataBuf->nAllocSize); - dataBuf->ordered = true; - dataBuf->prevTS = INT64_MIN; - return dataBuf; -} - void tscDestroyDataBlock(STableDataBlocks* pDataBlock) { if (pDataBlock == NULL) { return; @@ -467,6 +458,9 @@ void tscDestroyDataBlock(STableDataBlocks* pDataBlock) { tfree(pDataBlock->pData); tfree(pDataBlock->params); + + // free the refcount for metermeta + taosRemoveDataFromCache(tscCacheHandle, (void**) &(pDataBlock->pMeterMeta), false); tfree(pDataBlock); } @@ -513,11 +507,11 @@ SDataBlockList* tscCreateBlockArrayList() { void tscAppendDataBlock(SDataBlockList* pList, STableDataBlocks* pBlocks) { if (pList->nSize >= pList->nAlloc) { - pList->nAlloc = pList->nAlloc << 1; - pList->pData = realloc(pList->pData, sizeof(void*) * (size_t)pList->nAlloc); + pList->nAlloc = (pList->nAlloc) << 1U; + pList->pData = realloc(pList->pData, POINTER_BYTES * (size_t)pList->nAlloc); // reset allocated memory - memset(pList->pData + pList->nSize, 0, sizeof(void*) * (pList->nAlloc - pList->nSize)); + memset(pList->pData + pList->nSize, 0, POINTER_BYTES * (pList->nAlloc - pList->nSize)); } pList->pData[pList->nSize++] = pBlocks; @@ -539,29 +533,43 @@ void* tscDestroyBlockArrayList(SDataBlockList* pList) { } int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock) { - SSqlCmd* pCmd = &pSql->cmd; - + SSqlCmd *pCmd = &pSql->cmd; + assert(pDataBlock->pMeterMeta != NULL); + pCmd->count = pDataBlock->numOfMeters; - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); - strcpy(pMeterMetaInfo->name, pDataBlock->meterId); - + SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + + //set the correct metermeta object, the metermeta has been locked in pDataBlocks, so it must be in the cache + if (pMeterMetaInfo->pMeterMeta != pDataBlock->pMeterMeta) { + strcpy(pMeterMetaInfo->name, pDataBlock->meterId); + taosRemoveDataFromCache(tscCacheHandle, (void**) &(pMeterMetaInfo->pMeterMeta), false); + + pMeterMetaInfo->pMeterMeta = pDataBlock->pMeterMeta; + pDataBlock->pMeterMeta = NULL; // delegate the ownership of metermeta to pMeterMetaInfo + } else { + assert(strncmp(pMeterMetaInfo->name, pDataBlock->meterId, tListLen(pDataBlock->meterId)) == 0); + } + /* * the submit message consists of : [RPC header|message body|digest] * the dataBlock only includes the RPC Header buffer and actual submit messsage body, space for digest needs * additional space. */ int ret = tscAllocPayload(pCmd, pDataBlock->nAllocSize + sizeof(STaosDigest)); - if (TSDB_CODE_SUCCESS != ret) return ret; + if (TSDB_CODE_SUCCESS != ret) { + return ret; + } + memcpy(pCmd->payload, pDataBlock->pData, pDataBlock->nAllocSize); - + /* * the payloadLen should be actual message body size * the old value of payloadLen is the allocated payload size */ pCmd->payloadLen = pDataBlock->nAllocSize - tsRpcHeadSize; - + assert(pCmd->allocSize >= pCmd->payloadLen + tsRpcHeadSize + sizeof(STaosDigest)); - return tscGetMeterMeta(pSql, pMeterMetaInfo->name, 0); + return TSDB_CODE_SUCCESS; } void tscFreeUnusedDataBlocks(SDataBlockList* pList) { @@ -573,19 +581,38 @@ void tscFreeUnusedDataBlocks(SDataBlockList* pList) { } } -STableDataBlocks* tscCreateDataBlockEx(size_t size, int32_t rowSize, int32_t startOffset, char* name) { - STableDataBlocks* dataBuf = tscCreateDataBlock(size); +/** + * create the in-memory buffer for each table to keep the submitted data block + * @param initialSize + * @param rowSize + * @param startOffset + * @param name + * @param pMeterMeta the ownership of pMeterMeta should be transfer to STableDataBlocks + * @return + */ +STableDataBlocks* tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOffset, const char* name) { + + STableDataBlocks* dataBuf = (STableDataBlocks*)calloc(1, sizeof(STableDataBlocks)); + dataBuf->nAllocSize = (uint32_t) initialSize; + dataBuf->pData = calloc(1, dataBuf->nAllocSize); + dataBuf->ordered = true; + dataBuf->prevTS = INT64_MIN; dataBuf->rowSize = rowSize; dataBuf->size = startOffset; dataBuf->tsSource = -1; strncpy(dataBuf->meterId, name, TSDB_METER_ID_LEN); + + // sure that the metermeta must be in the local client cache + dataBuf->pMeterMeta = taosGetDataFromCache(tscCacheHandle, dataBuf->meterId); + assert(dataBuf->pMeterMeta != NULL && initialSize > 0); + return dataBuf; } STableDataBlocks* tscGetDataBlockFromList(void* pHashList, SDataBlockList* pDataBlockList, int64_t id, int32_t size, - int32_t startOffset, int32_t rowSize, char* tableId) { + int32_t startOffset, int32_t rowSize, const char* tableId) { STableDataBlocks* dataBuf = NULL; STableDataBlocks** t1 = (STableDataBlocks**)taosGetIntHashData(pHashList, id); @@ -594,7 +621,7 @@ STableDataBlocks* tscGetDataBlockFromList(void* pHashList, SDataBlockList* pData } if (dataBuf == NULL) { - dataBuf = tscCreateDataBlockEx((size_t)size, rowSize, startOffset, tableId); + dataBuf = tscCreateDataBlock((size_t)size, rowSize, startOffset, tableId); dataBuf = *(STableDataBlocks**)taosAddIntHash(pHashList, id, (char*)&dataBuf); tscAppendDataBlock(pDataBlockList, dataBuf); } From 6c860c7393ebd8bed91691468b5ab7050049b455 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Tue, 10 Dec 2019 12:35:45 +0800 Subject: [PATCH 092/108] [tbase-1317] --- src/client/src/tscSystem.c | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index 60b90ac328..6efe344719 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -198,7 +198,9 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) { switch (option) { case TSDB_OPTION_CONFIGDIR: cfg = tsGetConfigOption("configDir"); - if (cfg && cfg->cfgStatus <= TSDB_CFG_CSTATUS_OPTION) { + assert(cfg != NULL); + + if (cfg->cfgStatus <= TSDB_CFG_CSTATUS_OPTION) { strncpy(configDir, pStr, TSDB_FILENAME_LEN); cfg->cfgStatus = TSDB_CFG_CSTATUS_OPTION; tscPrint("set config file directory:%s", pStr); @@ -210,7 +212,9 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) { case TSDB_OPTION_SHELL_ACTIVITY_TIMER: cfg = tsGetConfigOption("shellActivityTimer"); - if (cfg && cfg->cfgStatus <= TSDB_CFG_CSTATUS_OPTION) { + assert(cfg != NULL); + + if (cfg->cfgStatus <= TSDB_CFG_CSTATUS_OPTION) { tsShellActivityTimer = atoi(pStr); if (tsShellActivityTimer < 1) tsShellActivityTimer = 1; if (tsShellActivityTimer > 3600) tsShellActivityTimer = 3600; @@ -224,13 +228,15 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) { case TSDB_OPTION_LOCALE: { // set locale cfg = tsGetConfigOption("locale"); + assert(cfg != NULL); + size_t len = strlen(pStr); if (len == 0 || len > TSDB_LOCALE_LEN) { tscPrint("Invalid locale:%s, use default", pStr); return -1; } - if (cfg && cfg && cfg->cfgStatus <= TSDB_CFG_CSTATUS_OPTION) { + if (cfg->cfgStatus <= TSDB_CFG_CSTATUS_OPTION) { char sep = '.'; if (strlen(tsLocale) == 0) { // locale does not set yet @@ -285,13 +291,15 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) { case TSDB_OPTION_CHARSET: { /* set charset will override the value of charset, assigned during system locale changed */ cfg = tsGetConfigOption("charset"); + assert(cfg != NULL); + size_t len = strlen(pStr); if (len == 0 || len > TSDB_LOCALE_LEN) { tscPrint("failed to set charset:%s", pStr); return -1; } - if (cfg && cfg->cfgStatus <= TSDB_CFG_CSTATUS_OPTION) { + if (cfg->cfgStatus <= TSDB_CFG_CSTATUS_OPTION) { if (taosValidateEncodec(pStr)) { if (strlen(tsCharset) == 0) { tscPrint("charset is set:%s", pStr); @@ -314,7 +322,9 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) { case TSDB_OPTION_TIMEZONE: cfg = tsGetConfigOption("timezone"); - if (cfg && cfg->cfgStatus <= TSDB_CFG_CSTATUS_OPTION) { + assert(cfg != NULL); + + if (cfg->cfgStatus <= TSDB_CFG_CSTATUS_OPTION) { strcpy(tsTimezone, pStr); tsSetTimeZone(); cfg->cfgStatus = TSDB_CFG_CSTATUS_OPTION; @@ -327,7 +337,9 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) { case TSDB_OPTION_SOCKET_TYPE: cfg = tsGetConfigOption("sockettype"); - if (cfg && cfg->cfgStatus <= TSDB_CFG_CSTATUS_OPTION) { + assert(cfg != NULL); + + if (cfg->cfgStatus <= TSDB_CFG_CSTATUS_OPTION) { if (strcasecmp(pStr, TAOS_SOCKET_TYPE_NAME_UDP) != 0 && strcasecmp(pStr, TAOS_SOCKET_TYPE_NAME_TCP) != 0) { tscError("only 'tcp' or 'udp' allowed for configuring the socket type"); return -1; @@ -340,6 +352,7 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) { break; default: + // TODO return the correct error code to client in the format for taos_errstr() tscError("Invalid option %d", option); return -1; } From 19d516169036eab7af561154e86a22b18c4ba519 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Tue, 10 Dec 2019 12:39:01 +0800 Subject: [PATCH 093/108] [tbase-1316] --- src/client/src/tscStream.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index 933960b893..31af78f618 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -447,7 +447,10 @@ static void setErrorInfo(STscObj* pObj, int32_t code, char* info) { SSqlCmd* pCmd = &pObj->pSql->cmd; pObj->pSql->res.code = code; - strncpy(pCmd->payload, info, pCmd->payloadLen); + + if (info != NULL) { + strncpy(pCmd->payload, info, pCmd->payloadLen); + } } TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *param, TAOS_RES *, TAOS_ROW row), From 085c2296c58bc85d0d3957adb977d7eda5805b43 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Tue, 10 Dec 2019 14:52:41 +0800 Subject: [PATCH 094/108] remove the compiler warnings --- src/client/src/tscFunctionImpl.c | 206 ++++++++++----------- src/client/src/tscSQLParser.c | 301 +++++++++++++------------------ src/client/src/tscSchemaUtil.c | 9 +- src/inc/tschemautil.h | 1 + src/inc/tsqlfunction.h | 2 - src/inc/ttypes.h | 2 +- src/util/src/ttypes.c | 2 +- 7 files changed, 244 insertions(+), 279 deletions(-) diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index d7f577fa4f..37bd46c75b 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -69,7 +69,8 @@ for (int32_t i = 0; i < (ctx)->tagInfo.numOfTagCols; ++i) { \ } \ } while(0); -void noop(SQLFunctionCtx *UNUSED_PARAM(pCtx)) {} +void noop1(SQLFunctionCtx *UNUSED_PARAM(pCtx)) {} +void noop2(SQLFunctionCtx *UNUSED_PARAM(pCtx), int32_t UNUSED_PARAM(index)) {} typedef struct tValuePair { tVariant v; @@ -507,10 +508,10 @@ static void do_sum(SQLFunctionCtx *pCtx) { assert(pCtx->size >= pCtx->preAggVals.numOfNull); if (pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_BIGINT) { - int64_t *retVal = pCtx->aOutputBuf; + int64_t *retVal = (int64_t*) pCtx->aOutputBuf; *retVal += pCtx->preAggVals.sum; } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE || pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - double *retVal = pCtx->aOutputBuf; + double *retVal = (double*) pCtx->aOutputBuf; *retVal += GET_DOUBLE_VAL(&(pCtx->preAggVals.sum)); } } else { // computing based on the true data block @@ -518,7 +519,7 @@ static void do_sum(SQLFunctionCtx *pCtx) { notNullElems = 0; if (pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_BIGINT) { - int64_t *retVal = pCtx->aOutputBuf; + int64_t *retVal = (int64_t*) pCtx->aOutputBuf; if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { LIST_ADD_N(*retVal, pCtx, pData, int8_t, notNullElems, pCtx->inputType); @@ -530,10 +531,10 @@ static void do_sum(SQLFunctionCtx *pCtx) { LIST_ADD_N(*retVal, pCtx, pData, int64_t, notNullElems, pCtx->inputType); } } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { - double *retVal = pCtx->aOutputBuf; + double *retVal = (double*) pCtx->aOutputBuf; LIST_ADD_N(*retVal, pCtx, pData, double, notNullElems, pCtx->inputType); } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - double *retVal = pCtx->aOutputBuf; + double *retVal = (double*) pCtx->aOutputBuf; LIST_ADD_N(*retVal, pCtx, pData, float, notNullElems, pCtx->inputType); } } @@ -553,7 +554,7 @@ static void do_sum_f(SQLFunctionCtx *pCtx, int32_t index) { } SET_VAL(pCtx, 1, 1); - int64_t *res = pCtx->aOutputBuf; + int64_t *res = (int64_t*) pCtx->aOutputBuf; if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { *res += GET_INT8_VAL(pData); @@ -564,10 +565,10 @@ static void do_sum_f(SQLFunctionCtx *pCtx, int32_t index) { } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT) { *res += GET_INT64_VAL(pData); } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { - double *retVal = pCtx->aOutputBuf; + double *retVal = (double*) pCtx->aOutputBuf; *retVal += GET_DOUBLE_VAL(pData); } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - double *retVal = pCtx->aOutputBuf; + double *retVal = (double*) pCtx->aOutputBuf; *retVal += GET_FLOAT_VAL(pData); } @@ -694,7 +695,7 @@ static int32_t first_dist_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY return BLK_DATA_NO_NEEDED; } - SFirstLastInfo *pInfo = (pCtx->aOutputBuf + pCtx->inputBytes); + SFirstLastInfo *pInfo = (SFirstLastInfo*) (pCtx->aOutputBuf + pCtx->inputBytes); if (pInfo->hasResult != DATA_SET_FLAG) { return BLK_DATA_ALL_NEEDED; } else { // data in current block is not earlier than current result @@ -708,7 +709,7 @@ static int32_t last_dist_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY return BLK_DATA_NO_NEEDED; } - SFirstLastInfo *pInfo = (pCtx->aOutputBuf + pCtx->inputBytes); + SFirstLastInfo *pInfo = (SFirstLastInfo*) (pCtx->aOutputBuf + pCtx->inputBytes); if (pInfo->hasResult != DATA_SET_FLAG) { return BLK_DATA_ALL_NEEDED; } else { @@ -842,7 +843,7 @@ static void avg_func_merge(SQLFunctionCtx *pCtx) { static void avg_func_second_merge(SQLFunctionCtx *pCtx) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); - double *sum = pCtx->aOutputBuf; + double *sum = (double*) pCtx->aOutputBuf; char * input = GET_INPUT_CHAR(pCtx); for (int32_t i = 0; i < pCtx->size; ++i, input += pCtx->inputBytes) { @@ -967,10 +968,10 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, TYPED_LOOPCHECK_N(int16_t, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems); } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { int32_t *pData = p; - int32_t *retVal = pOutput; + int32_t *retVal = (int32_t*) pOutput; for (int32_t i = 0; i < pCtx->size; ++i) { - if (pCtx->hasNull && isNull(&pData[i], pCtx->inputType)) { + if (pCtx->hasNull && isNull((const char*)&pData[i], pCtx->inputType)) { continue; } @@ -1215,27 +1216,27 @@ static void minMax_function_f(SQLFunctionCtx *pCtx, int32_t index, int32_t isMin UPDATE_DATA(pCtx, *output, i, num, isMin, key); } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { - int16_t *output = pCtx->aOutputBuf; + int16_t *output = (int16_t*) pCtx->aOutputBuf; int16_t i = GET_INT16_VAL(pData); UPDATE_DATA(pCtx, *output, i, num, isMin, key); } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { - int32_t *output = pCtx->aOutputBuf; + int32_t *output = (int32_t*) pCtx->aOutputBuf; int32_t i = GET_INT32_VAL(pData); UPDATE_DATA(pCtx, *output, i, num, isMin, key); } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT) { - int64_t *output = pCtx->aOutputBuf; + int64_t *output = (int64_t*) pCtx->aOutputBuf; int64_t i = GET_INT64_VAL(pData); UPDATE_DATA(pCtx, *output, i, num, isMin, key); } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - float *output = pCtx->aOutputBuf; + float *output = (float*) pCtx->aOutputBuf; float i = GET_FLOAT_VAL(pData); UPDATE_DATA(pCtx, *output, i, num, isMin, key); } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { - double *output = pCtx->aOutputBuf; + double *output = (double*) pCtx->aOutputBuf; double i = GET_DOUBLE_VAL(pData); UPDATE_DATA(pCtx, *output, i, num, isMin, key); @@ -1299,7 +1300,7 @@ static void stddev_function(SQLFunctionCtx *pCtx) { switch (pCtx->inputType) { case TSDB_DATA_TYPE_INT: { for (int32_t i = 0; i < pCtx->size; ++i) { - if (pCtx->hasNull && isNull(&((int32_t *)pData)[i], pCtx->inputType)) { + if (pCtx->hasNull && isNull((const char*) (&((int32_t *)pData)[i]), pCtx->inputType)) { continue; } *retVal += POW2(((int32_t *)pData)[i] - avg); @@ -1583,7 +1584,7 @@ static void first_dist_func_second_merge(SQLFunctionCtx *pCtx) { assert(pCtx->resultInfo->superTableQ); char * pData = GET_INPUT_CHAR(pCtx); - SFirstLastInfo *pInput = (pData + pCtx->outputBytes); + SFirstLastInfo *pInput = (SFirstLastInfo*) (pData + pCtx->outputBytes); if (pInput->hasResult != DATA_SET_FLAG) { return; } @@ -1761,7 +1762,7 @@ static void last_dist_func_merge(SQLFunctionCtx *pCtx) { static void last_dist_func_second_merge(SQLFunctionCtx *pCtx) { char *pData = GET_INPUT_CHAR(pCtx); - SFirstLastInfo *pInput = (pData + pCtx->outputBytes); + SFirstLastInfo *pInput = (SFirstLastInfo*) (pData + pCtx->outputBytes); if (pInput->hasResult != DATA_SET_FLAG) { return; } @@ -1869,7 +1870,7 @@ static void do_top_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, val.i64Key >= pList[pInfo->num - 1]->v.i64Key) || ((type >= TSDB_DATA_TYPE_FLOAT && type <= TSDB_DATA_TYPE_DOUBLE) && val.dKey >= pList[pInfo->num - 1]->v.dKey)) { - valuePairAssign(pList[pInfo->num], type, &val.i64Key, ts, pTags, pTagInfo, stage); + valuePairAssign(pList[pInfo->num], type, (const char*)&val.i64Key, ts, pTags, pTagInfo, stage); } else { int32_t i = pInfo->num - 1; @@ -1885,7 +1886,7 @@ static void do_top_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, } } - valuePairAssign(pList[i + 1], type, &val.i64Key, ts, pTags, pTagInfo, stage); + valuePairAssign(pList[i + 1], type, (const char*) &val.i64Key, ts, pTags, pTagInfo, stage); } pInfo->num++; @@ -1907,7 +1908,7 @@ static void do_top_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, } } - valuePairAssign(pList[i], type, &val.i64Key, ts, pTags, pTagInfo, stage); + valuePairAssign(pList[i], type, (const char*) &val.i64Key, ts, pTags, pTagInfo, stage); } } } @@ -1921,7 +1922,7 @@ static void do_bottom_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pDa if (pInfo->num < maxLen) { if (pInfo->num == 0) { - valuePairAssign(pList[pInfo->num], type, &val.i64Key, ts, pTags, pTagInfo, stage); + valuePairAssign(pList[pInfo->num], type, (const char*) &val.i64Key, ts, pTags, pTagInfo, stage); } else { int32_t i = pInfo->num - 1; @@ -1937,7 +1938,7 @@ static void do_bottom_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pDa } } - valuePairAssign(pList[i + 1], type, &val.i64Key, ts, pTags, pTagInfo, stage); + valuePairAssign(pList[i + 1], type, (const char*)&val.i64Key, ts, pTags, pTagInfo, stage); } pInfo->num++; @@ -1959,7 +1960,7 @@ static void do_bottom_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pDa } } - valuePairAssign(pList[i], type, &val.i64Key, ts, pTags, pTagInfo, stage); + valuePairAssign(pList[i], type, (const char*)&val.i64Key, ts, pTags, pTagInfo, stage); } } } @@ -2099,7 +2100,7 @@ bool top_bot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, char *mi return true; } - tValuePair *pRes = pTopBotInfo->res; + tValuePair *pRes = (tValuePair*) pTopBotInfo->res; if (functionId == TSDB_FUNC_TOP) { switch (pCtx->inputType) { @@ -2151,7 +2152,7 @@ static STopBotInfo *getTopBotOutputInfo(SQLFunctionCtx *pCtx) { // only the first_stage_merge is directly written data into final output buffer if (pResInfo->superTableQ && pCtx->currentStage != SECONDARY_STAGE_MERGE) { - return pCtx->aOutputBuf; + return (STopBotInfo*) pCtx->aOutputBuf; } else { // for normal table query and super table at the secondary_stage, result is written to intermediate buffer return pResInfo->interResultBuf; } @@ -2165,14 +2166,14 @@ static STopBotInfo *getTopBotOutputInfo(SQLFunctionCtx *pCtx) { */ static void buildTopBotStruct(STopBotInfo *pTopBotInfo, SQLFunctionCtx *pCtx) { char *tmp = (char *)pTopBotInfo + sizeof(STopBotInfo); - pTopBotInfo->res = tmp; + pTopBotInfo->res = (tValuePair**) tmp; tmp += POINTER_BYTES * pCtx->param[0].i64Key; size_t size = sizeof(tValuePair) + pCtx->tagInfo.tagsLen; for (int32_t i = 0; i < pCtx->param[0].i64Key; ++i) { - pTopBotInfo->res[i] = tmp; + pTopBotInfo->res[i] = (tValuePair*) tmp; pTopBotInfo->res[i]->pTags = tmp + sizeof(tValuePair); tmp += size; } @@ -2477,7 +2478,7 @@ static SAPercentileInfo *getAPerctInfo(SQLFunctionCtx *pCtx) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); if (pResInfo->superTableQ && pCtx->currentStage != SECONDARY_STAGE_MERGE) { - return pCtx->aOutputBuf; + return (SAPercentileInfo*) pCtx->aOutputBuf; } else { return pResInfo->interResultBuf; } @@ -2588,8 +2589,8 @@ static void apercentile_func_merge(SQLFunctionCtx *pCtx) { SAPercentileInfo *pInput = (SAPercentileInfo *)GET_INPUT_CHAR(pCtx); - pInput->pHisto = (char *)pInput + sizeof(SAPercentileInfo); - pInput->pHisto->elems = (char *)pInput->pHisto + sizeof(SHistogramInfo); + pInput->pHisto = (SHistogramInfo*) ((char *)pInput + sizeof(SAPercentileInfo)); + pInput->pHisto->elems = (SHistBin*) ((char *)pInput->pHisto + sizeof(SHistogramInfo)); if (pInput->pHisto->numOfElems <= 0) { return; @@ -2602,13 +2603,13 @@ static void apercentile_func_merge(SQLFunctionCtx *pCtx) { if (pHisto->numOfElems <= 0) { memcpy(pHisto, pInput->pHisto, size); - pHisto->elems = (char *)pHisto + sizeof(SHistogramInfo); + pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); } else { - pHisto->elems = (char *)pHisto + sizeof(SHistogramInfo); + pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); SHistogramInfo *pRes = tHistogramMerge(pHisto, pInput->pHisto, MAX_HISTOGRAM_BIN); memcpy(pHisto, pRes, sizeof(SHistogramInfo) + sizeof(SHistBin) * MAX_HISTOGRAM_BIN); - pHisto->elems = (char *)pHisto + sizeof(SHistogramInfo); + pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); tHistogramDestroy(&pRes); } @@ -2620,8 +2621,8 @@ static void apercentile_func_merge(SQLFunctionCtx *pCtx) { static void apercentile_func_second_merge(SQLFunctionCtx *pCtx) { SAPercentileInfo *pInput = (SAPercentileInfo *)GET_INPUT_CHAR(pCtx); - pInput->pHisto = (char *)pInput + sizeof(SAPercentileInfo); - pInput->pHisto->elems = (char *)pInput->pHisto + sizeof(SHistogramInfo); + pInput->pHisto = (SHistogramInfo*) ((char *)pInput + sizeof(SAPercentileInfo)); + pInput->pHisto->elems = (SHistBin*) ((char *)pInput->pHisto + sizeof(SHistogramInfo)); if (pInput->pHisto->numOfElems <= 0) { return; @@ -2632,9 +2633,9 @@ static void apercentile_func_second_merge(SQLFunctionCtx *pCtx) { if (pHisto->numOfElems <= 0) { memcpy(pHisto, pInput->pHisto, sizeof(SHistogramInfo) + sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1)); - pHisto->elems = (char *)pHisto + sizeof(SHistogramInfo); + pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); } else { - pHisto->elems = (char *)pHisto + sizeof(SHistogramInfo); + pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); SHistogramInfo *pRes = tHistogramMerge(pHisto, pInput->pHisto, MAX_HISTOGRAM_BIN); tHistogramDestroy(&pOutput->pHisto); @@ -2728,7 +2729,7 @@ static void leastsquares_function(SQLFunctionCtx *pCtx) { int32_t *p = pData; // LEASTSQR_CAL_LOOP(pCtx, param, pParamData, p); for (int32_t i = 0; i < pCtx->size; ++i) { - if (pCtx->hasNull && isNull(p, pCtx->inputType)) { + if (pCtx->hasNull && isNull((const char*) p, pCtx->inputType)) { continue; } @@ -2870,6 +2871,10 @@ static void date_col_output_function(SQLFunctionCtx *pCtx) { *(int64_t *)(pCtx->aOutputBuf) = pCtx->nStartQueryTimestamp; } +static FORCE_INLINE void date_col_output_function_f(SQLFunctionCtx *pCtx, int32_t index) { + date_col_output_function(pCtx); +} + static void col_project_function(SQLFunctionCtx *pCtx) { INC_INIT_VAL(pCtx, pCtx->size); @@ -2980,7 +2985,7 @@ static void diff_function(SQLFunctionCtx *pCtx) { int32_t *pOutput = (int32_t *)pCtx->aOutputBuf; for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull(&pData[i], pCtx->inputType)) { + if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) { continue; } @@ -3012,7 +3017,7 @@ static void diff_function(SQLFunctionCtx *pCtx) { int64_t *pOutput = (int64_t *)pCtx->aOutputBuf; for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull(&pData[i], pCtx->inputType)) { + if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) { continue; } @@ -3044,7 +3049,7 @@ static void diff_function(SQLFunctionCtx *pCtx) { double *pOutput = (double *)pCtx->aOutputBuf; for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull(&pData[i], pCtx->inputType)) { + if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) { continue; } @@ -3074,7 +3079,7 @@ static void diff_function(SQLFunctionCtx *pCtx) { float *pOutput = (float *)pCtx->aOutputBuf; for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull(&pData[i], pCtx->inputType)) { + if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) { continue; } @@ -3105,7 +3110,7 @@ static void diff_function(SQLFunctionCtx *pCtx) { int16_t *pOutput = (int16_t *)pCtx->aOutputBuf; for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull(&pData[i], pCtx->inputType)) { + if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) { continue; } @@ -3275,7 +3280,7 @@ static void arithmetic_function(SQLFunctionCtx *pCtx) { pCtx->aOutputBuf += pCtx->outputBytes * pCtx->size * GET_FORWARD_DIRECTION_FACTOR(pCtx->order); } -static bool arithmetic_function_f(SQLFunctionCtx *pCtx, int32_t index) { +static void arithmetic_function_f(SQLFunctionCtx *pCtx, int32_t index) { INC_INIT_VAL(pCtx, 1); SArithmeticSupport *sas = (SArithmeticSupport *)pCtx->param[0].pz; @@ -3284,7 +3289,6 @@ static bool arithmetic_function_f(SQLFunctionCtx *pCtx, int32_t index) { arithmetic_callback_function); pCtx->aOutputBuf += pCtx->outputBytes * GET_FORWARD_DIRECTION_FACTOR(pCtx->order); - return true; } #define LIST_MINMAX_N(ctx, minOutput, maxOutput, elemCnt, data, type, tsdbType, numOfNotNullElem) \ @@ -3707,7 +3711,7 @@ static void getStatics_i16(int64_t *primaryKey, int16_t *data, int32_t numOfRow, // int16_t lastVal = TSDB_DATA_SMALLINT_NULL; for (int32_t i = 0; i < numOfRow; ++i) { - if (isNull(&data[i], TSDB_DATA_TYPE_SMALLINT)) { + if (isNull((const char*) &data[i], TSDB_DATA_TYPE_SMALLINT)) { (*numOfNull) += 1; continue; } @@ -3747,7 +3751,7 @@ static void getStatics_i32(int64_t *primaryKey, int32_t *data, int32_t numOfRow, // int32_t lastVal = TSDB_DATA_INT_NULL; for (int32_t i = 0; i < numOfRow; ++i) { - if (isNull(&data[i], TSDB_DATA_TYPE_INT)) { + if (isNull((const char*) &data[i], TSDB_DATA_TYPE_INT)) { (*numOfNull) += 1; continue; } @@ -3784,7 +3788,7 @@ static void getStatics_i64(int64_t *primaryKey, int64_t *data, int32_t numOfRow, assert(numOfRow <= INT16_MAX); for (int32_t i = 0; i < numOfRow; ++i) { - if (isNull(&data[i], TSDB_DATA_TYPE_BIGINT)) { + if (isNull((const char*) &data[i], TSDB_DATA_TYPE_BIGINT)) { (*numOfNull) += 1; continue; } @@ -3822,7 +3826,7 @@ static void getStatics_f(int64_t *primaryKey, float *data, int32_t numOfRow, dou assert(numOfRow <= INT16_MAX); for (int32_t i = 0; i < numOfRow; ++i) { - if (isNull(&data[i], TSDB_DATA_TYPE_FLOAT)) { + if (isNull((const char*) &data[i], TSDB_DATA_TYPE_FLOAT)) { (*numOfNull) += 1; continue; } @@ -3875,7 +3879,7 @@ static void getStatics_d(int64_t *primaryKey, double *data, int32_t numOfRow, do assert(numOfRow <= INT16_MAX); for (int32_t i = 0; i < numOfRow; ++i) { - if (isNull(&data[i], TSDB_DATA_TYPE_DOUBLE)) { + if (isNull((const char*) &data[i], TSDB_DATA_TYPE_DOUBLE)) { (*numOfNull) += 1; continue; } @@ -3936,9 +3940,9 @@ void getStatistics(char *priData, char *data, int32_t size, int32_t numOfRow, in } else if (type == TSDB_DATA_TYPE_BIGINT || type == TSDB_DATA_TYPE_TIMESTAMP) { getStatics_i64(primaryKey, (int64_t *)data, numOfRow, min, max, sum, minIndex, maxIndex, numOfNull); } else if (type == TSDB_DATA_TYPE_DOUBLE) { - getStatics_d(primaryKey, (double *)data, numOfRow, min, max, sum, minIndex, maxIndex, numOfNull); + getStatics_d(primaryKey, (double *)data, numOfRow, (double*) min, (double*) max, (double*) sum, minIndex, maxIndex, numOfNull); } else if (type == TSDB_DATA_TYPE_FLOAT) { - getStatics_f(primaryKey, (float *)data, numOfRow, min, max, sum, minIndex, maxIndex, numOfNull); + getStatics_f(primaryKey, (float *)data, numOfRow, (double*) min, (double*) max, (double*) sum, minIndex, maxIndex, numOfNull); } } } @@ -4056,44 +4060,42 @@ static void twa_function(SQLFunctionCtx *pCtx) { // pCtx->numOfIteratedElems += notNullElems; } -static bool twa_function_f(SQLFunctionCtx *pCtx, int32_t index) { +static void twa_function_f(SQLFunctionCtx *pCtx, int32_t index) { void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { - return true; + return; } - + SET_VAL(pCtx, 1, 1); - + TSKEY *primaryKey = pCtx->ptsList; - + SResultInfo *pResInfo = GET_RES_INFO(pCtx); - STwaInfo * pInfo = pResInfo->interResultBuf; - + STwaInfo *pInfo = pResInfo->interResultBuf; + if (pInfo->lastKey == INT64_MIN) { pInfo->lastKey = pCtx->nStartQueryTimestamp; setTWALastVal(pCtx, pData, 0, pInfo); - + pInfo->hasResult = DATA_SET_FLAG; } - + if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT || pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { pInfo->dOutput += pInfo->dLastValue * (primaryKey[index] - pInfo->lastKey); } else { pInfo->iOutput += pInfo->iLastValue * (primaryKey[index] - pInfo->lastKey); } - + // record the last key/value pInfo->lastKey = primaryKey[index]; setTWALastVal(pCtx, pData, 0, pInfo); - + // pCtx->numOfIteratedElems += 1; pResInfo->hasResult = DATA_SET_FLAG; - + if (pResInfo->superTableQ) { memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(STwaInfo)); } - - return true; } static void twa_func_merge(SQLFunctionCtx *pCtx) { @@ -4105,7 +4107,7 @@ static void twa_func_merge(SQLFunctionCtx *pCtx) { int32_t numOfNotNull = 0; for (int32_t i = 0; i < pCtx->size; ++i, indicator += sizeof(STwaInfo)) { - STwaInfo *pInput = indicator; + STwaInfo *pInput = (STwaInfo*) indicator; if (pInput->hasResult != DATA_SET_FLAG) { continue; @@ -4207,7 +4209,7 @@ static void interp_function(SQLFunctionCtx *pCtx) { if (pCtx->outputType == TSDB_DATA_TYPE_FLOAT) { float v = GET_DOUBLE_VAL(pVal); - assignVal(pCtx->aOutputBuf, &v, pCtx->outputBytes, pCtx->outputType); + assignVal(pCtx->aOutputBuf, (const char*) &v, pCtx->outputBytes, pCtx->outputType); } else { assignVal(pCtx->aOutputBuf, pVal, pCtx->outputBytes, pCtx->outputType); } @@ -4368,7 +4370,7 @@ SQLAggFuncElem aAggs[28] = {{ count_function, count_function_f, no_next_step, - noop, + noop1, count_func_merge, count_func_merge, count_load_data_info, @@ -4444,8 +4446,8 @@ SQLAggFuncElem aAggs[28] = {{ stddev_function_f, stddev_next_step, stddev_finalizer, - noop, - noop, + noop1, + noop1, data_req_load_info, }, { @@ -4459,8 +4461,8 @@ SQLAggFuncElem aAggs[28] = {{ percentile_function_f, no_next_step, percentile_finalizer, - noop, - noop, + noop1, + noop1, data_req_load_info, }, { @@ -4489,8 +4491,8 @@ SQLAggFuncElem aAggs[28] = {{ first_function_f, no_next_step, function_finalizer, - noop, - noop, + noop1, + noop1, first_data_req_info, }, { @@ -4504,8 +4506,8 @@ SQLAggFuncElem aAggs[28] = {{ last_function_f, no_next_step, function_finalizer, - noop, - noop, + noop1, + noop1, last_data_req_info, }, { @@ -4517,10 +4519,10 @@ SQLAggFuncElem aAggs[28] = {{ TSDB_FUNCSTATE_SELECTIVITY, first_last_function_setup, last_row_function, - noop, + noop2, no_next_step, last_row_finalizer, - noop, + noop1, last_dist_func_second_merge, data_req_load_info, }, @@ -4597,8 +4599,8 @@ SQLAggFuncElem aAggs[28] = {{ leastsquares_function_f, no_next_step, leastsquares_finalizer, - noop, - noop, + noop1, + noop1, data_req_load_info, }, { @@ -4609,9 +4611,9 @@ SQLAggFuncElem aAggs[28] = {{ TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, function_setup, date_col_output_function, - date_col_output_function, + date_col_output_function_f, no_next_step, - noop, + noop1, copy_function, copy_function, no_data_info, @@ -4623,10 +4625,10 @@ SQLAggFuncElem aAggs[28] = {{ TSDB_FUNC_TS_DUMMY, TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, function_setup, - noop, - noop, + noop1, + noop2, no_next_step, - noop, + noop1, copy_function, copy_function, data_req_load_info, @@ -4639,9 +4641,9 @@ SQLAggFuncElem aAggs[28] = {{ TSDB_BASE_FUNC_SO, function_setup, tag_function, - noop, + noop2, no_next_step, - noop, + noop1, copy_function, copy_function, no_data_info, @@ -4671,7 +4673,7 @@ SQLAggFuncElem aAggs[28] = {{ tag_function, tag_function_f, no_next_step, - noop, + noop1, copy_function, copy_function, no_data_info, @@ -4686,7 +4688,7 @@ SQLAggFuncElem aAggs[28] = {{ col_project_function, col_project_function_f, no_next_step, - noop, + noop1, copy_function, copy_function, data_req_load_info, @@ -4701,7 +4703,7 @@ SQLAggFuncElem aAggs[28] = {{ tag_project_function, tag_project_function_f, no_next_step, - noop, + noop1, copy_function, copy_function, no_data_info, @@ -4716,7 +4718,7 @@ SQLAggFuncElem aAggs[28] = {{ arithmetic_function, arithmetic_function_f, no_next_step, - noop, + noop1, copy_function, copy_function, data_req_load_info, @@ -4731,9 +4733,9 @@ SQLAggFuncElem aAggs[28] = {{ diff_function, diff_function_f, no_next_step, - noop, - noop, - noop, + noop1, + noop1, + noop1, data_req_load_info, }, // distributed version used in two-stage aggregation processes @@ -4777,8 +4779,8 @@ SQLAggFuncElem aAggs[28] = {{ interp_function, do_sum_f, // todo filter handle no_next_step, - noop, - noop, + noop1, + noop1, copy_function, no_data_info, }}; diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index bc3079e09a..d96e15eb5b 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -20,13 +20,13 @@ #include "taos.h" #include "taosmsg.h" #include "tstoken.h" -#include "ttime.h" #include "tstrbuild.h" +#include "ttime.h" +#include "tscSQLParser.h" #include "tscUtil.h" #include "tschemautil.h" #include "tsclient.h" -#include "tscSQLParser.h" #define DEFAULT_PRIMARY_TIMESTAMP_COL_NAME "_c0" @@ -51,7 +51,7 @@ typedef struct SColumnIdListRes { static SSqlExpr* doAddProjectCol(SSqlCmd* pCmd, int32_t outputIndex, int32_t colIdx, int32_t tableIndex); static int32_t setShowInfo(SSqlObj* pSql, SSqlInfo* pInfo); -static char* getAccountId(SSqlObj* pSql); +static char* getAccountId(SSqlObj* pSql); static bool has(tFieldList* pFieldList, int32_t startIdx, const char* name); static void getCurrentDBName(SSqlObj* pSql, SSQLToken* pDBToken); @@ -110,7 +110,7 @@ static int32_t optrToString(tSQLExpr* pExpr, char** exprString); static int32_t getMeterIndex(SSQLToken* pTableToken, SSqlCmd* pCmd, SColumnIndex* pIndex); static int32_t doFunctionsCompatibleCheck(SSqlObj* pSql); static int32_t doLocalQueryProcess(SQuerySQL* pQuerySql, SSqlCmd* pCmd); -static int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCreateDbMsg *pCreate); +static int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCreateDbMsg* pCreate); static SColumnList getColumnList(int32_t num, int16_t tableIndex, int32_t columnIndex); @@ -118,7 +118,7 @@ static SColumnList getColumnList(int32_t num, int16_t tableIndex, int32_t column * Used during parsing query sql. Since the query sql usually small in length, error position * is not needed in the final error message. */ -static int32_t invalidSqlErrMsg(SSqlCmd *pCmd, const char* errMsg) { +static int32_t invalidSqlErrMsg(SSqlCmd* pCmd, const char* errMsg) { return tscInvalidSQLErrMsg(pCmd->payload, errMsg, NULL); } @@ -255,7 +255,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { } if (pToken->n > TSDB_DB_NAME_LEN) { - const char* msg = "db name too long"; + const char* msg = "db name too long"; return invalidSqlErrMsg(pCmd, msg); } @@ -286,14 +286,13 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { case SHOW_STREAMS: case SHOW_SCORES: case SHOW_GRANTS: - case SHOW_CONFIGS: + case SHOW_CONFIGS: case SHOW_VNODES: { return setShowInfo(pSql, pInfo); } case ALTER_DATABASE: case CREATE_DATABASE: { - if (pInfo->sqlType == ALTER_DATABASE) { pCmd->command = TSDB_SQL_ALTER_DB; } else { @@ -309,7 +308,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { int32_t ret = setObjFullName(pMeterMetaInfo->name, getAccountId(pSql), &(pCreateDB->dbname), NULL, NULL); if (ret != TSDB_CODE_SUCCESS) { - const char* msg2 = "name too long"; + const char* msg2 = "name too long"; return invalidSqlErrMsg(pCmd, msg2); } @@ -361,12 +360,12 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { if (pInfo->pDCLInfo->a[0].n > TSDB_USER_LEN || pInfo->pDCLInfo->a[1].n > TSDB_PASSWORD_LEN) { const char* msg = "name or password too long"; - return invalidSqlErrMsg(pCmd, msg); + return invalidSqlErrMsg(pCmd, msg); } if (tscValidateName(&pInfo->pDCLInfo->a[0]) != TSDB_CODE_SUCCESS) { const char* msg2 = "invalid user/account name"; - return invalidSqlErrMsg(pCmd, msg2); + return invalidSqlErrMsg(pCmd, msg2); } strncpy(pMeterMetaInfo->name, pInfo->pDCLInfo->a[0].z, pInfo->pDCLInfo->a[0].n); // name @@ -400,7 +399,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { pCmd->defaultVal[8] = 0; } else { const char* msg4 = "invalid state option, available options[no, r, w, all]"; - return invalidSqlErrMsg(pCmd, msg4); + return invalidSqlErrMsg(pCmd, msg4); } } } @@ -836,7 +835,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { const char* msg7 = "illegal number of tables in from clause"; const char* msg8 = "too many columns in selection clause"; const char* msg9 = "TWA query requires both the start and end time"; - + int32_t code = TSDB_CODE_SUCCESS; // too many result columns not support order by in query @@ -1017,8 +1016,8 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { } setColumnOffsetValueInResultset(pCmd); - - for(int32_t i = 0; i < pCmd->numOfTables; ++i) { + + for (int32_t i = 0; i < pCmd->numOfTables; ++i) { updateTagColumnIndex(pCmd, i); } @@ -1796,7 +1795,7 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, tSQLExprItem* pItem) { } if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { - SSchema colSchema = {.type = TSDB_DATA_TYPE_BINARY, .bytes = TSDB_METER_NAME_LEN}; + SSchema colSchema = {.type = TSDB_DATA_TYPE_BINARY, .bytes = TSDB_METER_NAME_LEN}; strcpy(colSchema.name, TSQL_TBNAME_L); pCmd->type = TSDB_QUERY_TYPE_STABLE_QUERY; @@ -2160,8 +2159,8 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, int32_t colIdx, tSQLExprItem* pItem int8_t resultType = pSchema[index.columnIndex].type; int16_t resultSize = pSchema[index.columnIndex].bytes; - char val[8] = {0}; - int32_t numOfAddedColumn = 1; + char val[8] = {0}; + int32_t numOfAddedColumn = 1; if (optr == TK_PERCENTILE || optr == TK_APERCENTILE) { tVariantDump(pVariant, val, TSDB_DATA_TYPE_DOUBLE); @@ -2186,7 +2185,6 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, int32_t colIdx, tSQLExprItem* pItem SSqlExpr* pExpr = tscSqlExprInsert(pCmd, colIdx, functionId, &index, resultType, resultSize, resultSize); addExprParams(pExpr, val, TSDB_DATA_TYPE_DOUBLE, sizeof(double), 0); } else { - tVariantDump(pVariant, val, TSDB_DATA_TYPE_BIGINT); int64_t nTop = *((int32_t*)val); @@ -2545,7 +2543,7 @@ int32_t setShowInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { } } } - }else if (type == SHOW_VNODES) { + } else if (type == SHOW_VNODES) { if (NULL == pInfo->pDCLInfo) { return invalidSqlErrMsg(pCmd, "No specified ip of dnode"); } @@ -2674,8 +2672,7 @@ void tscRestoreSQLFunctionForMetricQuery(SSqlCmd* pCmd) { bool hasUnsupportFunctionsForMetricQuery(SSqlCmd* pCmd) { const char* msg1 = "TWA not allowed to apply to super table directly"; - const char* msg2 = "functions not supported for super table"; - const char* msg3 = "TWA only support group by tbname for super table query"; + const char* msg2 = "TWA only support group by tbname for super table query"; // filter sql function not supported by metric query yet. for (int32_t i = 0; i < pCmd->fieldsInfo.numOfOutputCols; ++i) { @@ -2692,7 +2689,7 @@ bool hasUnsupportFunctionsForMetricQuery(SSqlCmd* pCmd) { } if (pCmd->groupbyExpr.numOfGroupCols != 1 || pCmd->groupbyExpr.columnInfo[0].colIdx != TSDB_TBNAME_COLUMN_INDEX) { - invalidSqlErrMsg(pCmd, msg3); + invalidSqlErrMsg(pCmd, msg2); return true; } } @@ -2701,8 +2698,6 @@ bool hasUnsupportFunctionsForMetricQuery(SSqlCmd* pCmd) { } static bool functionCompatibleCheck(SSqlCmd* pCmd) { - const char* msg1 = "column on select clause not allowed"; - int32_t startIdx = 0; int32_t functionID = tscSqlExprGet(pCmd, startIdx)->functionId; @@ -2745,7 +2740,7 @@ void updateTagColumnIndex(SSqlCmd* pCmd, int32_t tableIndex) { if (pCmd->groupbyExpr.numOfGroupCols > 0 && pCmd->groupbyExpr.tableIndex == tableIndex) { for (int32_t i = 0; i < pCmd->groupbyExpr.numOfGroupCols; ++i) { int32_t index = pCmd->groupbyExpr.columnInfo[i].colIdx; - + for (int32_t j = 0; j < pMeterMetaInfo->numOfTags; ++j) { int32_t tagColIndex = pMeterMetaInfo->tagColumnIndex[j]; if (tagColIndex == index) { @@ -2759,11 +2754,11 @@ void updateTagColumnIndex(SSqlCmd* pCmd, int32_t tableIndex) { // update tags column index for expression for (int32_t i = 0; i < pCmd->exprsInfo.numOfExprs; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pCmd, i); - + if (!TSDB_COL_IS_TAG(pExpr->colInfo.flag)) { // not tags, continue continue; } - + // not belongs to this table if (pExpr->uid != pMeterMetaInfo->pMeterMeta->uid) { continue; @@ -2776,40 +2771,37 @@ void updateTagColumnIndex(SSqlCmd* pCmd, int32_t tableIndex) { } } } - + // update join condition tag column index SJoinInfo* pJoinInfo = &pCmd->tagCond.joinInfo; if (!pJoinInfo->hasJoin) { // not join query return; } - + assert(pJoinInfo->left.uid != pJoinInfo->right.uid); - + // the join condition expression node belongs to this table(super table) if (pMeterMetaInfo->pMeterMeta->uid == pJoinInfo->left.uid) { - for(int32_t i = 0; i < pMeterMetaInfo->numOfTags; ++i) { + for (int32_t i = 0; i < pMeterMetaInfo->numOfTags; ++i) { if (pJoinInfo->left.tagCol == pMeterMetaInfo->tagColumnIndex[i]) { pJoinInfo->left.tagCol = i; } } } - + if (pMeterMetaInfo->pMeterMeta->uid == pJoinInfo->right.uid) { - for(int32_t i = 0; i < pMeterMetaInfo->numOfTags; ++i) { + for (int32_t i = 0; i < pMeterMetaInfo->numOfTags; ++i) { if (pJoinInfo->right.tagCol == pMeterMetaInfo->tagColumnIndex[i]) { pJoinInfo->right.tagCol = i; } } } - } int32_t parseGroupbyClause(SSqlCmd* pCmd, tVariantList* pList) { const char* msg1 = "too many columns in group by clause"; const char* msg2 = "invalid column name in group by clause"; - const char* msg4 = "group by only available for STable query"; - const char* msg5 = "group by columns must belong to one table"; - const char* msg6 = "only support group by one ordinary column"; + const char* msg3 = "group by columns must belong to one table"; const char* msg7 = "not support group by expression"; const char* msg8 = "not allowed column type for group by"; const char* msg9 = "tags not allowed for table query"; @@ -2828,9 +2820,8 @@ int32_t parseGroupbyClause(SSqlCmd* pCmd, tVariantList* pList) { SMeterMeta* pMeterMeta = NULL; SSchema* pSchema = NULL; + SSchema s = tsGetTbnameColumnSchema(); - SSchema s = {0}; - int32_t numOfReqTags = 0; int32_t tableIndex = COLUMN_INDEX_INITIAL_VAL; for (int32_t i = 0; i < pList->nExpr; ++i) { @@ -2844,7 +2835,7 @@ int32_t parseGroupbyClause(SSqlCmd* pCmd, tVariantList* pList) { } if (tableIndex != index.tableIndex && tableIndex >= 0) { - return invalidSqlErrMsg(pCmd, msg5); + return invalidSqlErrMsg(pCmd, msg3); } tableIndex = index.tableIndex; @@ -2852,22 +2843,12 @@ int32_t parseGroupbyClause(SSqlCmd* pCmd, tVariantList* pList) { pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, index.tableIndex); pMeterMeta = pMeterMetaInfo->pMeterMeta; - // TODO refactor!!!!!!!!!!!!!!1 if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { - s.colId = TSDB_TBNAME_COLUMN_INDEX; - s.type = TSDB_DATA_TYPE_BINARY; - s.bytes = TSDB_METER_NAME_LEN; - strcpy(s.name, TSQL_TBNAME_L); - pSchema = &s; } else { pSchema = tsGetColumnSchema(pMeterMeta, index.columnIndex); } - int16_t type = 0; - int16_t bytes = 0; - char* name = NULL; - bool groupTag = false; if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX || index.columnIndex >= pMeterMeta->numOfColumns) { groupTag = true; @@ -3138,7 +3119,7 @@ static int32_t optrToString(tSQLExpr* pExpr, char** exprString) { return TSDB_CODE_SUCCESS; } -static int32_t tablenameListToString(tSQLExpr* pExpr, /*char* str*/SStringBuilder* sb) { +static int32_t tablenameListToString(tSQLExpr* pExpr, /*char* str*/ SStringBuilder* sb) { tSQLExprList* pList = pExpr->pParam; if (pList->nExpr <= 0) { return TSDB_CODE_INVALID_SQL; @@ -3148,7 +3129,6 @@ static int32_t tablenameListToString(tSQLExpr* pExpr, /*char* str*/SStringBuilde taosStringBuilderAppendStringLen(sb, QUERY_COND_REL_PREFIX_IN, QUERY_COND_REL_PREFIX_IN_LEN); } - int32_t len = 0; for (int32_t i = 0; i < pList->nExpr; ++i) { tSQLExpr* pSub = pList->a[i].pNode; taosStringBuilderAppendStringLen(sb, pSub->val.pz, pSub->val.nLen); @@ -3165,7 +3145,7 @@ static int32_t tablenameListToString(tSQLExpr* pExpr, /*char* str*/SStringBuilde return TSDB_CODE_SUCCESS; } -static int32_t tablenameCondToString(tSQLExpr* pExpr, /*char* str*/SStringBuilder* sb) { +static int32_t tablenameCondToString(tSQLExpr* pExpr, /*char* str*/ SStringBuilder* sb) { taosStringBuilderAppendStringLen(sb, QUERY_COND_REL_PREFIX_LIKE, QUERY_COND_REL_PREFIX_LIKE_LEN); taosStringBuilderAppendString(sb, pExpr->val.pz); @@ -3187,7 +3167,6 @@ static int32_t extractColumnFilterInfo(SSqlCmd* pCmd, SColumnIndex* pIndex, tSQL const char* msg1 = "non binary column not support like operator"; const char* msg2 = "binary column not support this operator"; - const char* msg3 = "OR is not supported on different column filter"; SColumnBase* pColumn = tscColumnBaseInfoInsert(pCmd, pIndex); SColumnFilterInfo* pColFilter = NULL; @@ -3271,7 +3250,7 @@ static int32_t getTagCondString(SSqlCmd* pCmd, tSQLExpr* pExpr, char** str) { return tSQLExprLeafToString(pExpr, true, str); } -static int32_t getTablenameCond(SSqlCmd* pCmd, tSQLExpr* pTableCond, /*char* str*/SStringBuilder* sb) { +static int32_t getTablenameCond(SSqlCmd* pCmd, tSQLExpr* pTableCond, /*char* str*/ SStringBuilder* sb) { const char* msg0 = "invalid table name list"; if (pTableCond == NULL) { @@ -3615,10 +3594,9 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, tSQLExpr** pExpr, SCondExpr* const char* msg2 = "illegal column name"; const char* msg3 = "only one query time range allowed"; const char* msg4 = "only one join condition allowed"; - const char* msg5 = "AND is allowed to filter on different ordinary columns"; - const char* msg6 = "not support ordinary column join"; - const char* msg7 = "only one query condition on tbname allowed"; - const char* msg8 = "only in/like allowed in filter table name"; + const char* msg5 = "not support ordinary column join"; + const char* msg6 = "only one query condition on tbname allowed"; + const char* msg7 = "only in/like allowed in filter table name"; tSQLExpr* pLeft = (*pExpr)->pLeft; tSQLExpr* pRight = (*pExpr)->pRight; @@ -3680,7 +3658,7 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, tSQLExpr** pExpr, SCondExpr* // in case of in operator, keep it in a seperate attribute if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { if (!validTableNameOptr(*pExpr)) { - return invalidSqlErrMsg(pCmd, msg8); + return invalidSqlErrMsg(pCmd, msg7); } if (pCondExpr->pTableCond == NULL) { @@ -3688,7 +3666,7 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, tSQLExpr** pExpr, SCondExpr* pCondExpr->relType = parentOptr; pCondExpr->tableCondIndex = index.tableIndex; } else { - return invalidSqlErrMsg(pCmd, msg7); + return invalidSqlErrMsg(pCmd, msg6); } *type = TSQL_EXPR_TBNAME; @@ -3719,7 +3697,7 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, tSQLExpr** pExpr, SCondExpr* *type = TSQL_EXPR_COLUMN; if (pRight->nSQLOptr == TK_ID) { // other column cannot be served as the join column - return invalidSqlErrMsg(pCmd, msg6); + return invalidSqlErrMsg(pCmd, msg5); } ret = setExprToCond(pCmd, &pCondExpr->pColumnCond, *pExpr, NULL, parentOptr); @@ -3886,9 +3864,9 @@ static int32_t setTableCondForMetricQuery(SSqlObj* pSql, tSQLExpr* pExpr, int16_ // remove the duplicated input table names int32_t num = 0; - char* tableNameString = taosStringBuilderGetResult(sb, NULL); - - char** segments = strsplit(tableNameString + QUERY_COND_REL_PREFIX_IN_LEN, TBNAME_LIST_SEP, &num); + char* tableNameString = taosStringBuilderGetResult(sb, NULL); + + char** segments = strsplit(tableNameString + QUERY_COND_REL_PREFIX_IN_LEN, TBNAME_LIST_SEP, &num); qsort(segments, num, POINTER_BYTES, tableNameCompar); int32_t j = 1; @@ -3906,8 +3884,8 @@ static int32_t setTableCondForMetricQuery(SSqlObj* pSql, tSQLExpr* pExpr, int16_ if (i >= 1) { taosStringBuilderAppendStringLen(&sb1, TBNAME_LIST_SEP, 1); } - - char idBuf[TSDB_METER_ID_LEN + 1] = {0}; + + char idBuf[TSDB_METER_ID_LEN + 1] = {0}; int32_t xlen = strlen(segments[i]); SSQLToken t = {.z = segments[i], .n = xlen, .type = TK_STRING}; @@ -3915,17 +3893,17 @@ static int32_t setTableCondForMetricQuery(SSqlObj* pSql, tSQLExpr* pExpr, int16_ if (ret != TSDB_CODE_SUCCESS) { taosStringBuilderDestroy(&sb1); tfree(segments); - + invalidSqlErrMsg(pCmd, msg); return ret; } - + taosStringBuilderAppendString(&sb1, idBuf); } - + char* str = taosStringBuilderGetResult(&sb1, NULL); pCmd->tagCond.tbnameCond.cond = strdup(str); - + taosStringBuilderDestroy(&sb1); tfree(segments); return TSDB_CODE_SUCCESS; @@ -4052,21 +4030,20 @@ static void cleanQueryExpr(SCondExpr* pCondExpr) { } } - static void doAddJoinTagsColumnsIntoTagList(SSqlCmd* pCmd, SCondExpr* pCondExpr) { SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); if (QUERY_IS_JOIN_QUERY(pCmd->type) && UTIL_METER_IS_METRIC(pMeterMetaInfo)) { SColumnIndex index = {0}; - + getColumnIndexByNameEx(&pCondExpr->pJoinExpr->pLeft->colInfo, pCmd, &index); pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, index.tableIndex); - + int32_t columnInfo = index.columnIndex - pMeterMetaInfo->pMeterMeta->numOfColumns; addRequiredTagColumn(pCmd, columnInfo, index.tableIndex); - + getColumnIndexByNameEx(&pCondExpr->pJoinExpr->pRight->colInfo, pCmd, &index); pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, index.tableIndex); - + columnInfo = index.columnIndex - pMeterMetaInfo->pMeterMeta->numOfColumns; addRequiredTagColumn(pCmd, columnInfo, index.tableIndex); } @@ -4074,48 +4051,48 @@ static void doAddJoinTagsColumnsIntoTagList(SSqlCmd* pCmd, SCondExpr* pCondExpr) static int32_t getTagQueryCondExpr(SSqlCmd* pCmd, SCondExpr* pCondExpr, tSQLExpr** pExpr) { int32_t ret = TSDB_CODE_SUCCESS; - + if (pCondExpr->pTagCond != NULL) { for (int32_t i = 0; i < pCmd->numOfTables; ++i) { tSQLExpr* p1 = extractExprForSTable(pExpr, pCmd, i); - + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, i); - + char c[TSDB_MAX_TAGS_LEN] = {0}; char* str = c; - + if ((ret = getTagCondString(pCmd, p1, &str)) != TSDB_CODE_SUCCESS) { return ret; } - + tsSetMetricQueryCond(&pCmd->tagCond, pMeterMetaInfo->pMeterMeta->uid, c); - + doCompactQueryExpr(pExpr); tSQLExprDestroy(p1); } - + pCondExpr->pTagCond = NULL; } - + return ret; } int32_t parseWhereClause(SSqlObj* pSql, tSQLExpr** pExpr) { if (pExpr == NULL) { return TSDB_CODE_SUCCESS; } - + const char* msg = "invalid filter expression"; const char* msg1 = "invalid expression"; - + int32_t ret = TSDB_CODE_SUCCESS; - + SSqlCmd* pCmd = &pSql->cmd; pCmd->stime = 0; pCmd->etime = INT64_MAX; - //tags query condition may be larger than 512bytes, therefore, we need to prepare enough large space + // tags query condition may be larger than 512bytes, therefore, we need to prepare enough large space SStringBuilder sb = {0}; - SCondExpr condExpr = {0}; + SCondExpr condExpr = {0}; if ((*pExpr)->pLeft == NULL || (*pExpr)->pRight == NULL) { return invalidSqlErrMsg(pCmd, msg1); @@ -4125,54 +4102,54 @@ int32_t parseWhereClause(SSqlObj* pSql, tSQLExpr** pExpr) { if ((ret = getQueryCondExpr(pCmd, pExpr, &condExpr, &type, (*pExpr)->nSQLOptr)) != TSDB_CODE_SUCCESS) { return ret; } - + doCompactQueryExpr(pExpr); - + // after expression compact, the expression tree is only include tag query condition condExpr.pTagCond = (*pExpr); - + // 1. check if it is a join query if ((ret = validateJoinExpr(pCmd, &condExpr)) != TSDB_CODE_SUCCESS) { return ret; } - + // 2. get the query time range if ((ret = getTimeRangeFromExpr(pCmd, condExpr.pTimewindow)) != TSDB_CODE_SUCCESS) { return ret; } - + // 3. get the tag query condition if ((ret = getTagQueryCondExpr(pCmd, &condExpr, pExpr)) != TSDB_CODE_SUCCESS) { return ret; } - + // 4. get the table name query condition if ((ret = getTablenameCond(pCmd, condExpr.pTableCond, &sb)) != TSDB_CODE_SUCCESS) { return ret; } - + // 5. other column query condition if ((ret = getColumnQueryCondInfo(pCmd, condExpr.pColumnCond, TK_AND)) != TSDB_CODE_SUCCESS) { return ret; } - + // 6. join condition if ((ret = getJoinCondInfo(pSql, condExpr.pJoinExpr)) != TSDB_CODE_SUCCESS) { return ret; } - + // 7. query condition for table name pCmd->tagCond.relType = (condExpr.relType == TK_AND) ? TSDB_RELATION_AND : TSDB_RELATION_OR; - + ret = setTableCondForMetricQuery(pSql, condExpr.pTableCond, condExpr.tableCondIndex, &sb); taosStringBuilderDestroy(&sb); - + if (!validateFilterExpr(pCmd)) { return invalidSqlErrMsg(pCmd, msg); } - + doAddJoinTagsColumnsIntoTagList(pCmd, &condExpr); - + cleanQueryExpr(&condExpr); return ret; } @@ -4447,7 +4424,6 @@ int32_t parseOrderbyClause(SSqlCmd* pCmd, SQuerySQL* pQuerySql, SSchema* pSchema bool orderByTags = false; bool orderByTS = false; - bool orderByCol = false; if (index.columnIndex >= pMeterMetaInfo->pMeterMeta->numOfColumns) { int32_t relTagIndex = index.columnIndex - pMeterMetaInfo->pMeterMeta->numOfColumns; @@ -4824,7 +4800,6 @@ int32_t validateSqlFunctionInStreamSql(SSqlCmd* pCmd) { int32_t validateFunctionsInIntervalOrGroupbyQuery(SSqlCmd* pCmd) { bool isProjectionFunction = false; const char* msg1 = "column projection is not compatible with interval"; - const char* msg2 = "interval not allowed for tag queries"; // multi-output set/ todo refactor for (int32_t k = 0; k < pCmd->fieldsInfo.numOfOutputCols; ++k) { @@ -5006,7 +4981,7 @@ int32_t parseLimitClause(SSqlObj* pSql, SQuerySQL* pQuerySql) { // handle the limit offset value, validate the limit pCmd->limit = pQuerySql->limit; pCmd->globalLimit = pCmd->limit.limit; - + pCmd->slimit = pQuerySql->slimit; if (pCmd->slimit.offset < 0 || pCmd->limit.offset < 0) { @@ -5076,11 +5051,11 @@ int32_t parseLimitClause(SSqlObj* pSql, SQuerySQL* pQuerySql) { static int32_t setKeepOption(SSqlCmd* pCmd, SCreateDbMsg* pMsg, SCreateDBInfo* pCreateDb) { const char* msg = "invalid number of options"; - + pMsg->daysToKeep = htonl(-1); pMsg->daysToKeep1 = htonl(-1); pMsg->daysToKeep2 = htonl(-1); - + tVariantList* pKeep = pCreateDb->keep; if (pKeep != NULL) { switch (pKeep->nExpr) { @@ -5098,36 +5073,34 @@ static int32_t setKeepOption(SSqlCmd* pCmd, SCreateDbMsg* pMsg, SCreateDBInfo* p pMsg->daysToKeep2 = htonl(pKeep->a[2].pVar.i64Key); break; } - default: { - return invalidSqlErrMsg(pCmd, msg); - } + default: { return invalidSqlErrMsg(pCmd, msg); } } } - + return TSDB_CODE_SUCCESS; } static int32_t setTimePrecisionOption(SSqlCmd* pCmd, SCreateDbMsg* pMsg, SCreateDBInfo* pCreateDbInfo) { const char* msg = "invalid time precision"; - + pMsg->precision = TSDB_TIME_PRECISION_MILLI; // millisecond by default - + SSQLToken* pToken = &pCreateDbInfo->precision; if (pToken->n > 0) { pToken->n = strdequote(pToken->z); - + if (strncmp(pToken->z, TSDB_TIME_PRECISION_MILLI_STR, pToken->n) == 0 && strlen(TSDB_TIME_PRECISION_MILLI_STR) == pToken->n) { // time precision for this db: million second pMsg->precision = TSDB_TIME_PRECISION_MILLI; } else if (strncmp(pToken->z, TSDB_TIME_PRECISION_MICRO_STR, pToken->n) == 0 && - strlen(TSDB_TIME_PRECISION_MICRO_STR) == pToken->n) { + strlen(TSDB_TIME_PRECISION_MICRO_STR) == pToken->n) { pMsg->precision = TSDB_TIME_PRECISION_MICRO; } else { return invalidSqlErrMsg(pCmd, msg); } } - + return TSDB_CODE_SUCCESS; } @@ -5135,7 +5108,7 @@ static void setCreateDBOption(SCreateDbMsg* pMsg, SCreateDBInfo* pCreateDb) { pMsg->blocksPerMeter = htons(pCreateDb->numOfBlocksPerTable); pMsg->compression = pCreateDb->compressionLevel; - pMsg->commitLog = (char) pCreateDb->commitLog; + pMsg->commitLog = (char)pCreateDb->commitLog; pMsg->commitTime = htonl(pCreateDb->commitTime); pMsg->maxSessions = htonl(pCreateDb->tablesPerVnode); pMsg->cacheNumOfBlocks.fraction = pCreateDb->numOfAvgCacheBlocks; @@ -5148,19 +5121,19 @@ static void setCreateDBOption(SCreateDbMsg* pMsg, SCreateDBInfo* pCreateDb) { int32_t parseCreateDBOptions(SSqlCmd* pCmd, SCreateDBInfo* pCreateDbSql) { SCreateDbMsg* pMsg = (SCreateDbMsg*)(pCmd->payload + tsRpcHeadSize + sizeof(SMgmtHead)); setCreateDBOption(pMsg, pCreateDbSql); - + if (setKeepOption(pCmd, pMsg, pCreateDbSql) != TSDB_CODE_SUCCESS) { return TSDB_CODE_INVALID_SQL; } - + if (setTimePrecisionOption(pCmd, pMsg, pCreateDbSql) != TSDB_CODE_SUCCESS) { return TSDB_CODE_INVALID_SQL; } - + if (tscCheckCreateDbParams(pCmd, pMsg) != TSDB_CODE_SUCCESS) { return TSDB_CODE_INVALID_SQL; } - + return TSDB_CODE_SUCCESS; } @@ -5254,9 +5227,6 @@ static void doUpdateSqlFunctionForTagPrj(SSqlCmd* pCmd) { } } - int16_t resType = 0; - int16_t resBytes = 0; - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); SSchema* pSchema = tsGetSchema(pMeterMetaInfo->pMeterMeta); @@ -5353,11 +5323,9 @@ static void updateTagPrjFunction(SSqlCmd* pCmd) { */ static int32_t checkUpdateTagPrjFunctions(SSqlCmd* pCmd) { const char* msg1 = "only one selectivity function allowed in presence of tags function"; - const char* msg2 = "functions not allowed"; const char* msg3 = "aggregation function should not be mixed up with projection"; bool tagColExists = false; - int16_t numOfTimestamp = 0; // primary timestamp column int16_t numOfSelectivity = 0; int16_t numOfAggregation = 0; @@ -5499,13 +5467,10 @@ static int32_t doAddGroupbyColumnsOnDemand(SSqlCmd* pCmd) { int32_t doFunctionsCompatibleCheck(SSqlObj* pSql) { const char* msg1 = "functions/columns not allowed in group by query"; - const char* msg2 = "interval not allowed in group by normal column"; const char* msg3 = "group by not allowed on projection query"; - const char* msg4 = "tags retrieve not compatible with group by"; const char* msg5 = "retrieve tags not compatible with group by or interval query"; - SSqlCmd* pCmd = &pSql->cmd; - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + SSqlCmd* pCmd = &pSql->cmd; // only retrieve tags, group by is not supportted if (pCmd->command == TSDB_SQL_RETRIEVE_TAGS) { @@ -5517,11 +5482,6 @@ int32_t doFunctionsCompatibleCheck(SSqlObj* pSql) { } if (pCmd->groupbyExpr.numOfGroupCols > 0) { - SSchema* pSchema = tsGetSchema(pMeterMetaInfo->pMeterMeta); - int16_t bytes = 0; - int16_t type = 0; - char* name = NULL; - // check if all the tags prj columns belongs to the group by columns if (onlyTagPrjFunction(pCmd) && allTagPrjInGroupby(pCmd)) { updateTagPrjFunction(pCmd); @@ -5639,84 +5599,81 @@ int32_t doLocalQueryProcess(SQuerySQL* pQuerySql, SSqlCmd* pCmd) { case 4: pCmd->command = TSDB_SQL_CURRENT_USER; return TSDB_CODE_SUCCESS; - default: { - return invalidSqlErrMsg(pCmd, msg3); - } + default: { return invalidSqlErrMsg(pCmd, msg3); } } } // can only perform the parameters based on the macro definitation -int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCreateDbMsg *pCreate) { +int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCreateDbMsg* pCreate) { char msg[512] = {0}; - + if (pCreate->commitLog != -1 && (pCreate->commitLog < 0 || pCreate->commitLog > 1)) { snprintf(msg, tListLen(msg), "invalid db option commitLog: %d, only 0 or 1 allowed", pCreate->commitLog); return invalidSqlErrMsg(pCmd, msg); } - + if (pCreate->replications != -1 && (pCreate->replications < TSDB_REPLICA_MIN_NUM || pCreate->replications > TSDB_REPLICA_MAX_NUM)) { - snprintf(msg, tListLen(msg), "invalid db option replications: %d valid range: [%d, %d]", pCreate->replications, TSDB_REPLICA_MIN_NUM, - TSDB_REPLICA_MAX_NUM); + snprintf(msg, tListLen(msg), "invalid db option replications: %d valid range: [%d, %d]", pCreate->replications, + TSDB_REPLICA_MIN_NUM, TSDB_REPLICA_MAX_NUM); return invalidSqlErrMsg(pCmd, msg); } - + int32_t val = htonl(pCreate->daysPerFile); if (val != -1 && (val < TSDB_FILE_MIN_PARTITION_RANGE || val > TSDB_FILE_MAX_PARTITION_RANGE)) { snprintf(msg, tListLen(msg), "invalid db option daysPerFile: %d valid range: [%d, %d]", val, TSDB_FILE_MIN_PARTITION_RANGE, TSDB_FILE_MAX_PARTITION_RANGE); return invalidSqlErrMsg(pCmd, msg); } - + val = htonl(pCreate->rowsInFileBlock); if (val != -1 && (val < TSDB_MIN_ROWS_IN_FILEBLOCK || val > TSDB_MAX_ROWS_IN_FILEBLOCK)) { snprintf(msg, tListLen(msg), "invalid db option rowsInFileBlock: %d valid range: [%d, %d]", val, TSDB_MIN_ROWS_IN_FILEBLOCK, TSDB_MAX_ROWS_IN_FILEBLOCK); return invalidSqlErrMsg(pCmd, msg); } - + val = htonl(pCreate->cacheBlockSize); if (val != -1 && (val < TSDB_MIN_CACHE_BLOCK_SIZE || val > TSDB_MAX_CACHE_BLOCK_SIZE)) { snprintf(msg, tListLen(msg), "invalid db option cacheBlockSize: %d valid range: [%d, %d]", val, TSDB_MIN_CACHE_BLOCK_SIZE, TSDB_MAX_CACHE_BLOCK_SIZE); return invalidSqlErrMsg(pCmd, msg); } - + val = htonl(pCreate->maxSessions); if (val != -1 && (val < TSDB_MIN_TABLES_PER_VNODE || val > TSDB_MAX_TABLES_PER_VNODE)) { - snprintf(msg, tListLen(msg), "invalid db option maxSessions: %d valid range: [%d, %d]", val, TSDB_MIN_TABLES_PER_VNODE, - TSDB_MAX_TABLES_PER_VNODE); + snprintf(msg, tListLen(msg), "invalid db option maxSessions: %d valid range: [%d, %d]", val, + TSDB_MIN_TABLES_PER_VNODE, TSDB_MAX_TABLES_PER_VNODE); return invalidSqlErrMsg(pCmd, msg); } - - if (pCreate->precision != -1 && - (pCreate->precision != TSDB_TIME_PRECISION_MILLI && pCreate->precision != TSDB_TIME_PRECISION_MICRO)) { - snprintf(msg, tListLen(msg), "invalid db option timePrecision: %d valid value: [%d, %d]", pCreate->precision, TSDB_TIME_PRECISION_MILLI, - TSDB_TIME_PRECISION_MICRO); + + if (pCreate->precision != TSDB_TIME_PRECISION_MILLI && pCreate->precision != TSDB_TIME_PRECISION_MICRO) { + snprintf(msg, tListLen(msg), "invalid db option timePrecision: %d valid value: [%d, %d]", pCreate->precision, + TSDB_TIME_PRECISION_MILLI, TSDB_TIME_PRECISION_MICRO); return invalidSqlErrMsg(pCmd, msg); } - + if (pCreate->cacheNumOfBlocks.fraction != -1 && (pCreate->cacheNumOfBlocks.fraction < TSDB_MIN_AVG_BLOCKS || - pCreate->cacheNumOfBlocks.fraction > TSDB_MAX_AVG_BLOCKS)) { - snprintf(msg, tListLen(msg), "invalid db option ablocks: %f valid value: [%d, %d]", pCreate->cacheNumOfBlocks.fraction, - TSDB_MIN_AVG_BLOCKS, TSDB_MAX_AVG_BLOCKS); + pCreate->cacheNumOfBlocks.fraction > TSDB_MAX_AVG_BLOCKS)) { + snprintf(msg, tListLen(msg), "invalid db option ablocks: %f valid value: [%d, %d]", + pCreate->cacheNumOfBlocks.fraction, TSDB_MIN_AVG_BLOCKS, TSDB_MAX_AVG_BLOCKS); return invalidSqlErrMsg(pCmd, msg); } - + val = htonl(pCreate->commitTime); if (val != -1 && (val < TSDB_MIN_COMMIT_TIME_INTERVAL || val > TSDB_MAX_COMMIT_TIME_INTERVAL)) { snprintf(msg, tListLen(msg), "invalid db option commitTime: %d valid range: [%d, %d]", val, TSDB_MIN_COMMIT_TIME_INTERVAL, TSDB_MAX_COMMIT_TIME_INTERVAL); return invalidSqlErrMsg(pCmd, msg); } - + if (pCreate->compression != -1 && (pCreate->compression < TSDB_MIN_COMPRESSION_LEVEL || pCreate->compression > TSDB_MAX_COMPRESSION_LEVEL)) { - snprintf(msg, tListLen(msg), "invalid db option compression: %d valid range: [%d, %d]", pCreate->compression, TSDB_MIN_COMPRESSION_LEVEL, - TSDB_MAX_COMPRESSION_LEVEL); + snprintf(msg, tListLen(msg), "invalid db option compression: %d valid range: [%d, %d]", pCreate->compression, + TSDB_MIN_COMPRESSION_LEVEL, TSDB_MAX_COMPRESSION_LEVEL); return invalidSqlErrMsg(pCmd, msg); } - + return TSDB_CODE_SUCCESS; } @@ -5725,24 +5682,24 @@ void tscPrintSelectClause(SSqlCmd* pCmd) { if (pCmd == NULL || pCmd->exprsInfo.numOfExprs == 0) { return; } - - char* str = calloc(1, 10240); + + char* str = calloc(1, 10240); int32_t offset = 0; - + offset += sprintf(str, "%d [", pCmd->exprsInfo.numOfExprs); - for(int32_t i = 0; i < pCmd->exprsInfo.numOfExprs; ++i) { + for (int32_t i = 0; i < pCmd->exprsInfo.numOfExprs; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pCmd, i); - + int32_t size = sprintf(str + offset, "%s(%d)", aAggs[pExpr->functionId].aName, pExpr->colInfo.colId); offset += size; - + if (i < pCmd->exprsInfo.numOfExprs - 1) { str[offset++] = ','; } } - + str[offset] = ']'; printf("%s\n", str); - + free(str); } diff --git a/src/client/src/tscSchemaUtil.c b/src/client/src/tscSchemaUtil.c index fdbad2bbf8..85ca3eb863 100644 --- a/src/client/src/tscSchemaUtil.c +++ b/src/client/src/tscSchemaUtil.c @@ -83,6 +83,13 @@ struct SSchema* tsGetColumnSchema(SMeterMeta* pMeta, int32_t startCol) { return (SSchema*)(((char*)pMeta + sizeof(SMeterMeta)) + startCol * sizeof(SSchema)); } +struct SSchema tsGetTbnameColumnSchema() { + struct SSchema s = {.colId = TSDB_TBNAME_COLUMN_INDEX, .type = TSDB_DATA_TYPE_BINARY, .bytes = TSDB_METER_NAME_LEN}; + strcpy(s.name, TSQL_TBNAME_L); + + return s; +} + /** * the MeterMeta data format in memory is as follows: * @@ -123,7 +130,7 @@ bool tsMeterMetaIdentical(SMeterMeta* p1, SMeterMeta* p2) { return memcmp(p1, p2, size) == 0; } -//todo refactor +// todo refactor static FORCE_INLINE char* skipSegments(char* input, char delimiter, int32_t num) { for (int32_t i = 0; i < num; ++i) { while (*input != 0 && *input++ != delimiter) { diff --git a/src/inc/tschemautil.h b/src/inc/tschemautil.h index 0b8a2d6a93..7706bcd3a4 100644 --- a/src/inc/tschemautil.h +++ b/src/inc/tschemautil.h @@ -47,6 +47,7 @@ struct SSchema *tsGetSchema(SMeterMeta *pMeta); struct SSchema *tsGetTagSchema(SMeterMeta *pMeta); struct SSchema *tsGetColumnSchema(SMeterMeta *pMeta, int32_t startCol); +struct SSchema tsGetTbnameColumnSchema(); char *tsGetTagsValue(SMeterMeta *pMeta); diff --git a/src/inc/tsqlfunction.h b/src/inc/tsqlfunction.h index a5734ed60e..0ed6a9952e 100644 --- a/src/inc/tsqlfunction.h +++ b/src/inc/tsqlfunction.h @@ -227,8 +227,6 @@ typedef struct SPatternCompareInfo { int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionId, int32_t param, int16_t *type, int16_t *len, int16_t *interResBytes, int16_t extLength, bool isSuperTable); -SResultInfo *getResultSupportInfo(SQLFunctionCtx *pCtx); - int patternMatch(const char *zPattern, const char *zString, size_t size, const SPatternCompareInfo *pInfo); int WCSPatternMatch(const wchar_t *zPattern, const wchar_t *zString, size_t size, const SPatternCompareInfo *pInfo); diff --git a/src/inc/ttypes.h b/src/inc/ttypes.h index b2ea8e918a..0f8eb2d58c 100644 --- a/src/inc/ttypes.h +++ b/src/inc/ttypes.h @@ -50,7 +50,7 @@ bool isNull(const char *val, int32_t type); void setNull(char *val, int32_t type, int32_t bytes); void setNullN(char *val, int32_t type, int32_t bytes, int32_t numOfElems); -void assignVal(char *val, char *src, int32_t len, int32_t type); +void assignVal(char *val, const char *src, int32_t len, int32_t type); void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); // variant, each number/string/field_id has a corresponding struct during parsing sql diff --git a/src/util/src/ttypes.c b/src/util/src/ttypes.c index ad3f98beb6..96237e34ae 100644 --- a/src/util/src/ttypes.c +++ b/src/util/src/ttypes.c @@ -970,7 +970,7 @@ void setNullN(char *val, int32_t type, int32_t bytes, int32_t numOfElems) { } } -void assignVal(char *val, char *src, int32_t len, int32_t type) { +void assignVal(char *val, const char *src, int32_t len, int32_t type) { switch (type) { case TSDB_DATA_TYPE_INT: { *((int32_t *)val) = GET_INT32_VAL(src); From 7568052d031e42add32051f01ffa7ae46b5e21c9 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Tue, 10 Dec 2019 16:21:58 +0800 Subject: [PATCH 095/108] remove the compiler warnings --- src/system/detail/src/vnodeQueryImpl.c | 30 ++++++++++++++++++++------ 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/src/system/detail/src/vnodeQueryImpl.c b/src/system/detail/src/vnodeQueryImpl.c index fb8ff7ef19..45c3385ed0 100644 --- a/src/system/detail/src/vnodeQueryImpl.c +++ b/src/system/detail/src/vnodeQueryImpl.c @@ -252,15 +252,31 @@ static void vnodeSetOpenedFileNames(SQueryFilesInfo* pVnodeFilesInfo) { SHeaderFileInfo* pCurrentFileInfo = &pVnodeFilesInfo->pFileInfo[pVnodeFilesInfo->current]; - // set the full file path for current opened files - snprintf(pVnodeFilesInfo->headerFilePath, PATH_MAX, "%sv%df%d.head", pVnodeFilesInfo->dbFilePathPrefix, - pVnodeFilesInfo->vnodeId, pCurrentFileInfo->fileID); + /* + * set the full file path for current opened files + * the maximum allowed path string length is PATH_MAX in Linux, 100 bytes is used to + * suppress the compiler warnings + */ + char str[PATH_MAX + 100] = {0}; + int32_t PATH_WITH_EXTRA = PATH_MAX + 100; - snprintf(pVnodeFilesInfo->dataFilePath, PATH_MAX, "%sv%df%d.data", pVnodeFilesInfo->dbFilePathPrefix, - pVnodeFilesInfo->vnodeId, pCurrentFileInfo->fileID); + int32_t vnodeId = pVnodeFilesInfo->vnodeId; + int32_t fileId = pCurrentFileInfo->fileID; - snprintf(pVnodeFilesInfo->lastFilePath, PATH_MAX, "%sv%df%d.last", pVnodeFilesInfo->dbFilePathPrefix, - pVnodeFilesInfo->vnodeId, pCurrentFileInfo->fileID); + int32_t len = snprintf(str, PATH_WITH_EXTRA, "%sv%df%d.head", pVnodeFilesInfo->dbFilePathPrefix, vnodeId, fileId); + assert(len <= PATH_MAX); + + strncpy(pVnodeFilesInfo->headerFilePath, str, PATH_MAX); + + len = snprintf(str, PATH_WITH_EXTRA, "%sv%df%d.data", pVnodeFilesInfo->dbFilePathPrefix, vnodeId, fileId); + assert(len <= PATH_MAX); + + strncpy(pVnodeFilesInfo->dataFilePath, str, PATH_MAX); + + len = snprintf(str, PATH_WITH_EXTRA, "%sv%df%d.last", pVnodeFilesInfo->dbFilePathPrefix, vnodeId, fileId); + assert(len <= PATH_MAX); + + strncpy(pVnodeFilesInfo->lastFilePath, str, PATH_MAX); } /** From 9f164ab063457ea52a1f42b72c10a60835458f9c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 10 Dec 2019 16:31:44 +0800 Subject: [PATCH 096/108] resolve warnings in kit --- src/kit/shell/src/shellCommand.c | 2 ++ src/kit/shell/src/shellEngine.c | 13 +++++----- src/kit/shell/src/shellLinux.c | 1 + src/kit/taosdemo/taosdemo.c | 41 ++++++++++++++++++-------------- src/os/linux/inc/os.h | 1 + 5 files changed, 34 insertions(+), 24 deletions(-) diff --git a/src/kit/shell/src/shellCommand.c b/src/kit/shell/src/shellCommand.c index 46aa04c1d7..16545a5fe8 100644 --- a/src/kit/shell/src/shellCommand.c +++ b/src/kit/shell/src/shellCommand.c @@ -19,6 +19,8 @@ #include "shell.h" #include "shellCommand.h" +extern int wcwidth(wchar_t c); +extern int wcswidth(const wchar_t *s, size_t n); typedef struct { char widthInString; char widthOnScreen; diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c index ed3b71bfcd..8dd9fe36c5 100644 --- a/src/kit/shell/src/shellEngine.c +++ b/src/kit/shell/src/shellEngine.c @@ -16,6 +16,7 @@ #define _XOPEN_SOURCE #define _DEFAULT_SOURCE +#include #include "os.h" #include "shell.h" #include "shellCommand.h" @@ -446,7 +447,7 @@ int shellDumpResult(TAOS *con, char *fname, int *error_no, bool printMode) { printf("%*d|", l[i], *((int *)row[i])); break; case TSDB_DATA_TYPE_BIGINT: - printf("%*lld|", l[i], *((int64_t *)row[i])); + printf("%*" PRId64 "|", l[i], *((int64_t *)row[i])); break; case TSDB_DATA_TYPE_FLOAT: { #ifdef _TD_ARM_32_ @@ -481,7 +482,7 @@ int shellDumpResult(TAOS *con, char *fname, int *error_no, bool printMode) { break; case TSDB_DATA_TYPE_TIMESTAMP: if (args.is_raw_time) { - printf(" %lld|", *(int64_t *)row[i]); + printf(" %" PRId64 "|", *(int64_t *)row[i]); } else { if (taos_result_precision(result) == TSDB_TIME_PRECISION_MICRO) { tt = (time_t)((*(int64_t *)row[i]) / 1000000); @@ -531,7 +532,7 @@ int shellDumpResult(TAOS *con, char *fname, int *error_no, bool printMode) { printf("%d\n", *((int *)row[i])); break; case TSDB_DATA_TYPE_BIGINT: - printf("%lld\n", *((int64_t *)row[i])); + printf("%" PRId64 "\n", *((int64_t *)row[i])); break; case TSDB_DATA_TYPE_FLOAT: { #ifdef _TD_ARM_32_ @@ -564,7 +565,7 @@ int shellDumpResult(TAOS *con, char *fname, int *error_no, bool printMode) { break; case TSDB_DATA_TYPE_TIMESTAMP: if (args.is_raw_time) { - printf("%lld\n", *(int64_t *)row[i]); + printf("%" PRId64 "\n", *(int64_t *)row[i]); } else { if (taos_result_precision(result) == TSDB_TIME_PRECISION_MICRO) { tt = (time_t)((*(int64_t *)row[i]) / 1000000); @@ -619,7 +620,7 @@ int shellDumpResult(TAOS *con, char *fname, int *error_no, bool printMode) { fprintf(fp, "%d", *((int *)row[i])); break; case TSDB_DATA_TYPE_BIGINT: - fprintf(fp, "%lld", *((int64_t *)row[i])); + fprintf(fp, "%" PRId64, *((int64_t *)row[i])); break; case TSDB_DATA_TYPE_FLOAT: { #ifdef _TD_ARM_32_ @@ -651,7 +652,7 @@ int shellDumpResult(TAOS *con, char *fname, int *error_no, bool printMode) { break; case TSDB_DATA_TYPE_TIMESTAMP: if (args.is_raw_time) { - fprintf(fp, "%lld", *(int64_t *)row[i]); + fprintf(fp, "%" PRId64, *(int64_t *)row[i]); } else { if (taos_result_precision(result) == TSDB_TIME_PRECISION_MICRO) { tt = (time_t)((*(int64_t *)row[i]) / 1000000); diff --git a/src/kit/shell/src/shellLinux.c b/src/kit/shell/src/shellLinux.c index 2e2c44af1a..70e8752525 100644 --- a/src/kit/shell/src/shellLinux.c +++ b/src/kit/shell/src/shellLinux.c @@ -26,6 +26,7 @@ int indicator = 1; struct termios oldtio; +extern int wcwidth(wchar_t c); void insertChar(Command *cmd, char *c, int size); const char *argp_program_version = version; const char *argp_program_bug_address = ""; diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 22aa71bf6c..40fbabe1f7 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -265,30 +265,35 @@ double getCurrentTime(); void callBack(void *param, TAOS_RES *res, int code); int main(int argc, char *argv[]) { - struct arguments arguments = {NULL, - 0, - "root", - "taosdata", - "test", - "t", - false, - false, - "./output.txt", - 0, - "int", + struct arguments arguments = {NULL, // host + 0, // port + "root", // user + "taosdata", // password + "test", // database + "t", // tb_prefix + false, // use_metric + false, // insert_only + "./output.txt", // output_file + 0, // mode + { + "int", // datatype "", "", "", "", "", "", - "", - 8, - 1, - 1, - 1, - 1, - 50000}; + "" + }, + 8, // len_of_binary + 1, // num_of_CPR + 1, // num_of_connections + 1, // num_of_RPR + 1, // num_of_tables + 50000, // num_of_DPT + 0, // abort + NULL // arg_list + }; /* Parse our arguments; every option seen by parse_opt will be reflected in arguments. */ diff --git a/src/os/linux/inc/os.h b/src/os/linux/inc/os.h index 8d63e69cd6..acfd284737 100644 --- a/src/os/linux/inc/os.h +++ b/src/os/linux/inc/os.h @@ -23,6 +23,7 @@ extern "C" { #include #include +#include #include #include #include From eb6533a96eeacafece40e662bbe4c56c54295ec3 Mon Sep 17 00:00:00 2001 From: lihui Date: Tue, 10 Dec 2019 16:42:29 +0800 Subject: [PATCH 097/108] [TBASE-1323] --- src/util/src/ttypes.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/util/src/ttypes.c b/src/util/src/ttypes.c index 96237e34ae..b048748d95 100644 --- a/src/util/src/ttypes.c +++ b/src/util/src/ttypes.c @@ -139,7 +139,7 @@ void tVariantCreateFromBinary(tVariant *pVar, char *pz, uint32_t len, uint32_t t } case TSDB_DATA_TYPE_NCHAR: { // here we get the nchar length from raw binary bits length pVar->nLen = len / TSDB_NCHAR_SIZE; - pVar->wpz = malloc((pVar->nLen + 1) * TSDB_NCHAR_SIZE); + pVar->wpz = calloc(1, (pVar->nLen + 1) * TSDB_NCHAR_SIZE); wcsncpy(pVar->wpz, (wchar_t *)pz, pVar->nLen); pVar->wpz[pVar->nLen] = 0; @@ -998,6 +998,14 @@ void assignVal(char *val, const char *src, int32_t len, int32_t type) { *((int8_t *)val) = GET_INT8_VAL(src); break; }; + case TSDB_DATA_TYPE_BINARY: { + strncpy(val, src, len); + break; + }; + case TSDB_DATA_TYPE_NCHAR: { + wcsncpy((wchar_t*)val, (wchar_t*)src, len / TSDB_NCHAR_SIZE); + break; + }; default: { memcpy(val, src, len); break; From 7a74561f6f3e3941c0fbe1292a2db9fe14fe8a1e Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Tue, 10 Dec 2019 16:48:47 +0800 Subject: [PATCH 098/108] [tbase-1324] --- src/client/src/tscSQLParser.c | 3 ++- src/client/src/tscUtil.c | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index d96e15eb5b..9df50b0fa1 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -5340,7 +5340,8 @@ static int32_t checkUpdateTagPrjFunctions(SSqlCmd* pCmd) { for (int32_t i = 0; i < pCmd->fieldsInfo.numOfOutputCols; ++i) { int16_t functionId = tscSqlExprGet(pCmd, i)->functionId; - if (functionId == TSDB_FUNC_TAGPRJ || functionId == TSDB_FUNC_PRJ || functionId == TSDB_FUNC_TS) { + if (functionId == TSDB_FUNC_TAGPRJ || functionId == TSDB_FUNC_PRJ || functionId == TSDB_FUNC_TS || + functionId == TSDB_FUNC_ARITHM) { continue; } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 7c7310a1c7..9876dad906 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -246,7 +246,7 @@ bool tscProjectionQueryOnMetric(SSqlCmd* pCmd) { for (int32_t i = 0; i < pCmd->fieldsInfo.numOfOutputCols; ++i) { int32_t functionId = tscSqlExprGet(pCmd, i)->functionId; if (functionId != TSDB_FUNC_PRJ && functionId != TSDB_FUNC_TAGPRJ && - functionId != TSDB_FUNC_TAG && functionId != TSDB_FUNC_TS) { + functionId != TSDB_FUNC_TAG && functionId != TSDB_FUNC_TS && functionId != TSDB_FUNC_ARITHM) { return false; } } From e08aa1ab038aa8d2f493d4c407a7280c99935c1c Mon Sep 17 00:00:00 2001 From: haojun Liao Date: Tue, 10 Dec 2019 17:45:41 +0800 Subject: [PATCH 099/108] Update tscUtil.c fix bugs in issue #898 --- src/client/src/tscUtil.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index baafb57f6a..52e19ce244 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -236,7 +236,7 @@ bool tscProjectionQueryOnMetric(SSqlCmd* pCmd) { SSqlExpr* pExpr = tscSqlExprGet(pCmd, i); int32_t functionId = pExpr->functionId; if (functionId != TSDB_FUNC_PRJ && functionId != TSDB_FUNC_TAGPRJ && - functionId != TSDB_FUNC_TAG && functionId != TSDB_FUNC_TS) { + functionId != TSDB_FUNC_TAG && functionId != TSDB_FUNC_TS && functionId != TSDB_FUNC_ARITHM) { return false; } } From 7538f4e7553469b28c7a86c9656720e20d75c785 Mon Sep 17 00:00:00 2001 From: haojun Liao Date: Tue, 10 Dec 2019 17:57:06 +0800 Subject: [PATCH 100/108] Update tscSQLParser.c --- src/client/src/tscSQLParser.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index ba465c28b9..b6b2c0fd18 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -5336,7 +5336,8 @@ static int32_t checkUpdateTagPrjFunctions(SSqlCmd* pCmd) { for (int32_t i = 0; i < pCmd->fieldsInfo.numOfOutputCols; ++i) { int16_t functionId = tscSqlExprGet(pCmd, i)->functionId; - if (functionId == TSDB_FUNC_TAGPRJ || functionId == TSDB_FUNC_PRJ || functionId == TSDB_FUNC_TS) { + if (functionId == TSDB_FUNC_TAGPRJ || functionId == TSDB_FUNC_PRJ || functionId == TSDB_FUNC_TS || + functionId == TSDB_FUNC_ARITHM) { continue; } From e1b11fbb3bd8398103f8608b45aa6c8cecc0261f Mon Sep 17 00:00:00 2001 From: lihui Date: Tue, 10 Dec 2019 18:42:07 +0800 Subject: [PATCH 101/108] [TBASE-816] --- src/modules/monitor/src/monitorSystem.c | 25 +++++++++++++------------ src/os/linux/src/tsystem.c | 20 ++++++++++---------- 2 files changed, 23 insertions(+), 22 deletions(-) diff --git a/src/modules/monitor/src/monitorSystem.c b/src/modules/monitor/src/monitorSystem.c index c5f65eef36..f89f259688 100644 --- a/src/modules/monitor/src/monitorSystem.c +++ b/src/modules/monitor/src/monitorSystem.c @@ -14,6 +14,7 @@ */ #include "monitor.h" +#include #include #include #include @@ -376,7 +377,7 @@ void monitorSaveSystemInfo() { int64_t ts = taosGetTimestampUs(); char * sql = monitor->sql; - int pos = snprintf(sql, SQL_LENGTH, "insert into %s.dn_%s values(%ld", tsMonitorDbName, monitor->privateIpStr, ts); + int pos = snprintf(sql, SQL_LENGTH, "insert into %s.dn_%s values(%" PRId64, tsMonitorDbName, monitor->privateIpStr, ts); pos += monitorBuildCpuSql(sql + pos); pos += monitorBuildMemorySql(sql + pos); @@ -402,16 +403,16 @@ void monitorSaveAcctLog(char *acctId, int64_t currentPointsPerSecond, int64_t ma char sql[1024] = {0}; sprintf(sql, "insert into %s.acct_%s using %s.acct tags('%s') values(now" - ", %ld, %ld " - ", %ld, %ld" - ", %ld, %ld" - ", %ld, %ld" - ", %ld, %ld" - ", %ld, %ld" - ", %ld, %ld" - ", %ld, %ld" - ", %ld, %ld" - ", %ld, %ld" + ", %" PRId64, "%" PRId64 + ", %" PRId64, "%" PRId64 + ", %" PRId64, "%" PRId64 + ", %" PRId64, "%" PRId64 + ", %" PRId64, "%" PRId64 + ", %" PRId64, "%" PRId64 + ", %" PRId64, "%" PRId64 + ", %" PRId64, "%" PRId64 + ", %" PRId64, "%" PRId64 + ", %" PRId64, "%" PRId64 ", %d)", tsMonitorDbName, acctId, tsMonitorDbName, acctId, currentPointsPerSecond, maxPointsPerSecond, totalTimeSeries, maxTimeSeries, totalStorage, maxStorage, totalQueryTime, maxQueryTime, totalInbound, maxInbound, @@ -431,7 +432,7 @@ void monitorSaveLog(int level, const char *const format, ...) { return; } - int len = snprintf(sql, (size_t)max_length, "import into %s.log values(%ld, %d,'", tsMonitorDbName, + int len = snprintf(sql, (size_t)max_length, "import into %s.log values(%\" PRId64 \", %d,'", tsMonitorDbName, taosGetTimestampUs(), level); va_start(argpointer, format); diff --git a/src/os/linux/src/tsystem.c b/src/os/linux/src/tsystem.c index 0a9d97b3cf..611ede73b5 100644 --- a/src/os/linux/src/tsystem.c +++ b/src/os/linux/src/tsystem.c @@ -12,7 +12,7 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ - +#include #include #include #include @@ -99,7 +99,7 @@ bool taosGetProcMemory(float *memoryUsedMB) { int64_t memKB = 0; char tmp[10]; - sscanf(line, "%s %ld", tmp, &memKB); + sscanf(line, "%s %" PRId64, tmp, &memKB); *memoryUsedMB = (float)((double)memKB / 1024); tfree(line); @@ -124,7 +124,7 @@ bool taosGetSysCpuInfo(SysCpuInfo *cpuInfo) { } char cpu[10] = {0}; - sscanf(line, "%s %lu %lu %lu %lu", cpu, &cpuInfo->user, &cpuInfo->nice, &cpuInfo->system, &cpuInfo->idle); + sscanf(line, "%s %" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64, cpu, &cpuInfo->user, &cpuInfo->nice, &cpuInfo->system, &cpuInfo->idle); tfree(line); fclose(fp); @@ -150,7 +150,7 @@ bool taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) { for (int i = 0, blank = 0; line[i] != 0; ++i) { if (line[i] == ' ') blank++; if (blank == PROCESS_ITEM) { - sscanf(line + i + 1, "%lu %lu %lu %lu", &cpuInfo->utime, &cpuInfo->stime, &cpuInfo->cutime, &cpuInfo->cstime); + sscanf(line + i + 1, "%" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64, &cpuInfo->utime, &cpuInfo->stime, &cpuInfo->cutime, &cpuInfo->cstime); break; } } @@ -420,7 +420,7 @@ bool taosGetCardInfo(int64_t *bytes) { } } if (line != NULL) { - sscanf(line, "%s %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld", nouse0, &rbytes, &rpackts, &nouse1, &nouse2, &nouse3, + sscanf(line, "%s %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64, nouse0, &rbytes, &rpackts, &nouse1, &nouse2, &nouse3, &nouse4, &nouse5, &nouse6, &tbytes, &tpackets); *bytes = rbytes + tbytes; tfree(line); @@ -488,10 +488,10 @@ bool taosReadProcIO(int64_t *readbyte, int64_t *writebyte) { break; } if (strstr(line, "rchar:") != NULL) { - sscanf(line, "%s %ld", tmp, readbyte); + sscanf(line, "%s %" PRId64, tmp, readbyte); readIndex++; } else if (strstr(line, "wchar:") != NULL) { - sscanf(line, "%s %ld", tmp, writebyte); + sscanf(line, "%s %" PRId64, tmp, writebyte); readIndex++; } else { } @@ -564,9 +564,9 @@ void taosGetSystemInfo() { } void tsPrintOsInfo() { - pPrint(" os pageSize: %ld(KB)", tsPageSize); - pPrint(" os openMax: %ld", tsOpenMax); - pPrint(" os streamMax: %ld", tsStreamMax); + pPrint(" os pageSize: %" PRId64 "(KB)", tsPageSize); + pPrint(" os openMax: %" PRId64, tsOpenMax); + pPrint(" os streamMax: %" PRId64, tsStreamMax); pPrint(" os numOfCores: %d", tsNumOfCores); pPrint(" os totalDisk: %f(GB)", tsTotalDataDirGB); pPrint(" os totalMemory: %d(MB)", tsTotalMemoryMB); From a26f368c21fbfe48a0fffc7885f5b5b89d4690ba Mon Sep 17 00:00:00 2001 From: slguan Date: Tue, 10 Dec 2019 19:18:16 +0800 Subject: [PATCH 102/108] [TBASE-1325] --- src/system/detail/src/mgmtDnode.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/system/detail/src/mgmtDnode.c b/src/system/detail/src/mgmtDnode.c index 86550b46e7..8ab8cc13ba 100644 --- a/src/system/detail/src/mgmtDnode.c +++ b/src/system/detail/src/mgmtDnode.c @@ -59,7 +59,7 @@ void mgmtCalcNumOfFreeVnodes(SDnodeObj *pDnode) { if (pVload->vgId != 0) { mTrace("%d-dnode:%s, calc free vnodes, exist vnode:%d, vgroup:%d, state:%d %s, dropstate:%d %s, syncstatus:%d %s", totalVnodes, taosIpStr(pDnode->privateIp), i, pVload->vgId, - pVload->status, taosGetDnodeStatusStr(pVload->status), + pVload->status, taosGetVnodeStatusStr(pVload->status), pVload->dropStatus, taosGetVnodeDropStatusStr(pVload->dropStatus), pVload->syncStatus, taosGetVnodeSyncStatusStr(pVload->syncStatus)); totalVnodes++; From 1570fadf99cb6fb76c8d059f8237cd0b7b7a1b2c Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Wed, 11 Dec 2019 11:55:22 +0800 Subject: [PATCH 103/108] [tbase-1326] --- src/system/detail/inc/vnode.h | 20 +++-- src/system/detail/inc/vnodeRead.h | 83 ++++++++++----------- src/system/detail/src/vnodeQueryProcess.c | 26 ++++++- src/system/detail/src/vnodeRead.c | 91 ++++++++++++++++------- src/system/detail/src/vnodeShell.c | 4 +- 5 files changed, 142 insertions(+), 82 deletions(-) diff --git a/src/system/detail/inc/vnode.h b/src/system/detail/inc/vnode.h index 6e366b9b7f..4ac0848427 100644 --- a/src/system/detail/inc/vnode.h +++ b/src/system/detail/inc/vnode.h @@ -213,18 +213,20 @@ typedef struct { * Only the QInfo.signature == QInfo, this structure can be released safely. */ #define TSDB_QINFO_QUERY_FLAG 0x1 -#define TSDB_QINFO_RESET_SIG(x) atomic_store_64(&((x)->signature), (uint64_t)(x)) -#define TSDB_QINFO_SET_QUERY_FLAG(x) \ - atomic_val_compare_exchange_64(&((x)->signature), (uint64_t)(x), TSDB_QINFO_QUERY_FLAG); +//#define TSDB_QINFO_RESET_SIG(x) atomic_store_64(&((x)->signature), (uint64_t)(x)) +#define TSDB_QINFO_RESET_SIG(x) +#define TSDB_QINFO_SET_QUERY_FLAG(x) +//#define TSDB_QINFO_SET_QUERY_FLAG(x) \ +// atomic_val_compare_exchange_64(&((x)->signature), (uint64_t)(x), TSDB_QINFO_QUERY_FLAG); // live lock: wait for query reaching a safe-point, release all resources // belongs to this query #define TSDB_WAIT_TO_SAFE_DROP_QINFO(x) \ - { \ - while (atomic_val_compare_exchange_64(&((x)->signature), (x), 0) == TSDB_QINFO_QUERY_FLAG) { \ - taosMsleep(1); \ - } \ - } +// { \ +// while (atomic_val_compare_exchange_64(&((x)->signature), (x), 0) == TSDB_QINFO_QUERY_FLAG) { \ +// taosMsleep(1); \ +// } \ +// } struct tSQLBinaryExpr; @@ -370,6 +372,8 @@ void vnodeFreeQInfo(void *, bool); void vnodeFreeQInfoInQueue(void *param); bool vnodeIsQInfoValid(void *param); +void vnodeDecRefCount(void *param); +void vnodeAddRefCount(void *param); int32_t vnodeConvertQueryMeterMsg(SQueryMeterMsg *pQuery); diff --git a/src/system/detail/inc/vnodeRead.h b/src/system/detail/inc/vnodeRead.h index d059075142..3a1b6f7949 100644 --- a/src/system/detail/inc/vnodeRead.h +++ b/src/system/detail/inc/vnodeRead.h @@ -52,7 +52,7 @@ typedef struct SQueryLoadCompBlockInfo { * the header file info for one vnode */ typedef struct SHeaderFileInfo { - int32_t fileID; // file id + int32_t fileID; // file id } SHeaderFileInfo; typedef struct SQueryCostSummary { @@ -99,17 +99,17 @@ typedef struct SQueryFilesInfo { uint32_t numOfFiles; // the total available number of files for this virtual node during query execution int32_t current; // the memory mapped header file, NOTE: only one header file can be mmap. int32_t vnodeId; - - int32_t headerFd; // header file fd - char* pHeaderFileData; // mmap header files - int64_t headFileSize; - int32_t dataFd; - int32_t lastFd; - - char headerFilePath[PATH_MAX]; // current opened header file name - char dataFilePath[PATH_MAX]; // current opened data file name - char lastFilePath[PATH_MAX]; // current opened last file path - char dbFilePathPrefix[PATH_MAX]; + + int32_t headerFd; // header file fd + char* pHeaderFileData; // mmap header files + int64_t headFileSize; + int32_t dataFd; + int32_t lastFd; + + char headerFilePath[PATH_MAX]; // current opened header file name + char dataFilePath[PATH_MAX]; // current opened data file name + char lastFilePath[PATH_MAX]; // current opened last file path + char dbFilePathPrefix[PATH_MAX]; } SQueryFilesInfo; typedef struct RuntimeEnvironment { @@ -129,17 +129,17 @@ typedef struct RuntimeEnvironment { SQueryLoadBlockInfo loadBlockInfo; /* record current block load information */ SQueryLoadCompBlockInfo loadCompBlockInfo; /* record current compblock information in SQuery */ SQueryFilesInfo vnodeFileInfo; - int16_t numOfRowsPerPage; - int16_t offset[TSDB_MAX_COLUMNS]; - int16_t scanFlag; // denotes reversed scan of data or not - SInterpolationInfo interpoInfo; - SData** pInterpoBuf; - SOutputRes* pResult; // reference to SQuerySupporter->pResult - void* hashList; - int32_t usedIndex; // assigned SOutputRes in list - STSBuf* pTSBuf; - STSCursor cur; - SQueryCostSummary summary; + int16_t numOfRowsPerPage; + int16_t offset[TSDB_MAX_COLUMNS]; + int16_t scanFlag; // denotes reversed scan of data or not + SInterpolationInfo interpoInfo; + SData** pInterpoBuf; + SOutputRes* pResult; // reference to SQuerySupporter->pResult + void* hashList; + int32_t usedIndex; // assigned SOutputRes in list + STSBuf* pTSBuf; + STSCursor cur; + SQueryCostSummary summary; } SQueryRuntimeEnv; /* intermediate result during multimeter query involves interval */ @@ -214,14 +214,12 @@ typedef struct SMeterQuerySupportObj { SMeterDataInfo* pMeterDataInfo; - TSKEY* tsList; - int32_t tsNum; - + TSKEY* tsList; } SMeterQuerySupportObj; typedef struct _qinfo { - uint64_t signature; - + uint64_t signature; + int32_t refCount; // QInfo reference count, when the value is 0, it can be released safely char user[TSDB_METER_ID_LEN + 1]; char sql[TSDB_SHOW_SQL_LEN]; uint8_t stream; @@ -231,24 +229,21 @@ typedef struct _qinfo { int64_t useconds; int killed; struct _qinfo *prev, *next; + SQuery query; + int num; + int totalPoints; + int pointsRead; + int pointsReturned; + int pointsInterpo; + int code; + char bufIndex; + char changed; + char over; + SMeterObj* pObj; + sem_t dataReady; - SQuery query; - int num; - int totalPoints; - int pointsRead; - int pointsReturned; - int pointsInterpo; - int code; - char bufIndex; - char changed; - char over; - SMeterObj* pObj; - - int (*fp)(SMeterObj*, SQuery*); - - sem_t dataReady; SMeterQuerySupportObj* pMeterQuerySupporter; - + int (*fp)(SMeterObj*, SQuery*); } SQInfo; int32_t vnodeQuerySingleMeterPrepare(SQInfo* pQInfo, SMeterObj* pMeterObj, SMeterQuerySupportObj* pSMultiMeterObj, diff --git a/src/system/detail/src/vnodeQueryProcess.c b/src/system/detail/src/vnodeQueryProcess.c index 6fa0891baa..f39ca0dea1 100644 --- a/src/system/detail/src/vnodeQueryProcess.c +++ b/src/system/detail/src/vnodeQueryProcess.c @@ -890,12 +890,16 @@ static void vnodeMultiMeterQueryProcessor(SQInfo *pQInfo) { dTrace("QInfo:%p points returned:%d, totalRead:%d totalReturn:%d", pQInfo, pQuery->pointsRead, pQInfo->pointsRead, pQInfo->pointsReturned); + + vnodeDecRefCount(pQInfo); return; } pSupporter->pMeterDataInfo = (SMeterDataInfo *)calloc(1, sizeof(SMeterDataInfo) * pSupporter->numOfMeters); if (pSupporter->pMeterDataInfo == NULL) { dError("QInfo:%p failed to allocate memory, %s", pQInfo, strerror(errno)); + pQInfo->code = -TSDB_CODE_SERV_OUT_OF_MEMORY; + vnodeDecRefCount(pQInfo); return; } @@ -912,6 +916,7 @@ static void vnodeMultiMeterQueryProcessor(SQInfo *pQInfo) { // failed to save all intermediate results into disk, abort further query processing if (doCloseAllOpenedResults(pSupporter) != TSDB_CODE_SUCCESS) { dError("QInfo:%p failed to save intermediate results, abort further query processing", pQInfo); + vnodeDecRefCount(pQInfo); return; } @@ -919,6 +924,7 @@ static void vnodeMultiMeterQueryProcessor(SQInfo *pQInfo) { if (isQueryKilled(pQuery)) { dTrace("QInfo:%p query killed, abort", pQInfo); + vnodeDecRefCount(pQInfo); return; } @@ -940,6 +946,7 @@ static void vnodeMultiMeterQueryProcessor(SQInfo *pQInfo) { pQInfo->pointsRead += pQuery->pointsRead; dTrace("QInfo:%p points returned:%d, totalRead:%d totalReturn:%d", pQInfo, pQuery->pointsRead, pQInfo->pointsRead, pQInfo->pointsReturned); + vnodeDecRefCount(pQInfo); } /* @@ -1174,10 +1181,13 @@ void vnodeSingleMeterQuery(SSchedMsg *pMsg) { if (pQInfo->killed) { TSDB_QINFO_RESET_SIG(pQInfo); dTrace("QInfo:%p it is already killed, reset signature and abort", pQInfo); + vnodeDecRefCount(pQInfo); + return; } - assert(pQInfo->signature == TSDB_QINFO_QUERY_FLAG); + assert(pQInfo->refCount >= 1); +// assert(pQInfo->signature == TSDB_QINFO_QUERY_FLAG); SQuery * pQuery = &pQInfo->query; SMeterObj *pMeterObj = pQInfo->pObj; @@ -1215,6 +1225,7 @@ void vnodeSingleMeterQuery(SSchedMsg *pMsg) { sem_post(&pQInfo->dataReady); TSDB_QINFO_RESET_SIG(pQInfo); + vnodeDecRefCount(pQInfo); return; } @@ -1237,6 +1248,8 @@ void vnodeSingleMeterQuery(SSchedMsg *pMsg) { sem_post(&pQInfo->dataReady); TSDB_QINFO_RESET_SIG(pQInfo); + vnodeDecRefCount(pQInfo); + return; } } @@ -1249,7 +1262,7 @@ void vnodeSingleMeterQuery(SSchedMsg *pMsg) { vnodePrintQueryStatistics(pQInfo->pMeterQuerySupporter); sem_post(&pQInfo->dataReady); TSDB_QINFO_RESET_SIG(pQInfo); - + vnodeDecRefCount(pQInfo); return; } @@ -1284,8 +1297,10 @@ void vnodeSingleMeterQuery(SSchedMsg *pMsg) { pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->pointsRead); } - sem_post(&pQInfo->dataReady); TSDB_QINFO_RESET_SIG(pQInfo); + sem_post(&pQInfo->dataReady); + + vnodeDecRefCount(pQInfo); } void vnodeMultiMeterQuery(SSchedMsg *pMsg) { @@ -1297,11 +1312,13 @@ void vnodeMultiMeterQuery(SSchedMsg *pMsg) { if (pQInfo->killed) { TSDB_QINFO_RESET_SIG(pQInfo); + vnodeDecRefCount(pQInfo); dTrace("QInfo:%p it is already killed, reset signature and abort", pQInfo); return; } - assert(pQInfo->signature == TSDB_QINFO_QUERY_FLAG); + assert(pQInfo->refCount >= 1); +// assert(pQInfo->signature == TSDB_QINFO_QUERY_FLAG); SQuery *pQuery = &pQInfo->query; pQuery->pointsRead = 0; @@ -1337,4 +1354,5 @@ void vnodeMultiMeterQuery(SSchedMsg *pMsg) { sem_post(&pQInfo->dataReady); TSDB_QINFO_RESET_SIG(pQInfo); + vnodeDecRefCount(pQInfo); } diff --git a/src/system/detail/src/vnodeRead.c b/src/system/detail/src/vnodeRead.c index 21e83c5198..2ea41d9d58 100644 --- a/src/system/detail/src/vnodeRead.c +++ b/src/system/detail/src/vnodeRead.c @@ -392,10 +392,10 @@ __clean_memory: return NULL; } -static void vnodeFreeQInfoInQueueImpl(SSchedMsg *pMsg) { - SQInfo *pQInfo = (SQInfo *)pMsg->ahandle; - vnodeFreeQInfo(pQInfo, true); -} +//static void vnodeFreeQInfoInQueueImpl(SSchedMsg *pMsg) { +// SQInfo *pQInfo = (SQInfo *)pMsg->ahandle; +// vnodeFreeQInfo(pQInfo, true); +//} void vnodeFreeQInfoInQueue(void *param) { SQInfo *pQInfo = (SQInfo *)param; @@ -403,15 +403,18 @@ void vnodeFreeQInfoInQueue(void *param) { if (!vnodeIsQInfoValid(pQInfo)) return; pQInfo->killed = 1; + dTrace("QInfo:%p set kill flag to free QInfo"); + + vnodeDecRefCount(pQInfo); + +// dTrace("QInfo:%p set kill flag and add to queue, stop query ASAP", pQInfo); +// SSchedMsg schedMsg = {0}; +// schedMsg.fp = vnodeFreeQInfoInQueueImpl; - dTrace("QInfo:%p set kill flag and add to queue, stop query ASAP", pQInfo); - SSchedMsg schedMsg = {0}; - schedMsg.fp = vnodeFreeQInfoInQueueImpl; - - schedMsg.msg = NULL; - schedMsg.thandle = (void *)1; - schedMsg.ahandle = param; - taosScheduleTask(queryQhandle, &schedMsg); +// schedMsg.msg = NULL; +// schedMsg.thandle = (void *)1; +// schedMsg.ahandle = param; +// taosScheduleTask(queryQhandle, &schedMsg); } void vnodeFreeQInfo(void *param, bool decQueryRef) { @@ -499,7 +502,30 @@ bool vnodeIsQInfoValid(void *param) { * into local variable, then compare by using local variable */ uint64_t sig = pQInfo->signature; - return (sig == (uint64_t)pQInfo) || (sig == TSDB_QINFO_QUERY_FLAG); +// return (sig == (uint64_t)pQInfo) || (sig == TSDB_QINFO_QUERY_FLAG); + return (sig == (uint64_t)pQInfo); +} + +void vnodeDecRefCount(void *param) { + SQInfo *pQInfo = (SQInfo*) param; + + assert(vnodeIsQInfoValid(pQInfo)); + + int32_t ref = atomic_sub_fetch_32(&pQInfo->refCount, 1); + dTrace("QInfo:%p decrease obj refcount, %d", pQInfo, ref); + + if (ref == 0) { + vnodeFreeQInfo(pQInfo, true); + } +} + +void vnodeAddRefCount(void *param) { + SQInfo *pQInfo = (SQInfo*) param; + + assert(vnodeIsQInfoValid(pQInfo)); + + int32_t ref = atomic_add_fetch_32(&pQInfo->refCount, 1); + dTrace("QInfo:%p add refcount, %d", pQInfo, ref); } void vnodeQueryData(SSchedMsg *pMsg) { @@ -511,10 +537,11 @@ void vnodeQueryData(SSchedMsg *pMsg) { if (pQInfo->killed) { TSDB_QINFO_RESET_SIG(pQInfo); dTrace("QInfo:%p it is already killed, reset signature and abort", pQInfo); + vnodeDecRefCount(pQInfo); return; } - assert(pQInfo->signature == TSDB_QINFO_QUERY_FLAG); +// assert(pQInfo->signature == TSDB_QINFO_QUERY_FLAG); pQuery = &(pQInfo->query); SMeterObj *pObj = pQInfo->pObj; @@ -586,6 +613,7 @@ void vnodeQueryData(SSchedMsg *pMsg) { dTrace("QInfo:%p reset signature", pQInfo); TSDB_QINFO_RESET_SIG(pQInfo); sem_post(&pQInfo->dataReady); + vnodeDecRefCount(pQInfo); } void *vnodeQueryInTimeRange(SMeterObj **pMetersObj, SSqlGroupbyExpr *pGroupbyExpr, SSqlFunctionExpr *pSqlExprs, @@ -668,14 +696,22 @@ void *vnodeQueryInTimeRange(SMeterObj **pMetersObj, SSqlGroupbyExpr *pGroupbyExp } // set in query flag - pQInfo->signature = TSDB_QINFO_QUERY_FLAG; - +// pQInfo->signature = TSDB_QINFO_QUERY_FLAG; + + /* + * The reference count, which is 2, is for both the current query thread and the future retrieve request, + * which will always be issued by client to acquire data or free SQInfo struct. + */ + vnodeAddRefCount(pQInfo); + vnodeAddRefCount(pQInfo); + schedMsg.msg = NULL; schedMsg.thandle = (void *)1; schedMsg.ahandle = pQInfo; - dTrace("QInfo:%p set query flag and prepare runtime environment completed, wait for schedule", pQInfo); - + dTrace("QInfo:%p set query flag and prepare runtime environment completed, ref:%d, wait for schedule", pQInfo, + pQInfo->refCount); + taosScheduleTask(queryQhandle, &schedMsg); return pQInfo; @@ -779,8 +815,10 @@ void *vnodeQueryOnMultiMeters(SMeterObj **pMetersObj, SSqlGroupbyExpr *pGroupbyE return pQInfo; } - pQInfo->signature = TSDB_QINFO_QUERY_FLAG; - +// pQInfo->signature = TSDB_QINFO_QUERY_FLAG; + vnodeAddRefCount(pQInfo); + vnodeAddRefCount(pQInfo); + schedMsg.msg = NULL; schedMsg.thandle = (void *)1; schedMsg.ahandle = pQInfo; @@ -862,19 +900,22 @@ int vnodeSaveQueryResult(void *handle, char *data, int32_t *size) { if (pQInfo->over == 0) { //dTrace("QInfo:%p set query flag, oldSig:%p, func:%s", pQInfo, pQInfo->signature, __FUNCTION__); dTrace("QInfo:%p set query flag, oldSig:%p", pQInfo, pQInfo->signature); - uint64_t oldSignature = TSDB_QINFO_SET_QUERY_FLAG(pQInfo); +// uint64_t oldSignature = TSDB_QINFO_SET_QUERY_FLAG(pQInfo); /* * If SQInfo has been released, the value of signature cannot be equalled to the address of pQInfo, * since in release function, the original value has been destroyed. However, this memory area may be reused * by another function. It may be 0 or any value, but it is rarely still be equalled to the address of SQInfo. */ - if (oldSignature == 0 || oldSignature != (uint64_t)pQInfo) { - dTrace("%p freed or killed, old sig:%p abort query", pQInfo, oldSignature); +// if (oldSignature == 0 || oldSignature != (uint64_t)pQInfo) { + if (pQInfo->killed == 1) { +// dTrace("%p freed or killed, old sig:%p abort query", pQInfo, oldSignature); + dTrace("%p freed or killed, abort query", pQInfo); } else { + vnodeAddRefCount(pQInfo); dTrace("%p add query into task queue for schedule", pQInfo); - - SSchedMsg schedMsg; + + SSchedMsg schedMsg = {0}; if (pQInfo->pMeterQuerySupporter != NULL) { if (pQInfo->pMeterQuerySupporter->pSidSet == NULL) { diff --git a/src/system/detail/src/vnodeShell.c b/src/system/detail/src/vnodeShell.c index cfdd956768..6ff3f4d27c 100644 --- a/src/system/detail/src/vnodeShell.c +++ b/src/system/detail/src/vnodeShell.c @@ -470,9 +470,11 @@ void vnodeExecuteRetrieveReq(SSchedMsg *pSched) { pMsg += size; msgLen = pMsg - pStart; + assert(code != TSDB_CODE_ACTION_IN_PROGRESS); + if (numOfRows == 0 && (pRetrieve->qhandle == (uint64_t)pObj->qhandle) && (code != TSDB_CODE_ACTION_IN_PROGRESS)) { dTrace("QInfo:%p %s free qhandle code:%d", pObj->qhandle, __FUNCTION__, code); - vnodeFreeQInfo(pObj->qhandle, true); + vnodeDecRefCount(pObj->qhandle); pObj->qhandle = NULL; } From 992a449ed6167c3144b990accd4626d415def36d Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Wed, 11 Dec 2019 15:30:54 +0800 Subject: [PATCH 104/108] [tbase-1327] --- src/client/src/tscServer.c | 18 ++++++++---------- src/system/detail/src/mgmtMeter.c | 7 +++++-- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 17d870fcb2..e10865b642 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1507,7 +1507,6 @@ int tscBuildQueryMsg(SSqlObj *pSql) { pQueryMsg->uid = pMeterMeta->uid; pQueryMsg->numOfTagsCols = 0; } else { // query on metric - SMetricMeta *pMetricMeta = pMeterMetaInfo->pMetricMeta; if (pMeterMetaInfo->vnodeIndex < 0) { tscError("%p error vnodeIdx:%d", pSql, pMeterMetaInfo->vnodeIndex); return -1; @@ -2872,15 +2871,14 @@ int tscBuildMetricMetaMsg(SSqlObj *pSql) { pElem->groupbyTagColumnList = htonl(offset); for (int32_t j = 0; j < pCmd->groupbyExpr.numOfGroupCols; ++j) { SColIndexEx *pCol = &pCmd->groupbyExpr.columnInfo[j]; - - *((int16_t *)pMsg) = pCol->colId; - pMsg += sizeof(pCol->colId); - - *((int16_t *)pMsg) += pCol->colIdx; - pMsg += sizeof(pCol->colIdx); - - *((int16_t *)pMsg) += pCol->flag; - pMsg += sizeof(pCol->flag); + SColIndexEx* pDestCol = (SColIndexEx*) pMsg; + + pDestCol->colIdxInBuf = 0; + pDestCol->colIdx = htons(pCol->colIdx); + pDestCol->colId = htons(pDestCol->colId); + pDestCol->flag = htons(pDestCol->flag); + + pMsg += sizeof(SColIndexEx); } } } diff --git a/src/system/detail/src/mgmtMeter.c b/src/system/detail/src/mgmtMeter.c index 5ef9f61b0b..be141f278d 100644 --- a/src/system/detail/src/mgmtMeter.c +++ b/src/system/detail/src/mgmtMeter.c @@ -1094,9 +1094,12 @@ static SMetricMetaElemMsg *doConvertMetricMetaMsg(SMetricMetaMsg *pMetricMetaMsg pElem->groupbyTagColumnList = htonl(pElem->groupbyTagColumnList); - int16_t *groupColIds = (int16_t*) (((char *)pMetricMetaMsg) + pElem->groupbyTagColumnList); + SColIndexEx *groupColIds = (SColIndexEx*) (((char *)pMetricMetaMsg) + pElem->groupbyTagColumnList); for (int32_t i = 0; i < pElem->numOfGroupCols; ++i) { - groupColIds[i] = htons(groupColIds[i]); + groupColIds[i].colId = htons(groupColIds[i].colId); + groupColIds[i].colIdx = htons(groupColIds[i].colIdx); + groupColIds[i].flag = htons(groupColIds[i].flag); + groupColIds[i].colIdxInBuf = 0; } return pElem; From 38d377b421fa99a06de872fa48c4abda9f24ef3c Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Wed, 11 Dec 2019 16:49:46 +0800 Subject: [PATCH 105/108] [tbase-1328] --- src/system/detail/src/mgmtSupertableQuery.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/system/detail/src/mgmtSupertableQuery.c b/src/system/detail/src/mgmtSupertableQuery.c index b56173f14f..10a64408ce 100644 --- a/src/system/detail/src/mgmtSupertableQuery.c +++ b/src/system/detail/src/mgmtSupertableQuery.c @@ -784,14 +784,15 @@ int mgmtRetrieveMetersFromMetric(SMetricMetaMsg* pMsg, int32_t tableIndex, tQuer } // todo refactor!!!!! -static char* getTagValueFromMeter(STabObj* pMeter, int32_t offset, void* param) { +static char* getTagValueFromMeter(STabObj* pMeter, int32_t offset, int32_t len, char* param) { if (offset == TSDB_TBNAME_COLUMN_INDEX) { extractMeterName(pMeter->meterId, param); - return param; } else { - char* tags = pMeter->pTagData + TSDB_METER_ID_LEN; // tag start position - return (tags + offset); + char* tags = pMeter->pTagData + offset + TSDB_METER_ID_LEN; // tag start position + memcpy(param, tags, len); // make sure the value is null-terminated string } + + return param; } bool tSkipListNodeFilterCallback(const void* pNode, void* param) { @@ -799,8 +800,9 @@ bool tSkipListNodeFilterCallback(const void* pNode, void* param) { tQueryInfo* pInfo = (tQueryInfo*)param; STabObj* pMeter = (STabObj*)(((tSkipListNode*)pNode)->pData); - char name[TSDB_METER_NAME_LEN + 1] = {0}; - char* val = getTagValueFromMeter(pMeter, pInfo->offset, name); + char buf[TSDB_MAX_TAGS_LEN] = {0}; + + char* val = getTagValueFromMeter(pMeter, pInfo->offset, pInfo->sch.bytes, buf); int8_t type = pInfo->sch.type; int32_t ret = 0; From 8636bc95aaf7aac1596abfdc8d45c8b6da4b250d Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Wed, 11 Dec 2019 20:23:34 +0800 Subject: [PATCH 106/108] [tbase-1326] --- src/client/src/tscJoinProcess.c | 3 +-- src/system/detail/inc/vnode.h | 2 +- src/system/detail/src/vnodeQueryProcess.c | 6 ------ src/system/detail/src/vnodeRead.c | 8 +++++--- src/system/detail/src/vnodeShell.c | 2 +- 5 files changed, 8 insertions(+), 13 deletions(-) diff --git a/src/client/src/tscJoinProcess.c b/src/client/src/tscJoinProcess.c index 375e0066b1..a94b308e87 100644 --- a/src/client/src/tscJoinProcess.c +++ b/src/client/src/tscJoinProcess.c @@ -528,8 +528,7 @@ void tscFetchDatablockFromSubquery(SSqlObj* pSql) { numOfFetch++; } } else { - if ((pRes->row >= pRes->numOfRows && (!tscHasReachLimitation(pSql->pSubs[i])) && - tscProjectionQueryOnTable(&pSql->cmd)) || (pRes->numOfRows == 0)) { + if (pRes->row >= pRes->numOfRows && (!tscHasReachLimitation(pSql->pSubs[i]))) { numOfFetch++; } } diff --git a/src/system/detail/inc/vnode.h b/src/system/detail/inc/vnode.h index 4ac0848427..37e31803d0 100644 --- a/src/system/detail/inc/vnode.h +++ b/src/system/detail/inc/vnode.h @@ -339,7 +339,7 @@ extern void * vnodeTmrCtrl; // read API extern int (*vnodeSearchKeyFunc[])(char *pValue, int num, TSKEY key, int order); -void *vnodeQueryInTimeRange(SMeterObj **pMeterObj, SSqlGroupbyExpr *pGroupbyExpr, SSqlFunctionExpr *sqlExprs, +void *vnodeQueryOnSingleTable(SMeterObj **pMeterObj, SSqlGroupbyExpr *pGroupbyExpr, SSqlFunctionExpr *sqlExprs, SQueryMeterMsg *pQueryMsg, int *code); void *vnodeQueryOnMultiMeters(SMeterObj **pMeterObj, SSqlGroupbyExpr *pGroupbyExpr, SSqlFunctionExpr *pSqlExprs, diff --git a/src/system/detail/src/vnodeQueryProcess.c b/src/system/detail/src/vnodeQueryProcess.c index f39ca0dea1..231b47206e 100644 --- a/src/system/detail/src/vnodeQueryProcess.c +++ b/src/system/detail/src/vnodeQueryProcess.c @@ -890,8 +890,6 @@ static void vnodeMultiMeterQueryProcessor(SQInfo *pQInfo) { dTrace("QInfo:%p points returned:%d, totalRead:%d totalReturn:%d", pQInfo, pQuery->pointsRead, pQInfo->pointsRead, pQInfo->pointsReturned); - - vnodeDecRefCount(pQInfo); return; } @@ -899,7 +897,6 @@ static void vnodeMultiMeterQueryProcessor(SQInfo *pQInfo) { if (pSupporter->pMeterDataInfo == NULL) { dError("QInfo:%p failed to allocate memory, %s", pQInfo, strerror(errno)); pQInfo->code = -TSDB_CODE_SERV_OUT_OF_MEMORY; - vnodeDecRefCount(pQInfo); return; } @@ -916,7 +913,6 @@ static void vnodeMultiMeterQueryProcessor(SQInfo *pQInfo) { // failed to save all intermediate results into disk, abort further query processing if (doCloseAllOpenedResults(pSupporter) != TSDB_CODE_SUCCESS) { dError("QInfo:%p failed to save intermediate results, abort further query processing", pQInfo); - vnodeDecRefCount(pQInfo); return; } @@ -924,7 +920,6 @@ static void vnodeMultiMeterQueryProcessor(SQInfo *pQInfo) { if (isQueryKilled(pQuery)) { dTrace("QInfo:%p query killed, abort", pQInfo); - vnodeDecRefCount(pQInfo); return; } @@ -946,7 +941,6 @@ static void vnodeMultiMeterQueryProcessor(SQInfo *pQInfo) { pQInfo->pointsRead += pQuery->pointsRead; dTrace("QInfo:%p points returned:%d, totalRead:%d totalReturn:%d", pQInfo, pQuery->pointsRead, pQInfo->pointsRead, pQInfo->pointsReturned); - vnodeDecRefCount(pQInfo); } /* diff --git a/src/system/detail/src/vnodeRead.c b/src/system/detail/src/vnodeRead.c index 2ea41d9d58..5640c6bc0d 100644 --- a/src/system/detail/src/vnodeRead.c +++ b/src/system/detail/src/vnodeRead.c @@ -512,8 +512,9 @@ void vnodeDecRefCount(void *param) { assert(vnodeIsQInfoValid(pQInfo)); int32_t ref = atomic_sub_fetch_32(&pQInfo->refCount, 1); - dTrace("QInfo:%p decrease obj refcount, %d", pQInfo, ref); + assert(ref >= 0); + dTrace("QInfo:%p decrease obj refcount, %d", pQInfo, ref); if (ref == 0) { vnodeFreeQInfo(pQInfo, true); } @@ -616,7 +617,7 @@ void vnodeQueryData(SSchedMsg *pMsg) { vnodeDecRefCount(pQInfo); } -void *vnodeQueryInTimeRange(SMeterObj **pMetersObj, SSqlGroupbyExpr *pGroupbyExpr, SSqlFunctionExpr *pSqlExprs, +void *vnodeQueryOnSingleTable(SMeterObj **pMetersObj, SSqlGroupbyExpr *pGroupbyExpr, SSqlFunctionExpr *pSqlExprs, SQueryMeterMsg *pQueryMsg, int32_t *code) { SQInfo *pQInfo; SQuery *pQuery; @@ -687,6 +688,7 @@ void *vnodeQueryInTimeRange(SMeterObj **pMetersObj, SSqlGroupbyExpr *pGroupbyExp } if (pQInfo->over == 1) { + vnodeAddRefCount(pQInfo); // for retrieve procedure return pQInfo; } @@ -811,13 +813,13 @@ void *vnodeQueryOnMultiMeters(SMeterObj **pMetersObj, SSqlGroupbyExpr *pGroupbyE goto _error; } + vnodeAddRefCount(pQInfo); if (pQInfo->over == 1) { return pQInfo; } // pQInfo->signature = TSDB_QINFO_QUERY_FLAG; vnodeAddRefCount(pQInfo); - vnodeAddRefCount(pQInfo); schedMsg.msg = NULL; schedMsg.thandle = (void *)1; diff --git a/src/system/detail/src/vnodeShell.c b/src/system/detail/src/vnodeShell.c index 6ff3f4d27c..5efbb41014 100644 --- a/src/system/detail/src/vnodeShell.c +++ b/src/system/detail/src/vnodeShell.c @@ -376,7 +376,7 @@ int vnodeProcessQueryRequest(char *pMsg, int msgLen, SShellObj *pObj) { if (QUERY_IS_STABLE_QUERY(pQueryMsg->queryType)) { pObj->qhandle = vnodeQueryOnMultiMeters(pMeterObjList, pGroupbyExpr, pExprs, pQueryMsg, &code); } else { - pObj->qhandle = vnodeQueryInTimeRange(pMeterObjList, pGroupbyExpr, pExprs, pQueryMsg, &code); + pObj->qhandle = vnodeQueryOnSingleTable(pMeterObjList, pGroupbyExpr, pExprs, pQueryMsg, &code); } _query_over: From c060be25b937f946eabcfceaac6342e022398733 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Wed, 11 Dec 2019 21:02:32 +0800 Subject: [PATCH 107/108] [tbase-1327] --- src/client/src/tscServer.c | 6 ++++++ src/client/src/tscUtil.c | 8 +++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index e10865b642..40399d85b7 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1290,6 +1290,12 @@ static SSqlObj *tscCreateSqlObjForSubquery(SSqlObj *pSql, SRetrieveSupport *trsu SSqlObj *pNew = createSubqueryObj(pSql, 0, tscRetrieveDataRes, trsupport, prevSqlObj); if (pNew != NULL) { // the sub query of two-stage super table query pNew->cmd.type |= TSDB_QUERY_TYPE_STABLE_SUBQUERY; + assert(pNew->cmd.numOfTables == 1); + + //launch subquery for each vnode, so the subquery index equals to the vnodeIndex. + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(&pNew->cmd, 0); + pMeterMetaInfo->vnodeIndex = trsupport->subqueryIndex; + pSql->pSubs[trsupport->subqueryIndex] = pNew; } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 9876dad906..4b0df76784 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1706,12 +1706,10 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void } pNew->fp = fp; - pNew->param = param; - SMeterMetaInfo* pMetermetaInfo = tscGetMeterMetaInfo(pCmd, tableIndex); - + char key[TSDB_MAX_TAGS_LEN + 1] = {0}; - tscGetMetricMetaCacheKey(pCmd, key, pMetermetaInfo->pMeterMeta->uid); + tscGetMetricMetaCacheKey(pCmd, key, uid); #ifdef _DEBUG_VIEW printf("the metricmeta key is:%s\n", key); @@ -1736,7 +1734,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void } assert(pFinalInfo->pMeterMeta != NULL); - if (UTIL_METER_IS_METRIC(pMetermetaInfo)) { + if (UTIL_METER_IS_METRIC(pMeterMetaInfo)) { assert(pFinalInfo->pMetricMeta != NULL); } From e03f3364b80e7b7ba7a4b1479390216c826df6d2 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Wed, 11 Dec 2019 21:03:52 +0800 Subject: [PATCH 108/108] [tbase-1326] --- src/system/detail/inc/vnode.h | 22 ------------------ src/system/detail/src/vnodeQueryProcess.c | 28 ++++++----------------- src/system/detail/src/vnodeRead.c | 25 ++------------------ 3 files changed, 9 insertions(+), 66 deletions(-) diff --git a/src/system/detail/inc/vnode.h b/src/system/detail/inc/vnode.h index 37e31803d0..3a5e7e7260 100644 --- a/src/system/detail/inc/vnode.h +++ b/src/system/detail/inc/vnode.h @@ -206,28 +206,6 @@ typedef struct { char cont[]; } SVMsgHeader; -/* - * The value of QInfo.signature is used to denote that a query is executing, it isn't safe to release QInfo yet. - * The release operations will be blocked in a busy-waiting until the query operation reach a safepoint. - * Then it will reset the signature in a atomic operation, followed by release operation. - * Only the QInfo.signature == QInfo, this structure can be released safely. - */ -#define TSDB_QINFO_QUERY_FLAG 0x1 -//#define TSDB_QINFO_RESET_SIG(x) atomic_store_64(&((x)->signature), (uint64_t)(x)) -#define TSDB_QINFO_RESET_SIG(x) -#define TSDB_QINFO_SET_QUERY_FLAG(x) -//#define TSDB_QINFO_SET_QUERY_FLAG(x) \ -// atomic_val_compare_exchange_64(&((x)->signature), (uint64_t)(x), TSDB_QINFO_QUERY_FLAG); - -// live lock: wait for query reaching a safe-point, release all resources -// belongs to this query -#define TSDB_WAIT_TO_SAFE_DROP_QINFO(x) \ -// { \ -// while (atomic_val_compare_exchange_64(&((x)->signature), (x), 0) == TSDB_QINFO_QUERY_FLAG) { \ -// taosMsleep(1); \ -// } \ -// } - struct tSQLBinaryExpr; typedef struct SColumnInfoEx { diff --git a/src/system/detail/src/vnodeQueryProcess.c b/src/system/detail/src/vnodeQueryProcess.c index 231b47206e..a545645a09 100644 --- a/src/system/detail/src/vnodeQueryProcess.c +++ b/src/system/detail/src/vnodeQueryProcess.c @@ -1173,15 +1173,13 @@ void vnodeSingleMeterQuery(SSchedMsg *pMsg) { } if (pQInfo->killed) { - TSDB_QINFO_RESET_SIG(pQInfo); - dTrace("QInfo:%p it is already killed, reset signature and abort", pQInfo); + dTrace("QInfo:%p it is already killed, abort", pQInfo); vnodeDecRefCount(pQInfo); return; } assert(pQInfo->refCount >= 1); -// assert(pQInfo->signature == TSDB_QINFO_QUERY_FLAG); SQuery * pQuery = &pQInfo->query; SMeterObj *pMeterObj = pQInfo->pObj; @@ -1215,10 +1213,7 @@ void vnodeSingleMeterQuery(SSchedMsg *pMsg) { pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->pointsRead, numOfInterpo, pQInfo->pointsRead, pQInfo->pointsInterpo, pQInfo->pointsReturned); - dTrace("QInfo:%p reset signature", pQInfo); - sem_post(&pQInfo->dataReady); - TSDB_QINFO_RESET_SIG(pQInfo); vnodeDecRefCount(pQInfo); return; @@ -1238,10 +1233,7 @@ void vnodeSingleMeterQuery(SSchedMsg *pMsg) { pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->pointsRead, pQInfo->pointsRead, pQInfo->pointsInterpo, pQInfo->pointsReturned); - dTrace("QInfo:%p reset signature", pQInfo); - sem_post(&pQInfo->dataReady); - TSDB_QINFO_RESET_SIG(pQInfo); vnodeDecRefCount(pQInfo); return; @@ -1250,12 +1242,12 @@ void vnodeSingleMeterQuery(SSchedMsg *pMsg) { } pQInfo->over = 1; - dTrace("QInfo:%p vid:%d sid:%d id:%s, query over, %d points are returned, reset signature", pQInfo, + dTrace("QInfo:%p vid:%d sid:%d id:%s, query over, %d points are returned", pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQInfo->pointsRead); vnodePrintQueryStatistics(pQInfo->pMeterQuerySupporter); sem_post(&pQInfo->dataReady); - TSDB_QINFO_RESET_SIG(pQInfo); + vnodeDecRefCount(pQInfo); return; } @@ -1284,16 +1276,14 @@ void vnodeSingleMeterQuery(SSchedMsg *pMsg) { /* check if query is killed or not */ if (isQueryKilled(pQuery)) { - dTrace("QInfo:%p query is killed, reset signature", pQInfo); + dTrace("QInfo:%p query is killed", pQInfo); pQInfo->over = 1; } else { - dTrace("QInfo:%p vid:%d sid:%d id:%s, meter query thread completed, %d points are returned, reset signature", + dTrace("QInfo:%p vid:%d sid:%d id:%s, meter query thread completed, %d points are returned", pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->pointsRead); } - TSDB_QINFO_RESET_SIG(pQInfo); sem_post(&pQInfo->dataReady); - vnodeDecRefCount(pQInfo); } @@ -1305,14 +1295,12 @@ void vnodeMultiMeterQuery(SSchedMsg *pMsg) { } if (pQInfo->killed) { - TSDB_QINFO_RESET_SIG(pQInfo); vnodeDecRefCount(pQInfo); - dTrace("QInfo:%p it is already killed, reset signature and abort", pQInfo); + dTrace("QInfo:%p it is already killed, abort", pQInfo); return; } assert(pQInfo->refCount >= 1); -// assert(pQInfo->signature == TSDB_QINFO_QUERY_FLAG); SQuery *pQuery = &pQInfo->query; pQuery->pointsRead = 0; @@ -1333,7 +1321,6 @@ void vnodeMultiMeterQuery(SSchedMsg *pMsg) { pQInfo->useconds += (taosGetTimestampUs() - st); pQInfo->over = isQueryKilled(pQuery) ? 1 : 0; - dTrace("QInfo:%p reset signature", pQInfo); taosInterpoSetStartInfo(&pQInfo->pMeterQuerySupporter->runtimeEnv.interpoInfo, pQuery->pointsRead, pQInfo->query.interpoType); @@ -1341,12 +1328,11 @@ void vnodeMultiMeterQuery(SSchedMsg *pMsg) { if (pQuery->pointsRead == 0) { pQInfo->over = 1; - dTrace("QInfo:%p over, %d meters queried, %d points are returned, reset signature", pQInfo, pSupporter->numOfMeters, + dTrace("QInfo:%p over, %d meters queried, %d points are returned", pQInfo, pSupporter->numOfMeters, pQInfo->pointsRead); vnodePrintQueryStatistics(pSupporter); } sem_post(&pQInfo->dataReady); - TSDB_QINFO_RESET_SIG(pQInfo); vnodeDecRefCount(pQInfo); } diff --git a/src/system/detail/src/vnodeRead.c b/src/system/detail/src/vnodeRead.c index 5640c6bc0d..ccd59f356b 100644 --- a/src/system/detail/src/vnodeRead.c +++ b/src/system/detail/src/vnodeRead.c @@ -422,8 +422,6 @@ void vnodeFreeQInfo(void *param, bool decQueryRef) { if (!vnodeIsQInfoValid(param)) return; pQInfo->killed = 1; - TSDB_WAIT_TO_SAFE_DROP_QINFO(pQInfo); - SMeterObj *pObj = pQInfo->pObj; dTrace("QInfo:%p start to free SQInfo", pQInfo); @@ -502,7 +500,6 @@ bool vnodeIsQInfoValid(void *param) { * into local variable, then compare by using local variable */ uint64_t sig = pQInfo->signature; -// return (sig == (uint64_t)pQInfo) || (sig == TSDB_QINFO_QUERY_FLAG); return (sig == (uint64_t)pQInfo); } @@ -536,13 +533,11 @@ void vnodeQueryData(SSchedMsg *pMsg) { pQInfo = (SQInfo *)pMsg->ahandle; if (pQInfo->killed) { - TSDB_QINFO_RESET_SIG(pQInfo); - dTrace("QInfo:%p it is already killed, reset signature and abort", pQInfo); + dTrace("QInfo:%p it is already killed, abort", pQInfo); vnodeDecRefCount(pQInfo); return; } -// assert(pQInfo->signature == TSDB_QINFO_QUERY_FLAG); pQuery = &(pQInfo->query); SMeterObj *pObj = pQInfo->pObj; @@ -610,9 +605,6 @@ void vnodeQueryData(SSchedMsg *pMsg) { tclose(pQInfo->query.lfd); } - /* reset QInfo signature */ - dTrace("QInfo:%p reset signature", pQInfo); - TSDB_QINFO_RESET_SIG(pQInfo); sem_post(&pQInfo->dataReady); vnodeDecRefCount(pQInfo); } @@ -697,9 +689,6 @@ void *vnodeQueryOnSingleTable(SMeterObj **pMetersObj, SSqlGroupbyExpr *pGroupbyE schedMsg.fp = vnodeQueryData; } - // set in query flag -// pQInfo->signature = TSDB_QINFO_QUERY_FLAG; - /* * The reference count, which is 2, is for both the current query thread and the future retrieve request, * which will always be issued by client to acquire data or free SQInfo struct. @@ -818,7 +807,6 @@ void *vnodeQueryOnMultiMeters(SMeterObj **pMetersObj, SSqlGroupbyExpr *pGroupbyE return pQInfo; } -// pQInfo->signature = TSDB_QINFO_QUERY_FLAG; vnodeAddRefCount(pQInfo); schedMsg.msg = NULL; @@ -900,18 +888,9 @@ int vnodeSaveQueryResult(void *handle, char *data, int32_t *size) { pQInfo->pointsRead); if (pQInfo->over == 0) { - //dTrace("QInfo:%p set query flag, oldSig:%p, func:%s", pQInfo, pQInfo->signature, __FUNCTION__); - dTrace("QInfo:%p set query flag, oldSig:%p", pQInfo, pQInfo->signature); -// uint64_t oldSignature = TSDB_QINFO_SET_QUERY_FLAG(pQInfo); + dTrace("QInfo:%p set query flag, sig:%p, func:%s", pQInfo, pQInfo->signature, __FUNCTION__); - /* - * If SQInfo has been released, the value of signature cannot be equalled to the address of pQInfo, - * since in release function, the original value has been destroyed. However, this memory area may be reused - * by another function. It may be 0 or any value, but it is rarely still be equalled to the address of SQInfo. - */ -// if (oldSignature == 0 || oldSignature != (uint64_t)pQInfo) { if (pQInfo->killed == 1) { -// dTrace("%p freed or killed, old sig:%p abort query", pQInfo, oldSignature); dTrace("%p freed or killed, abort query", pQInfo); } else { vnodeAddRefCount(pQInfo);