From e593c12cef6b7247d5d7ed24d372182fdbc31ee4 Mon Sep 17 00:00:00 2001 From: slguan Date: Mon, 2 Dec 2019 16:18:32 +0800 Subject: [PATCH 001/123] Instructions for using the RESTful interface --- .../webdocs/markdowndocs/connector-ch.md | 126 ++++++++++++++++-- 1 file changed, 116 insertions(+), 10 deletions(-) diff --git a/documentation/webdocs/markdowndocs/connector-ch.md b/documentation/webdocs/markdowndocs/connector-ch.md index 9261b9eef6..e629a44bf1 100644 --- a/documentation/webdocs/markdowndocs/connector-ch.md +++ b/documentation/webdocs/markdowndocs/connector-ch.md @@ -316,11 +316,23 @@ import taos 如：http://192.168.0.1:6020/rest/sql 是指向IP地址为192.168.0.1的URL. -HTTP请求的Header里需带有身份认证信息，TDengine单机版仅支持Basic认证机制。 +HTTP请求的Header里需带有身份认证信息，TDengine支持Basic认证与自定义认证两种机制，后续版本将提供标准安全的数字签名机制来做身份验证。 + +- 自定义身份认证信息如下所示（稍后介绍） + +``` +Authorization: Taosd +``` + +- Basic身份认证信息如下所示 + +``` +Authorization: Basic +``` HTTP请求的BODY里就是一个完整的SQL语句，SQL语句中的数据表应提供数据库前缀，例如\.\。如果表名不带数据库前缀，系统会返回错误。因为HTTP模块只是一个简单的转发，没有当前DB的概念。 -使用curl来发起一个HTTP Request, 语法如下： +使用curl通过自定义身份认证方式来发起一个HTTP Request, 语法如下： ``` curl -H 'Authorization: Basic ' -d '' :/rest/sql @@ -332,7 +344,7 @@ curl -H 'Authorization: Basic ' -d '' :/rest/sql curl -u username:password -d '' :/rest/sql ``` -其中，`TOKEN`为`{username}:{password}`经过Base64编码之后的字符串，例如`root:taosdata`编码后为`cm9vdDp0YW9zZGF0YQ==` +其中，`TOKEN`为`{username}:{password}`经过Base64编码之后的字符串, 例如`root:taosdata`编码后为`cm9vdDp0YW9zZGF0YQ==` ### HTTP返回格式 @@ -356,21 +368,55 @@ curl -u username:password -d '' :/rest/sql - 第三行是具体返回的数据，一排一排的呈现。如果不返回结果集，仅[[affected_rows]] - 第四行”rows”表明总共多少行数据 +### 自定义授权码 + +HTTP请求中需要带有授权码``, 用于身份识别。授权码通常由管理员提供, 可简单的通过发送`HTTP GET`请求来获取授权码, 操作如下： + +``` +curl http://:6020/rest/login// +``` + +其中, `ip`是TDengine数据库的IP地址, `username`为数据库用户名, `password`为数据库密码, 返回值为`JSON`格式, 各字段含义如下： + +- status：请求结果的标志位 + +- code：返回值代码 + +- desc: 授权码 + +获取授权码示例： + +``` +curl http://192.168.0.1:6020/rest/login/root/taosdata +``` + +返回值： + +``` +{ + "status": "succ", + "code": 0, + "desc": +"/KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04" +} +``` + ### 使用示例 - 在demo库里查询表t1的所有记录, curl如下： - `curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.t1' 192.168.0.1:6020/rest/sql` - - 返回值： +``` +curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.t1' 192.168.0.1:6020/rest/sql` +``` +返回值： ``` { "status": "succ", "head": ["column1","column2","column3"], "data": [ - ["2017-12-12 23:44:25.730", 1, 2.3], - ["2017-12-12 22:44:25.728", 4, 5.6] + ["2017-12-12 22:44:25.728",4,5.60000], + ["2017-12-12 23:44:25.730",1,2.30000] ], "rows": 2 } @@ -378,9 +424,11 @@ curl -u username:password -d '' :/rest/sql - 创建库demo： - `curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'create database demo' 192.168.0.1:6020/rest/sql` +``` +curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'create database demo' 192.168.0.1:6020/rest/sql` +``` - 返回值： +返回值： ``` { "status": "succ", @@ -390,6 +438,64 @@ curl -u username:password -d '' :/rest/sql } ``` +### 其他用法 + +#### 结果集采用Unix时间戳 + +HTTP请求URL采用`sqlt`时，返回结果集的时间戳将采用Unix时间戳格式表示，例如 + +``` +curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.t1' 192.168.0.1:6020/rest/sqlt` +``` + +返回值： + +``` +{ + "status": "succ", + "head": ["column1","column2","column3"], + "data": [ + [1513089865728,4,5.60000], + [1513093465730,1,2.30000] + ], + "rows": 2 +} +``` + +#### 结果集采用UTC时间字符串 + +HTTP请求URL采用`sqlutc`时，返回结果集的时间戳将采用UTC时间字符串表示，例如 +``` + curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.t1' 192.168.0.1:6020/rest/sqlutc` +``` + +返回值： + +``` +{ + "status": "succ", + "head": ["column1","column2","column3"], + "data": [ + ["2017-12-12T22:44:25.728+0800",4,5.60000], + ["2017-12-12T23:44:25.730+0800",1,2.30000] + ], + "rows": 2 +} +``` + +### 重要配置项 + +下面仅列出一些与RESTFul接口有关的配置参数，其他系统参数请看配置文件里的说明。注意：配置修改后，需要重启taosd服务才能生效 + +- httpIp: 对外提供RESTFul服务的IP地址，默认绑定到0.0.0.0 +- httpPort: 对外提供RESTFul服务的端口号，默认绑定到6020 +- httpMaxThreads: 启动的线程数量，默认为2 +- httpCacheSessions: 缓存连接的数量，并发请求数目需小于此数值的10倍，默认值为100 +- restfulRowLimit: 返回结果集（JSON格式）的最大条数，默认值为10240 +- httpEnableCompress: 是否支持压缩，默认不支持，目前TDengine仅支持gzip压缩格式 +- httpDebugFlag: 日志开关，131：仅错误和报警信息，135：所有，默认131 + + ## Go Connector #### 安装TDengine From 0517feb1fd1c52733ca8683b78c913a7eb594a43 Mon Sep 17 00:00:00 2001 From: slguan Date: Mon, 2 Dec 2019 16:31:38 +0800 Subject: [PATCH 002/123] Instructions for using the RESTful interface --- .../webdocs/markdowndocs/connector-ch.md | 30 ++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/documentation/webdocs/markdowndocs/connector-ch.md b/documentation/webdocs/markdowndocs/connector-ch.md index e629a44bf1..85596f2053 100644 --- a/documentation/webdocs/markdowndocs/connector-ch.md +++ b/documentation/webdocs/markdowndocs/connector-ch.md @@ -306,15 +306,16 @@ import taos ### HTTP请求格式 -​ `http://:/rest/sql` +``` +http://:/rest/sql +``` -​ 参数说明： +​参数说明： -​ IP: 集群中的任一台主机 +- IP: 集群中的任一台主机 +- PORT: 配置文件中httpPort配置项，缺省为6020 -​ PORT: 配置文件中httpPort配置项，缺省为6020 - -如：http://192.168.0.1:6020/rest/sql 是指向IP地址为192.168.0.1的URL. +例如：http://192.168.0.1:6020/rest/sql 是指向IP地址为192.168.0.1的URL. HTTP请求的Header里需带有身份认证信息，TDengine支持Basic认证与自定义认证两种机制，后续版本将提供标准安全的数字签名机制来做身份验证。 @@ -348,7 +349,8 @@ curl -u username:password -d '' :/rest/sql ### HTTP返回格式 -返回值为JSON格式，如下： +返回值为JSON格式，如下: + ``` { "status": "succ", @@ -363,10 +365,10 @@ curl -u username:password -d '' :/rest/sql 说明： -- 第一行”status”告知操作结果是成功还是失败; -- 第二行”head”是表的定义，如果不返回结果集，仅有一列“affected_rows”; -- 第三行是具体返回的数据，一排一排的呈现。如果不返回结果集，仅[[affected_rows]] -- 第四行”rows”表明总共多少行数据 +- status: 告知操作结果是成功还是失败 +- head: 表的定义，如果不返回结果集，仅有一列“affected_rows” +- data: 具体返回的数据，一排一排的呈现,如果不返回结果集，仅[[affected_rows]] +- rows: 表明总共多少行数据 ### 自定义授权码 @@ -403,7 +405,7 @@ curl http://192.168.0.1:6020/rest/login/root/taosdata ### 使用示例 -- 在demo库里查询表t1的所有记录, curl如下： +- 在demo库里查询表t1的所有记录： ``` curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.t1' 192.168.0.1:6020/rest/sql` @@ -445,7 +447,7 @@ curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'create database demo' 19 HTTP请求URL采用`sqlt`时，返回结果集的时间戳将采用Unix时间戳格式表示，例如 ``` -curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.t1' 192.168.0.1:6020/rest/sqlt` +curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.t1' 192.168.0.1:6020/rest/sqlt ``` 返回值： @@ -466,7 +468,7 @@ curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.t1' 1 HTTP请求URL采用`sqlutc`时，返回结果集的时间戳将采用UTC时间字符串表示，例如 ``` - curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.t1' 192.168.0.1:6020/rest/sqlutc` + curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.t1' 192.168.0.1:6020/rest/sqlutc ``` 返回值： From e5418dd3e13370be8a5d5b7518e5069472389e58 Mon Sep 17 00:00:00 2001 From: haojun Liao Date: Mon, 2 Dec 2019 18:37:51 +0800 Subject: [PATCH 003/123] Update TAOS SQL-ch.md Add some detailed description with respect to the time unit conversion for interval clause. --- documentation/webdocs/markdowndocs/TAOS SQL-ch.md | 1 + 1 file changed, 1 insertion(+) diff --git a/documentation/webdocs/markdowndocs/TAOS SQL-ch.md b/documentation/webdocs/markdowndocs/TAOS SQL-ch.md index 6a8549bbd2..347ac4f21f 100644 --- a/documentation/webdocs/markdowndocs/TAOS SQL-ch.md +++ b/documentation/webdocs/markdowndocs/TAOS SQL-ch.md @@ -18,6 +18,7 @@ TDengine提供类似SQL语法，用户可以在TDengine Shell中使用SQL语句 - 插入记录时，如果时间戳为0，插入数据时使用服务器当前时间 - Epoch Time: 时间戳也可以是一个长整数，表示从1970-01-01 08:00:00.000开始的毫秒数 - 时间可以加减，比如 now-2h，表明查询时刻向前推2个小时(最近2小时)。数字后面的时间单位：a(毫秒), s(秒), m(分), h(小时), d(天)，w(周), n(月), y(年)。比如select * from t1 where ts > now-2w and ts <= now-1w, 表示查询两周前整整一周的数据 +- TDengine暂不支持时间窗口按照自然年和自然月切分。Where条件中的时间窗口单位的换算关系如下：interval(1y) 等效于 interval(365d), interval(1n) 等效于 interval(30d), interval(1w) 等效于 interval(7d) TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMicrosecond就可支持微秒。 From 5d25f776a4bcb49b432e9809301eb623ec98cc2f Mon Sep 17 00:00:00 2001 From: fang Date: Mon, 2 Dec 2019 22:02:11 +0800 Subject: [PATCH 004/123] python connector user guide --- .../tdenginedocs-cn/connector/index.html | 90 +++++++++++++++---- .../tdenginedocs-en/connector/index.html | 66 +++++++++++++- 2 files changed, 138 insertions(+), 18 deletions(-) diff --git a/documentation/tdenginedocs-cn/connector/index.html b/documentation/tdenginedocs-cn/connector/index.html index fca343e977..d808e42dda 100644 --- a/documentation/tdenginedocs-cn/connector/index.html +++ b/documentation/tdenginedocs-cn/connector/index.html @@ -114,23 +114,83 @@ public Connection getConn() throws Exception{

对于TDengine操作的报错信息，用户可使用JDBCDriver包里提供的枚举类TSDBError.java来获取error message和error code的列表。对于更多的具体操作的相关代码，请参考TDengine提供的使用示范项目JDBCDemo。

Python Connector

-

Python客户端安装

-

用户可以在源代码的src/connector/python文件夹下找到python2和python3的安装包。用户可以通过pip命令安装：

-

​ pip install src/connector/python/python2/

-

或

-

​ pip install src/connector/python/python3/

-

如果机器上没有pip命令，用户可将src/connector/python/python3或src/connector/python/python2下的taos文件夹拷贝到应用程序的目录使用。

-

Python客户端接口

+

安装准备

+
  • 已安装TDengine, 如果客户端在Windows上，需要安装Windows 版本的TDengine客户端
    • +
  • 已安装python 2.7 or >= 3.4
    • +
  • 已安装pip
    • +

    安装

    +

    Linux

    +

    用户可以在源代码的src/connector/python文件夹下找到python2和python3的安装包， 然后通过pip命令安装

    +
    pip install src/connector/python/linux/python2/
    +

    或者

    +
    pip install src/connector/python/linux/python3/
    +

    Windows

    +

    在已安装Windows 客户端的情况下， 将文件"C:\TDengine\driver\taos.dll" 拷贝到 "C:\windows\system32" 目录下, 然后进入Windwos cmd 命令行界面

    +
    cd C:\TDengine\connector\python\windows
    +
    pip install python2\
    +

    或者

    +
    cd C:\TDengine\connector\python\windows
    +
    pip install python3\
    +

    * 如果机器上没有pip命令，用户可将src/connector/python/windows/python3或src/connector/python/windows/python2下的taos文件夹拷贝到应用程序的目录使用。

    +

    Python client interfaces

    在使用TDengine的python接口时，需导入TDengine客户端模块：

    -
    import taos
    -

    用户可通过python的帮助信息直接查看模块的使用信息，或者参考code/examples/python中的示例程序。以下为部分常用类和方法：

    +
    import taos
    +

    代码示例

    +
  • 获取连接
    • +
    
+conn = taos.connect(host="127.0.0.1", user="root", password="taosdata", config="/etc/taos")
+c1 = conn.cursor()
+
    +
  • 写入数据
    • +
    
+import datetime
+ 
+# create a database
+c1.execute('create database db')
+c1.execute('use db')
+# create a table
+c1.execute('create table tb (ts timestamp, temperature int, humidity float)')
+# insert a record
+start_time = datetime.datetime(2019, 11, 1)
+affected_rows = c1.execute('insert into tb values (\'%s\', 0, 0.0)' %start_time)
+# insert multiple records in a batch
+time_interval = datetime.timedelta(seconds=60)
+sqlcmd = ['insert into tb values']
+for irow in range(1,11):
+  start_time += time_interval
+  sqlcmd.append('(\'%s\', %d, %f)' %(start_time, irow, irow*1.2))
+affected_rows = c1.execute(' '.join(sqlcmd))
+
    +
  • 写入数据
    • +
    +c1.execute('select * from tb')
+# fetch all returned results
+data = c1.fetchall()
+# data is a list of returned rows with each row being a tuple
+numOfRows = c1.rowcount
+numOfCols = c1.descriptions
+for irow in range(numOfRows):
+  print("Row%d: ts=%s, temperature=%d, humidity=%f" %(irow, data[irow][0], data[irow][1],data[irow][2])
+  
+# use the cursor as an iterator to retrieve all returned results
+c1.execute('select * from tb')
+for data in c1:
+  print("ts=%s, temperature=%d, humidity=%f" %(data[0], data[1],data[2])
+
     +
  • 关闭连接
    • +
    +c1.close()
+conn.close()
+
     +

    帮助信息

    +

    用户可通过python的帮助信息直接查看模块的使用信息，或者参考code/examples/python中的示例程序。以下为部分常用类和方法:

      -

    • TaosConnection类

      -

      参考python中help(taos.TaosConnection)。

      • -

    • TaosCursor类

      -

      参考python中help(taos.TaosCursor)。

      • -

    • connect方法

      -

      用于生成taos.TaosConnection的实例。

      • +

    • TaosConnection 类

      +

      参考python中help(taos.TDengineConnection)

      • +

    • TaosCursor 类

      +

      参考python中help(taos.TDengineCursor)

      • +

    • connect 方法

      +

      用于生成taos.TDengineConnection的实例。

    RESTful Connector

    为支持各种不同类型平台的开发，TDengine提供符合REST设计标准的API，即RESTful API。为最大程度降低学习成本，不同于其他数据库RESTful API的设计方法，TDengine直接通过HTTP POST 请求BODY中包含的SQL语句来操作数据库，仅需要一个URL。

    diff --git a/documentation/tdenginedocs-en/connector/index.html b/documentation/tdenginedocs-en/connector/index.html index ce32c062ff..ba1cbcaf85 100644 --- a/documentation/tdenginedocs-en/connector/index.html +++ b/documentation/tdenginedocs-en/connector/index.html @@ -122,15 +122,75 @@ public Connection getConn() throws Exception{

    All the error codes and error messages can be found in TSDBError.java . For a more detailed coding example, please refer to the demo project JDBCDemo in TDengine's code examples.

    Python Connector

    +

    Pre-requirement

    +
  • TDengine installed, TDengine-client installed if on Windows
    • +
  • python 2.7 or >= 3.4
    • +
  • pip installed

    • Install TDengine Python client

    +

    Linux

    Users can find python client packages in our source code directory src/connector/python. There are two directories corresponding two python versions. Please choose the correct package to install. Users can use pip command to install:

    -
    pip install src/connector/python/python2/
    +
    pip install src/connector/python/linux/python2/

    or

    -
    pip install src/connector/python/python3/
    -

    If pip command is not installed on the system, users can choose to install pip or just copy the taos directory in the python client directory to the application directory to use.

    +
    pip install src/connector/python/linux/python3/
    +

    Windows

    +

    Assume you have installed the Windows client , copy the file "C:\TDengine\driver\taos.dll" to the folder "C:\windows\system32", and then enter the cmd command interface

    +
    cd C:\TDengine\connector\python\windows
    +
    pip install python2\
    +

    or

    +
    cd C:\TDengine\connector\python\windows
    +
    pip install python3\
    +

    * If pip command is not installed on the system, users can choose to install pip or just copy the taos directory in the python client directory to the application directory to use.

    Python client interfaces

    To use TDengine Python client, import TDengine module at first:

    import taos
    +

    Examples

    +
  • get the connection
    • +
    
+conn = taos.connect(host="127.0.0.1", user="root", password="taosdata", config="/etc/taos")
+c1 = conn.cursor()
+
    +
  • insert records into the database
    • +
    
+import datetime
+ 
+# create a database
+c1.execute('create database db')
+c1.execute('use db')
+# create a table
+c1.execute('create table tb (ts timestamp, temperature int, humidity float)')
+# insert a record
+start_time = datetime.datetime(2019, 11, 1)
+affected_rows = c1.execute('insert into tb values (\'%s\', 0, 0.0)' %start_time)
+# insert multiple records in a batch
+time_interval = datetime.timedelta(seconds=60)
+sqlcmd = ['insert into tb values']
+for irow in range(1,11):
+  start_time += time_interval
+  sqlcmd.append('(\'%s\', %d, %f)' %(start_time, irow, irow*1.2))
+affected_rows = c1.execute(' '.join(sqlcmd))
+
    +
  • query the database
    • +
    +c1.execute('select * from tb')
+# fetch all returned results
+data = c1.fetchall()
+# data is a list of returned rows with each row being a tuple
+numOfRows = c1.rowcount
+numOfCols = c1.descriptions
+for irow in range(numOfRows):
+  print("Row%d: ts=%s, temperature=%d, humidity=%f" %(irow, data[irow][0], data[irow][1],data[irow][2])
+  
+# use the cursor as an iterator to retrieve all returned results
+c1.execute('select * from tb')
+for data in c1:
+  print("ts=%s, temperature=%d, humidity=%f" %(data[0], data[1],data[2])
+
     +
  • close the connection
    • +
    +c1.close()
+conn.close()
+
     +

    Help information

    Users can get module information from Python help interface or refer to our [python code example](). We list the main classes and methods below:

    TDengine is specially designed and optimized for time-series data processing in IoT, connected cars, Industrial IoT, IT infrastructure and application monitoring, and other scenarios. Compared with other solutions, it is 10x faster on insert/query speed. With a single-core machine, over 20K requestes can be processed, millions data points can be ingested, and over 10 million data points can be retrieved in a second. Via column-based storage and tuned compression algorithm for different data types, less than 1/10 storage space is required.

    Explore More on TDengine

    -

    Please read through the whole documentation to learn more about TDengine.

    Back \ No newline at end of file +

    Please read through the whole documentation to learn more about TDengine.

    Back From 327eebb51ba0b76f343cb03356237f5871ba195f Mon Sep 17 00:00:00 2001 From: ATR Date: Mon, 9 Dec 2019 19:18:21 +0800 Subject: [PATCH 084/123] Update index.html create table t (ts timestamp, speed int); // fix 'cdata' to 'speed' --- documentation/tdenginedocs-cn/getting-started/index.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation/tdenginedocs-cn/getting-started/index.html b/documentation/tdenginedocs-cn/getting-started/index.html index 65ef667d00..d7d5d8540c 100644 --- a/documentation/tdenginedocs-cn/getting-started/index.html +++ b/documentation/tdenginedocs-cn/getting-started/index.html @@ -28,7 +28,7 @@

    在TDengine终端中，用户可以通过SQL命令来创建/删除数据库、表等，并进行插入查询操作。在终端中运行的SQL语句需要以分号结束来运行。示例：

    create database db;
 use db;
-create table t (ts timestamp, cdata int);
+create table t (ts timestamp, speed int);
 insert into t values ('2019-07-15 00:00:00', 10);
 insert into t values ('2019-07-15 01:00:00', 20);
 select * from t;
@@ -85,4 +85,4 @@ Query OK, 2 row(s) in set (0.001700s)

    TDengine是专为物联网、车联网、工业互联网、运维监测等场景优化设计的时序数据处理引擎。与其他方案相比，它的插入查询速度都快10倍以上。单核一秒钟就能插入100万数据点，读出1000万数据点。由于采用列式存储和优化的压缩算法，存储空间不及普通数据库的1/10.

    深入了解TDengine

    -

    请继续阅读文档来深入了解TDengine。

    回去 \ No newline at end of file +

    请继续阅读文档来深入了解TDengine。

    回去 From 4c7a815fdd29e1e8763231c924aeb7df667f6d72 Mon Sep 17 00:00:00 2001 From: lihui Date: Mon, 9 Dec 2019 19:19:13 +0800 Subject: [PATCH 085/123] [NONE] --- src/client/src/tscParseInsert.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 6317decbe0..0aa04f518c 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -444,7 +444,7 @@ int tsParseOneRowData(char **str, STableDataBlocks *pDataBlocks, SSchema schema[ } } - tmpTokenBuf[j] = sToken.z[i]; + tmpTokenBuf[j] = sToken.z[k]; j++; } tmpTokenBuf[j] = 0; From fe4fb35392798a86663519883d39f7013fc9901c Mon Sep 17 00:00:00 2001 From: slguan Date: Mon, 9 Dec 2019 22:08:17 +0800 Subject: [PATCH 086/123] [TBASE-1241] --- src/inc/tglobalcfg.h | 2 +- src/modules/monitor/src/monitorSystem.c | 20 +------------------- src/os/linux/src/tsystem.c | 4 ++-- src/system/detail/src/dnodeSystem.c | 2 +- src/system/detail/src/mgmtShell.c | 7 ++----- src/system/detail/src/vnodeShell.c | 8 +++----- src/system/lite/src/mgmtDnode.spec.c | 2 +- src/util/src/tglobalcfg.c | 21 +++++++++------------ 8 files changed, 20 insertions(+), 46 deletions(-) diff --git a/src/inc/tglobalcfg.h b/src/inc/tglobalcfg.h index ede3c97ce9..99bc3431d6 100644 --- a/src/inc/tglobalcfg.h +++ b/src/inc/tglobalcfg.h @@ -74,7 +74,6 @@ extern int tsMetricMetaKeepTimer; extern float tsNumOfThreadsPerCore; extern float tsRatioOfQueryThreads; extern char tsPublicIp[]; -extern char tsInternalIp[]; extern char tsPrivateIp[]; extern char tsServerIpStr[]; extern short tsNumOfVnodesPerCore; @@ -151,6 +150,7 @@ extern int tsTelegrafUseFieldNum; extern int tsAdminRowLimit; extern int tsTscEnableRecordSql; +extern int tsAnyIp; extern char tsMonitorDbName[]; extern char tsInternalPass[]; diff --git a/src/modules/monitor/src/monitorSystem.c b/src/modules/monitor/src/monitorSystem.c index c5f65eef36..86f075cb2f 100644 --- a/src/modules/monitor/src/monitorSystem.c +++ b/src/modules/monitor/src/monitorSystem.c @@ -113,11 +113,7 @@ void monitorInitConn(void *para, void *unused) { monitor->state = MONITOR_STATE_INITIALIZING; if (monitor->privateIpStr[0] == 0) { -#ifdef CLUSTER strcpy(monitor->privateIpStr, tsPrivateIp); -#else - strcpy(monitor->privateIpStr, tsInternalIp); -#endif for (int i = 0; i < TSDB_IPv4ADDR_LEN; ++i) { if (monitor->privateIpStr[i] == '.') { monitor->privateIpStr[i] = '_'; @@ -167,11 +163,7 @@ void dnodeBuildMonitorSql(char *sql, int cmd) { tsMonitorDbName, IP_LEN_STR + 1); } else if (cmd == MONITOR_CMD_CREATE_TB_DN) { snprintf(sql, SQL_LENGTH, "create table if not exists %s.dn_%s using %s.dn tags('%s')", tsMonitorDbName, -#ifdef CLUSTER monitor->privateIpStr, tsMonitorDbName, tsPrivateIp); -#else - monitor->privateIpStr, tsMonitorDbName, tsInternalIp); -#endif } else if (cmd == MONITOR_CMD_CREATE_MT_ACCT) { snprintf(sql, SQL_LENGTH, "create table if not exists %s.acct(ts timestamp " @@ -226,10 +218,8 @@ void monitorInitDatabaseCb(void *param, TAOS_RES *result, int code) { taosLogSqlFp = monitorExecuteSQL; #ifdef CLUSTER taosLogAcctFp = monitorSaveAcctLog; - monitorLPrint("dnode:%s is started", tsPrivateIp); -#else - monitorLPrint("dnode:%s is started", tsInternalIp); #endif + monitorLPrint("dnode:%s is started", tsPrivateIp); } monitor->cmdIndex++; monitorInitDatabase(); @@ -245,11 +235,7 @@ void monitorStopSystem() { return; } -#ifdef CLUSTER monitorLPrint("dnode:%s monitor module is stopped", tsPrivateIp); -#else - monitorLPrint("dnode:%s monitor module is stopped", tsInternalIp); -#endif monitor->state = MONITOR_STATE_STOPPED; taosLogFp = NULL; if (monitor->initTimer != NULL) { @@ -439,11 +425,7 @@ void monitorSaveLog(int level, const char *const format, ...) { va_end(argpointer); if (len > max_length) len = max_length; -#ifdef CLUSTER len += sprintf(sql + len, "', '%s')", tsPrivateIp); -#else - len += sprintf(sql + len, "', '%s')", tsInternalIp); -#endif sql[len++] = 0; monitorTrace("monitor:%p, save log, sql: %s", monitor->conn, sql); diff --git a/src/os/linux/src/tsystem.c b/src/os/linux/src/tsystem.c index 03192bb9cc..3b46a559a1 100644 --- a/src/os/linux/src/tsystem.c +++ b/src/os/linux/src/tsystem.c @@ -381,8 +381,8 @@ bool taosGetCardName(char *ip, char *name) { bool taosGetCardInfo(int64_t *bytes) { static char tsPublicCard[1000] = {0}; if (tsPublicCard[0] == 0) { - if (!taosGetCardName(tsInternalIp, tsPublicCard)) { - pError("can't get card name from ip:%s", tsInternalIp); + if (!taosGetCardName(tsPrivateIp, tsPublicCard)) { + pError("can't get card name from ip:%s", tsPrivateIp); return false; } int cardNameLen = (int)strlen(tsPublicCard); diff --git a/src/system/detail/src/dnodeSystem.c b/src/system/detail/src/dnodeSystem.c index 05a7bef486..f4bea50fc7 100644 --- a/src/system/detail/src/dnodeSystem.c +++ b/src/system/detail/src/dnodeSystem.c @@ -139,7 +139,7 @@ int dnodeInitSystem() { vnodeInitMgmtIp(); tsPrintGlobalConfig(); - dPrint("Server IP address is:%s", tsInternalIp); + dPrint("Server IP address is:%s", tsPrivateIp); taosSetCoreDump(); diff --git a/src/system/detail/src/mgmtShell.c b/src/system/detail/src/mgmtShell.c index 6084c5489d..e6afe11855 100644 --- a/src/system/detail/src/mgmtShell.c +++ b/src/system/detail/src/mgmtShell.c @@ -79,11 +79,8 @@ int mgmtInitShell() { if (numOfThreads < 1) numOfThreads = 1; memset(&rpcInit, 0, sizeof(rpcInit)); -#ifdef CLUSTER - rpcInit.localIp = tsInternalIp; -#else - rpcInit.localIp = "0.0.0.0"; -#endif + + rpcInit.localIp = tsAnyIp ? "0.0.0.0" : tsPrivateIp;; rpcInit.localPort = tsMgmtShellPort; rpcInit.label = "MND-shell"; rpcInit.numOfThreads = numOfThreads; diff --git a/src/system/detail/src/vnodeShell.c b/src/system/detail/src/vnodeShell.c index 66bede89b7..45d6cc64b5 100644 --- a/src/system/detail/src/vnodeShell.c +++ b/src/system/detail/src/vnodeShell.c @@ -143,11 +143,9 @@ int vnodeInitShell() { if (numOfThreads < 1) numOfThreads = 1; memset(&rpcInit, 0, sizeof(rpcInit)); -#ifdef CLUSTER - rpcInit.localIp = tsInternalIp; -#else - rpcInit.localIp = "0.0.0.0"; -#endif + + rpcInit.localIp = tsAnyIp ? "0.0.0.0" : tsPrivateIp; + rpcInit.localPort = tsVnodeShellPort; rpcInit.label = "DND-shell"; rpcInit.numOfThreads = numOfThreads; diff --git a/src/system/lite/src/mgmtDnode.spec.c b/src/system/lite/src/mgmtDnode.spec.c index dc7dd7d472..170683a998 100644 --- a/src/system/lite/src/mgmtDnode.spec.c +++ b/src/system/lite/src/mgmtDnode.spec.c @@ -27,7 +27,7 @@ int mgmtUpdateDnode(SDnodeObj *pDnode) { return 0; } void mgmtCleanUpDnodes() {} int mgmtInitDnodes() { - dnodeObj.privateIp = inet_addr(tsInternalIp);; + dnodeObj.privateIp = inet_addr(tsPrivateIp);; dnodeObj.createdTime = (int64_t)tsRebootTime * 1000; dnodeObj.lastReboot = tsRebootTime; dnodeObj.numOfCores = (uint16_t)tsNumOfCores; diff --git a/src/util/src/tglobalcfg.c b/src/util/src/tglobalcfg.c index cef11d30cb..aabc722519 100644 --- a/src/util/src/tglobalcfg.c +++ b/src/util/src/tglobalcfg.c @@ -75,7 +75,6 @@ int tsMetricMetaKeepTimer = 600; // second float tsNumOfThreadsPerCore = 1.0; float tsRatioOfQueryThreads = 0.5; char tsPublicIp[TSDB_IPv4ADDR_LEN] = {0}; -char tsInternalIp[TSDB_IPv4ADDR_LEN] = {0}; char tsPrivateIp[TSDB_IPv4ADDR_LEN] = {0}; char tsServerIpStr[TSDB_IPv4ADDR_LEN] = "127.0.0.1"; short tsNumOfVnodesPerCore = 8; @@ -164,6 +163,7 @@ int tsAdminRowLimit = 10240; int tsTscEnableRecordSql = 0; int tsEnableCoreFile = 0; +int tsAnyIp = 1; int tsRpcTimer = 300; int tsRpcMaxTime = 600; // seconds; @@ -448,9 +448,6 @@ static void doInitGlobalConfig() { tsInitConfigOption(cfg++, "privateIp", tsPrivateIp, TSDB_CFG_VTYPE_IPSTR, TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLUSTER, 0, 0, TSDB_IPv4ADDR_LEN, TSDB_CFG_UTYPE_NONE); - tsInitConfigOption(cfg++, "internalIp", tsInternalIp, TSDB_CFG_VTYPE_IPSTR, - TSDB_CFG_CTYPE_B_CONFIG, - 0, 0, TSDB_IPv4ADDR_LEN, TSDB_CFG_UTYPE_NONE); tsInitConfigOption(cfg++, "localIp", tsLocalIp, TSDB_CFG_VTYPE_IPSTR, TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT, 0, 0, TSDB_IPv4ADDR_LEN, TSDB_CFG_UTYPE_NONE); @@ -709,7 +706,7 @@ static void doInitGlobalConfig() { 1, 100000, 0, TSDB_CFG_UTYPE_NONE); tsInitConfigOption(cfg++, "httpEnableRecordSql", &tsHttpEnableRecordSql, TSDB_CFG_VTYPE_INT, TSDB_CFG_CTYPE_B_CONFIG, - 1, 100000, 0, TSDB_CFG_UTYPE_NONE); + 0, 1, 0, TSDB_CFG_UTYPE_NONE); tsInitConfigOption(cfg++, "telegrafUseFieldNum", &tsTelegrafUseFieldNum, TSDB_CFG_VTYPE_INT, TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW, 0, 1, 1, TSDB_CFG_UTYPE_NONE); @@ -773,12 +770,16 @@ static void doInitGlobalConfig() { tsInitConfigOption(cfg++, "tscEnableRecordSql", &tsTscEnableRecordSql, TSDB_CFG_VTYPE_INT, TSDB_CFG_CTYPE_B_CONFIG, - 1, 100000, 0, TSDB_CFG_UTYPE_NONE); + 0, 1, 0, TSDB_CFG_UTYPE_NONE); tsInitConfigOption(cfg++, "enableCoreFile", &tsEnableCoreFile, TSDB_CFG_VTYPE_INT, TSDB_CFG_CTYPE_B_CONFIG, - 1, 100000, 0, TSDB_CFG_UTYPE_NONE); - + 0, 1, 0, TSDB_CFG_UTYPE_NONE); + + tsInitConfigOption(cfg++, "anyIp", &tsAnyIp, TSDB_CFG_VTYPE_INT, + TSDB_CFG_CTYPE_B_CONFIG, + 0, 1, 0, TSDB_CFG_UTYPE_NONE); + // version info tsInitConfigOption(cfg++, "gitinfo", gitinfo, TSDB_CFG_VTYPE_STRING, TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT, @@ -904,10 +905,6 @@ bool tsReadGlobalConfig() { strcpy(tsPublicIp, tsPrivateIp); } - if (tsInternalIp[0] == 0) { - strcpy(tsInternalIp, tsPrivateIp); - } - if (tsLocalIp[0] == 0) { strcpy(tsLocalIp, tsPrivateIp); } From a5df5d938c991cb01cc7a0e968b30a1ae055f848 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Mon, 9 Dec 2019 22:19:35 +0800 Subject: [PATCH 087/123] suppress compile warnings --- src/client/src/tscAst.c | 7 +++---- src/client/src/tscServer.c | 9 ++++++--- src/client/src/tscSub.c | 4 ++-- src/client/src/tscUtil.c | 3 ++- 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/client/src/tscAst.c b/src/client/src/tscAst.c index a06a075248..d071358dbf 100644 --- a/src/client/src/tscAst.c +++ b/src/client/src/tscAst.c @@ -643,13 +643,12 @@ int32_t intersect(tQueryResultset *pLeft, tQueryResultset *pRight, tQueryResults } /* - * + * traverse the result and apply the function to each item to check if the item is qualified or not */ -void tSQLListTraverseOnResult(struct tSQLBinaryExpr *pExpr, bool (*fp)(tSkipListNode *, void *), - tQueryResultset * pResult) { +static void tSQLListTraverseOnResult(struct tSQLBinaryExpr *pExpr, __result_filter_fn_t fp, tQueryResultset *pResult) { assert(pExpr->pLeft->nodeType == TSQL_NODE_COL && pExpr->pRight->nodeType == TSQL_NODE_VALUE); - // brutal force search + // brutal force scan the result list and check for each item in the list int64_t num = pResult->num; for (int32_t i = 0, j = 0; i < pResult->num; ++i) { if (fp == NULL || (fp(pResult->pRes[i], pExpr->info) == true)) { diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 28cefeea73..0913cf956c 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -3350,7 +3350,7 @@ int tscProcessShowRsp(SSqlObj *pSql) { } int tscProcessConnectRsp(SSqlObj *pSql) { - char temp[TSDB_METER_ID_LEN]; + char temp[TSDB_METER_ID_LEN*2]; SConnectRsp *pConnect; STscObj *pObj = pSql->pTscObj; @@ -3358,8 +3358,11 @@ int tscProcessConnectRsp(SSqlObj *pSql) { pConnect = (SConnectRsp *)pRes->pRsp; strcpy(pObj->acctId, pConnect->acctId); // copy acctId from response - sprintf(temp, "%s%s%s", pObj->acctId, TS_PATH_DELIMITER, pObj->db); - strcpy(pObj->db, temp); + int32_t len =sprintf(temp, "%s%s%s", pObj->acctId, TS_PATH_DELIMITER, pObj->db); + + assert(len <= tListLen(pObj->db)); + strncpy(pObj->db, temp, tListLen(pObj->db)); + #ifdef CLUSTER SIpList * pIpList; char *rsp = pRes->pRsp + sizeof(SConnectRsp); diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c index d6bc1eebe9..f2e9395c68 100644 --- a/src/client/src/tscSub.c +++ b/src/client/src/tscSub.c @@ -56,7 +56,7 @@ TAOS_SUB *taos_subscribe(const char *host, const char *user, const char *pass, c if (pSub->taos == NULL) { tfree(pSub); } else { - char qstr[128]; + char qstr[256] = {0}; sprintf(qstr, "use %s", db); int res = taos_query(pSub->taos, qstr); if (res != 0) { @@ -64,7 +64,7 @@ TAOS_SUB *taos_subscribe(const char *host, const char *user, const char *pass, c taos_close(pSub->taos); tfree(pSub); } else { - sprintf(qstr, "select * from %s where _c0 > now+1000d", pSub->name); + snprintf(qstr, tListLen(qstr), "select * from %s where _c0 > now+1000d", pSub->name); if (taos_query(pSub->taos, qstr)) { tscTrace("failed to select, reason:%s", taos_errstr(pSub->taos)); taos_close(pSub->taos); diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 3ff262bf68..288e564cb0 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1138,7 +1138,8 @@ void tscColumnFilterInfoCopy(SColumnFilterInfo* dst, const SColumnFilterInfo* sr *dst = *src; if (dst->filterOnBinary) { size_t len = (size_t) dst->len + 1; - dst->pz = calloc(1, len); + char* pTmp = calloc(1, len); + dst->pz = (int64_t) pTmp; memcpy((char*) dst->pz, (char*) src->pz, (size_t) len); } } From 2ab2e2831e77ef51edfd5f9fff373db97ee603ab Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Mon, 9 Dec 2019 23:23:05 +0800 Subject: [PATCH 088/123] suppress some compile warnings --- src/inc/taosmsg.h | 2 +- src/system/detail/inc/mgmtUtil.h | 2 +- src/system/detail/src/mgmtSupertableQuery.c | 12 +++++++----- src/system/detail/src/vnodeShell.c | 2 +- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index f39786da2c..895cf23cf8 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -622,7 +622,7 @@ typedef struct { char repStrategy; char loadLatest; // load into mem or not - char precision; // time resoluation + uint8_t precision; // time resolution char reserved[16]; } SVnodeCfg, SCreateDbMsg, SDbCfg, SAlterDbMsg; diff --git a/src/system/detail/inc/mgmtUtil.h b/src/system/detail/inc/mgmtUtil.h index 04bacbe1db..1f70485894 100644 --- a/src/system/detail/inc/mgmtUtil.h +++ b/src/system/detail/inc/mgmtUtil.h @@ -37,6 +37,6 @@ int32_t mgmtRetrieveMetersFromMetric(SMetricMetaMsg* pInfo, int32_t tableIndex, int32_t mgmtDoJoin(SMetricMetaMsg* pMetricMetaMsg, tQueryResultset* pRes); void mgmtReorganizeMetersInMetricMeta(SMetricMetaMsg* pInfo, int32_t index, tQueryResultset* pRes); -bool tSkipListNodeFilterCallback(struct tSkipListNode *pNode, void *param); +bool tSkipListNodeFilterCallback(const void *pNode, void *param); #endif //TBASE_MGMTUTIL_H diff --git a/src/system/detail/src/mgmtSupertableQuery.c b/src/system/detail/src/mgmtSupertableQuery.c index 4dc7760d89..9b39532276 100644 --- a/src/system/detail/src/mgmtSupertableQuery.c +++ b/src/system/detail/src/mgmtSupertableQuery.c @@ -203,7 +203,7 @@ static bool mgmtTablenameFilterCallback(tSkipListNode* pNode, void* param) { static void mgmtRetrieveFromLikeOptr(tQueryResultset* pRes, const char* str, STabObj* pMetric) { SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER; - SMeterNameFilterSupporter supporter = {info, str}; + SMeterNameFilterSupporter supporter = {info, (char*) str}; pRes->num = tSkipListIterateList(pMetric->pSkipList, (tSkipListNode***)&pRes->pRes, mgmtTablenameFilterCallback, &supporter); @@ -230,7 +230,7 @@ static void mgmtFilterByTableNameCond(tQueryResultset* pRes, char* condStr, int3 free(str); } -static bool mgmtJoinFilterCallback(tSkipListNode* pNode, void* param) { +UNUSED_FUNC static bool mgmtJoinFilterCallback(tSkipListNode* pNode, void* param) { SJoinSupporter* pSupporter = (SJoinSupporter*)param; SSchema s = {0}; @@ -639,7 +639,8 @@ static void getTagColumnInfo(SSyntaxTreeFilterSupporter* pSupporter, SSchema* pS } } -void filterPrepare(tSQLBinaryExpr* pExpr, void* param) { +void filterPrepare(void* expr, void* param) { + tSQLBinaryExpr *pExpr = (tSQLBinaryExpr*) expr; if (pExpr->info != NULL) { return; } @@ -791,9 +792,10 @@ static char* getTagValueFromMeter(STabObj* pMeter, int32_t offset, void* param) } } -bool tSkipListNodeFilterCallback(tSkipListNode* pNode, void* param) { +bool tSkipListNodeFilterCallback(const void* pNode, void* param) { + tQueryInfo* pInfo = (tQueryInfo*)param; - STabObj* pMeter = (STabObj*)pNode->pData; + STabObj* pMeter = (STabObj*)(((tSkipListNode*)pNode)->pData); char name[TSDB_METER_NAME_LEN + 1] = {0}; char* val = getTagValueFromMeter(pMeter, pInfo->offset, name); diff --git a/src/system/detail/src/vnodeShell.c b/src/system/detail/src/vnodeShell.c index 5eea6632af..9ae33c43b7 100644 --- a/src/system/detail/src/vnodeShell.c +++ b/src/system/detail/src/vnodeShell.c @@ -473,7 +473,7 @@ void vnodeExecuteRetrieveReq(SSchedMsg *pSched) { if (numOfRows == 0 && (pRetrieve->qhandle == (uint64_t)pObj->qhandle) && (code != TSDB_CODE_ACTION_IN_PROGRESS)) { dTrace("QInfo:%p %s free qhandle code:%d", pObj->qhandle, __FUNCTION__, code); - vnodeFreeQInfo(pObj->qhandle); + vnodeFreeQInfo(pObj->qhandle, true); pObj->qhandle = NULL; } From c2a2c575a114378a2cc8f6f992b778dbef1215ef Mon Sep 17 00:00:00 2001 From: fang Date: Tue, 10 Dec 2019 08:46:01 +0800 Subject: [PATCH 089/123] solve some statich check questions --- src/client/src/tscPrepare.c | 1 - src/client/src/tscSQLParser.c | 1 - 2 files changed, 2 deletions(-) diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index 2f1db58117..ef8ddfd211 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -65,7 +65,6 @@ static int normalStmtAddPart(SNormalStmt* stmt, bool isParam, char* str, uint32_ } stmt->sizeParts = size; stmt->parts = (SNormalStmtPart*)tmp; - free(tmp); } stmt->parts[stmt->numParts].isParam = isParam; diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index cb73a6e9bd..ac2b958be7 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -2927,7 +2927,6 @@ static SColumnFilterInfo* addColumnFilterInfo(SColumnBase* pColumn) { char* tmp = realloc(pColumn->filterInfo, sizeof(SColumnFilterInfo) * (size)); if (tmp != NULL) { pColumn->filterInfo = (SColumnFilterInfo*)tmp; - free(tmp); } pColumn->numOfFilters++; From 959b6ec95ce9722d1698930d02f97453301f948e Mon Sep 17 00:00:00 2001 From: fang Date: Tue, 10 Dec 2019 09:41:10 +0800 Subject: [PATCH 090/123] solve some static check questions --- src/util/src/tutil.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/util/src/tutil.c b/src/util/src/tutil.c index e0dcb0aa3f..ee7795863a 100644 --- a/src/util/src/tutil.c +++ b/src/util/src/tutil.c @@ -106,6 +106,7 @@ char **strsplit(char *z, const char *delim, int32_t *num) { if ((*num) >= size) { size = (size << 1); split = realloc(split, POINTER_BYTES * size); + assert(NULL != split); } } From d2ef1dad53744f8297b0b338cb390cbc8304e2c5 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Tue, 10 Dec 2019 12:16:10 +0800 Subject: [PATCH 091/123] [tbase-1287] --- src/client/inc/tscUtil.h | 6 +-- src/client/inc/tsclient.h | 25 ++++++----- src/client/src/tscParseInsert.c | 6 +-- src/client/src/tscUtil.c | 77 ++++++++++++++++++++++----------- 4 files changed, 72 insertions(+), 42 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 3deb4c463f..5d828d7cf0 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -67,7 +67,7 @@ typedef struct SJoinSubquerySupporter { } SJoinSubquerySupporter; void tscDestroyDataBlock(STableDataBlocks* pDataBlock); -STableDataBlocks* tscCreateDataBlock(int32_t size); +STableDataBlocks* tscCreateDataBlock(size_t initialBufSize, int32_t rowSize, int32_t startOffset, const char* name); void tscAppendDataBlock(SDataBlockList* pList, STableDataBlocks* pBlocks); SParamInfo* tscAddParamToDataBlock(STableDataBlocks* pDataBlock, char type, uint8_t timePrec, short bytes, uint32_t offset); @@ -78,9 +78,7 @@ int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDa void tscFreeUnusedDataBlocks(SDataBlockList* pList); int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SDataBlockList* pDataList); STableDataBlocks* tscGetDataBlockFromList(void* pHashList, SDataBlockList* pDataBlockList, int64_t id, int32_t size, - int32_t startOffset, int32_t rowSize, char* tableId); -STableDataBlocks* tscCreateDataBlockEx(size_t size, int32_t rowSize, int32_t startOffset, char* name); - + int32_t startOffset, int32_t rowSize, const char* tableId); SVnodeSidList* tscGetVnodeSidList(SMetricMeta* pMetricmeta, int32_t vnodeIdx); SMeterSidExtInfo* tscGetMeterSidInfo(SVnodeSidList* pSidList, int32_t idx); diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 4de33f5ac4..be9ba47f2a 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -231,17 +231,22 @@ typedef struct SParamInfo { typedef struct STableDataBlocks { char meterId[TSDB_METER_ID_LEN]; - int8_t tsSource; - bool ordered; + int8_t tsSource; // where does the UNIX timestamp come from, server or client + bool ordered; // if current rows are ordered or not + int64_t vgid; // virtual group id + int64_t prevTS; // previous timestamp, recorded to decide if the records array is ts ascending + int32_t numOfMeters; // number of tables in current submit block - int64_t vgid; - int64_t prevTS; - - int32_t numOfMeters; - - int32_t rowSize; + int32_t rowSize; // row size for current table uint32_t nAllocSize; uint32_t size; + + /* + * the metermeta for current table, the metermeta will be used during submit stage, keep a ref + * to avoid it to be removed from cache + */ + SMeterMeta* pMeterMeta; + union { char *filename; char *pData; @@ -255,8 +260,8 @@ typedef struct STableDataBlocks { typedef struct SDataBlockList { int32_t idx; - int32_t nSize; - int32_t nAlloc; + uint32_t nSize; + uint32_t nAlloc; char * userParam; /* user assigned parameters for async query */ void * udfp; /* user defined function pointer, used in async model */ STableDataBlocks **pData; diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 6317decbe0..a07ad6f211 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -985,7 +985,7 @@ int doParserInsertSql(SSqlObj *pSql, char *str) { strcpy(fname, full_path.we_wordv[0]); wordfree(&full_path); - STableDataBlocks *pDataBlock = tscCreateDataBlockEx(PATH_MAX, pMeterMetaInfo->pMeterMeta->rowSize, + STableDataBlocks *pDataBlock = tscCreateDataBlock(PATH_MAX, pMeterMetaInfo->pMeterMeta->rowSize, sizeof(SShellSubmitBlock), pMeterMetaInfo->name); tscAppendDataBlock(pCmd->pDataBlocks, pDataBlock); @@ -1222,8 +1222,8 @@ static int tscInsertDataFromFile(SSqlObj *pSql, FILE *fp, char *tmpTokenBuf) { int32_t rowSize = pMeterMeta->rowSize; pCmd->pDataBlocks = tscCreateBlockArrayList(); - STableDataBlocks *pTableDataBlock = - tscCreateDataBlockEx(TSDB_PAYLOAD_SIZE, pMeterMeta->rowSize, sizeof(SShellSubmitBlock), pMeterMetaInfo->name); + STableDataBlocks *pTableDataBlock = tscCreateDataBlock(TSDB_PAYLOAD_SIZE, pMeterMeta->rowSize, + sizeof(SShellSubmitBlock), pMeterMetaInfo->name); tscAppendDataBlock(pCmd->pDataBlocks, pTableDataBlock); diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 288e564cb0..7c7310a1c7 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -451,15 +451,6 @@ void tscFreeSqlObj(SSqlObj* pSql) { free(pSql); } -STableDataBlocks* tscCreateDataBlock(int32_t size) { - STableDataBlocks* dataBuf = (STableDataBlocks*)calloc(1, sizeof(STableDataBlocks)); - dataBuf->nAllocSize = (uint32_t)size; - dataBuf->pData = calloc(1, dataBuf->nAllocSize); - dataBuf->ordered = true; - dataBuf->prevTS = INT64_MIN; - return dataBuf; -} - void tscDestroyDataBlock(STableDataBlocks* pDataBlock) { if (pDataBlock == NULL) { return; @@ -467,6 +458,9 @@ void tscDestroyDataBlock(STableDataBlocks* pDataBlock) { tfree(pDataBlock->pData); tfree(pDataBlock->params); + + // free the refcount for metermeta + taosRemoveDataFromCache(tscCacheHandle, (void**) &(pDataBlock->pMeterMeta), false); tfree(pDataBlock); } @@ -513,11 +507,11 @@ SDataBlockList* tscCreateBlockArrayList() { void tscAppendDataBlock(SDataBlockList* pList, STableDataBlocks* pBlocks) { if (pList->nSize >= pList->nAlloc) { - pList->nAlloc = pList->nAlloc << 1; - pList->pData = realloc(pList->pData, sizeof(void*) * (size_t)pList->nAlloc); + pList->nAlloc = (pList->nAlloc) << 1U; + pList->pData = realloc(pList->pData, POINTER_BYTES * (size_t)pList->nAlloc); // reset allocated memory - memset(pList->pData + pList->nSize, 0, sizeof(void*) * (pList->nAlloc - pList->nSize)); + memset(pList->pData + pList->nSize, 0, POINTER_BYTES * (pList->nAlloc - pList->nSize)); } pList->pData[pList->nSize++] = pBlocks; @@ -539,29 +533,43 @@ void* tscDestroyBlockArrayList(SDataBlockList* pList) { } int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock) { - SSqlCmd* pCmd = &pSql->cmd; - + SSqlCmd *pCmd = &pSql->cmd; + assert(pDataBlock->pMeterMeta != NULL); + pCmd->count = pDataBlock->numOfMeters; - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); - strcpy(pMeterMetaInfo->name, pDataBlock->meterId); - + SMeterMetaInfo *pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + + //set the correct metermeta object, the metermeta has been locked in pDataBlocks, so it must be in the cache + if (pMeterMetaInfo->pMeterMeta != pDataBlock->pMeterMeta) { + strcpy(pMeterMetaInfo->name, pDataBlock->meterId); + taosRemoveDataFromCache(tscCacheHandle, (void**) &(pMeterMetaInfo->pMeterMeta), false); + + pMeterMetaInfo->pMeterMeta = pDataBlock->pMeterMeta; + pDataBlock->pMeterMeta = NULL; // delegate the ownership of metermeta to pMeterMetaInfo + } else { + assert(strncmp(pMeterMetaInfo->name, pDataBlock->meterId, tListLen(pDataBlock->meterId)) == 0); + } + /* * the submit message consists of : [RPC header|message body|digest] * the dataBlock only includes the RPC Header buffer and actual submit messsage body, space for digest needs * additional space. */ int ret = tscAllocPayload(pCmd, pDataBlock->nAllocSize + sizeof(STaosDigest)); - if (TSDB_CODE_SUCCESS != ret) return ret; + if (TSDB_CODE_SUCCESS != ret) { + return ret; + } + memcpy(pCmd->payload, pDataBlock->pData, pDataBlock->nAllocSize); - + /* * the payloadLen should be actual message body size * the old value of payloadLen is the allocated payload size */ pCmd->payloadLen = pDataBlock->nAllocSize - tsRpcHeadSize; - + assert(pCmd->allocSize >= pCmd->payloadLen + tsRpcHeadSize + sizeof(STaosDigest)); - return tscGetMeterMeta(pSql, pMeterMetaInfo->name, 0); + return TSDB_CODE_SUCCESS; } void tscFreeUnusedDataBlocks(SDataBlockList* pList) { @@ -573,19 +581,38 @@ void tscFreeUnusedDataBlocks(SDataBlockList* pList) { } } -STableDataBlocks* tscCreateDataBlockEx(size_t size, int32_t rowSize, int32_t startOffset, char* name) { - STableDataBlocks* dataBuf = tscCreateDataBlock(size); +/** + * create the in-memory buffer for each table to keep the submitted data block + * @param initialSize + * @param rowSize + * @param startOffset + * @param name + * @param pMeterMeta the ownership of pMeterMeta should be transfer to STableDataBlocks + * @return + */ +STableDataBlocks* tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOffset, const char* name) { + + STableDataBlocks* dataBuf = (STableDataBlocks*)calloc(1, sizeof(STableDataBlocks)); + dataBuf->nAllocSize = (uint32_t) initialSize; + dataBuf->pData = calloc(1, dataBuf->nAllocSize); + dataBuf->ordered = true; + dataBuf->prevTS = INT64_MIN; dataBuf->rowSize = rowSize; dataBuf->size = startOffset; dataBuf->tsSource = -1; strncpy(dataBuf->meterId, name, TSDB_METER_ID_LEN); + + // sure that the metermeta must be in the local client cache + dataBuf->pMeterMeta = taosGetDataFromCache(tscCacheHandle, dataBuf->meterId); + assert(dataBuf->pMeterMeta != NULL && initialSize > 0); + return dataBuf; } STableDataBlocks* tscGetDataBlockFromList(void* pHashList, SDataBlockList* pDataBlockList, int64_t id, int32_t size, - int32_t startOffset, int32_t rowSize, char* tableId) { + int32_t startOffset, int32_t rowSize, const char* tableId) { STableDataBlocks* dataBuf = NULL; STableDataBlocks** t1 = (STableDataBlocks**)taosGetIntHashData(pHashList, id); @@ -594,7 +621,7 @@ STableDataBlocks* tscGetDataBlockFromList(void* pHashList, SDataBlockList* pData } if (dataBuf == NULL) { - dataBuf = tscCreateDataBlockEx((size_t)size, rowSize, startOffset, tableId); + dataBuf = tscCreateDataBlock((size_t)size, rowSize, startOffset, tableId); dataBuf = *(STableDataBlocks**)taosAddIntHash(pHashList, id, (char*)&dataBuf); tscAppendDataBlock(pDataBlockList, dataBuf); } From 6c860c7393ebd8bed91691468b5ab7050049b455 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Tue, 10 Dec 2019 12:35:45 +0800 Subject: [PATCH 092/123] [tbase-1317] --- src/client/src/tscSystem.c | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index 60b90ac328..6efe344719 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -198,7 +198,9 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) { switch (option) { case TSDB_OPTION_CONFIGDIR: cfg = tsGetConfigOption("configDir"); - if (cfg && cfg->cfgStatus <= TSDB_CFG_CSTATUS_OPTION) { + assert(cfg != NULL); + + if (cfg->cfgStatus <= TSDB_CFG_CSTATUS_OPTION) { strncpy(configDir, pStr, TSDB_FILENAME_LEN); cfg->cfgStatus = TSDB_CFG_CSTATUS_OPTION; tscPrint("set config file directory:%s", pStr); @@ -210,7 +212,9 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) { case TSDB_OPTION_SHELL_ACTIVITY_TIMER: cfg = tsGetConfigOption("shellActivityTimer"); - if (cfg && cfg->cfgStatus <= TSDB_CFG_CSTATUS_OPTION) { + assert(cfg != NULL); + + if (cfg->cfgStatus <= TSDB_CFG_CSTATUS_OPTION) { tsShellActivityTimer = atoi(pStr); if (tsShellActivityTimer < 1) tsShellActivityTimer = 1; if (tsShellActivityTimer > 3600) tsShellActivityTimer = 3600; @@ -224,13 +228,15 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) { case TSDB_OPTION_LOCALE: { // set locale cfg = tsGetConfigOption("locale"); + assert(cfg != NULL); + size_t len = strlen(pStr); if (len == 0 || len > TSDB_LOCALE_LEN) { tscPrint("Invalid locale:%s, use default", pStr); return -1; } - if (cfg && cfg && cfg->cfgStatus <= TSDB_CFG_CSTATUS_OPTION) { + if (cfg->cfgStatus <= TSDB_CFG_CSTATUS_OPTION) { char sep = '.'; if (strlen(tsLocale) == 0) { // locale does not set yet @@ -285,13 +291,15 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) { case TSDB_OPTION_CHARSET: { /* set charset will override the value of charset, assigned during system locale changed */ cfg = tsGetConfigOption("charset"); + assert(cfg != NULL); + size_t len = strlen(pStr); if (len == 0 || len > TSDB_LOCALE_LEN) { tscPrint("failed to set charset:%s", pStr); return -1; } - if (cfg && cfg->cfgStatus <= TSDB_CFG_CSTATUS_OPTION) { + if (cfg->cfgStatus <= TSDB_CFG_CSTATUS_OPTION) { if (taosValidateEncodec(pStr)) { if (strlen(tsCharset) == 0) { tscPrint("charset is set:%s", pStr); @@ -314,7 +322,9 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) { case TSDB_OPTION_TIMEZONE: cfg = tsGetConfigOption("timezone"); - if (cfg && cfg->cfgStatus <= TSDB_CFG_CSTATUS_OPTION) { + assert(cfg != NULL); + + if (cfg->cfgStatus <= TSDB_CFG_CSTATUS_OPTION) { strcpy(tsTimezone, pStr); tsSetTimeZone(); cfg->cfgStatus = TSDB_CFG_CSTATUS_OPTION; @@ -327,7 +337,9 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) { case TSDB_OPTION_SOCKET_TYPE: cfg = tsGetConfigOption("sockettype"); - if (cfg && cfg->cfgStatus <= TSDB_CFG_CSTATUS_OPTION) { + assert(cfg != NULL); + + if (cfg->cfgStatus <= TSDB_CFG_CSTATUS_OPTION) { if (strcasecmp(pStr, TAOS_SOCKET_TYPE_NAME_UDP) != 0 && strcasecmp(pStr, TAOS_SOCKET_TYPE_NAME_TCP) != 0) { tscError("only 'tcp' or 'udp' allowed for configuring the socket type"); return -1; @@ -340,6 +352,7 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) { break; default: + // TODO return the correct error code to client in the format for taos_errstr() tscError("Invalid option %d", option); return -1; } From 19d516169036eab7af561154e86a22b18c4ba519 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Tue, 10 Dec 2019 12:39:01 +0800 Subject: [PATCH 093/123] [tbase-1316] --- src/client/src/tscStream.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index 933960b893..31af78f618 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -447,7 +447,10 @@ static void setErrorInfo(STscObj* pObj, int32_t code, char* info) { SSqlCmd* pCmd = &pObj->pSql->cmd; pObj->pSql->res.code = code; - strncpy(pCmd->payload, info, pCmd->payloadLen); + + if (info != NULL) { + strncpy(pCmd->payload, info, pCmd->payloadLen); + } } TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sqlstr, void (*fp)(void *param, TAOS_RES *, TAOS_ROW row), From 2bf71e71079ed55d68aecad4040afcde1914313f Mon Sep 17 00:00:00 2001 From: slguan Date: Tue, 10 Dec 2019 13:00:00 +0800 Subject: [PATCH 094/123] [TBASE-1241] --- src/client/src/tscServer.c | 5 +++ src/client/src/tscSql.c | 32 ++------------- src/inc/taosmsg.h | 4 ++ src/inc/tglobalcfg.h | 2 + src/inc/tutil.h | 2 + src/system/detail/inc/mgmt.h | 4 +- src/system/detail/src/dnodeService.c | 8 +--- src/system/detail/src/mgmtMeter.c | 22 ++++++---- src/system/detail/src/mgmtShell.c | 61 ++++++++++++++++++++++------ src/system/detail/src/mgmtVgroup.c | 16 +++++--- src/util/src/tglobalcfg.c | 10 +++++ src/util/src/tutil.c | 36 ++++++++++++++-- 12 files changed, 136 insertions(+), 66 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index d25b2dde53..3fac0c74f2 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -2642,6 +2642,11 @@ int tscBuildConnectMsg(SSqlObj *pSql) { db = (db == NULL) ? pObj->db : db + 1; strcpy(pConnect->db, db); + strcpy(pConnect->clientVersion, version); + pConnect->usePublicIp = (int8_t)tsUsePublicIp; + pConnect->isCluster = (int8_t)tsIsCluster; + memset(pConnect->reserved, 0, sizeof(pConnect->reserved)); + pMsg += sizeof(SConnectMsg); msgLen = pMsg - pStart; diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 233a37dc59..381b342fdc 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -165,38 +165,12 @@ TAOS *taos_connect(const char *ip, const char *user, const char *pass, const cha STscObj* pObj = (STscObj*) taos; // version compare only requires the first 3 segments of the version string - int32_t comparedSegments = 3; - char client_version[64] = {0}; - char server_version[64] = {0}; - int clientVersionNumber[4] = {0}; - int serverVersionNumber[4] = {0}; - - strcpy(client_version, version); - strcpy(server_version, taos_get_server_info(taos)); - - if (!taosGetVersionNumber(client_version, clientVersionNumber)) { - tscError("taos:%p, invalid client version:%s", taos, client_version); - pObj->pSql->res.code = TSDB_CODE_INVALID_CLIENT_VERSION; + int code = taosCheckVersion(version, taos_get_server_info(taos), 3); + if (code != 0) { + pObj->pSql->res.code = code; taos_close(taos); return NULL; } - - if (!taosGetVersionNumber(server_version, serverVersionNumber)) { - tscError("taos:%p, invalid server version:%s", taos, server_version); - pObj->pSql->res.code = TSDB_CODE_INVALID_CLIENT_VERSION; - taos_close(taos); - return NULL; - } - - for(int32_t i = 0; i < comparedSegments; ++i) { - if (clientVersionNumber[i] != serverVersionNumber[i]) { - tscError("taos:%p, the %d-th number of server version:%s not matched with client version:%s, close connection", - taos, i, server_version, version); - pObj->pSql->res.code = TSDB_CODE_INVALID_CLIENT_VERSION; - taos_close(taos); - return NULL; - } - } } return taos; diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index c1820a5b9c..ae58c43477 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -350,7 +350,11 @@ typedef struct { } SAlterTableMsg; typedef struct { + char clientVersion[TSDB_VERSION_LEN]; char db[TSDB_METER_ID_LEN]; + int8_t usePublicIp; + int8_t isCluster; + int8_t reserved[14]; } SConnectMsg; typedef struct { diff --git a/src/inc/tglobalcfg.h b/src/inc/tglobalcfg.h index 99bc3431d6..0bf47d75ad 100644 --- a/src/inc/tglobalcfg.h +++ b/src/inc/tglobalcfg.h @@ -151,6 +151,8 @@ extern int tsAdminRowLimit; extern int tsTscEnableRecordSql; extern int tsAnyIp; +extern int tsUsePublicIp; +extern int tsIsCluster; extern char tsMonitorDbName[]; extern char tsInternalPass[]; diff --git a/src/inc/tutil.h b/src/inc/tutil.h index bdf9df63ee..df82c2113a 100644 --- a/src/inc/tutil.h +++ b/src/inc/tutil.h @@ -187,6 +187,8 @@ static FORCE_INLINE void taosEncryptPass(uint8_t *inBuf, unsigned int inLen, cha memcpy(target, context.digest, TSDB_KEY_LEN); } +int taosCheckVersion(char *input_client_version, char *input_server_version, int compared_segments); + char *taosIpStr(uint32_t ipInt); #define TAOS_ALLOC_MODE_DEFAULT 0 diff --git a/src/system/detail/inc/mgmt.h b/src/system/detail/inc/mgmt.h index 75ec841076..b59c5cf61c 100644 --- a/src/system/detail/inc/mgmt.h +++ b/src/system/detail/inc/mgmt.h @@ -222,6 +222,8 @@ typedef struct _connObj { char superAuth : 1; // super user flag char writeAuth : 1; // write flag char killConnection : 1; // kill the connection flag + char usePublicIp : 1; // if the connection request is publicIp + char reserved : 4; uint32_t queryId; // query ID to be killed uint32_t streamId; // stream ID to be killed uint32_t ip; // shell IP @@ -343,7 +345,7 @@ void mgmtCleanUpVgroups(); int mgmtInitMeters(); STabObj *mgmtGetMeter(char *meterId); STabObj *mgmtGetMeterInfo(char *src, char *tags[]); -int mgmtRetrieveMetricMeta(void *thandle, char **pStart, SMetricMetaMsg *pInfo); +int mgmtRetrieveMetricMeta(SConnObj *pConn, char **pStart, SMetricMetaMsg *pInfo); int mgmtCreateMeter(SDbObj *pDb, SCreateTableMsg *pCreate); int mgmtDropMeter(SDbObj *pDb, char *meterId, int ignore); int mgmtAlterMeter(SDbObj *pDb, SAlterTableMsg *pAlter); diff --git a/src/system/detail/src/dnodeService.c b/src/system/detail/src/dnodeService.c index f03bd5f3bb..eec918acd0 100644 --- a/src/system/detail/src/dnodeService.c +++ b/src/system/detail/src/dnodeService.c @@ -55,12 +55,8 @@ int main(int argc, char *argv[]) { exit(EXIT_FAILURE); } } else if (strcmp(argv[i], "-V") == 0) { - #ifdef CLUSTER - printf("enterprise version: %s compatible_version: %s\n", version, compatible_version); - #else - printf("community version: %s compatible_version: %s\n", version, compatible_version); - #endif - + char *versionStr = tsIsCluster ? "enterprise" : "community"; + printf("%s version: %s compatible_version: %s\n", versionStr, version, compatible_version); printf("gitinfo: %s\n", gitinfo); printf("buildinfo: %s\n", buildinfo); return 0; diff --git a/src/system/detail/src/mgmtMeter.c b/src/system/detail/src/mgmtMeter.c index 4e1d62e60e..9acd4f1326 100644 --- a/src/system/detail/src/mgmtMeter.c +++ b/src/system/detail/src/mgmtMeter.c @@ -987,7 +987,7 @@ SSchema *mgmtGetMeterSchema(STabObj *pMeter) { /* * serialize SVnodeSidList to byte array */ -static char *mgmtBuildMetricMetaMsg(STabObj *pMeter, int32_t *ovgId, SVnodeSidList **pList, SMetricMeta *pMeta, +static char *mgmtBuildMetricMetaMsg(SConnObj *pConn, STabObj *pMeter, int32_t *ovgId, SVnodeSidList **pList, SMetricMeta *pMeta, int32_t tagLen, int16_t numOfTags, int16_t *tagsId, int32_t maxNumOfMeters, char *pMsg) { if (pMeter->gid.vgId != *ovgId || ((*pList) != NULL && (*pList)->numOfSids >= maxNumOfMeters)) { @@ -1004,8 +1004,13 @@ static char *mgmtBuildMetricMetaMsg(STabObj *pMeter, int32_t *ovgId, SVnodeSidLi SVgObj *pVgroup = mgmtGetVgroup(pMeter->gid.vgId); for (int i = 0; i < TSDB_VNODES_SUPPORT; ++i) { - (*pList)->vpeerDesc[i].ip = pVgroup->vnodeGid[i].publicIp; - (*pList)->vpeerDesc[i].vnode = pVgroup->vnodeGid[i].vnode; + if (pConn->usePublicIp) { + (*pList)->vpeerDesc[i].ip = pVgroup->vnodeGid[i].publicIp; + (*pList)->vpeerDesc[i].vnode = pVgroup->vnodeGid[i].vnode; + } else { + (*pList)->vpeerDesc[i].ip = pVgroup->vnodeGid[i].ip; + (*pList)->vpeerDesc[i].vnode = pVgroup->vnodeGid[i].vnode; + } } pMsg += sizeof(SVnodeSidList); @@ -1102,10 +1107,10 @@ static SMetricMetaElemMsg *doConvertMetricMetaMsg(SMetricMetaMsg *pMetricMetaMsg return pElem; } -static int32_t mgmtBuildMetricMetaRspMsg(void *thandle, SMetricMetaMsg *pMetricMetaMsg, tQueryResultset *pResult, +static int32_t mgmtBuildMetricMetaRspMsg(SConnObj *pConn, SMetricMetaMsg *pMetricMetaMsg, tQueryResultset *pResult, char **pStart, int32_t *tagLen, int32_t rspMsgSize, int32_t maxTablePerVnode, int32_t code) { - *pStart = taosBuildRspMsgWithSize(thandle, TSDB_MSG_TYPE_METRIC_META_RSP, rspMsgSize); + *pStart = taosBuildRspMsgWithSize(pConn->thandle, TSDB_MSG_TYPE_METRIC_META_RSP, rspMsgSize); if (*pStart == NULL) { return 0; } @@ -1143,7 +1148,7 @@ static int32_t mgmtBuildMetricMetaRspMsg(void *thandle, SMetricMetaMsg *pMetricM for (int32_t i = 0; i < pResult[j].num; ++i) { STabObj *pMeter = pResult[j].pRes[i]; - pMsg = mgmtBuildMetricMetaMsg(pMeter, &ovgId, &pList, pMeta, tagLen[j], pElem->numOfTags, pElem->tagCols, + pMsg = mgmtBuildMetricMetaMsg(pConn, pMeter, &ovgId, &pList, pMeta, tagLen[j], pElem->numOfTags, pElem->tagCols, maxTablePerVnode, pMsg); } @@ -1159,7 +1164,7 @@ static int32_t mgmtBuildMetricMetaRspMsg(void *thandle, SMetricMetaMsg *pMetricM return msgLen; } -int mgmtRetrieveMetricMeta(void *thandle, char **pStart, SMetricMetaMsg *pMetricMetaMsg) { +int mgmtRetrieveMetricMeta(SConnObj *pConn, char **pStart, SMetricMetaMsg *pMetricMetaMsg) { /* * naive method: Do not limit the maximum number of meters in each * vnode(subquery), split the result according to vnodes @@ -1233,8 +1238,7 @@ int mgmtRetrieveMetricMeta(void *thandle, char **pStart, SMetricMetaMsg *pMetric msgLen = 512; } - msgLen = mgmtBuildMetricMetaRspMsg(thandle, pMetricMetaMsg, result, pStart, tagLen, msgLen, maxMetersPerVNodeForQuery, - ret); + msgLen = mgmtBuildMetricMetaRspMsg(pConn, pMetricMetaMsg, result, pStart, tagLen, msgLen, maxMetersPerVNodeForQuery, ret); for (int32_t i = 0; i < pMetricMetaMsg->numOfMeters; ++i) { tQueryResultClean(&result[i]); diff --git a/src/system/detail/src/mgmtShell.c b/src/system/detail/src/mgmtShell.c index e6afe11855..576353d758 100644 --- a/src/system/detail/src/mgmtShell.c +++ b/src/system/detail/src/mgmtShell.c @@ -314,8 +314,13 @@ int mgmtProcessMeterMetaMsg(char *pMsg, int msgLen, SConnObj *pConn) { goto _exit_code; } for (int i = 0; i < TSDB_VNODES_SUPPORT; ++i) { - pMeta->vpeerDesc[i].ip = pVgroup->vnodeGid[i].publicIp; - pMeta->vpeerDesc[i].vnode = htonl(pVgroup->vnodeGid[i].vnode); + if (pConn->usePublicIp) { + pMeta->vpeerDesc[i].ip = pVgroup->vnodeGid[i].publicIp; + pMeta->vpeerDesc[i].vnode = htonl(pVgroup->vnodeGid[i].vnode); + } else { + pMeta->vpeerDesc[i].ip = pVgroup->vnodeGid[i].ip; + pMeta->vpeerDesc[i].vnode = htonl(pVgroup->vnodeGid[i].vnode); + } } } } @@ -453,8 +458,13 @@ int mgmtProcessMultiMeterMetaMsg(char *pMsg, int msgLen, SConnObj *pConn) { } for (int i = 0; i < TSDB_VNODES_SUPPORT; ++i) { - pMeta->meta.vpeerDesc[i].ip = pVgroup->vnodeGid[i].publicIp; - pMeta->meta.vpeerDesc[i].vnode = htonl(pVgroup->vnodeGid[i].vnode); + if (pConn->usePublicIp) { + pMeta->meta.vpeerDesc[i].ip = pVgroup->vnodeGid[i].publicIp; + pMeta->meta.vpeerDesc[i].vnode = htonl(pVgroup->vnodeGid[i].vnode); + } else { + pMeta->meta.vpeerDesc[i].ip = pVgroup->vnodeGid[i].ip; + pMeta->meta.vpeerDesc[i].vnode = htonl(pVgroup->vnodeGid[i].vnode); + } } } } @@ -526,7 +536,7 @@ int mgmtProcessMetricMetaMsg(char *pMsg, int msgLen, SConnObj *pConn) { msgLen = pMsg - pStart; } else { - msgLen = mgmtRetrieveMetricMeta(pConn->thandle, &pStart, pMetricMetaMsg); + msgLen = mgmtRetrieveMetricMeta(pConn, &pStart, pMetricMetaMsg); if (msgLen <= 0) { taosSendSimpleRsp(pConn->thandle, TSDB_MSG_TYPE_METRIC_META_RSP, TSDB_CODE_SERV_OUT_OF_MEMORY); return 0; @@ -1099,10 +1109,17 @@ int mgmtProcessHeartBeatMsg(char *cont, int contLen, SConnObj *pConn) { pHBRsp->killConnection = pConn->killConnection; #ifdef CLUSTER - int size = pSdbPublicIpList->numOfIps * 4; - pHBRsp->ipList.numOfIps = pSdbPublicIpList->numOfIps; - memcpy(pHBRsp->ipList.ip, pSdbPublicIpList->ip, size); - pMsg += sizeof(SHeartBeatRsp) + size; + if (pConn->usePublicIp) { + int size = pSdbPublicIpList->numOfIps * 4; + pHBRsp->ipList.numOfIps = pSdbPublicIpList->numOfIps; + memcpy(pHBRsp->ipList.ip, pSdbPublicIpList->ip, size); + pMsg += sizeof(SHeartBeatRsp) + size; + } else { + int size = pSdbIpList->numOfIps * 4; + pHBRsp->ipList.numOfIps = pSdbIpList->numOfIps; + memcpy(pHBRsp->ipList.ip, pSdbIpList->ip, size); + pMsg += sizeof(SHeartBeatRsp) + size; + } #else pMsg += sizeof(SHeartBeatRsp); #endif @@ -1178,6 +1195,18 @@ int mgmtProcessConnectMsg(char *pMsg, int msgLen, SConnObj *pConn) { pAcct = mgmtGetAcct(pUser->acct); + code = taosCheckVersion(pConnectMsg->clientVersion, version, 3); + if (code != 0) { + mError("invalid client version:%s", pConnectMsg->clientVersion); + goto _rsp; + } + + if (pConnectMsg->isCluster != tsIsCluster) { + mError("Cluster Edition and lite Edition cannot be interconnected, client:%d server:%d", pConnectMsg->isCluster, tsIsCluster); + code = TSDB_CODE_INVALID_CLIENT_VERSION; + goto _rsp; + } + if (pConnectMsg->db[0]) { sprintf(dbName, "%x%s%s", pAcct->acctId, TS_PATH_DELIMITER, pConnectMsg->db); pDb = mgmtGetDb(dbName); @@ -1217,9 +1246,17 @@ _rsp: pMsg += sizeof(SConnectRsp); #ifdef CLUSTER - int size = pSdbPublicIpList->numOfIps * 4 + sizeof(SIpList); - memcpy(pMsg, pSdbPublicIpList, size); - pMsg += size; + if (pConnectMsg->usePublicIp) { + pConn->usePublicIp = 1; + int size = pSdbPublicIpList->numOfIps * 4 + sizeof(SIpList); + memcpy(pMsg, pSdbPublicIpList, size); + pMsg += size; + } + else { + int size = pSdbIpList->numOfIps * 4 + sizeof(SIpList); + memcpy(pMsg, pSdbIpList, size); + pMsg += size; + } #endif // set the time resolution: millisecond or microsecond diff --git a/src/system/detail/src/mgmtVgroup.c b/src/system/detail/src/mgmtVgroup.c index e3bed57b33..d7e935241a 100644 --- a/src/system/detail/src/mgmtVgroup.c +++ b/src/system/detail/src/mgmtVgroup.c @@ -102,13 +102,17 @@ int mgmtInitVgroups() { } taosIdPoolReinit(pVgroup->idPool); -#ifdef CLUSTER - if (pVgroup->vnodeGid[0].publicIp == 0) { - pVgroup->vnodeGid[0].publicIp = inet_addr(tsPublicIp); - pVgroup->vnodeGid[0].ip = inet_addr(tsPrivateIp); - sdbUpdateRow(vgSdb, pVgroup, tsVgUpdateSize, 1); + + if (tsIsCluster) { + /* + * Upgrade from open source version to cluster version for the first time + */ + if (pVgroup->vnodeGid[0].publicIp == 0) { + pVgroup->vnodeGid[0].publicIp = inet_addr(tsPublicIp); + pVgroup->vnodeGid[0].ip = inet_addr(tsPrivateIp); + sdbUpdateRow(vgSdb, pVgroup, tsVgUpdateSize, 1); + } } -#endif mgmtSetDnodeVgid(pVgroup->vnodeGid, pVgroup->numOfVnodes, pVgroup->vgId); } diff --git a/src/util/src/tglobalcfg.c b/src/util/src/tglobalcfg.c index aabc722519..345dbdd4db 100644 --- a/src/util/src/tglobalcfg.c +++ b/src/util/src/tglobalcfg.c @@ -164,6 +164,13 @@ int tsAdminRowLimit = 10240; int tsTscEnableRecordSql = 0; int tsEnableCoreFile = 0; int tsAnyIp = 1; +int tsUsePublicIp = 0; + +#ifdef CLUSTER +int tsIsCluster = 1; +#else +int tsIsCluster = 0; +#endif int tsRpcTimer = 300; int tsRpcMaxTime = 600; // seconds; @@ -780,6 +787,9 @@ static void doInitGlobalConfig() { TSDB_CFG_CTYPE_B_CONFIG, 0, 1, 0, TSDB_CFG_UTYPE_NONE); + tsInitConfigOption(cfg++, "usePublicIp", &tsUsePublicIp, TSDB_CFG_VTYPE_INT, + TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT, + 0, 1, 0, TSDB_CFG_UTYPE_NONE); // version info tsInitConfigOption(cfg++, "gitinfo", gitinfo, TSDB_CFG_VTYPE_STRING, TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT, diff --git a/src/util/src/tutil.c b/src/util/src/tutil.c index e0dcb0aa3f..f21ca96a81 100644 --- a/src/util/src/tutil.c +++ b/src/util/src/tutil.c @@ -24,6 +24,8 @@ #include "ttime.h" #include "ttypes.h" #include "tutil.h" +#include "tlog.h" +#include "taoserror.h" int32_t strdequote(char *z) { if (z == NULL) { @@ -450,10 +452,8 @@ bool taosValidateEncodec(char *encodec) { return false; } iconv_close(cd); - return true; -#else - return true; #endif + return true; } bool taosGetVersionNumber(char *versionStr, int *versionNubmer) { @@ -485,6 +485,36 @@ bool taosGetVersionNumber(char *versionStr, int *versionNubmer) { return true; } +int taosCheckVersion(char *input_client_version, char *input_server_version, int comparedSegments) { + char client_version[64] = {0}; + char server_version[64] = {0}; + int clientVersionNumber[4] = {0}; + int serverVersionNumber[4] = {0}; + + strcpy(client_version, input_client_version); + strcpy(server_version, input_server_version); + + if (!taosGetVersionNumber(client_version, clientVersionNumber)) { + pError("invalid client version:%s", client_version); + return TSDB_CODE_INVALID_CLIENT_VERSION; + } + + if (!taosGetVersionNumber(server_version, serverVersionNumber)) { + pError("invalid server version:%s", server_version); + return TSDB_CODE_INVALID_CLIENT_VERSION; + return NULL; + } + + for(int32_t i = 0; i < comparedSegments; ++i) { + if (clientVersionNumber[i] != serverVersionNumber[i]) { + tscError("the %d-th number of server version:%s not matched with client version:%s", i, server_version, version); + return TSDB_CODE_INVALID_CLIENT_VERSION; + } + } + + return 0; +} + char *taosIpStr(uint32_t ipInt) { static char ipStrArray[3][30]; static int ipStrIndex = 0; From b07e3dfd070d4563f513f6df4a673ddc895afe15 Mon Sep 17 00:00:00 2001 From: slguan Date: Tue, 10 Dec 2019 14:52:16 +0800 Subject: [PATCH 095/123] [TBASE-1241] --- src/util/src/tglobalcfg.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/util/src/tglobalcfg.c b/src/util/src/tglobalcfg.c index 345dbdd4db..d1a61633ae 100644 --- a/src/util/src/tglobalcfg.c +++ b/src/util/src/tglobalcfg.c @@ -783,9 +783,12 @@ static void doInitGlobalConfig() { TSDB_CFG_CTYPE_B_CONFIG, 0, 1, 0, TSDB_CFG_UTYPE_NONE); +#ifdef CLUSTER tsInitConfigOption(cfg++, "anyIp", &tsAnyIp, TSDB_CFG_VTYPE_INT, TSDB_CFG_CTYPE_B_CONFIG, 0, 1, 0, TSDB_CFG_UTYPE_NONE); + } +#endif tsInitConfigOption(cfg++, "usePublicIp", &tsUsePublicIp, TSDB_CFG_VTYPE_INT, TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT, From 085c2296c58bc85d0d3957adb977d7eda5805b43 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Tue, 10 Dec 2019 14:52:41 +0800 Subject: [PATCH 096/123] remove the compiler warnings --- src/client/src/tscFunctionImpl.c | 206 ++++++++++----------- src/client/src/tscSQLParser.c | 301 +++++++++++++------------------ src/client/src/tscSchemaUtil.c | 9 +- src/inc/tschemautil.h | 1 + src/inc/tsqlfunction.h | 2 - src/inc/ttypes.h | 2 +- src/util/src/ttypes.c | 2 +- 7 files changed, 244 insertions(+), 279 deletions(-) diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index d7f577fa4f..37bd46c75b 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -69,7 +69,8 @@ for (int32_t i = 0; i < (ctx)->tagInfo.numOfTagCols; ++i) { \ } \ } while(0); -void noop(SQLFunctionCtx *UNUSED_PARAM(pCtx)) {} +void noop1(SQLFunctionCtx *UNUSED_PARAM(pCtx)) {} +void noop2(SQLFunctionCtx *UNUSED_PARAM(pCtx), int32_t UNUSED_PARAM(index)) {} typedef struct tValuePair { tVariant v; @@ -507,10 +508,10 @@ static void do_sum(SQLFunctionCtx *pCtx) { assert(pCtx->size >= pCtx->preAggVals.numOfNull); if (pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_BIGINT) { - int64_t *retVal = pCtx->aOutputBuf; + int64_t *retVal = (int64_t*) pCtx->aOutputBuf; *retVal += pCtx->preAggVals.sum; } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE || pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - double *retVal = pCtx->aOutputBuf; + double *retVal = (double*) pCtx->aOutputBuf; *retVal += GET_DOUBLE_VAL(&(pCtx->preAggVals.sum)); } } else { // computing based on the true data block @@ -518,7 +519,7 @@ static void do_sum(SQLFunctionCtx *pCtx) { notNullElems = 0; if (pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_BIGINT) { - int64_t *retVal = pCtx->aOutputBuf; + int64_t *retVal = (int64_t*) pCtx->aOutputBuf; if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { LIST_ADD_N(*retVal, pCtx, pData, int8_t, notNullElems, pCtx->inputType); @@ -530,10 +531,10 @@ static void do_sum(SQLFunctionCtx *pCtx) { LIST_ADD_N(*retVal, pCtx, pData, int64_t, notNullElems, pCtx->inputType); } } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { - double *retVal = pCtx->aOutputBuf; + double *retVal = (double*) pCtx->aOutputBuf; LIST_ADD_N(*retVal, pCtx, pData, double, notNullElems, pCtx->inputType); } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - double *retVal = pCtx->aOutputBuf; + double *retVal = (double*) pCtx->aOutputBuf; LIST_ADD_N(*retVal, pCtx, pData, float, notNullElems, pCtx->inputType); } } @@ -553,7 +554,7 @@ static void do_sum_f(SQLFunctionCtx *pCtx, int32_t index) { } SET_VAL(pCtx, 1, 1); - int64_t *res = pCtx->aOutputBuf; + int64_t *res = (int64_t*) pCtx->aOutputBuf; if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { *res += GET_INT8_VAL(pData); @@ -564,10 +565,10 @@ static void do_sum_f(SQLFunctionCtx *pCtx, int32_t index) { } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT) { *res += GET_INT64_VAL(pData); } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { - double *retVal = pCtx->aOutputBuf; + double *retVal = (double*) pCtx->aOutputBuf; *retVal += GET_DOUBLE_VAL(pData); } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - double *retVal = pCtx->aOutputBuf; + double *retVal = (double*) pCtx->aOutputBuf; *retVal += GET_FLOAT_VAL(pData); } @@ -694,7 +695,7 @@ static int32_t first_dist_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY return BLK_DATA_NO_NEEDED; } - SFirstLastInfo *pInfo = (pCtx->aOutputBuf + pCtx->inputBytes); + SFirstLastInfo *pInfo = (SFirstLastInfo*) (pCtx->aOutputBuf + pCtx->inputBytes); if (pInfo->hasResult != DATA_SET_FLAG) { return BLK_DATA_ALL_NEEDED; } else { // data in current block is not earlier than current result @@ -708,7 +709,7 @@ static int32_t last_dist_data_req_info(SQLFunctionCtx *pCtx, TSKEY start, TSKEY return BLK_DATA_NO_NEEDED; } - SFirstLastInfo *pInfo = (pCtx->aOutputBuf + pCtx->inputBytes); + SFirstLastInfo *pInfo = (SFirstLastInfo*) (pCtx->aOutputBuf + pCtx->inputBytes); if (pInfo->hasResult != DATA_SET_FLAG) { return BLK_DATA_ALL_NEEDED; } else { @@ -842,7 +843,7 @@ static void avg_func_merge(SQLFunctionCtx *pCtx) { static void avg_func_second_merge(SQLFunctionCtx *pCtx) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); - double *sum = pCtx->aOutputBuf; + double *sum = (double*) pCtx->aOutputBuf; char * input = GET_INPUT_CHAR(pCtx); for (int32_t i = 0; i < pCtx->size; ++i, input += pCtx->inputBytes) { @@ -967,10 +968,10 @@ static void minMax_function(SQLFunctionCtx *pCtx, char *pOutput, int32_t isMin, TYPED_LOOPCHECK_N(int16_t, pOutput, p, pCtx, pCtx->inputType, isMin, *notNullElems); } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { int32_t *pData = p; - int32_t *retVal = pOutput; + int32_t *retVal = (int32_t*) pOutput; for (int32_t i = 0; i < pCtx->size; ++i) { - if (pCtx->hasNull && isNull(&pData[i], pCtx->inputType)) { + if (pCtx->hasNull && isNull((const char*)&pData[i], pCtx->inputType)) { continue; } @@ -1215,27 +1216,27 @@ static void minMax_function_f(SQLFunctionCtx *pCtx, int32_t index, int32_t isMin UPDATE_DATA(pCtx, *output, i, num, isMin, key); } else if (pCtx->inputType == TSDB_DATA_TYPE_SMALLINT) { - int16_t *output = pCtx->aOutputBuf; + int16_t *output = (int16_t*) pCtx->aOutputBuf; int16_t i = GET_INT16_VAL(pData); UPDATE_DATA(pCtx, *output, i, num, isMin, key); } else if (pCtx->inputType == TSDB_DATA_TYPE_INT) { - int32_t *output = pCtx->aOutputBuf; + int32_t *output = (int32_t*) pCtx->aOutputBuf; int32_t i = GET_INT32_VAL(pData); UPDATE_DATA(pCtx, *output, i, num, isMin, key); } else if (pCtx->inputType == TSDB_DATA_TYPE_BIGINT) { - int64_t *output = pCtx->aOutputBuf; + int64_t *output = (int64_t*) pCtx->aOutputBuf; int64_t i = GET_INT64_VAL(pData); UPDATE_DATA(pCtx, *output, i, num, isMin, key); } else if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT) { - float *output = pCtx->aOutputBuf; + float *output = (float*) pCtx->aOutputBuf; float i = GET_FLOAT_VAL(pData); UPDATE_DATA(pCtx, *output, i, num, isMin, key); } else if (pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { - double *output = pCtx->aOutputBuf; + double *output = (double*) pCtx->aOutputBuf; double i = GET_DOUBLE_VAL(pData); UPDATE_DATA(pCtx, *output, i, num, isMin, key); @@ -1299,7 +1300,7 @@ static void stddev_function(SQLFunctionCtx *pCtx) { switch (pCtx->inputType) { case TSDB_DATA_TYPE_INT: { for (int32_t i = 0; i < pCtx->size; ++i) { - if (pCtx->hasNull && isNull(&((int32_t *)pData)[i], pCtx->inputType)) { + if (pCtx->hasNull && isNull((const char*) (&((int32_t *)pData)[i]), pCtx->inputType)) { continue; } *retVal += POW2(((int32_t *)pData)[i] - avg); @@ -1583,7 +1584,7 @@ static void first_dist_func_second_merge(SQLFunctionCtx *pCtx) { assert(pCtx->resultInfo->superTableQ); char * pData = GET_INPUT_CHAR(pCtx); - SFirstLastInfo *pInput = (pData + pCtx->outputBytes); + SFirstLastInfo *pInput = (SFirstLastInfo*) (pData + pCtx->outputBytes); if (pInput->hasResult != DATA_SET_FLAG) { return; } @@ -1761,7 +1762,7 @@ static void last_dist_func_merge(SQLFunctionCtx *pCtx) { static void last_dist_func_second_merge(SQLFunctionCtx *pCtx) { char *pData = GET_INPUT_CHAR(pCtx); - SFirstLastInfo *pInput = (pData + pCtx->outputBytes); + SFirstLastInfo *pInput = (SFirstLastInfo*) (pData + pCtx->outputBytes); if (pInput->hasResult != DATA_SET_FLAG) { return; } @@ -1869,7 +1870,7 @@ static void do_top_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, val.i64Key >= pList[pInfo->num - 1]->v.i64Key) || ((type >= TSDB_DATA_TYPE_FLOAT && type <= TSDB_DATA_TYPE_DOUBLE) && val.dKey >= pList[pInfo->num - 1]->v.dKey)) { - valuePairAssign(pList[pInfo->num], type, &val.i64Key, ts, pTags, pTagInfo, stage); + valuePairAssign(pList[pInfo->num], type, (const char*)&val.i64Key, ts, pTags, pTagInfo, stage); } else { int32_t i = pInfo->num - 1; @@ -1885,7 +1886,7 @@ static void do_top_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, } } - valuePairAssign(pList[i + 1], type, &val.i64Key, ts, pTags, pTagInfo, stage); + valuePairAssign(pList[i + 1], type, (const char*) &val.i64Key, ts, pTags, pTagInfo, stage); } pInfo->num++; @@ -1907,7 +1908,7 @@ static void do_top_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, } } - valuePairAssign(pList[i], type, &val.i64Key, ts, pTags, pTagInfo, stage); + valuePairAssign(pList[i], type, (const char*) &val.i64Key, ts, pTags, pTagInfo, stage); } } } @@ -1921,7 +1922,7 @@ static void do_bottom_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pDa if (pInfo->num < maxLen) { if (pInfo->num == 0) { - valuePairAssign(pList[pInfo->num], type, &val.i64Key, ts, pTags, pTagInfo, stage); + valuePairAssign(pList[pInfo->num], type, (const char*) &val.i64Key, ts, pTags, pTagInfo, stage); } else { int32_t i = pInfo->num - 1; @@ -1937,7 +1938,7 @@ static void do_bottom_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pDa } } - valuePairAssign(pList[i + 1], type, &val.i64Key, ts, pTags, pTagInfo, stage); + valuePairAssign(pList[i + 1], type, (const char*)&val.i64Key, ts, pTags, pTagInfo, stage); } pInfo->num++; @@ -1959,7 +1960,7 @@ static void do_bottom_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pDa } } - valuePairAssign(pList[i], type, &val.i64Key, ts, pTags, pTagInfo, stage); + valuePairAssign(pList[i], type, (const char*)&val.i64Key, ts, pTags, pTagInfo, stage); } } } @@ -2099,7 +2100,7 @@ bool top_bot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, char *mi return true; } - tValuePair *pRes = pTopBotInfo->res; + tValuePair *pRes = (tValuePair*) pTopBotInfo->res; if (functionId == TSDB_FUNC_TOP) { switch (pCtx->inputType) { @@ -2151,7 +2152,7 @@ static STopBotInfo *getTopBotOutputInfo(SQLFunctionCtx *pCtx) { // only the first_stage_merge is directly written data into final output buffer if (pResInfo->superTableQ && pCtx->currentStage != SECONDARY_STAGE_MERGE) { - return pCtx->aOutputBuf; + return (STopBotInfo*) pCtx->aOutputBuf; } else { // for normal table query and super table at the secondary_stage, result is written to intermediate buffer return pResInfo->interResultBuf; } @@ -2165,14 +2166,14 @@ static STopBotInfo *getTopBotOutputInfo(SQLFunctionCtx *pCtx) { */ static void buildTopBotStruct(STopBotInfo *pTopBotInfo, SQLFunctionCtx *pCtx) { char *tmp = (char *)pTopBotInfo + sizeof(STopBotInfo); - pTopBotInfo->res = tmp; + pTopBotInfo->res = (tValuePair**) tmp; tmp += POINTER_BYTES * pCtx->param[0].i64Key; size_t size = sizeof(tValuePair) + pCtx->tagInfo.tagsLen; for (int32_t i = 0; i < pCtx->param[0].i64Key; ++i) { - pTopBotInfo->res[i] = tmp; + pTopBotInfo->res[i] = (tValuePair*) tmp; pTopBotInfo->res[i]->pTags = tmp + sizeof(tValuePair); tmp += size; } @@ -2477,7 +2478,7 @@ static SAPercentileInfo *getAPerctInfo(SQLFunctionCtx *pCtx) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); if (pResInfo->superTableQ && pCtx->currentStage != SECONDARY_STAGE_MERGE) { - return pCtx->aOutputBuf; + return (SAPercentileInfo*) pCtx->aOutputBuf; } else { return pResInfo->interResultBuf; } @@ -2588,8 +2589,8 @@ static void apercentile_func_merge(SQLFunctionCtx *pCtx) { SAPercentileInfo *pInput = (SAPercentileInfo *)GET_INPUT_CHAR(pCtx); - pInput->pHisto = (char *)pInput + sizeof(SAPercentileInfo); - pInput->pHisto->elems = (char *)pInput->pHisto + sizeof(SHistogramInfo); + pInput->pHisto = (SHistogramInfo*) ((char *)pInput + sizeof(SAPercentileInfo)); + pInput->pHisto->elems = (SHistBin*) ((char *)pInput->pHisto + sizeof(SHistogramInfo)); if (pInput->pHisto->numOfElems <= 0) { return; @@ -2602,13 +2603,13 @@ static void apercentile_func_merge(SQLFunctionCtx *pCtx) { if (pHisto->numOfElems <= 0) { memcpy(pHisto, pInput->pHisto, size); - pHisto->elems = (char *)pHisto + sizeof(SHistogramInfo); + pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); } else { - pHisto->elems = (char *)pHisto + sizeof(SHistogramInfo); + pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); SHistogramInfo *pRes = tHistogramMerge(pHisto, pInput->pHisto, MAX_HISTOGRAM_BIN); memcpy(pHisto, pRes, sizeof(SHistogramInfo) + sizeof(SHistBin) * MAX_HISTOGRAM_BIN); - pHisto->elems = (char *)pHisto + sizeof(SHistogramInfo); + pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); tHistogramDestroy(&pRes); } @@ -2620,8 +2621,8 @@ static void apercentile_func_merge(SQLFunctionCtx *pCtx) { static void apercentile_func_second_merge(SQLFunctionCtx *pCtx) { SAPercentileInfo *pInput = (SAPercentileInfo *)GET_INPUT_CHAR(pCtx); - pInput->pHisto = (char *)pInput + sizeof(SAPercentileInfo); - pInput->pHisto->elems = (char *)pInput->pHisto + sizeof(SHistogramInfo); + pInput->pHisto = (SHistogramInfo*) ((char *)pInput + sizeof(SAPercentileInfo)); + pInput->pHisto->elems = (SHistBin*) ((char *)pInput->pHisto + sizeof(SHistogramInfo)); if (pInput->pHisto->numOfElems <= 0) { return; @@ -2632,9 +2633,9 @@ static void apercentile_func_second_merge(SQLFunctionCtx *pCtx) { if (pHisto->numOfElems <= 0) { memcpy(pHisto, pInput->pHisto, sizeof(SHistogramInfo) + sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1)); - pHisto->elems = (char *)pHisto + sizeof(SHistogramInfo); + pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); } else { - pHisto->elems = (char *)pHisto + sizeof(SHistogramInfo); + pHisto->elems = (SHistBin*) ((char *)pHisto + sizeof(SHistogramInfo)); SHistogramInfo *pRes = tHistogramMerge(pHisto, pInput->pHisto, MAX_HISTOGRAM_BIN); tHistogramDestroy(&pOutput->pHisto); @@ -2728,7 +2729,7 @@ static void leastsquares_function(SQLFunctionCtx *pCtx) { int32_t *p = pData; // LEASTSQR_CAL_LOOP(pCtx, param, pParamData, p); for (int32_t i = 0; i < pCtx->size; ++i) { - if (pCtx->hasNull && isNull(p, pCtx->inputType)) { + if (pCtx->hasNull && isNull((const char*) p, pCtx->inputType)) { continue; } @@ -2870,6 +2871,10 @@ static void date_col_output_function(SQLFunctionCtx *pCtx) { *(int64_t *)(pCtx->aOutputBuf) = pCtx->nStartQueryTimestamp; } +static FORCE_INLINE void date_col_output_function_f(SQLFunctionCtx *pCtx, int32_t index) { + date_col_output_function(pCtx); +} + static void col_project_function(SQLFunctionCtx *pCtx) { INC_INIT_VAL(pCtx, pCtx->size); @@ -2980,7 +2985,7 @@ static void diff_function(SQLFunctionCtx *pCtx) { int32_t *pOutput = (int32_t *)pCtx->aOutputBuf; for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull(&pData[i], pCtx->inputType)) { + if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) { continue; } @@ -3012,7 +3017,7 @@ static void diff_function(SQLFunctionCtx *pCtx) { int64_t *pOutput = (int64_t *)pCtx->aOutputBuf; for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull(&pData[i], pCtx->inputType)) { + if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) { continue; } @@ -3044,7 +3049,7 @@ static void diff_function(SQLFunctionCtx *pCtx) { double *pOutput = (double *)pCtx->aOutputBuf; for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull(&pData[i], pCtx->inputType)) { + if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) { continue; } @@ -3074,7 +3079,7 @@ static void diff_function(SQLFunctionCtx *pCtx) { float *pOutput = (float *)pCtx->aOutputBuf; for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull(&pData[i], pCtx->inputType)) { + if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) { continue; } @@ -3105,7 +3110,7 @@ static void diff_function(SQLFunctionCtx *pCtx) { int16_t *pOutput = (int16_t *)pCtx->aOutputBuf; for (; i < pCtx->size && i >= 0; i += step) { - if (pCtx->hasNull && isNull(&pData[i], pCtx->inputType)) { + if (pCtx->hasNull && isNull((const char*) &pData[i], pCtx->inputType)) { continue; } @@ -3275,7 +3280,7 @@ static void arithmetic_function(SQLFunctionCtx *pCtx) { pCtx->aOutputBuf += pCtx->outputBytes * pCtx->size * GET_FORWARD_DIRECTION_FACTOR(pCtx->order); } -static bool arithmetic_function_f(SQLFunctionCtx *pCtx, int32_t index) { +static void arithmetic_function_f(SQLFunctionCtx *pCtx, int32_t index) { INC_INIT_VAL(pCtx, 1); SArithmeticSupport *sas = (SArithmeticSupport *)pCtx->param[0].pz; @@ -3284,7 +3289,6 @@ static bool arithmetic_function_f(SQLFunctionCtx *pCtx, int32_t index) { arithmetic_callback_function); pCtx->aOutputBuf += pCtx->outputBytes * GET_FORWARD_DIRECTION_FACTOR(pCtx->order); - return true; } #define LIST_MINMAX_N(ctx, minOutput, maxOutput, elemCnt, data, type, tsdbType, numOfNotNullElem) \ @@ -3707,7 +3711,7 @@ static void getStatics_i16(int64_t *primaryKey, int16_t *data, int32_t numOfRow, // int16_t lastVal = TSDB_DATA_SMALLINT_NULL; for (int32_t i = 0; i < numOfRow; ++i) { - if (isNull(&data[i], TSDB_DATA_TYPE_SMALLINT)) { + if (isNull((const char*) &data[i], TSDB_DATA_TYPE_SMALLINT)) { (*numOfNull) += 1; continue; } @@ -3747,7 +3751,7 @@ static void getStatics_i32(int64_t *primaryKey, int32_t *data, int32_t numOfRow, // int32_t lastVal = TSDB_DATA_INT_NULL; for (int32_t i = 0; i < numOfRow; ++i) { - if (isNull(&data[i], TSDB_DATA_TYPE_INT)) { + if (isNull((const char*) &data[i], TSDB_DATA_TYPE_INT)) { (*numOfNull) += 1; continue; } @@ -3784,7 +3788,7 @@ static void getStatics_i64(int64_t *primaryKey, int64_t *data, int32_t numOfRow, assert(numOfRow <= INT16_MAX); for (int32_t i = 0; i < numOfRow; ++i) { - if (isNull(&data[i], TSDB_DATA_TYPE_BIGINT)) { + if (isNull((const char*) &data[i], TSDB_DATA_TYPE_BIGINT)) { (*numOfNull) += 1; continue; } @@ -3822,7 +3826,7 @@ static void getStatics_f(int64_t *primaryKey, float *data, int32_t numOfRow, dou assert(numOfRow <= INT16_MAX); for (int32_t i = 0; i < numOfRow; ++i) { - if (isNull(&data[i], TSDB_DATA_TYPE_FLOAT)) { + if (isNull((const char*) &data[i], TSDB_DATA_TYPE_FLOAT)) { (*numOfNull) += 1; continue; } @@ -3875,7 +3879,7 @@ static void getStatics_d(int64_t *primaryKey, double *data, int32_t numOfRow, do assert(numOfRow <= INT16_MAX); for (int32_t i = 0; i < numOfRow; ++i) { - if (isNull(&data[i], TSDB_DATA_TYPE_DOUBLE)) { + if (isNull((const char*) &data[i], TSDB_DATA_TYPE_DOUBLE)) { (*numOfNull) += 1; continue; } @@ -3936,9 +3940,9 @@ void getStatistics(char *priData, char *data, int32_t size, int32_t numOfRow, in } else if (type == TSDB_DATA_TYPE_BIGINT || type == TSDB_DATA_TYPE_TIMESTAMP) { getStatics_i64(primaryKey, (int64_t *)data, numOfRow, min, max, sum, minIndex, maxIndex, numOfNull); } else if (type == TSDB_DATA_TYPE_DOUBLE) { - getStatics_d(primaryKey, (double *)data, numOfRow, min, max, sum, minIndex, maxIndex, numOfNull); + getStatics_d(primaryKey, (double *)data, numOfRow, (double*) min, (double*) max, (double*) sum, minIndex, maxIndex, numOfNull); } else if (type == TSDB_DATA_TYPE_FLOAT) { - getStatics_f(primaryKey, (float *)data, numOfRow, min, max, sum, minIndex, maxIndex, numOfNull); + getStatics_f(primaryKey, (float *)data, numOfRow, (double*) min, (double*) max, (double*) sum, minIndex, maxIndex, numOfNull); } } } @@ -4056,44 +4060,42 @@ static void twa_function(SQLFunctionCtx *pCtx) { // pCtx->numOfIteratedElems += notNullElems; } -static bool twa_function_f(SQLFunctionCtx *pCtx, int32_t index) { +static void twa_function_f(SQLFunctionCtx *pCtx, int32_t index) { void *pData = GET_INPUT_CHAR_INDEX(pCtx, index); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { - return true; + return; } - + SET_VAL(pCtx, 1, 1); - + TSKEY *primaryKey = pCtx->ptsList; - + SResultInfo *pResInfo = GET_RES_INFO(pCtx); - STwaInfo * pInfo = pResInfo->interResultBuf; - + STwaInfo *pInfo = pResInfo->interResultBuf; + if (pInfo->lastKey == INT64_MIN) { pInfo->lastKey = pCtx->nStartQueryTimestamp; setTWALastVal(pCtx, pData, 0, pInfo); - + pInfo->hasResult = DATA_SET_FLAG; } - + if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT || pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { pInfo->dOutput += pInfo->dLastValue * (primaryKey[index] - pInfo->lastKey); } else { pInfo->iOutput += pInfo->iLastValue * (primaryKey[index] - pInfo->lastKey); } - + // record the last key/value pInfo->lastKey = primaryKey[index]; setTWALastVal(pCtx, pData, 0, pInfo); - + // pCtx->numOfIteratedElems += 1; pResInfo->hasResult = DATA_SET_FLAG; - + if (pResInfo->superTableQ) { memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(STwaInfo)); } - - return true; } static void twa_func_merge(SQLFunctionCtx *pCtx) { @@ -4105,7 +4107,7 @@ static void twa_func_merge(SQLFunctionCtx *pCtx) { int32_t numOfNotNull = 0; for (int32_t i = 0; i < pCtx->size; ++i, indicator += sizeof(STwaInfo)) { - STwaInfo *pInput = indicator; + STwaInfo *pInput = (STwaInfo*) indicator; if (pInput->hasResult != DATA_SET_FLAG) { continue; @@ -4207,7 +4209,7 @@ static void interp_function(SQLFunctionCtx *pCtx) { if (pCtx->outputType == TSDB_DATA_TYPE_FLOAT) { float v = GET_DOUBLE_VAL(pVal); - assignVal(pCtx->aOutputBuf, &v, pCtx->outputBytes, pCtx->outputType); + assignVal(pCtx->aOutputBuf, (const char*) &v, pCtx->outputBytes, pCtx->outputType); } else { assignVal(pCtx->aOutputBuf, pVal, pCtx->outputBytes, pCtx->outputType); } @@ -4368,7 +4370,7 @@ SQLAggFuncElem aAggs[28] = {{ count_function, count_function_f, no_next_step, - noop, + noop1, count_func_merge, count_func_merge, count_load_data_info, @@ -4444,8 +4446,8 @@ SQLAggFuncElem aAggs[28] = {{ stddev_function_f, stddev_next_step, stddev_finalizer, - noop, - noop, + noop1, + noop1, data_req_load_info, }, { @@ -4459,8 +4461,8 @@ SQLAggFuncElem aAggs[28] = {{ percentile_function_f, no_next_step, percentile_finalizer, - noop, - noop, + noop1, + noop1, data_req_load_info, }, { @@ -4489,8 +4491,8 @@ SQLAggFuncElem aAggs[28] = {{ first_function_f, no_next_step, function_finalizer, - noop, - noop, + noop1, + noop1, first_data_req_info, }, { @@ -4504,8 +4506,8 @@ SQLAggFuncElem aAggs[28] = {{ last_function_f, no_next_step, function_finalizer, - noop, - noop, + noop1, + noop1, last_data_req_info, }, { @@ -4517,10 +4519,10 @@ SQLAggFuncElem aAggs[28] = {{ TSDB_FUNCSTATE_SELECTIVITY, first_last_function_setup, last_row_function, - noop, + noop2, no_next_step, last_row_finalizer, - noop, + noop1, last_dist_func_second_merge, data_req_load_info, }, @@ -4597,8 +4599,8 @@ SQLAggFuncElem aAggs[28] = {{ leastsquares_function_f, no_next_step, leastsquares_finalizer, - noop, - noop, + noop1, + noop1, data_req_load_info, }, { @@ -4609,9 +4611,9 @@ SQLAggFuncElem aAggs[28] = {{ TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, function_setup, date_col_output_function, - date_col_output_function, + date_col_output_function_f, no_next_step, - noop, + noop1, copy_function, copy_function, no_data_info, @@ -4623,10 +4625,10 @@ SQLAggFuncElem aAggs[28] = {{ TSDB_FUNC_TS_DUMMY, TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, function_setup, - noop, - noop, + noop1, + noop2, no_next_step, - noop, + noop1, copy_function, copy_function, data_req_load_info, @@ -4639,9 +4641,9 @@ SQLAggFuncElem aAggs[28] = {{ TSDB_BASE_FUNC_SO, function_setup, tag_function, - noop, + noop2, no_next_step, - noop, + noop1, copy_function, copy_function, no_data_info, @@ -4671,7 +4673,7 @@ SQLAggFuncElem aAggs[28] = {{ tag_function, tag_function_f, no_next_step, - noop, + noop1, copy_function, copy_function, no_data_info, @@ -4686,7 +4688,7 @@ SQLAggFuncElem aAggs[28] = {{ col_project_function, col_project_function_f, no_next_step, - noop, + noop1, copy_function, copy_function, data_req_load_info, @@ -4701,7 +4703,7 @@ SQLAggFuncElem aAggs[28] = {{ tag_project_function, tag_project_function_f, no_next_step, - noop, + noop1, copy_function, copy_function, no_data_info, @@ -4716,7 +4718,7 @@ SQLAggFuncElem aAggs[28] = {{ arithmetic_function, arithmetic_function_f, no_next_step, - noop, + noop1, copy_function, copy_function, data_req_load_info, @@ -4731,9 +4733,9 @@ SQLAggFuncElem aAggs[28] = {{ diff_function, diff_function_f, no_next_step, - noop, - noop, - noop, + noop1, + noop1, + noop1, data_req_load_info, }, // distributed version used in two-stage aggregation processes @@ -4777,8 +4779,8 @@ SQLAggFuncElem aAggs[28] = {{ interp_function, do_sum_f, // todo filter handle no_next_step, - noop, - noop, + noop1, + noop1, copy_function, no_data_info, }}; diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index bc3079e09a..d96e15eb5b 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -20,13 +20,13 @@ #include "taos.h" #include "taosmsg.h" #include "tstoken.h" -#include "ttime.h" #include "tstrbuild.h" +#include "ttime.h" +#include "tscSQLParser.h" #include "tscUtil.h" #include "tschemautil.h" #include "tsclient.h" -#include "tscSQLParser.h" #define DEFAULT_PRIMARY_TIMESTAMP_COL_NAME "_c0" @@ -51,7 +51,7 @@ typedef struct SColumnIdListRes { static SSqlExpr* doAddProjectCol(SSqlCmd* pCmd, int32_t outputIndex, int32_t colIdx, int32_t tableIndex); static int32_t setShowInfo(SSqlObj* pSql, SSqlInfo* pInfo); -static char* getAccountId(SSqlObj* pSql); +static char* getAccountId(SSqlObj* pSql); static bool has(tFieldList* pFieldList, int32_t startIdx, const char* name); static void getCurrentDBName(SSqlObj* pSql, SSQLToken* pDBToken); @@ -110,7 +110,7 @@ static int32_t optrToString(tSQLExpr* pExpr, char** exprString); static int32_t getMeterIndex(SSQLToken* pTableToken, SSqlCmd* pCmd, SColumnIndex* pIndex); static int32_t doFunctionsCompatibleCheck(SSqlObj* pSql); static int32_t doLocalQueryProcess(SQuerySQL* pQuerySql, SSqlCmd* pCmd); -static int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCreateDbMsg *pCreate); +static int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCreateDbMsg* pCreate); static SColumnList getColumnList(int32_t num, int16_t tableIndex, int32_t columnIndex); @@ -118,7 +118,7 @@ static SColumnList getColumnList(int32_t num, int16_t tableIndex, int32_t column * Used during parsing query sql. Since the query sql usually small in length, error position * is not needed in the final error message. */ -static int32_t invalidSqlErrMsg(SSqlCmd *pCmd, const char* errMsg) { +static int32_t invalidSqlErrMsg(SSqlCmd* pCmd, const char* errMsg) { return tscInvalidSQLErrMsg(pCmd->payload, errMsg, NULL); } @@ -255,7 +255,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { } if (pToken->n > TSDB_DB_NAME_LEN) { - const char* msg = "db name too long"; + const char* msg = "db name too long"; return invalidSqlErrMsg(pCmd, msg); } @@ -286,14 +286,13 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { case SHOW_STREAMS: case SHOW_SCORES: case SHOW_GRANTS: - case SHOW_CONFIGS: + case SHOW_CONFIGS: case SHOW_VNODES: { return setShowInfo(pSql, pInfo); } case ALTER_DATABASE: case CREATE_DATABASE: { - if (pInfo->sqlType == ALTER_DATABASE) { pCmd->command = TSDB_SQL_ALTER_DB; } else { @@ -309,7 +308,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { int32_t ret = setObjFullName(pMeterMetaInfo->name, getAccountId(pSql), &(pCreateDB->dbname), NULL, NULL); if (ret != TSDB_CODE_SUCCESS) { - const char* msg2 = "name too long"; + const char* msg2 = "name too long"; return invalidSqlErrMsg(pCmd, msg2); } @@ -361,12 +360,12 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { if (pInfo->pDCLInfo->a[0].n > TSDB_USER_LEN || pInfo->pDCLInfo->a[1].n > TSDB_PASSWORD_LEN) { const char* msg = "name or password too long"; - return invalidSqlErrMsg(pCmd, msg); + return invalidSqlErrMsg(pCmd, msg); } if (tscValidateName(&pInfo->pDCLInfo->a[0]) != TSDB_CODE_SUCCESS) { const char* msg2 = "invalid user/account name"; - return invalidSqlErrMsg(pCmd, msg2); + return invalidSqlErrMsg(pCmd, msg2); } strncpy(pMeterMetaInfo->name, pInfo->pDCLInfo->a[0].z, pInfo->pDCLInfo->a[0].n); // name @@ -400,7 +399,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { pCmd->defaultVal[8] = 0; } else { const char* msg4 = "invalid state option, available options[no, r, w, all]"; - return invalidSqlErrMsg(pCmd, msg4); + return invalidSqlErrMsg(pCmd, msg4); } } } @@ -836,7 +835,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { const char* msg7 = "illegal number of tables in from clause"; const char* msg8 = "too many columns in selection clause"; const char* msg9 = "TWA query requires both the start and end time"; - + int32_t code = TSDB_CODE_SUCCESS; // too many result columns not support order by in query @@ -1017,8 +1016,8 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { } setColumnOffsetValueInResultset(pCmd); - - for(int32_t i = 0; i < pCmd->numOfTables; ++i) { + + for (int32_t i = 0; i < pCmd->numOfTables; ++i) { updateTagColumnIndex(pCmd, i); } @@ -1796,7 +1795,7 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, tSQLExprItem* pItem) { } if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { - SSchema colSchema = {.type = TSDB_DATA_TYPE_BINARY, .bytes = TSDB_METER_NAME_LEN}; + SSchema colSchema = {.type = TSDB_DATA_TYPE_BINARY, .bytes = TSDB_METER_NAME_LEN}; strcpy(colSchema.name, TSQL_TBNAME_L); pCmd->type = TSDB_QUERY_TYPE_STABLE_QUERY; @@ -2160,8 +2159,8 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, int32_t colIdx, tSQLExprItem* pItem int8_t resultType = pSchema[index.columnIndex].type; int16_t resultSize = pSchema[index.columnIndex].bytes; - char val[8] = {0}; - int32_t numOfAddedColumn = 1; + char val[8] = {0}; + int32_t numOfAddedColumn = 1; if (optr == TK_PERCENTILE || optr == TK_APERCENTILE) { tVariantDump(pVariant, val, TSDB_DATA_TYPE_DOUBLE); @@ -2186,7 +2185,6 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, int32_t colIdx, tSQLExprItem* pItem SSqlExpr* pExpr = tscSqlExprInsert(pCmd, colIdx, functionId, &index, resultType, resultSize, resultSize); addExprParams(pExpr, val, TSDB_DATA_TYPE_DOUBLE, sizeof(double), 0); } else { - tVariantDump(pVariant, val, TSDB_DATA_TYPE_BIGINT); int64_t nTop = *((int32_t*)val); @@ -2545,7 +2543,7 @@ int32_t setShowInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { } } } - }else if (type == SHOW_VNODES) { + } else if (type == SHOW_VNODES) { if (NULL == pInfo->pDCLInfo) { return invalidSqlErrMsg(pCmd, "No specified ip of dnode"); } @@ -2674,8 +2672,7 @@ void tscRestoreSQLFunctionForMetricQuery(SSqlCmd* pCmd) { bool hasUnsupportFunctionsForMetricQuery(SSqlCmd* pCmd) { const char* msg1 = "TWA not allowed to apply to super table directly"; - const char* msg2 = "functions not supported for super table"; - const char* msg3 = "TWA only support group by tbname for super table query"; + const char* msg2 = "TWA only support group by tbname for super table query"; // filter sql function not supported by metric query yet. for (int32_t i = 0; i < pCmd->fieldsInfo.numOfOutputCols; ++i) { @@ -2692,7 +2689,7 @@ bool hasUnsupportFunctionsForMetricQuery(SSqlCmd* pCmd) { } if (pCmd->groupbyExpr.numOfGroupCols != 1 || pCmd->groupbyExpr.columnInfo[0].colIdx != TSDB_TBNAME_COLUMN_INDEX) { - invalidSqlErrMsg(pCmd, msg3); + invalidSqlErrMsg(pCmd, msg2); return true; } } @@ -2701,8 +2698,6 @@ bool hasUnsupportFunctionsForMetricQuery(SSqlCmd* pCmd) { } static bool functionCompatibleCheck(SSqlCmd* pCmd) { - const char* msg1 = "column on select clause not allowed"; - int32_t startIdx = 0; int32_t functionID = tscSqlExprGet(pCmd, startIdx)->functionId; @@ -2745,7 +2740,7 @@ void updateTagColumnIndex(SSqlCmd* pCmd, int32_t tableIndex) { if (pCmd->groupbyExpr.numOfGroupCols > 0 && pCmd->groupbyExpr.tableIndex == tableIndex) { for (int32_t i = 0; i < pCmd->groupbyExpr.numOfGroupCols; ++i) { int32_t index = pCmd->groupbyExpr.columnInfo[i].colIdx; - + for (int32_t j = 0; j < pMeterMetaInfo->numOfTags; ++j) { int32_t tagColIndex = pMeterMetaInfo->tagColumnIndex[j]; if (tagColIndex == index) { @@ -2759,11 +2754,11 @@ void updateTagColumnIndex(SSqlCmd* pCmd, int32_t tableIndex) { // update tags column index for expression for (int32_t i = 0; i < pCmd->exprsInfo.numOfExprs; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pCmd, i); - + if (!TSDB_COL_IS_TAG(pExpr->colInfo.flag)) { // not tags, continue continue; } - + // not belongs to this table if (pExpr->uid != pMeterMetaInfo->pMeterMeta->uid) { continue; @@ -2776,40 +2771,37 @@ void updateTagColumnIndex(SSqlCmd* pCmd, int32_t tableIndex) { } } } - + // update join condition tag column index SJoinInfo* pJoinInfo = &pCmd->tagCond.joinInfo; if (!pJoinInfo->hasJoin) { // not join query return; } - + assert(pJoinInfo->left.uid != pJoinInfo->right.uid); - + // the join condition expression node belongs to this table(super table) if (pMeterMetaInfo->pMeterMeta->uid == pJoinInfo->left.uid) { - for(int32_t i = 0; i < pMeterMetaInfo->numOfTags; ++i) { + for (int32_t i = 0; i < pMeterMetaInfo->numOfTags; ++i) { if (pJoinInfo->left.tagCol == pMeterMetaInfo->tagColumnIndex[i]) { pJoinInfo->left.tagCol = i; } } } - + if (pMeterMetaInfo->pMeterMeta->uid == pJoinInfo->right.uid) { - for(int32_t i = 0; i < pMeterMetaInfo->numOfTags; ++i) { + for (int32_t i = 0; i < pMeterMetaInfo->numOfTags; ++i) { if (pJoinInfo->right.tagCol == pMeterMetaInfo->tagColumnIndex[i]) { pJoinInfo->right.tagCol = i; } } } - } int32_t parseGroupbyClause(SSqlCmd* pCmd, tVariantList* pList) { const char* msg1 = "too many columns in group by clause"; const char* msg2 = "invalid column name in group by clause"; - const char* msg4 = "group by only available for STable query"; - const char* msg5 = "group by columns must belong to one table"; - const char* msg6 = "only support group by one ordinary column"; + const char* msg3 = "group by columns must belong to one table"; const char* msg7 = "not support group by expression"; const char* msg8 = "not allowed column type for group by"; const char* msg9 = "tags not allowed for table query"; @@ -2828,9 +2820,8 @@ int32_t parseGroupbyClause(SSqlCmd* pCmd, tVariantList* pList) { SMeterMeta* pMeterMeta = NULL; SSchema* pSchema = NULL; + SSchema s = tsGetTbnameColumnSchema(); - SSchema s = {0}; - int32_t numOfReqTags = 0; int32_t tableIndex = COLUMN_INDEX_INITIAL_VAL; for (int32_t i = 0; i < pList->nExpr; ++i) { @@ -2844,7 +2835,7 @@ int32_t parseGroupbyClause(SSqlCmd* pCmd, tVariantList* pList) { } if (tableIndex != index.tableIndex && tableIndex >= 0) { - return invalidSqlErrMsg(pCmd, msg5); + return invalidSqlErrMsg(pCmd, msg3); } tableIndex = index.tableIndex; @@ -2852,22 +2843,12 @@ int32_t parseGroupbyClause(SSqlCmd* pCmd, tVariantList* pList) { pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, index.tableIndex); pMeterMeta = pMeterMetaInfo->pMeterMeta; - // TODO refactor!!!!!!!!!!!!!!1 if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { - s.colId = TSDB_TBNAME_COLUMN_INDEX; - s.type = TSDB_DATA_TYPE_BINARY; - s.bytes = TSDB_METER_NAME_LEN; - strcpy(s.name, TSQL_TBNAME_L); - pSchema = &s; } else { pSchema = tsGetColumnSchema(pMeterMeta, index.columnIndex); } - int16_t type = 0; - int16_t bytes = 0; - char* name = NULL; - bool groupTag = false; if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX || index.columnIndex >= pMeterMeta->numOfColumns) { groupTag = true; @@ -3138,7 +3119,7 @@ static int32_t optrToString(tSQLExpr* pExpr, char** exprString) { return TSDB_CODE_SUCCESS; } -static int32_t tablenameListToString(tSQLExpr* pExpr, /*char* str*/SStringBuilder* sb) { +static int32_t tablenameListToString(tSQLExpr* pExpr, /*char* str*/ SStringBuilder* sb) { tSQLExprList* pList = pExpr->pParam; if (pList->nExpr <= 0) { return TSDB_CODE_INVALID_SQL; @@ -3148,7 +3129,6 @@ static int32_t tablenameListToString(tSQLExpr* pExpr, /*char* str*/SStringBuilde taosStringBuilderAppendStringLen(sb, QUERY_COND_REL_PREFIX_IN, QUERY_COND_REL_PREFIX_IN_LEN); } - int32_t len = 0; for (int32_t i = 0; i < pList->nExpr; ++i) { tSQLExpr* pSub = pList->a[i].pNode; taosStringBuilderAppendStringLen(sb, pSub->val.pz, pSub->val.nLen); @@ -3165,7 +3145,7 @@ static int32_t tablenameListToString(tSQLExpr* pExpr, /*char* str*/SStringBuilde return TSDB_CODE_SUCCESS; } -static int32_t tablenameCondToString(tSQLExpr* pExpr, /*char* str*/SStringBuilder* sb) { +static int32_t tablenameCondToString(tSQLExpr* pExpr, /*char* str*/ SStringBuilder* sb) { taosStringBuilderAppendStringLen(sb, QUERY_COND_REL_PREFIX_LIKE, QUERY_COND_REL_PREFIX_LIKE_LEN); taosStringBuilderAppendString(sb, pExpr->val.pz); @@ -3187,7 +3167,6 @@ static int32_t extractColumnFilterInfo(SSqlCmd* pCmd, SColumnIndex* pIndex, tSQL const char* msg1 = "non binary column not support like operator"; const char* msg2 = "binary column not support this operator"; - const char* msg3 = "OR is not supported on different column filter"; SColumnBase* pColumn = tscColumnBaseInfoInsert(pCmd, pIndex); SColumnFilterInfo* pColFilter = NULL; @@ -3271,7 +3250,7 @@ static int32_t getTagCondString(SSqlCmd* pCmd, tSQLExpr* pExpr, char** str) { return tSQLExprLeafToString(pExpr, true, str); } -static int32_t getTablenameCond(SSqlCmd* pCmd, tSQLExpr* pTableCond, /*char* str*/SStringBuilder* sb) { +static int32_t getTablenameCond(SSqlCmd* pCmd, tSQLExpr* pTableCond, /*char* str*/ SStringBuilder* sb) { const char* msg0 = "invalid table name list"; if (pTableCond == NULL) { @@ -3615,10 +3594,9 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, tSQLExpr** pExpr, SCondExpr* const char* msg2 = "illegal column name"; const char* msg3 = "only one query time range allowed"; const char* msg4 = "only one join condition allowed"; - const char* msg5 = "AND is allowed to filter on different ordinary columns"; - const char* msg6 = "not support ordinary column join"; - const char* msg7 = "only one query condition on tbname allowed"; - const char* msg8 = "only in/like allowed in filter table name"; + const char* msg5 = "not support ordinary column join"; + const char* msg6 = "only one query condition on tbname allowed"; + const char* msg7 = "only in/like allowed in filter table name"; tSQLExpr* pLeft = (*pExpr)->pLeft; tSQLExpr* pRight = (*pExpr)->pRight; @@ -3680,7 +3658,7 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, tSQLExpr** pExpr, SCondExpr* // in case of in operator, keep it in a seperate attribute if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { if (!validTableNameOptr(*pExpr)) { - return invalidSqlErrMsg(pCmd, msg8); + return invalidSqlErrMsg(pCmd, msg7); } if (pCondExpr->pTableCond == NULL) { @@ -3688,7 +3666,7 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, tSQLExpr** pExpr, SCondExpr* pCondExpr->relType = parentOptr; pCondExpr->tableCondIndex = index.tableIndex; } else { - return invalidSqlErrMsg(pCmd, msg7); + return invalidSqlErrMsg(pCmd, msg6); } *type = TSQL_EXPR_TBNAME; @@ -3719,7 +3697,7 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, tSQLExpr** pExpr, SCondExpr* *type = TSQL_EXPR_COLUMN; if (pRight->nSQLOptr == TK_ID) { // other column cannot be served as the join column - return invalidSqlErrMsg(pCmd, msg6); + return invalidSqlErrMsg(pCmd, msg5); } ret = setExprToCond(pCmd, &pCondExpr->pColumnCond, *pExpr, NULL, parentOptr); @@ -3886,9 +3864,9 @@ static int32_t setTableCondForMetricQuery(SSqlObj* pSql, tSQLExpr* pExpr, int16_ // remove the duplicated input table names int32_t num = 0; - char* tableNameString = taosStringBuilderGetResult(sb, NULL); - - char** segments = strsplit(tableNameString + QUERY_COND_REL_PREFIX_IN_LEN, TBNAME_LIST_SEP, &num); + char* tableNameString = taosStringBuilderGetResult(sb, NULL); + + char** segments = strsplit(tableNameString + QUERY_COND_REL_PREFIX_IN_LEN, TBNAME_LIST_SEP, &num); qsort(segments, num, POINTER_BYTES, tableNameCompar); int32_t j = 1; @@ -3906,8 +3884,8 @@ static int32_t setTableCondForMetricQuery(SSqlObj* pSql, tSQLExpr* pExpr, int16_ if (i >= 1) { taosStringBuilderAppendStringLen(&sb1, TBNAME_LIST_SEP, 1); } - - char idBuf[TSDB_METER_ID_LEN + 1] = {0}; + + char idBuf[TSDB_METER_ID_LEN + 1] = {0}; int32_t xlen = strlen(segments[i]); SSQLToken t = {.z = segments[i], .n = xlen, .type = TK_STRING}; @@ -3915,17 +3893,17 @@ static int32_t setTableCondForMetricQuery(SSqlObj* pSql, tSQLExpr* pExpr, int16_ if (ret != TSDB_CODE_SUCCESS) { taosStringBuilderDestroy(&sb1); tfree(segments); - + invalidSqlErrMsg(pCmd, msg); return ret; } - + taosStringBuilderAppendString(&sb1, idBuf); } - + char* str = taosStringBuilderGetResult(&sb1, NULL); pCmd->tagCond.tbnameCond.cond = strdup(str); - + taosStringBuilderDestroy(&sb1); tfree(segments); return TSDB_CODE_SUCCESS; @@ -4052,21 +4030,20 @@ static void cleanQueryExpr(SCondExpr* pCondExpr) { } } - static void doAddJoinTagsColumnsIntoTagList(SSqlCmd* pCmd, SCondExpr* pCondExpr) { SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); if (QUERY_IS_JOIN_QUERY(pCmd->type) && UTIL_METER_IS_METRIC(pMeterMetaInfo)) { SColumnIndex index = {0}; - + getColumnIndexByNameEx(&pCondExpr->pJoinExpr->pLeft->colInfo, pCmd, &index); pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, index.tableIndex); - + int32_t columnInfo = index.columnIndex - pMeterMetaInfo->pMeterMeta->numOfColumns; addRequiredTagColumn(pCmd, columnInfo, index.tableIndex); - + getColumnIndexByNameEx(&pCondExpr->pJoinExpr->pRight->colInfo, pCmd, &index); pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, index.tableIndex); - + columnInfo = index.columnIndex - pMeterMetaInfo->pMeterMeta->numOfColumns; addRequiredTagColumn(pCmd, columnInfo, index.tableIndex); } @@ -4074,48 +4051,48 @@ static void doAddJoinTagsColumnsIntoTagList(SSqlCmd* pCmd, SCondExpr* pCondExpr) static int32_t getTagQueryCondExpr(SSqlCmd* pCmd, SCondExpr* pCondExpr, tSQLExpr** pExpr) { int32_t ret = TSDB_CODE_SUCCESS; - + if (pCondExpr->pTagCond != NULL) { for (int32_t i = 0; i < pCmd->numOfTables; ++i) { tSQLExpr* p1 = extractExprForSTable(pExpr, pCmd, i); - + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, i); - + char c[TSDB_MAX_TAGS_LEN] = {0}; char* str = c; - + if ((ret = getTagCondString(pCmd, p1, &str)) != TSDB_CODE_SUCCESS) { return ret; } - + tsSetMetricQueryCond(&pCmd->tagCond, pMeterMetaInfo->pMeterMeta->uid, c); - + doCompactQueryExpr(pExpr); tSQLExprDestroy(p1); } - + pCondExpr->pTagCond = NULL; } - + return ret; } int32_t parseWhereClause(SSqlObj* pSql, tSQLExpr** pExpr) { if (pExpr == NULL) { return TSDB_CODE_SUCCESS; } - + const char* msg = "invalid filter expression"; const char* msg1 = "invalid expression"; - + int32_t ret = TSDB_CODE_SUCCESS; - + SSqlCmd* pCmd = &pSql->cmd; pCmd->stime = 0; pCmd->etime = INT64_MAX; - //tags query condition may be larger than 512bytes, therefore, we need to prepare enough large space + // tags query condition may be larger than 512bytes, therefore, we need to prepare enough large space SStringBuilder sb = {0}; - SCondExpr condExpr = {0}; + SCondExpr condExpr = {0}; if ((*pExpr)->pLeft == NULL || (*pExpr)->pRight == NULL) { return invalidSqlErrMsg(pCmd, msg1); @@ -4125,54 +4102,54 @@ int32_t parseWhereClause(SSqlObj* pSql, tSQLExpr** pExpr) { if ((ret = getQueryCondExpr(pCmd, pExpr, &condExpr, &type, (*pExpr)->nSQLOptr)) != TSDB_CODE_SUCCESS) { return ret; } - + doCompactQueryExpr(pExpr); - + // after expression compact, the expression tree is only include tag query condition condExpr.pTagCond = (*pExpr); - + // 1. check if it is a join query if ((ret = validateJoinExpr(pCmd, &condExpr)) != TSDB_CODE_SUCCESS) { return ret; } - + // 2. get the query time range if ((ret = getTimeRangeFromExpr(pCmd, condExpr.pTimewindow)) != TSDB_CODE_SUCCESS) { return ret; } - + // 3. get the tag query condition if ((ret = getTagQueryCondExpr(pCmd, &condExpr, pExpr)) != TSDB_CODE_SUCCESS) { return ret; } - + // 4. get the table name query condition if ((ret = getTablenameCond(pCmd, condExpr.pTableCond, &sb)) != TSDB_CODE_SUCCESS) { return ret; } - + // 5. other column query condition if ((ret = getColumnQueryCondInfo(pCmd, condExpr.pColumnCond, TK_AND)) != TSDB_CODE_SUCCESS) { return ret; } - + // 6. join condition if ((ret = getJoinCondInfo(pSql, condExpr.pJoinExpr)) != TSDB_CODE_SUCCESS) { return ret; } - + // 7. query condition for table name pCmd->tagCond.relType = (condExpr.relType == TK_AND) ? TSDB_RELATION_AND : TSDB_RELATION_OR; - + ret = setTableCondForMetricQuery(pSql, condExpr.pTableCond, condExpr.tableCondIndex, &sb); taosStringBuilderDestroy(&sb); - + if (!validateFilterExpr(pCmd)) { return invalidSqlErrMsg(pCmd, msg); } - + doAddJoinTagsColumnsIntoTagList(pCmd, &condExpr); - + cleanQueryExpr(&condExpr); return ret; } @@ -4447,7 +4424,6 @@ int32_t parseOrderbyClause(SSqlCmd* pCmd, SQuerySQL* pQuerySql, SSchema* pSchema bool orderByTags = false; bool orderByTS = false; - bool orderByCol = false; if (index.columnIndex >= pMeterMetaInfo->pMeterMeta->numOfColumns) { int32_t relTagIndex = index.columnIndex - pMeterMetaInfo->pMeterMeta->numOfColumns; @@ -4824,7 +4800,6 @@ int32_t validateSqlFunctionInStreamSql(SSqlCmd* pCmd) { int32_t validateFunctionsInIntervalOrGroupbyQuery(SSqlCmd* pCmd) { bool isProjectionFunction = false; const char* msg1 = "column projection is not compatible with interval"; - const char* msg2 = "interval not allowed for tag queries"; // multi-output set/ todo refactor for (int32_t k = 0; k < pCmd->fieldsInfo.numOfOutputCols; ++k) { @@ -5006,7 +4981,7 @@ int32_t parseLimitClause(SSqlObj* pSql, SQuerySQL* pQuerySql) { // handle the limit offset value, validate the limit pCmd->limit = pQuerySql->limit; pCmd->globalLimit = pCmd->limit.limit; - + pCmd->slimit = pQuerySql->slimit; if (pCmd->slimit.offset < 0 || pCmd->limit.offset < 0) { @@ -5076,11 +5051,11 @@ int32_t parseLimitClause(SSqlObj* pSql, SQuerySQL* pQuerySql) { static int32_t setKeepOption(SSqlCmd* pCmd, SCreateDbMsg* pMsg, SCreateDBInfo* pCreateDb) { const char* msg = "invalid number of options"; - + pMsg->daysToKeep = htonl(-1); pMsg->daysToKeep1 = htonl(-1); pMsg->daysToKeep2 = htonl(-1); - + tVariantList* pKeep = pCreateDb->keep; if (pKeep != NULL) { switch (pKeep->nExpr) { @@ -5098,36 +5073,34 @@ static int32_t setKeepOption(SSqlCmd* pCmd, SCreateDbMsg* pMsg, SCreateDBInfo* p pMsg->daysToKeep2 = htonl(pKeep->a[2].pVar.i64Key); break; } - default: { - return invalidSqlErrMsg(pCmd, msg); - } + default: { return invalidSqlErrMsg(pCmd, msg); } } } - + return TSDB_CODE_SUCCESS; } static int32_t setTimePrecisionOption(SSqlCmd* pCmd, SCreateDbMsg* pMsg, SCreateDBInfo* pCreateDbInfo) { const char* msg = "invalid time precision"; - + pMsg->precision = TSDB_TIME_PRECISION_MILLI; // millisecond by default - + SSQLToken* pToken = &pCreateDbInfo->precision; if (pToken->n > 0) { pToken->n = strdequote(pToken->z); - + if (strncmp(pToken->z, TSDB_TIME_PRECISION_MILLI_STR, pToken->n) == 0 && strlen(TSDB_TIME_PRECISION_MILLI_STR) == pToken->n) { // time precision for this db: million second pMsg->precision = TSDB_TIME_PRECISION_MILLI; } else if (strncmp(pToken->z, TSDB_TIME_PRECISION_MICRO_STR, pToken->n) == 0 && - strlen(TSDB_TIME_PRECISION_MICRO_STR) == pToken->n) { + strlen(TSDB_TIME_PRECISION_MICRO_STR) == pToken->n) { pMsg->precision = TSDB_TIME_PRECISION_MICRO; } else { return invalidSqlErrMsg(pCmd, msg); } } - + return TSDB_CODE_SUCCESS; } @@ -5135,7 +5108,7 @@ static void setCreateDBOption(SCreateDbMsg* pMsg, SCreateDBInfo* pCreateDb) { pMsg->blocksPerMeter = htons(pCreateDb->numOfBlocksPerTable); pMsg->compression = pCreateDb->compressionLevel; - pMsg->commitLog = (char) pCreateDb->commitLog; + pMsg->commitLog = (char)pCreateDb->commitLog; pMsg->commitTime = htonl(pCreateDb->commitTime); pMsg->maxSessions = htonl(pCreateDb->tablesPerVnode); pMsg->cacheNumOfBlocks.fraction = pCreateDb->numOfAvgCacheBlocks; @@ -5148,19 +5121,19 @@ static void setCreateDBOption(SCreateDbMsg* pMsg, SCreateDBInfo* pCreateDb) { int32_t parseCreateDBOptions(SSqlCmd* pCmd, SCreateDBInfo* pCreateDbSql) { SCreateDbMsg* pMsg = (SCreateDbMsg*)(pCmd->payload + tsRpcHeadSize + sizeof(SMgmtHead)); setCreateDBOption(pMsg, pCreateDbSql); - + if (setKeepOption(pCmd, pMsg, pCreateDbSql) != TSDB_CODE_SUCCESS) { return TSDB_CODE_INVALID_SQL; } - + if (setTimePrecisionOption(pCmd, pMsg, pCreateDbSql) != TSDB_CODE_SUCCESS) { return TSDB_CODE_INVALID_SQL; } - + if (tscCheckCreateDbParams(pCmd, pMsg) != TSDB_CODE_SUCCESS) { return TSDB_CODE_INVALID_SQL; } - + return TSDB_CODE_SUCCESS; } @@ -5254,9 +5227,6 @@ static void doUpdateSqlFunctionForTagPrj(SSqlCmd* pCmd) { } } - int16_t resType = 0; - int16_t resBytes = 0; - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); SSchema* pSchema = tsGetSchema(pMeterMetaInfo->pMeterMeta); @@ -5353,11 +5323,9 @@ static void updateTagPrjFunction(SSqlCmd* pCmd) { */ static int32_t checkUpdateTagPrjFunctions(SSqlCmd* pCmd) { const char* msg1 = "only one selectivity function allowed in presence of tags function"; - const char* msg2 = "functions not allowed"; const char* msg3 = "aggregation function should not be mixed up with projection"; bool tagColExists = false; - int16_t numOfTimestamp = 0; // primary timestamp column int16_t numOfSelectivity = 0; int16_t numOfAggregation = 0; @@ -5499,13 +5467,10 @@ static int32_t doAddGroupbyColumnsOnDemand(SSqlCmd* pCmd) { int32_t doFunctionsCompatibleCheck(SSqlObj* pSql) { const char* msg1 = "functions/columns not allowed in group by query"; - const char* msg2 = "interval not allowed in group by normal column"; const char* msg3 = "group by not allowed on projection query"; - const char* msg4 = "tags retrieve not compatible with group by"; const char* msg5 = "retrieve tags not compatible with group by or interval query"; - SSqlCmd* pCmd = &pSql->cmd; - SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0); + SSqlCmd* pCmd = &pSql->cmd; // only retrieve tags, group by is not supportted if (pCmd->command == TSDB_SQL_RETRIEVE_TAGS) { @@ -5517,11 +5482,6 @@ int32_t doFunctionsCompatibleCheck(SSqlObj* pSql) { } if (pCmd->groupbyExpr.numOfGroupCols > 0) { - SSchema* pSchema = tsGetSchema(pMeterMetaInfo->pMeterMeta); - int16_t bytes = 0; - int16_t type = 0; - char* name = NULL; - // check if all the tags prj columns belongs to the group by columns if (onlyTagPrjFunction(pCmd) && allTagPrjInGroupby(pCmd)) { updateTagPrjFunction(pCmd); @@ -5639,84 +5599,81 @@ int32_t doLocalQueryProcess(SQuerySQL* pQuerySql, SSqlCmd* pCmd) { case 4: pCmd->command = TSDB_SQL_CURRENT_USER; return TSDB_CODE_SUCCESS; - default: { - return invalidSqlErrMsg(pCmd, msg3); - } + default: { return invalidSqlErrMsg(pCmd, msg3); } } } // can only perform the parameters based on the macro definitation -int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCreateDbMsg *pCreate) { +int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCreateDbMsg* pCreate) { char msg[512] = {0}; - + if (pCreate->commitLog != -1 && (pCreate->commitLog < 0 || pCreate->commitLog > 1)) { snprintf(msg, tListLen(msg), "invalid db option commitLog: %d, only 0 or 1 allowed", pCreate->commitLog); return invalidSqlErrMsg(pCmd, msg); } - + if (pCreate->replications != -1 && (pCreate->replications < TSDB_REPLICA_MIN_NUM || pCreate->replications > TSDB_REPLICA_MAX_NUM)) { - snprintf(msg, tListLen(msg), "invalid db option replications: %d valid range: [%d, %d]", pCreate->replications, TSDB_REPLICA_MIN_NUM, - TSDB_REPLICA_MAX_NUM); + snprintf(msg, tListLen(msg), "invalid db option replications: %d valid range: [%d, %d]", pCreate->replications, + TSDB_REPLICA_MIN_NUM, TSDB_REPLICA_MAX_NUM); return invalidSqlErrMsg(pCmd, msg); } - + int32_t val = htonl(pCreate->daysPerFile); if (val != -1 && (val < TSDB_FILE_MIN_PARTITION_RANGE || val > TSDB_FILE_MAX_PARTITION_RANGE)) { snprintf(msg, tListLen(msg), "invalid db option daysPerFile: %d valid range: [%d, %d]", val, TSDB_FILE_MIN_PARTITION_RANGE, TSDB_FILE_MAX_PARTITION_RANGE); return invalidSqlErrMsg(pCmd, msg); } - + val = htonl(pCreate->rowsInFileBlock); if (val != -1 && (val < TSDB_MIN_ROWS_IN_FILEBLOCK || val > TSDB_MAX_ROWS_IN_FILEBLOCK)) { snprintf(msg, tListLen(msg), "invalid db option rowsInFileBlock: %d valid range: [%d, %d]", val, TSDB_MIN_ROWS_IN_FILEBLOCK, TSDB_MAX_ROWS_IN_FILEBLOCK); return invalidSqlErrMsg(pCmd, msg); } - + val = htonl(pCreate->cacheBlockSize); if (val != -1 && (val < TSDB_MIN_CACHE_BLOCK_SIZE || val > TSDB_MAX_CACHE_BLOCK_SIZE)) { snprintf(msg, tListLen(msg), "invalid db option cacheBlockSize: %d valid range: [%d, %d]", val, TSDB_MIN_CACHE_BLOCK_SIZE, TSDB_MAX_CACHE_BLOCK_SIZE); return invalidSqlErrMsg(pCmd, msg); } - + val = htonl(pCreate->maxSessions); if (val != -1 && (val < TSDB_MIN_TABLES_PER_VNODE || val > TSDB_MAX_TABLES_PER_VNODE)) { - snprintf(msg, tListLen(msg), "invalid db option maxSessions: %d valid range: [%d, %d]", val, TSDB_MIN_TABLES_PER_VNODE, - TSDB_MAX_TABLES_PER_VNODE); + snprintf(msg, tListLen(msg), "invalid db option maxSessions: %d valid range: [%d, %d]", val, + TSDB_MIN_TABLES_PER_VNODE, TSDB_MAX_TABLES_PER_VNODE); return invalidSqlErrMsg(pCmd, msg); } - - if (pCreate->precision != -1 && - (pCreate->precision != TSDB_TIME_PRECISION_MILLI && pCreate->precision != TSDB_TIME_PRECISION_MICRO)) { - snprintf(msg, tListLen(msg), "invalid db option timePrecision: %d valid value: [%d, %d]", pCreate->precision, TSDB_TIME_PRECISION_MILLI, - TSDB_TIME_PRECISION_MICRO); + + if (pCreate->precision != TSDB_TIME_PRECISION_MILLI && pCreate->precision != TSDB_TIME_PRECISION_MICRO) { + snprintf(msg, tListLen(msg), "invalid db option timePrecision: %d valid value: [%d, %d]", pCreate->precision, + TSDB_TIME_PRECISION_MILLI, TSDB_TIME_PRECISION_MICRO); return invalidSqlErrMsg(pCmd, msg); } - + if (pCreate->cacheNumOfBlocks.fraction != -1 && (pCreate->cacheNumOfBlocks.fraction < TSDB_MIN_AVG_BLOCKS || - pCreate->cacheNumOfBlocks.fraction > TSDB_MAX_AVG_BLOCKS)) { - snprintf(msg, tListLen(msg), "invalid db option ablocks: %f valid value: [%d, %d]", pCreate->cacheNumOfBlocks.fraction, - TSDB_MIN_AVG_BLOCKS, TSDB_MAX_AVG_BLOCKS); + pCreate->cacheNumOfBlocks.fraction > TSDB_MAX_AVG_BLOCKS)) { + snprintf(msg, tListLen(msg), "invalid db option ablocks: %f valid value: [%d, %d]", + pCreate->cacheNumOfBlocks.fraction, TSDB_MIN_AVG_BLOCKS, TSDB_MAX_AVG_BLOCKS); return invalidSqlErrMsg(pCmd, msg); } - + val = htonl(pCreate->commitTime); if (val != -1 && (val < TSDB_MIN_COMMIT_TIME_INTERVAL || val > TSDB_MAX_COMMIT_TIME_INTERVAL)) { snprintf(msg, tListLen(msg), "invalid db option commitTime: %d valid range: [%d, %d]", val, TSDB_MIN_COMMIT_TIME_INTERVAL, TSDB_MAX_COMMIT_TIME_INTERVAL); return invalidSqlErrMsg(pCmd, msg); } - + if (pCreate->compression != -1 && (pCreate->compression < TSDB_MIN_COMPRESSION_LEVEL || pCreate->compression > TSDB_MAX_COMPRESSION_LEVEL)) { - snprintf(msg, tListLen(msg), "invalid db option compression: %d valid range: [%d, %d]", pCreate->compression, TSDB_MIN_COMPRESSION_LEVEL, - TSDB_MAX_COMPRESSION_LEVEL); + snprintf(msg, tListLen(msg), "invalid db option compression: %d valid range: [%d, %d]", pCreate->compression, + TSDB_MIN_COMPRESSION_LEVEL, TSDB_MAX_COMPRESSION_LEVEL); return invalidSqlErrMsg(pCmd, msg); } - + return TSDB_CODE_SUCCESS; } @@ -5725,24 +5682,24 @@ void tscPrintSelectClause(SSqlCmd* pCmd) { if (pCmd == NULL || pCmd->exprsInfo.numOfExprs == 0) { return; } - - char* str = calloc(1, 10240); + + char* str = calloc(1, 10240); int32_t offset = 0; - + offset += sprintf(str, "%d [", pCmd->exprsInfo.numOfExprs); - for(int32_t i = 0; i < pCmd->exprsInfo.numOfExprs; ++i) { + for (int32_t i = 0; i < pCmd->exprsInfo.numOfExprs; ++i) { SSqlExpr* pExpr = tscSqlExprGet(pCmd, i); - + int32_t size = sprintf(str + offset, "%s(%d)", aAggs[pExpr->functionId].aName, pExpr->colInfo.colId); offset += size; - + if (i < pCmd->exprsInfo.numOfExprs - 1) { str[offset++] = ','; } } - + str[offset] = ']'; printf("%s\n", str); - + free(str); } diff --git a/src/client/src/tscSchemaUtil.c b/src/client/src/tscSchemaUtil.c index fdbad2bbf8..85ca3eb863 100644 --- a/src/client/src/tscSchemaUtil.c +++ b/src/client/src/tscSchemaUtil.c @@ -83,6 +83,13 @@ struct SSchema* tsGetColumnSchema(SMeterMeta* pMeta, int32_t startCol) { return (SSchema*)(((char*)pMeta + sizeof(SMeterMeta)) + startCol * sizeof(SSchema)); } +struct SSchema tsGetTbnameColumnSchema() { + struct SSchema s = {.colId = TSDB_TBNAME_COLUMN_INDEX, .type = TSDB_DATA_TYPE_BINARY, .bytes = TSDB_METER_NAME_LEN}; + strcpy(s.name, TSQL_TBNAME_L); + + return s; +} + /** * the MeterMeta data format in memory is as follows: * @@ -123,7 +130,7 @@ bool tsMeterMetaIdentical(SMeterMeta* p1, SMeterMeta* p2) { return memcmp(p1, p2, size) == 0; } -//todo refactor +// todo refactor static FORCE_INLINE char* skipSegments(char* input, char delimiter, int32_t num) { for (int32_t i = 0; i < num; ++i) { while (*input != 0 && *input++ != delimiter) { diff --git a/src/inc/tschemautil.h b/src/inc/tschemautil.h index 0b8a2d6a93..7706bcd3a4 100644 --- a/src/inc/tschemautil.h +++ b/src/inc/tschemautil.h @@ -47,6 +47,7 @@ struct SSchema *tsGetSchema(SMeterMeta *pMeta); struct SSchema *tsGetTagSchema(SMeterMeta *pMeta); struct SSchema *tsGetColumnSchema(SMeterMeta *pMeta, int32_t startCol); +struct SSchema tsGetTbnameColumnSchema(); char *tsGetTagsValue(SMeterMeta *pMeta); diff --git a/src/inc/tsqlfunction.h b/src/inc/tsqlfunction.h index a5734ed60e..0ed6a9952e 100644 --- a/src/inc/tsqlfunction.h +++ b/src/inc/tsqlfunction.h @@ -227,8 +227,6 @@ typedef struct SPatternCompareInfo { int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionId, int32_t param, int16_t *type, int16_t *len, int16_t *interResBytes, int16_t extLength, bool isSuperTable); -SResultInfo *getResultSupportInfo(SQLFunctionCtx *pCtx); - int patternMatch(const char *zPattern, const char *zString, size_t size, const SPatternCompareInfo *pInfo); int WCSPatternMatch(const wchar_t *zPattern, const wchar_t *zString, size_t size, const SPatternCompareInfo *pInfo); diff --git a/src/inc/ttypes.h b/src/inc/ttypes.h index b2ea8e918a..0f8eb2d58c 100644 --- a/src/inc/ttypes.h +++ b/src/inc/ttypes.h @@ -50,7 +50,7 @@ bool isNull(const char *val, int32_t type); void setNull(char *val, int32_t type, int32_t bytes); void setNullN(char *val, int32_t type, int32_t bytes, int32_t numOfElems); -void assignVal(char *val, char *src, int32_t len, int32_t type); +void assignVal(char *val, const char *src, int32_t len, int32_t type); void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); // variant, each number/string/field_id has a corresponding struct during parsing sql diff --git a/src/util/src/ttypes.c b/src/util/src/ttypes.c index ad3f98beb6..96237e34ae 100644 --- a/src/util/src/ttypes.c +++ b/src/util/src/ttypes.c @@ -970,7 +970,7 @@ void setNullN(char *val, int32_t type, int32_t bytes, int32_t numOfElems) { } } -void assignVal(char *val, char *src, int32_t len, int32_t type) { +void assignVal(char *val, const char *src, int32_t len, int32_t type) { switch (type) { case TSDB_DATA_TYPE_INT: { *((int32_t *)val) = GET_INT32_VAL(src); From 7568052d031e42add32051f01ffa7ae46b5e21c9 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Tue, 10 Dec 2019 16:21:58 +0800 Subject: [PATCH 097/123] remove the compiler warnings --- src/system/detail/src/vnodeQueryImpl.c | 30 ++++++++++++++++++++------ 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/src/system/detail/src/vnodeQueryImpl.c b/src/system/detail/src/vnodeQueryImpl.c index fb8ff7ef19..45c3385ed0 100644 --- a/src/system/detail/src/vnodeQueryImpl.c +++ b/src/system/detail/src/vnodeQueryImpl.c @@ -252,15 +252,31 @@ static void vnodeSetOpenedFileNames(SQueryFilesInfo* pVnodeFilesInfo) { SHeaderFileInfo* pCurrentFileInfo = &pVnodeFilesInfo->pFileInfo[pVnodeFilesInfo->current]; - // set the full file path for current opened files - snprintf(pVnodeFilesInfo->headerFilePath, PATH_MAX, "%sv%df%d.head", pVnodeFilesInfo->dbFilePathPrefix, - pVnodeFilesInfo->vnodeId, pCurrentFileInfo->fileID); + /* + * set the full file path for current opened files + * the maximum allowed path string length is PATH_MAX in Linux, 100 bytes is used to + * suppress the compiler warnings + */ + char str[PATH_MAX + 100] = {0}; + int32_t PATH_WITH_EXTRA = PATH_MAX + 100; - snprintf(pVnodeFilesInfo->dataFilePath, PATH_MAX, "%sv%df%d.data", pVnodeFilesInfo->dbFilePathPrefix, - pVnodeFilesInfo->vnodeId, pCurrentFileInfo->fileID); + int32_t vnodeId = pVnodeFilesInfo->vnodeId; + int32_t fileId = pCurrentFileInfo->fileID; - snprintf(pVnodeFilesInfo->lastFilePath, PATH_MAX, "%sv%df%d.last", pVnodeFilesInfo->dbFilePathPrefix, - pVnodeFilesInfo->vnodeId, pCurrentFileInfo->fileID); + int32_t len = snprintf(str, PATH_WITH_EXTRA, "%sv%df%d.head", pVnodeFilesInfo->dbFilePathPrefix, vnodeId, fileId); + assert(len <= PATH_MAX); + + strncpy(pVnodeFilesInfo->headerFilePath, str, PATH_MAX); + + len = snprintf(str, PATH_WITH_EXTRA, "%sv%df%d.data", pVnodeFilesInfo->dbFilePathPrefix, vnodeId, fileId); + assert(len <= PATH_MAX); + + strncpy(pVnodeFilesInfo->dataFilePath, str, PATH_MAX); + + len = snprintf(str, PATH_WITH_EXTRA, "%sv%df%d.last", pVnodeFilesInfo->dbFilePathPrefix, vnodeId, fileId); + assert(len <= PATH_MAX); + + strncpy(pVnodeFilesInfo->lastFilePath, str, PATH_MAX); } /** From 9f164ab063457ea52a1f42b72c10a60835458f9c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 10 Dec 2019 16:31:44 +0800 Subject: [PATCH 098/123] resolve warnings in kit --- src/kit/shell/src/shellCommand.c | 2 ++ src/kit/shell/src/shellEngine.c | 13 +++++----- src/kit/shell/src/shellLinux.c | 1 + src/kit/taosdemo/taosdemo.c | 41 ++++++++++++++++++-------------- src/os/linux/inc/os.h | 1 + 5 files changed, 34 insertions(+), 24 deletions(-) diff --git a/src/kit/shell/src/shellCommand.c b/src/kit/shell/src/shellCommand.c index 46aa04c1d7..16545a5fe8 100644 --- a/src/kit/shell/src/shellCommand.c +++ b/src/kit/shell/src/shellCommand.c @@ -19,6 +19,8 @@ #include "shell.h" #include "shellCommand.h" +extern int wcwidth(wchar_t c); +extern int wcswidth(const wchar_t *s, size_t n); typedef struct { char widthInString; char widthOnScreen; diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c index ed3b71bfcd..8dd9fe36c5 100644 --- a/src/kit/shell/src/shellEngine.c +++ b/src/kit/shell/src/shellEngine.c @@ -16,6 +16,7 @@ #define _XOPEN_SOURCE #define _DEFAULT_SOURCE +#include #include "os.h" #include "shell.h" #include "shellCommand.h" @@ -446,7 +447,7 @@ int shellDumpResult(TAOS *con, char *fname, int *error_no, bool printMode) { printf("%*d|", l[i], *((int *)row[i])); break; case TSDB_DATA_TYPE_BIGINT: - printf("%*lld|", l[i], *((int64_t *)row[i])); + printf("%*" PRId64 "|", l[i], *((int64_t *)row[i])); break; case TSDB_DATA_TYPE_FLOAT: { #ifdef _TD_ARM_32_ @@ -481,7 +482,7 @@ int shellDumpResult(TAOS *con, char *fname, int *error_no, bool printMode) { break; case TSDB_DATA_TYPE_TIMESTAMP: if (args.is_raw_time) { - printf(" %lld|", *(int64_t *)row[i]); + printf(" %" PRId64 "|", *(int64_t *)row[i]); } else { if (taos_result_precision(result) == TSDB_TIME_PRECISION_MICRO) { tt = (time_t)((*(int64_t *)row[i]) / 1000000); @@ -531,7 +532,7 @@ int shellDumpResult(TAOS *con, char *fname, int *error_no, bool printMode) { printf("%d\n", *((int *)row[i])); break; case TSDB_DATA_TYPE_BIGINT: - printf("%lld\n", *((int64_t *)row[i])); + printf("%" PRId64 "\n", *((int64_t *)row[i])); break; case TSDB_DATA_TYPE_FLOAT: { #ifdef _TD_ARM_32_ @@ -564,7 +565,7 @@ int shellDumpResult(TAOS *con, char *fname, int *error_no, bool printMode) { break; case TSDB_DATA_TYPE_TIMESTAMP: if (args.is_raw_time) { - printf("%lld\n", *(int64_t *)row[i]); + printf("%" PRId64 "\n", *(int64_t *)row[i]); } else { if (taos_result_precision(result) == TSDB_TIME_PRECISION_MICRO) { tt = (time_t)((*(int64_t *)row[i]) / 1000000); @@ -619,7 +620,7 @@ int shellDumpResult(TAOS *con, char *fname, int *error_no, bool printMode) { fprintf(fp, "%d", *((int *)row[i])); break; case TSDB_DATA_TYPE_BIGINT: - fprintf(fp, "%lld", *((int64_t *)row[i])); + fprintf(fp, "%" PRId64, *((int64_t *)row[i])); break; case TSDB_DATA_TYPE_FLOAT: { #ifdef _TD_ARM_32_ @@ -651,7 +652,7 @@ int shellDumpResult(TAOS *con, char *fname, int *error_no, bool printMode) { break; case TSDB_DATA_TYPE_TIMESTAMP: if (args.is_raw_time) { - fprintf(fp, "%lld", *(int64_t *)row[i]); + fprintf(fp, "%" PRId64, *(int64_t *)row[i]); } else { if (taos_result_precision(result) == TSDB_TIME_PRECISION_MICRO) { tt = (time_t)((*(int64_t *)row[i]) / 1000000); diff --git a/src/kit/shell/src/shellLinux.c b/src/kit/shell/src/shellLinux.c index 2e2c44af1a..70e8752525 100644 --- a/src/kit/shell/src/shellLinux.c +++ b/src/kit/shell/src/shellLinux.c @@ -26,6 +26,7 @@ int indicator = 1; struct termios oldtio; +extern int wcwidth(wchar_t c); void insertChar(Command *cmd, char *c, int size); const char *argp_program_version = version; const char *argp_program_bug_address = ""; diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 22aa71bf6c..40fbabe1f7 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -265,30 +265,35 @@ double getCurrentTime(); void callBack(void *param, TAOS_RES *res, int code); int main(int argc, char *argv[]) { - struct arguments arguments = {NULL, - 0, - "root", - "taosdata", - "test", - "t", - false, - false, - "./output.txt", - 0, - "int", + struct arguments arguments = {NULL, // host + 0, // port + "root", // user + "taosdata", // password + "test", // database + "t", // tb_prefix + false, // use_metric + false, // insert_only + "./output.txt", // output_file + 0, // mode + { + "int", // datatype "", "", "", "", "", "", - "", - 8, - 1, - 1, - 1, - 1, - 50000}; + "" + }, + 8, // len_of_binary + 1, // num_of_CPR + 1, // num_of_connections + 1, // num_of_RPR + 1, // num_of_tables + 50000, // num_of_DPT + 0, // abort + NULL // arg_list + }; /* Parse our arguments; every option seen by parse_opt will be reflected in arguments. */ diff --git a/src/os/linux/inc/os.h b/src/os/linux/inc/os.h index 8d63e69cd6..acfd284737 100644 --- a/src/os/linux/inc/os.h +++ b/src/os/linux/inc/os.h @@ -23,6 +23,7 @@ extern "C" { #include #include +#include #include #include #include From eb6533a96eeacafece40e662bbe4c56c54295ec3 Mon Sep 17 00:00:00 2001 From: lihui Date: Tue, 10 Dec 2019 16:42:29 +0800 Subject: [PATCH 099/123] [TBASE-1323] --- src/util/src/ttypes.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/util/src/ttypes.c b/src/util/src/ttypes.c index 96237e34ae..b048748d95 100644 --- a/src/util/src/ttypes.c +++ b/src/util/src/ttypes.c @@ -139,7 +139,7 @@ void tVariantCreateFromBinary(tVariant *pVar, char *pz, uint32_t len, uint32_t t } case TSDB_DATA_TYPE_NCHAR: { // here we get the nchar length from raw binary bits length pVar->nLen = len / TSDB_NCHAR_SIZE; - pVar->wpz = malloc((pVar->nLen + 1) * TSDB_NCHAR_SIZE); + pVar->wpz = calloc(1, (pVar->nLen + 1) * TSDB_NCHAR_SIZE); wcsncpy(pVar->wpz, (wchar_t *)pz, pVar->nLen); pVar->wpz[pVar->nLen] = 0; @@ -998,6 +998,14 @@ void assignVal(char *val, const char *src, int32_t len, int32_t type) { *((int8_t *)val) = GET_INT8_VAL(src); break; }; + case TSDB_DATA_TYPE_BINARY: { + strncpy(val, src, len); + break; + }; + case TSDB_DATA_TYPE_NCHAR: { + wcsncpy((wchar_t*)val, (wchar_t*)src, len / TSDB_NCHAR_SIZE); + break; + }; default: { memcpy(val, src, len); break; From 7a74561f6f3e3941c0fbe1292a2db9fe14fe8a1e Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Tue, 10 Dec 2019 16:48:47 +0800 Subject: [PATCH 100/123] [tbase-1324] --- src/client/src/tscSQLParser.c | 3 ++- src/client/src/tscUtil.c | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index d96e15eb5b..9df50b0fa1 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -5340,7 +5340,8 @@ static int32_t checkUpdateTagPrjFunctions(SSqlCmd* pCmd) { for (int32_t i = 0; i < pCmd->fieldsInfo.numOfOutputCols; ++i) { int16_t functionId = tscSqlExprGet(pCmd, i)->functionId; - if (functionId == TSDB_FUNC_TAGPRJ || functionId == TSDB_FUNC_PRJ || functionId == TSDB_FUNC_TS) { + if (functionId == TSDB_FUNC_TAGPRJ || functionId == TSDB_FUNC_PRJ || functionId == TSDB_FUNC_TS || + functionId == TSDB_FUNC_ARITHM) { continue; } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 7c7310a1c7..9876dad906 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -246,7 +246,7 @@ bool tscProjectionQueryOnMetric(SSqlCmd* pCmd) { for (int32_t i = 0; i < pCmd->fieldsInfo.numOfOutputCols; ++i) { int32_t functionId = tscSqlExprGet(pCmd, i)->functionId; if (functionId != TSDB_FUNC_PRJ && functionId != TSDB_FUNC_TAGPRJ && - functionId != TSDB_FUNC_TAG && functionId != TSDB_FUNC_TS) { + functionId != TSDB_FUNC_TAG && functionId != TSDB_FUNC_TS && functionId != TSDB_FUNC_ARITHM) { return false; } } From 1475d520a95e1a5ccb0e778d9d54fc87f7a1b303 Mon Sep 17 00:00:00 2001 From: slguan Date: Tue, 10 Dec 2019 17:36:43 +0800 Subject: [PATCH 101/123] [TBASE-1241] --- src/util/src/tglobalcfg.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/util/src/tglobalcfg.c b/src/util/src/tglobalcfg.c index d1a61633ae..edff1c432b 100644 --- a/src/util/src/tglobalcfg.c +++ b/src/util/src/tglobalcfg.c @@ -787,7 +787,6 @@ static void doInitGlobalConfig() { tsInitConfigOption(cfg++, "anyIp", &tsAnyIp, TSDB_CFG_VTYPE_INT, TSDB_CFG_CTYPE_B_CONFIG, 0, 1, 0, TSDB_CFG_UTYPE_NONE); - } #endif tsInitConfigOption(cfg++, "usePublicIp", &tsUsePublicIp, TSDB_CFG_VTYPE_INT, From e1b11fbb3bd8398103f8608b45aa6c8cecc0261f Mon Sep 17 00:00:00 2001 From: lihui Date: Tue, 10 Dec 2019 18:42:07 +0800 Subject: [PATCH 102/123] [TBASE-816] --- src/modules/monitor/src/monitorSystem.c | 25 +++++++++++++------------ src/os/linux/src/tsystem.c | 20 ++++++++++---------- 2 files changed, 23 insertions(+), 22 deletions(-) diff --git a/src/modules/monitor/src/monitorSystem.c b/src/modules/monitor/src/monitorSystem.c index c5f65eef36..f89f259688 100644 --- a/src/modules/monitor/src/monitorSystem.c +++ b/src/modules/monitor/src/monitorSystem.c @@ -14,6 +14,7 @@ */ #include "monitor.h" +#include #include #include #include @@ -376,7 +377,7 @@ void monitorSaveSystemInfo() { int64_t ts = taosGetTimestampUs(); char * sql = monitor->sql; - int pos = snprintf(sql, SQL_LENGTH, "insert into %s.dn_%s values(%ld", tsMonitorDbName, monitor->privateIpStr, ts); + int pos = snprintf(sql, SQL_LENGTH, "insert into %s.dn_%s values(%" PRId64, tsMonitorDbName, monitor->privateIpStr, ts); pos += monitorBuildCpuSql(sql + pos); pos += monitorBuildMemorySql(sql + pos); @@ -402,16 +403,16 @@ void monitorSaveAcctLog(char *acctId, int64_t currentPointsPerSecond, int64_t ma char sql[1024] = {0}; sprintf(sql, "insert into %s.acct_%s using %s.acct tags('%s') values(now" - ", %ld, %ld " - ", %ld, %ld" - ", %ld, %ld" - ", %ld, %ld" - ", %ld, %ld" - ", %ld, %ld" - ", %ld, %ld" - ", %ld, %ld" - ", %ld, %ld" - ", %ld, %ld" + ", %" PRId64, "%" PRId64 + ", %" PRId64, "%" PRId64 + ", %" PRId64, "%" PRId64 + ", %" PRId64, "%" PRId64 + ", %" PRId64, "%" PRId64 + ", %" PRId64, "%" PRId64 + ", %" PRId64, "%" PRId64 + ", %" PRId64, "%" PRId64 + ", %" PRId64, "%" PRId64 + ", %" PRId64, "%" PRId64 ", %d)", tsMonitorDbName, acctId, tsMonitorDbName, acctId, currentPointsPerSecond, maxPointsPerSecond, totalTimeSeries, maxTimeSeries, totalStorage, maxStorage, totalQueryTime, maxQueryTime, totalInbound, maxInbound, @@ -431,7 +432,7 @@ void monitorSaveLog(int level, const char *const format, ...) { return; } - int len = snprintf(sql, (size_t)max_length, "import into %s.log values(%ld, %d,'", tsMonitorDbName, + int len = snprintf(sql, (size_t)max_length, "import into %s.log values(%\" PRId64 \", %d,'", tsMonitorDbName, taosGetTimestampUs(), level); va_start(argpointer, format); diff --git a/src/os/linux/src/tsystem.c b/src/os/linux/src/tsystem.c index 0a9d97b3cf..611ede73b5 100644 --- a/src/os/linux/src/tsystem.c +++ b/src/os/linux/src/tsystem.c @@ -12,7 +12,7 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ - +#include #include #include #include @@ -99,7 +99,7 @@ bool taosGetProcMemory(float *memoryUsedMB) { int64_t memKB = 0; char tmp[10]; - sscanf(line, "%s %ld", tmp, &memKB); + sscanf(line, "%s %" PRId64, tmp, &memKB); *memoryUsedMB = (float)((double)memKB / 1024); tfree(line); @@ -124,7 +124,7 @@ bool taosGetSysCpuInfo(SysCpuInfo *cpuInfo) { } char cpu[10] = {0}; - sscanf(line, "%s %lu %lu %lu %lu", cpu, &cpuInfo->user, &cpuInfo->nice, &cpuInfo->system, &cpuInfo->idle); + sscanf(line, "%s %" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64, cpu, &cpuInfo->user, &cpuInfo->nice, &cpuInfo->system, &cpuInfo->idle); tfree(line); fclose(fp); @@ -150,7 +150,7 @@ bool taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) { for (int i = 0, blank = 0; line[i] != 0; ++i) { if (line[i] == ' ') blank++; if (blank == PROCESS_ITEM) { - sscanf(line + i + 1, "%lu %lu %lu %lu", &cpuInfo->utime, &cpuInfo->stime, &cpuInfo->cutime, &cpuInfo->cstime); + sscanf(line + i + 1, "%" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64, &cpuInfo->utime, &cpuInfo->stime, &cpuInfo->cutime, &cpuInfo->cstime); break; } } @@ -420,7 +420,7 @@ bool taosGetCardInfo(int64_t *bytes) { } } if (line != NULL) { - sscanf(line, "%s %ld %ld %ld %ld %ld %ld %ld %ld %ld %ld", nouse0, &rbytes, &rpackts, &nouse1, &nouse2, &nouse3, + sscanf(line, "%s %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64, nouse0, &rbytes, &rpackts, &nouse1, &nouse2, &nouse3, &nouse4, &nouse5, &nouse6, &tbytes, &tpackets); *bytes = rbytes + tbytes; tfree(line); @@ -488,10 +488,10 @@ bool taosReadProcIO(int64_t *readbyte, int64_t *writebyte) { break; } if (strstr(line, "rchar:") != NULL) { - sscanf(line, "%s %ld", tmp, readbyte); + sscanf(line, "%s %" PRId64, tmp, readbyte); readIndex++; } else if (strstr(line, "wchar:") != NULL) { - sscanf(line, "%s %ld", tmp, writebyte); + sscanf(line, "%s %" PRId64, tmp, writebyte); readIndex++; } else { } @@ -564,9 +564,9 @@ void taosGetSystemInfo() { } void tsPrintOsInfo() { - pPrint(" os pageSize: %ld(KB)", tsPageSize); - pPrint(" os openMax: %ld", tsOpenMax); - pPrint(" os streamMax: %ld", tsStreamMax); + pPrint(" os pageSize: %" PRId64 "(KB)", tsPageSize); + pPrint(" os openMax: %" PRId64, tsOpenMax); + pPrint(" os streamMax: %" PRId64, tsStreamMax); pPrint(" os numOfCores: %d", tsNumOfCores); pPrint(" os totalDisk: %f(GB)", tsTotalDataDirGB); pPrint(" os totalMemory: %d(MB)", tsTotalMemoryMB); From 1570fadf99cb6fb76c8d059f8237cd0b7b7a1b2c Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Wed, 11 Dec 2019 11:55:22 +0800 Subject: [PATCH 103/123] [tbase-1326] --- src/system/detail/inc/vnode.h | 20 +++-- src/system/detail/inc/vnodeRead.h | 83 ++++++++++----------- src/system/detail/src/vnodeQueryProcess.c | 26 ++++++- src/system/detail/src/vnodeRead.c | 91 ++++++++++++++++------- src/system/detail/src/vnodeShell.c | 4 +- 5 files changed, 142 insertions(+), 82 deletions(-) diff --git a/src/system/detail/inc/vnode.h b/src/system/detail/inc/vnode.h index 6e366b9b7f..4ac0848427 100644 --- a/src/system/detail/inc/vnode.h +++ b/src/system/detail/inc/vnode.h @@ -213,18 +213,20 @@ typedef struct { * Only the QInfo.signature == QInfo, this structure can be released safely. */ #define TSDB_QINFO_QUERY_FLAG 0x1 -#define TSDB_QINFO_RESET_SIG(x) atomic_store_64(&((x)->signature), (uint64_t)(x)) -#define TSDB_QINFO_SET_QUERY_FLAG(x) \ - atomic_val_compare_exchange_64(&((x)->signature), (uint64_t)(x), TSDB_QINFO_QUERY_FLAG); +//#define TSDB_QINFO_RESET_SIG(x) atomic_store_64(&((x)->signature), (uint64_t)(x)) +#define TSDB_QINFO_RESET_SIG(x) +#define TSDB_QINFO_SET_QUERY_FLAG(x) +//#define TSDB_QINFO_SET_QUERY_FLAG(x) \ +// atomic_val_compare_exchange_64(&((x)->signature), (uint64_t)(x), TSDB_QINFO_QUERY_FLAG); // live lock: wait for query reaching a safe-point, release all resources // belongs to this query #define TSDB_WAIT_TO_SAFE_DROP_QINFO(x) \ - { \ - while (atomic_val_compare_exchange_64(&((x)->signature), (x), 0) == TSDB_QINFO_QUERY_FLAG) { \ - taosMsleep(1); \ - } \ - } +// { \ +// while (atomic_val_compare_exchange_64(&((x)->signature), (x), 0) == TSDB_QINFO_QUERY_FLAG) { \ +// taosMsleep(1); \ +// } \ +// } struct tSQLBinaryExpr; @@ -370,6 +372,8 @@ void vnodeFreeQInfo(void *, bool); void vnodeFreeQInfoInQueue(void *param); bool vnodeIsQInfoValid(void *param); +void vnodeDecRefCount(void *param); +void vnodeAddRefCount(void *param); int32_t vnodeConvertQueryMeterMsg(SQueryMeterMsg *pQuery); diff --git a/src/system/detail/inc/vnodeRead.h b/src/system/detail/inc/vnodeRead.h index d059075142..3a1b6f7949 100644 --- a/src/system/detail/inc/vnodeRead.h +++ b/src/system/detail/inc/vnodeRead.h @@ -52,7 +52,7 @@ typedef struct SQueryLoadCompBlockInfo { * the header file info for one vnode */ typedef struct SHeaderFileInfo { - int32_t fileID; // file id + int32_t fileID; // file id } SHeaderFileInfo; typedef struct SQueryCostSummary { @@ -99,17 +99,17 @@ typedef struct SQueryFilesInfo { uint32_t numOfFiles; // the total available number of files for this virtual node during query execution int32_t current; // the memory mapped header file, NOTE: only one header file can be mmap. int32_t vnodeId; - - int32_t headerFd; // header file fd - char* pHeaderFileData; // mmap header files - int64_t headFileSize; - int32_t dataFd; - int32_t lastFd; - - char headerFilePath[PATH_MAX]; // current opened header file name - char dataFilePath[PATH_MAX]; // current opened data file name - char lastFilePath[PATH_MAX]; // current opened last file path - char dbFilePathPrefix[PATH_MAX]; + + int32_t headerFd; // header file fd + char* pHeaderFileData; // mmap header files + int64_t headFileSize; + int32_t dataFd; + int32_t lastFd; + + char headerFilePath[PATH_MAX]; // current opened header file name + char dataFilePath[PATH_MAX]; // current opened data file name + char lastFilePath[PATH_MAX]; // current opened last file path + char dbFilePathPrefix[PATH_MAX]; } SQueryFilesInfo; typedef struct RuntimeEnvironment { @@ -129,17 +129,17 @@ typedef struct RuntimeEnvironment { SQueryLoadBlockInfo loadBlockInfo; /* record current block load information */ SQueryLoadCompBlockInfo loadCompBlockInfo; /* record current compblock information in SQuery */ SQueryFilesInfo vnodeFileInfo; - int16_t numOfRowsPerPage; - int16_t offset[TSDB_MAX_COLUMNS]; - int16_t scanFlag; // denotes reversed scan of data or not - SInterpolationInfo interpoInfo; - SData** pInterpoBuf; - SOutputRes* pResult; // reference to SQuerySupporter->pResult - void* hashList; - int32_t usedIndex; // assigned SOutputRes in list - STSBuf* pTSBuf; - STSCursor cur; - SQueryCostSummary summary; + int16_t numOfRowsPerPage; + int16_t offset[TSDB_MAX_COLUMNS]; + int16_t scanFlag; // denotes reversed scan of data or not + SInterpolationInfo interpoInfo; + SData** pInterpoBuf; + SOutputRes* pResult; // reference to SQuerySupporter->pResult + void* hashList; + int32_t usedIndex; // assigned SOutputRes in list + STSBuf* pTSBuf; + STSCursor cur; + SQueryCostSummary summary; } SQueryRuntimeEnv; /* intermediate result during multimeter query involves interval */ @@ -214,14 +214,12 @@ typedef struct SMeterQuerySupportObj { SMeterDataInfo* pMeterDataInfo; - TSKEY* tsList; - int32_t tsNum; - + TSKEY* tsList; } SMeterQuerySupportObj; typedef struct _qinfo { - uint64_t signature; - + uint64_t signature; + int32_t refCount; // QInfo reference count, when the value is 0, it can be released safely char user[TSDB_METER_ID_LEN + 1]; char sql[TSDB_SHOW_SQL_LEN]; uint8_t stream; @@ -231,24 +229,21 @@ typedef struct _qinfo { int64_t useconds; int killed; struct _qinfo *prev, *next; + SQuery query; + int num; + int totalPoints; + int pointsRead; + int pointsReturned; + int pointsInterpo; + int code; + char bufIndex; + char changed; + char over; + SMeterObj* pObj; + sem_t dataReady; - SQuery query; - int num; - int totalPoints; - int pointsRead; - int pointsReturned; - int pointsInterpo; - int code; - char bufIndex; - char changed; - char over; - SMeterObj* pObj; - - int (*fp)(SMeterObj*, SQuery*); - - sem_t dataReady; SMeterQuerySupportObj* pMeterQuerySupporter; - + int (*fp)(SMeterObj*, SQuery*); } SQInfo; int32_t vnodeQuerySingleMeterPrepare(SQInfo* pQInfo, SMeterObj* pMeterObj, SMeterQuerySupportObj* pSMultiMeterObj, diff --git a/src/system/detail/src/vnodeQueryProcess.c b/src/system/detail/src/vnodeQueryProcess.c index 6fa0891baa..f39ca0dea1 100644 --- a/src/system/detail/src/vnodeQueryProcess.c +++ b/src/system/detail/src/vnodeQueryProcess.c @@ -890,12 +890,16 @@ static void vnodeMultiMeterQueryProcessor(SQInfo *pQInfo) { dTrace("QInfo:%p points returned:%d, totalRead:%d totalReturn:%d", pQInfo, pQuery->pointsRead, pQInfo->pointsRead, pQInfo->pointsReturned); + + vnodeDecRefCount(pQInfo); return; } pSupporter->pMeterDataInfo = (SMeterDataInfo *)calloc(1, sizeof(SMeterDataInfo) * pSupporter->numOfMeters); if (pSupporter->pMeterDataInfo == NULL) { dError("QInfo:%p failed to allocate memory, %s", pQInfo, strerror(errno)); + pQInfo->code = -TSDB_CODE_SERV_OUT_OF_MEMORY; + vnodeDecRefCount(pQInfo); return; } @@ -912,6 +916,7 @@ static void vnodeMultiMeterQueryProcessor(SQInfo *pQInfo) { // failed to save all intermediate results into disk, abort further query processing if (doCloseAllOpenedResults(pSupporter) != TSDB_CODE_SUCCESS) { dError("QInfo:%p failed to save intermediate results, abort further query processing", pQInfo); + vnodeDecRefCount(pQInfo); return; } @@ -919,6 +924,7 @@ static void vnodeMultiMeterQueryProcessor(SQInfo *pQInfo) { if (isQueryKilled(pQuery)) { dTrace("QInfo:%p query killed, abort", pQInfo); + vnodeDecRefCount(pQInfo); return; } @@ -940,6 +946,7 @@ static void vnodeMultiMeterQueryProcessor(SQInfo *pQInfo) { pQInfo->pointsRead += pQuery->pointsRead; dTrace("QInfo:%p points returned:%d, totalRead:%d totalReturn:%d", pQInfo, pQuery->pointsRead, pQInfo->pointsRead, pQInfo->pointsReturned); + vnodeDecRefCount(pQInfo); } /* @@ -1174,10 +1181,13 @@ void vnodeSingleMeterQuery(SSchedMsg *pMsg) { if (pQInfo->killed) { TSDB_QINFO_RESET_SIG(pQInfo); dTrace("QInfo:%p it is already killed, reset signature and abort", pQInfo); + vnodeDecRefCount(pQInfo); + return; } - assert(pQInfo->signature == TSDB_QINFO_QUERY_FLAG); + assert(pQInfo->refCount >= 1); +// assert(pQInfo->signature == TSDB_QINFO_QUERY_FLAG); SQuery * pQuery = &pQInfo->query; SMeterObj *pMeterObj = pQInfo->pObj; @@ -1215,6 +1225,7 @@ void vnodeSingleMeterQuery(SSchedMsg *pMsg) { sem_post(&pQInfo->dataReady); TSDB_QINFO_RESET_SIG(pQInfo); + vnodeDecRefCount(pQInfo); return; } @@ -1237,6 +1248,8 @@ void vnodeSingleMeterQuery(SSchedMsg *pMsg) { sem_post(&pQInfo->dataReady); TSDB_QINFO_RESET_SIG(pQInfo); + vnodeDecRefCount(pQInfo); + return; } } @@ -1249,7 +1262,7 @@ void vnodeSingleMeterQuery(SSchedMsg *pMsg) { vnodePrintQueryStatistics(pQInfo->pMeterQuerySupporter); sem_post(&pQInfo->dataReady); TSDB_QINFO_RESET_SIG(pQInfo); - + vnodeDecRefCount(pQInfo); return; } @@ -1284,8 +1297,10 @@ void vnodeSingleMeterQuery(SSchedMsg *pMsg) { pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->pointsRead); } - sem_post(&pQInfo->dataReady); TSDB_QINFO_RESET_SIG(pQInfo); + sem_post(&pQInfo->dataReady); + + vnodeDecRefCount(pQInfo); } void vnodeMultiMeterQuery(SSchedMsg *pMsg) { @@ -1297,11 +1312,13 @@ void vnodeMultiMeterQuery(SSchedMsg *pMsg) { if (pQInfo->killed) { TSDB_QINFO_RESET_SIG(pQInfo); + vnodeDecRefCount(pQInfo); dTrace("QInfo:%p it is already killed, reset signature and abort", pQInfo); return; } - assert(pQInfo->signature == TSDB_QINFO_QUERY_FLAG); + assert(pQInfo->refCount >= 1); +// assert(pQInfo->signature == TSDB_QINFO_QUERY_FLAG); SQuery *pQuery = &pQInfo->query; pQuery->pointsRead = 0; @@ -1337,4 +1354,5 @@ void vnodeMultiMeterQuery(SSchedMsg *pMsg) { sem_post(&pQInfo->dataReady); TSDB_QINFO_RESET_SIG(pQInfo); + vnodeDecRefCount(pQInfo); } diff --git a/src/system/detail/src/vnodeRead.c b/src/system/detail/src/vnodeRead.c index 21e83c5198..2ea41d9d58 100644 --- a/src/system/detail/src/vnodeRead.c +++ b/src/system/detail/src/vnodeRead.c @@ -392,10 +392,10 @@ __clean_memory: return NULL; } -static void vnodeFreeQInfoInQueueImpl(SSchedMsg *pMsg) { - SQInfo *pQInfo = (SQInfo *)pMsg->ahandle; - vnodeFreeQInfo(pQInfo, true); -} +//static void vnodeFreeQInfoInQueueImpl(SSchedMsg *pMsg) { +// SQInfo *pQInfo = (SQInfo *)pMsg->ahandle; +// vnodeFreeQInfo(pQInfo, true); +//} void vnodeFreeQInfoInQueue(void *param) { SQInfo *pQInfo = (SQInfo *)param; @@ -403,15 +403,18 @@ void vnodeFreeQInfoInQueue(void *param) { if (!vnodeIsQInfoValid(pQInfo)) return; pQInfo->killed = 1; + dTrace("QInfo:%p set kill flag to free QInfo"); + + vnodeDecRefCount(pQInfo); + +// dTrace("QInfo:%p set kill flag and add to queue, stop query ASAP", pQInfo); +// SSchedMsg schedMsg = {0}; +// schedMsg.fp = vnodeFreeQInfoInQueueImpl; - dTrace("QInfo:%p set kill flag and add to queue, stop query ASAP", pQInfo); - SSchedMsg schedMsg = {0}; - schedMsg.fp = vnodeFreeQInfoInQueueImpl; - - schedMsg.msg = NULL; - schedMsg.thandle = (void *)1; - schedMsg.ahandle = param; - taosScheduleTask(queryQhandle, &schedMsg); +// schedMsg.msg = NULL; +// schedMsg.thandle = (void *)1; +// schedMsg.ahandle = param; +// taosScheduleTask(queryQhandle, &schedMsg); } void vnodeFreeQInfo(void *param, bool decQueryRef) { @@ -499,7 +502,30 @@ bool vnodeIsQInfoValid(void *param) { * into local variable, then compare by using local variable */ uint64_t sig = pQInfo->signature; - return (sig == (uint64_t)pQInfo) || (sig == TSDB_QINFO_QUERY_FLAG); +// return (sig == (uint64_t)pQInfo) || (sig == TSDB_QINFO_QUERY_FLAG); + return (sig == (uint64_t)pQInfo); +} + +void vnodeDecRefCount(void *param) { + SQInfo *pQInfo = (SQInfo*) param; + + assert(vnodeIsQInfoValid(pQInfo)); + + int32_t ref = atomic_sub_fetch_32(&pQInfo->refCount, 1); + dTrace("QInfo:%p decrease obj refcount, %d", pQInfo, ref); + + if (ref == 0) { + vnodeFreeQInfo(pQInfo, true); + } +} + +void vnodeAddRefCount(void *param) { + SQInfo *pQInfo = (SQInfo*) param; + + assert(vnodeIsQInfoValid(pQInfo)); + + int32_t ref = atomic_add_fetch_32(&pQInfo->refCount, 1); + dTrace("QInfo:%p add refcount, %d", pQInfo, ref); } void vnodeQueryData(SSchedMsg *pMsg) { @@ -511,10 +537,11 @@ void vnodeQueryData(SSchedMsg *pMsg) { if (pQInfo->killed) { TSDB_QINFO_RESET_SIG(pQInfo); dTrace("QInfo:%p it is already killed, reset signature and abort", pQInfo); + vnodeDecRefCount(pQInfo); return; } - assert(pQInfo->signature == TSDB_QINFO_QUERY_FLAG); +// assert(pQInfo->signature == TSDB_QINFO_QUERY_FLAG); pQuery = &(pQInfo->query); SMeterObj *pObj = pQInfo->pObj; @@ -586,6 +613,7 @@ void vnodeQueryData(SSchedMsg *pMsg) { dTrace("QInfo:%p reset signature", pQInfo); TSDB_QINFO_RESET_SIG(pQInfo); sem_post(&pQInfo->dataReady); + vnodeDecRefCount(pQInfo); } void *vnodeQueryInTimeRange(SMeterObj **pMetersObj, SSqlGroupbyExpr *pGroupbyExpr, SSqlFunctionExpr *pSqlExprs, @@ -668,14 +696,22 @@ void *vnodeQueryInTimeRange(SMeterObj **pMetersObj, SSqlGroupbyExpr *pGroupbyExp } // set in query flag - pQInfo->signature = TSDB_QINFO_QUERY_FLAG; - +// pQInfo->signature = TSDB_QINFO_QUERY_FLAG; + + /* + * The reference count, which is 2, is for both the current query thread and the future retrieve request, + * which will always be issued by client to acquire data or free SQInfo struct. + */ + vnodeAddRefCount(pQInfo); + vnodeAddRefCount(pQInfo); + schedMsg.msg = NULL; schedMsg.thandle = (void *)1; schedMsg.ahandle = pQInfo; - dTrace("QInfo:%p set query flag and prepare runtime environment completed, wait for schedule", pQInfo); - + dTrace("QInfo:%p set query flag and prepare runtime environment completed, ref:%d, wait for schedule", pQInfo, + pQInfo->refCount); + taosScheduleTask(queryQhandle, &schedMsg); return pQInfo; @@ -779,8 +815,10 @@ void *vnodeQueryOnMultiMeters(SMeterObj **pMetersObj, SSqlGroupbyExpr *pGroupbyE return pQInfo; } - pQInfo->signature = TSDB_QINFO_QUERY_FLAG; - +// pQInfo->signature = TSDB_QINFO_QUERY_FLAG; + vnodeAddRefCount(pQInfo); + vnodeAddRefCount(pQInfo); + schedMsg.msg = NULL; schedMsg.thandle = (void *)1; schedMsg.ahandle = pQInfo; @@ -862,19 +900,22 @@ int vnodeSaveQueryResult(void *handle, char *data, int32_t *size) { if (pQInfo->over == 0) { //dTrace("QInfo:%p set query flag, oldSig:%p, func:%s", pQInfo, pQInfo->signature, __FUNCTION__); dTrace("QInfo:%p set query flag, oldSig:%p", pQInfo, pQInfo->signature); - uint64_t oldSignature = TSDB_QINFO_SET_QUERY_FLAG(pQInfo); +// uint64_t oldSignature = TSDB_QINFO_SET_QUERY_FLAG(pQInfo); /* * If SQInfo has been released, the value of signature cannot be equalled to the address of pQInfo, * since in release function, the original value has been destroyed. However, this memory area may be reused * by another function. It may be 0 or any value, but it is rarely still be equalled to the address of SQInfo. */ - if (oldSignature == 0 || oldSignature != (uint64_t)pQInfo) { - dTrace("%p freed or killed, old sig:%p abort query", pQInfo, oldSignature); +// if (oldSignature == 0 || oldSignature != (uint64_t)pQInfo) { + if (pQInfo->killed == 1) { +// dTrace("%p freed or killed, old sig:%p abort query", pQInfo, oldSignature); + dTrace("%p freed or killed, abort query", pQInfo); } else { + vnodeAddRefCount(pQInfo); dTrace("%p add query into task queue for schedule", pQInfo); - - SSchedMsg schedMsg; + + SSchedMsg schedMsg = {0}; if (pQInfo->pMeterQuerySupporter != NULL) { if (pQInfo->pMeterQuerySupporter->pSidSet == NULL) { diff --git a/src/system/detail/src/vnodeShell.c b/src/system/detail/src/vnodeShell.c index cfdd956768..6ff3f4d27c 100644 --- a/src/system/detail/src/vnodeShell.c +++ b/src/system/detail/src/vnodeShell.c @@ -470,9 +470,11 @@ void vnodeExecuteRetrieveReq(SSchedMsg *pSched) { pMsg += size; msgLen = pMsg - pStart; + assert(code != TSDB_CODE_ACTION_IN_PROGRESS); + if (numOfRows == 0 && (pRetrieve->qhandle == (uint64_t)pObj->qhandle) && (code != TSDB_CODE_ACTION_IN_PROGRESS)) { dTrace("QInfo:%p %s free qhandle code:%d", pObj->qhandle, __FUNCTION__, code); - vnodeFreeQInfo(pObj->qhandle, true); + vnodeDecRefCount(pObj->qhandle); pObj->qhandle = NULL; } From d774aa3993542766e3551e93212000d7923ac48f Mon Sep 17 00:00:00 2001 From: slguan Date: Wed, 11 Dec 2019 14:39:48 +0800 Subject: [PATCH 104/123] [TBASE-1241] --- src/client/src/tscServer.c | 3 --- src/inc/taosmsg.h | 14 ++++++++------ src/rpc/src/trpc.c | 2 ++ src/system/detail/inc/mgmt.h | 3 ++- src/system/detail/src/mgmtShell.c | 24 ++++++++++++++---------- 5 files changed, 26 insertions(+), 20 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 3fac0c74f2..801256644c 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -2643,9 +2643,6 @@ int tscBuildConnectMsg(SSqlObj *pSql) { strcpy(pConnect->db, db); strcpy(pConnect->clientVersion, version); - pConnect->usePublicIp = (int8_t)tsUsePublicIp; - pConnect->isCluster = (int8_t)tsIsCluster; - memset(pConnect->reserved, 0, sizeof(pConnect->reserved)); pMsg += sizeof(SConnectMsg); diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index ae58c43477..1d20ad76ed 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -224,7 +224,9 @@ typedef struct { uint32_t destId; char meterId[TSDB_UNI_LEN]; uint16_t port; // for UDP only - char empty[1]; + uint8_t usePublicIp : 1; + uint8_t isCluster : 1; + uint8_t empty : 6; uint8_t msgType; int32_t msgLen; uint8_t content[0]; @@ -352,9 +354,6 @@ typedef struct { typedef struct { char clientVersion[TSDB_VERSION_LEN]; char db[TSDB_METER_ID_LEN]; - int8_t usePublicIp; - int8_t isCluster; - int8_t reserved[14]; } SConnectMsg; typedef struct { @@ -667,8 +666,11 @@ typedef struct { typedef struct { uint32_t destId; char meterId[TSDB_UNI_LEN]; - char empty[3]; - char msgType; + uint16_t port; // for UDP only + uint8_t usePublicIp : 1; + uint8_t isCluster : 1; + uint8_t empty : 6; + uint8_t msgType; int32_t msgLen; uint8_t content[0]; } SIntMsg; diff --git a/src/rpc/src/trpc.c b/src/rpc/src/trpc.c index 9f006ab05a..7b5229ddc7 100755 --- a/src/rpc/src/trpc.c +++ b/src/rpc/src/trpc.c @@ -246,6 +246,8 @@ char *taosBuildReqHeader(void *param, char type, char *msg) { pHeader->destId = pConn->peerId; pHeader->port = 0; pHeader->uid = (uint32_t)pConn + (uint32_t)getpid(); + pHeader->usePublicIp = (tsUsePublicIp == 0 ? 0 : 1); + pHeader->isCluster = (tsIsCluster == 0 ? 0 : 1); memcpy(pHeader->meterId, pConn->meterId, tListLen(pHeader->meterId)); diff --git a/src/system/detail/inc/mgmt.h b/src/system/detail/inc/mgmt.h index b59c5cf61c..e339274025 100644 --- a/src/system/detail/inc/mgmt.h +++ b/src/system/detail/inc/mgmt.h @@ -223,7 +223,8 @@ typedef struct _connObj { char writeAuth : 1; // write flag char killConnection : 1; // kill the connection flag char usePublicIp : 1; // if the connection request is publicIp - char reserved : 4; + char isCluster : 1; + char reserved : 3; uint32_t queryId; // query ID to be killed uint32_t streamId; // stream ID to be killed uint32_t ip; // shell IP diff --git a/src/system/detail/src/mgmtShell.c b/src/system/detail/src/mgmtShell.c index 576353d758..4334218530 100644 --- a/src/system/detail/src/mgmtShell.c +++ b/src/system/detail/src/mgmtShell.c @@ -1201,8 +1201,8 @@ int mgmtProcessConnectMsg(char *pMsg, int msgLen, SConnObj *pConn) { goto _rsp; } - if (pConnectMsg->isCluster != tsIsCluster) { - mError("Cluster Edition and lite Edition cannot be interconnected, client:%d server:%d", pConnectMsg->isCluster, tsIsCluster); + if (pConn->isCluster != tsIsCluster) { + mError("Cluster Edition and lite Edition cannot be interconnected, client:%d server:%d", pConn->isCluster, tsIsCluster); code = TSDB_CODE_INVALID_CLIENT_VERSION; goto _rsp; } @@ -1246,17 +1246,13 @@ _rsp: pMsg += sizeof(SConnectRsp); #ifdef CLUSTER - if (pConnectMsg->usePublicIp) { - pConn->usePublicIp = 1; - int size = pSdbPublicIpList->numOfIps * 4 + sizeof(SIpList); + int size = pSdbPublicIpList->numOfIps * 4 + sizeof(SIpList); + if (pConn->usePublicIp) { memcpy(pMsg, pSdbPublicIpList, size); - pMsg += size; - } - else { - int size = pSdbIpList->numOfIps * 4 + sizeof(SIpList); + } else { memcpy(pMsg, pSdbIpList, size); - pMsg += size; } + pMsg += size; #endif // set the time resolution: millisecond or microsecond @@ -1305,8 +1301,16 @@ void *mgmtProcessMsgFromShell(char *msg, void *ahandle, void *thandle) { pConn = connList + pMsg->destId; pConn->thandle = thandle; strcpy(pConn->user, pMsg->meterId); + + uint32_t ip = taosGetRpcLocalIp(thandle); + if (ip == tsPublicIp) { + pConn->usePublicIp = true; + } } + pConn->usePublicIp = pMsg->usePublicIp; + pConn->isCluster = pMsg->isCluster; + if (pMsg->msgType == TSDB_MSG_TYPE_CONNECT) { (*mgmtProcessShellMsg[pMsg->msgType])((char *)pMsg->content, pMsg->msgLen - sizeof(SIntMsg), pConn); } else { From 04b5df6273da6ed9fa307bf3cfc494da9a984478 Mon Sep 17 00:00:00 2001 From: slguan Date: Wed, 11 Dec 2019 15:12:17 +0800 Subject: [PATCH 105/123] [TBASE-1241] --- src/inc/taosmsg.h | 11 +++-------- src/inc/tglobalcfg.h | 2 +- src/inc/trpc.h | 2 ++ src/rpc/src/trpc.c | 7 +++++-- src/system/detail/inc/mgmt.h | 5 ++--- src/system/detail/src/mgmtShell.c | 22 +++++++++------------- src/util/src/tglobalcfg.c | 6 ++---- 7 files changed, 24 insertions(+), 31 deletions(-) diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 1d20ad76ed..b4baaff745 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -224,9 +224,7 @@ typedef struct { uint32_t destId; char meterId[TSDB_UNI_LEN]; uint16_t port; // for UDP only - uint8_t usePublicIp : 1; - uint8_t isCluster : 1; - uint8_t empty : 6; + char empty[1]; uint8_t msgType; int32_t msgLen; uint8_t content[0]; @@ -666,11 +664,8 @@ typedef struct { typedef struct { uint32_t destId; char meterId[TSDB_UNI_LEN]; - uint16_t port; // for UDP only - uint8_t usePublicIp : 1; - uint8_t isCluster : 1; - uint8_t empty : 6; - uint8_t msgType; + char empty[3]; + char msgType; int32_t msgLen; uint8_t content[0]; } SIntMsg; diff --git a/src/inc/tglobalcfg.h b/src/inc/tglobalcfg.h index 0bf47d75ad..60b18c77dc 100644 --- a/src/inc/tglobalcfg.h +++ b/src/inc/tglobalcfg.h @@ -80,6 +80,7 @@ extern short tsNumOfVnodesPerCore; extern short tsNumOfTotalVnodes; extern short tsCheckHeaderFile; extern uint32_t tsServerIp; +extern uint32_t tsPublicIpInt; extern int tsSessionsPerVnode; extern int tsAverageCacheBlocks; @@ -151,7 +152,6 @@ extern int tsAdminRowLimit; extern int tsTscEnableRecordSql; extern int tsAnyIp; -extern int tsUsePublicIp; extern int tsIsCluster; extern char tsMonitorDbName[]; diff --git a/src/inc/trpc.h b/src/inc/trpc.h index 97a0c905f8..6554d0d201 100644 --- a/src/inc/trpc.h +++ b/src/inc/trpc.h @@ -109,6 +109,8 @@ int taosSetSecurityInfo(int cid, int sid, char *id, int spi, int encrypt, char * void taosGetRpcConnInfo(void *thandle, uint32_t *peerId, uint32_t *peerIp, uint16_t *peerPort, int *cid, int *sid); +uint32_t taosGetRpcLocalIp(void *thandle); + int taosGetOutType(void *thandle); #ifdef __cplusplus diff --git a/src/rpc/src/trpc.c b/src/rpc/src/trpc.c index 7b5229ddc7..35740acc14 100755 --- a/src/rpc/src/trpc.c +++ b/src/rpc/src/trpc.c @@ -246,8 +246,6 @@ char *taosBuildReqHeader(void *param, char type, char *msg) { pHeader->destId = pConn->peerId; pHeader->port = 0; pHeader->uid = (uint32_t)pConn + (uint32_t)getpid(); - pHeader->usePublicIp = (tsUsePublicIp == 0 ? 0 : 1); - pHeader->isCluster = (tsIsCluster == 0 ? 0 : 1); memcpy(pHeader->meterId, pConn->meterId, tListLen(pHeader->meterId)); @@ -1412,6 +1410,11 @@ void taosGetRpcConnInfo(void *thandle, uint32_t *peerId, uint32_t *peerIp, uint1 *sid = pConn->sid; } +uint32_t taosGetRpcLocalIp(void *thandle) { + SRpcConn *pConn = (SRpcConn *)thandle; + return pConn->peerIp; +} + int taosGetOutType(void *thandle) { SRpcConn *pConn = (SRpcConn *)thandle; if (pConn == NULL) return -1; diff --git a/src/system/detail/inc/mgmt.h b/src/system/detail/inc/mgmt.h index e339274025..d5d604e313 100644 --- a/src/system/detail/inc/mgmt.h +++ b/src/system/detail/inc/mgmt.h @@ -222,9 +222,8 @@ typedef struct _connObj { char superAuth : 1; // super user flag char writeAuth : 1; // write flag char killConnection : 1; // kill the connection flag - char usePublicIp : 1; // if the connection request is publicIp - char isCluster : 1; - char reserved : 3; + uint8_t usePublicIp : 1; // if the connection request is publicIp + uint8_t reserved : 4; uint32_t queryId; // query ID to be killed uint32_t streamId; // stream ID to be killed uint32_t ip; // shell IP diff --git a/src/system/detail/src/mgmtShell.c b/src/system/detail/src/mgmtShell.c index 4334218530..d94c205625 100644 --- a/src/system/detail/src/mgmtShell.c +++ b/src/system/detail/src/mgmtShell.c @@ -1201,12 +1201,6 @@ int mgmtProcessConnectMsg(char *pMsg, int msgLen, SConnObj *pConn) { goto _rsp; } - if (pConn->isCluster != tsIsCluster) { - mError("Cluster Edition and lite Edition cannot be interconnected, client:%d server:%d", pConn->isCluster, tsIsCluster); - code = TSDB_CODE_INVALID_CLIENT_VERSION; - goto _rsp; - } - if (pConnectMsg->db[0]) { sprintf(dbName, "%x%s%s", pAcct->acctId, TS_PATH_DELIMITER, pConnectMsg->db); pDb = mgmtGetDb(dbName); @@ -1226,7 +1220,12 @@ int mgmtProcessConnectMsg(char *pMsg, int msgLen, SConnObj *pConn) { pConn->pAcct = pAcct; pConn->pDb = pDb; pConn->pUser = pUser; + + uint32_t peerIp = taosGetRpcLocalIp(pConn->thandle); + pConn->usePublicIp = (peerIp == tsPublicIpInt ? 1 : 0); mgmtEstablishConn(pConn); + mPrint("pConn:%p is created, peerIp:%s publicIp:%s usePublicIp:%u", + pConn, taosIpStr(peerIp), taosIpStr(tsPublicIpInt), pConn->usePublicIp); _rsp: pStart = taosBuildRspMsgWithSize(pConn->thandle, TSDB_MSG_TYPE_CONNECT_RSP, 128); @@ -1302,15 +1301,12 @@ void *mgmtProcessMsgFromShell(char *msg, void *ahandle, void *thandle) { pConn->thandle = thandle; strcpy(pConn->user, pMsg->meterId); - uint32_t ip = taosGetRpcLocalIp(thandle); - if (ip == tsPublicIp) { - pConn->usePublicIp = true; - } + uint32_t peerIp = taosGetRpcLocalIp(thandle); + pConn->usePublicIp = (peerIp == tsPublicIpInt ? 1 : 0); + mPrint("pConn:%p is rebuild, peerIp:%s publicIp:%s usePublicIp:%u", + pConn, taosIpStr(peerIp), taosIpStr(tsPublicIpInt), pConn->usePublicIp); } - pConn->usePublicIp = pMsg->usePublicIp; - pConn->isCluster = pMsg->isCluster; - if (pMsg->msgType == TSDB_MSG_TYPE_CONNECT) { (*mgmtProcessShellMsg[pMsg->msgType])((char *)pMsg->content, pMsg->msgLen - sizeof(SIntMsg), pConn); } else { diff --git a/src/util/src/tglobalcfg.c b/src/util/src/tglobalcfg.c index edff1c432b..748843cdcc 100644 --- a/src/util/src/tglobalcfg.c +++ b/src/util/src/tglobalcfg.c @@ -164,7 +164,7 @@ int tsAdminRowLimit = 10240; int tsTscEnableRecordSql = 0; int tsEnableCoreFile = 0; int tsAnyIp = 1; -int tsUsePublicIp = 0; +uint32_t tsPublicIpInt = 0; #ifdef CLUSTER int tsIsCluster = 1; @@ -789,9 +789,6 @@ static void doInitGlobalConfig() { 0, 1, 0, TSDB_CFG_UTYPE_NONE); #endif - tsInitConfigOption(cfg++, "usePublicIp", &tsUsePublicIp, TSDB_CFG_VTYPE_INT, - TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_CLIENT, - 0, 1, 0, TSDB_CFG_UTYPE_NONE); // version info tsInitConfigOption(cfg++, "gitinfo", gitinfo, TSDB_CFG_VTYPE_STRING, TSDB_CFG_CTYPE_B_SHOW | TSDB_CFG_CTYPE_B_CLIENT, @@ -916,6 +913,7 @@ bool tsReadGlobalConfig() { if (tsPublicIp[0] == 0) { strcpy(tsPublicIp, tsPrivateIp); } + tsPublicIpInt = inet_addr(tsPublicIp); if (tsLocalIp[0] == 0) { strcpy(tsLocalIp, tsPrivateIp); From 8314e22328d5872f763522aa0d7d49a31c19099c Mon Sep 17 00:00:00 2001 From: slguan Date: Wed, 11 Dec 2019 15:23:08 +0800 Subject: [PATCH 106/123] [TBASE-1241] --- src/system/detail/src/mgmtShell.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/system/detail/src/mgmtShell.c b/src/system/detail/src/mgmtShell.c index d94c205625..f78791ea75 100644 --- a/src/system/detail/src/mgmtShell.c +++ b/src/system/detail/src/mgmtShell.c @@ -1224,9 +1224,7 @@ int mgmtProcessConnectMsg(char *pMsg, int msgLen, SConnObj *pConn) { uint32_t peerIp = taosGetRpcLocalIp(pConn->thandle); pConn->usePublicIp = (peerIp == tsPublicIpInt ? 1 : 0); mgmtEstablishConn(pConn); - mPrint("pConn:%p is created, peerIp:%s publicIp:%s usePublicIp:%u", - pConn, taosIpStr(peerIp), taosIpStr(tsPublicIpInt), pConn->usePublicIp); - + _rsp: pStart = taosBuildRspMsgWithSize(pConn->thandle, TSDB_MSG_TYPE_CONNECT_RSP, 128); if (pStart == NULL) return 0; From 992a449ed6167c3144b990accd4626d415def36d Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Wed, 11 Dec 2019 15:30:54 +0800 Subject: [PATCH 107/123] [tbase-1327] --- src/client/src/tscServer.c | 18 ++++++++---------- src/system/detail/src/mgmtMeter.c | 7 +++++-- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 17d870fcb2..e10865b642 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1507,7 +1507,6 @@ int tscBuildQueryMsg(SSqlObj *pSql) { pQueryMsg->uid = pMeterMeta->uid; pQueryMsg->numOfTagsCols = 0; } else { // query on metric - SMetricMeta *pMetricMeta = pMeterMetaInfo->pMetricMeta; if (pMeterMetaInfo->vnodeIndex < 0) { tscError("%p error vnodeIdx:%d", pSql, pMeterMetaInfo->vnodeIndex); return -1; @@ -2872,15 +2871,14 @@ int tscBuildMetricMetaMsg(SSqlObj *pSql) { pElem->groupbyTagColumnList = htonl(offset); for (int32_t j = 0; j < pCmd->groupbyExpr.numOfGroupCols; ++j) { SColIndexEx *pCol = &pCmd->groupbyExpr.columnInfo[j]; - - *((int16_t *)pMsg) = pCol->colId; - pMsg += sizeof(pCol->colId); - - *((int16_t *)pMsg) += pCol->colIdx; - pMsg += sizeof(pCol->colIdx); - - *((int16_t *)pMsg) += pCol->flag; - pMsg += sizeof(pCol->flag); + SColIndexEx* pDestCol = (SColIndexEx*) pMsg; + + pDestCol->colIdxInBuf = 0; + pDestCol->colIdx = htons(pCol->colIdx); + pDestCol->colId = htons(pDestCol->colId); + pDestCol->flag = htons(pDestCol->flag); + + pMsg += sizeof(SColIndexEx); } } } diff --git a/src/system/detail/src/mgmtMeter.c b/src/system/detail/src/mgmtMeter.c index 5ef9f61b0b..be141f278d 100644 --- a/src/system/detail/src/mgmtMeter.c +++ b/src/system/detail/src/mgmtMeter.c @@ -1094,9 +1094,12 @@ static SMetricMetaElemMsg *doConvertMetricMetaMsg(SMetricMetaMsg *pMetricMetaMsg pElem->groupbyTagColumnList = htonl(pElem->groupbyTagColumnList); - int16_t *groupColIds = (int16_t*) (((char *)pMetricMetaMsg) + pElem->groupbyTagColumnList); + SColIndexEx *groupColIds = (SColIndexEx*) (((char *)pMetricMetaMsg) + pElem->groupbyTagColumnList); for (int32_t i = 0; i < pElem->numOfGroupCols; ++i) { - groupColIds[i] = htons(groupColIds[i]); + groupColIds[i].colId = htons(groupColIds[i].colId); + groupColIds[i].colIdx = htons(groupColIds[i].colIdx); + groupColIds[i].flag = htons(groupColIds[i].flag); + groupColIds[i].colIdxInBuf = 0; } return pElem; From 7d6db536d0f16b61eaf99e377f08b8464b0d5d06 Mon Sep 17 00:00:00 2001 From: slguan Date: Wed, 11 Dec 2019 16:19:44 +0800 Subject: [PATCH 108/123] [TBASE-1241] --- src/inc/taosmsg.h | 2 ++ src/rpc/src/trpc.c | 1 + src/system/detail/src/mgmtShell.c | 11 +++++------ 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index b4baaff745..db36309222 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -222,6 +222,7 @@ typedef struct { // internal part uint32_t destId; + uint32_t destIp; char meterId[TSDB_UNI_LEN]; uint16_t port; // for UDP only char empty[1]; @@ -663,6 +664,7 @@ typedef struct { // internal message typedef struct { uint32_t destId; + uint32_t destIp; char meterId[TSDB_UNI_LEN]; char empty[3]; char msgType; diff --git a/src/rpc/src/trpc.c b/src/rpc/src/trpc.c index 35740acc14..21f2ae4dc0 100755 --- a/src/rpc/src/trpc.c +++ b/src/rpc/src/trpc.c @@ -1219,6 +1219,7 @@ int taosSendMsgToPeerH(void *thandle, char *pCont, int contLen, void *ahandle) { pServer = pConn->pServer; pChann = pServer->channList + pConn->chann; pHeader = (STaosHeader *)(pCont - sizeof(STaosHeader)); + pHeader->destIp = pConn->peerIp; msg = (char *)pHeader; if ((pHeader->msgType & 1U) == 0 && pConn->localPort) pHeader->port = pConn->localPort; diff --git a/src/system/detail/src/mgmtShell.c b/src/system/detail/src/mgmtShell.c index f78791ea75..c56939e0dd 100644 --- a/src/system/detail/src/mgmtShell.c +++ b/src/system/detail/src/mgmtShell.c @@ -1224,7 +1224,7 @@ int mgmtProcessConnectMsg(char *pMsg, int msgLen, SConnObj *pConn) { uint32_t peerIp = taosGetRpcLocalIp(pConn->thandle); pConn->usePublicIp = (peerIp == tsPublicIpInt ? 1 : 0); mgmtEstablishConn(pConn); - + _rsp: pStart = taosBuildRspMsgWithSize(pConn->thandle, TSDB_MSG_TYPE_CONNECT_RSP, 128); if (pStart == NULL) return 0; @@ -1298,13 +1298,12 @@ void *mgmtProcessMsgFromShell(char *msg, void *ahandle, void *thandle) { pConn = connList + pMsg->destId; pConn->thandle = thandle; strcpy(pConn->user, pMsg->meterId); - - uint32_t peerIp = taosGetRpcLocalIp(thandle); - pConn->usePublicIp = (peerIp == tsPublicIpInt ? 1 : 0); - mPrint("pConn:%p is rebuild, peerIp:%s publicIp:%s usePublicIp:%u", - pConn, taosIpStr(peerIp), taosIpStr(tsPublicIpInt), pConn->usePublicIp); } + pConn->usePublicIp = (pMsg->destIp == tsPublicIpInt ? 1 : 0); + mTrace("pConn:%p, destIp:%s publicIp:%s usePublicIp:%u", + pConn, taosIpStr(pMsg->destIp), taosIpStr(tsPublicIpInt), pConn->usePublicIp); + if (pMsg->msgType == TSDB_MSG_TYPE_CONNECT) { (*mgmtProcessShellMsg[pMsg->msgType])((char *)pMsg->content, pMsg->msgLen - sizeof(SIntMsg), pConn); } else { From a3c0c50d9000078c6657f3db4a5bff4669f97123 Mon Sep 17 00:00:00 2001 From: slguan Date: Wed, 11 Dec 2019 16:22:11 +0800 Subject: [PATCH 109/123] [TBASE-1241] --- src/system/detail/src/mgmtShell.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/system/detail/src/mgmtShell.c b/src/system/detail/src/mgmtShell.c index c56939e0dd..c5fb87bcdb 100644 --- a/src/system/detail/src/mgmtShell.c +++ b/src/system/detail/src/mgmtShell.c @@ -1298,11 +1298,11 @@ void *mgmtProcessMsgFromShell(char *msg, void *ahandle, void *thandle) { pConn = connList + pMsg->destId; pConn->thandle = thandle; strcpy(pConn->user, pMsg->meterId); - } - pConn->usePublicIp = (pMsg->destIp == tsPublicIpInt ? 1 : 0); - mTrace("pConn:%p, destIp:%s publicIp:%s usePublicIp:%u", - pConn, taosIpStr(pMsg->destIp), taosIpStr(tsPublicIpInt), pConn->usePublicIp); + pConn->usePublicIp = (pMsg->destIp == tsPublicIpInt ? 1 : 0); + mPrint("pConn:%p is rebuild, destIp:%s publicIp:%s usePublicIp:%u", + pConn, taosIpStr(pMsg->destIp), taosIpStr(tsPublicIpInt), pConn->usePublicIp); + } if (pMsg->msgType == TSDB_MSG_TYPE_CONNECT) { (*mgmtProcessShellMsg[pMsg->msgType])((char *)pMsg->content, pMsg->msgLen - sizeof(SIntMsg), pConn); From 6dfc958cf4a36dcf07f042aaf559b38437eb6763 Mon Sep 17 00:00:00 2001 From: slguan Date: Wed, 11 Dec 2019 16:30:40 +0800 Subject: [PATCH 110/123] [TBASE-1241] --- src/inc/trpc.h | 2 -- src/rpc/src/trpc.c | 5 ----- src/system/detail/src/mgmtShell.c | 10 +++------- 3 files changed, 3 insertions(+), 14 deletions(-) diff --git a/src/inc/trpc.h b/src/inc/trpc.h index 6554d0d201..97a0c905f8 100644 --- a/src/inc/trpc.h +++ b/src/inc/trpc.h @@ -109,8 +109,6 @@ int taosSetSecurityInfo(int cid, int sid, char *id, int spi, int encrypt, char * void taosGetRpcConnInfo(void *thandle, uint32_t *peerId, uint32_t *peerIp, uint16_t *peerPort, int *cid, int *sid); -uint32_t taosGetRpcLocalIp(void *thandle); - int taosGetOutType(void *thandle); #ifdef __cplusplus diff --git a/src/rpc/src/trpc.c b/src/rpc/src/trpc.c index 21f2ae4dc0..88bfbc2c2d 100755 --- a/src/rpc/src/trpc.c +++ b/src/rpc/src/trpc.c @@ -1411,11 +1411,6 @@ void taosGetRpcConnInfo(void *thandle, uint32_t *peerId, uint32_t *peerIp, uint1 *sid = pConn->sid; } -uint32_t taosGetRpcLocalIp(void *thandle) { - SRpcConn *pConn = (SRpcConn *)thandle; - return pConn->peerIp; -} - int taosGetOutType(void *thandle) { SRpcConn *pConn = (SRpcConn *)thandle; if (pConn == NULL) return -1; diff --git a/src/system/detail/src/mgmtShell.c b/src/system/detail/src/mgmtShell.c index c5fb87bcdb..b738c0dab3 100644 --- a/src/system/detail/src/mgmtShell.c +++ b/src/system/detail/src/mgmtShell.c @@ -1220,11 +1220,8 @@ int mgmtProcessConnectMsg(char *pMsg, int msgLen, SConnObj *pConn) { pConn->pAcct = pAcct; pConn->pDb = pDb; pConn->pUser = pUser; - - uint32_t peerIp = taosGetRpcLocalIp(pConn->thandle); - pConn->usePublicIp = (peerIp == tsPublicIpInt ? 1 : 0); mgmtEstablishConn(pConn); - + _rsp: pStart = taosBuildRspMsgWithSize(pConn->thandle, TSDB_MSG_TYPE_CONNECT_RSP, 128); if (pStart == NULL) return 0; @@ -1298,10 +1295,9 @@ void *mgmtProcessMsgFromShell(char *msg, void *ahandle, void *thandle) { pConn = connList + pMsg->destId; pConn->thandle = thandle; strcpy(pConn->user, pMsg->meterId); - pConn->usePublicIp = (pMsg->destIp == tsPublicIpInt ? 1 : 0); - mPrint("pConn:%p is rebuild, destIp:%s publicIp:%s usePublicIp:%u", - pConn, taosIpStr(pMsg->destIp), taosIpStr(tsPublicIpInt), pConn->usePublicIp); + mTrace("pConn:%p is rebuild, destIp:%s publicIp:%s usePublicIp:%u", + pConn, taosIpStr(pMsg->destIp), taosIpStr(tsPublicIpInt), pConn->usePublicIp); } if (pMsg->msgType == TSDB_MSG_TYPE_CONNECT) { From 38d377b421fa99a06de872fa48c4abda9f24ef3c Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Wed, 11 Dec 2019 16:49:46 +0800 Subject: [PATCH 111/123] [tbase-1328] --- src/system/detail/src/mgmtSupertableQuery.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/system/detail/src/mgmtSupertableQuery.c b/src/system/detail/src/mgmtSupertableQuery.c index b56173f14f..10a64408ce 100644 --- a/src/system/detail/src/mgmtSupertableQuery.c +++ b/src/system/detail/src/mgmtSupertableQuery.c @@ -784,14 +784,15 @@ int mgmtRetrieveMetersFromMetric(SMetricMetaMsg* pMsg, int32_t tableIndex, tQuer } // todo refactor!!!!! -static char* getTagValueFromMeter(STabObj* pMeter, int32_t offset, void* param) { +static char* getTagValueFromMeter(STabObj* pMeter, int32_t offset, int32_t len, char* param) { if (offset == TSDB_TBNAME_COLUMN_INDEX) { extractMeterName(pMeter->meterId, param); - return param; } else { - char* tags = pMeter->pTagData + TSDB_METER_ID_LEN; // tag start position - return (tags + offset); + char* tags = pMeter->pTagData + offset + TSDB_METER_ID_LEN; // tag start position + memcpy(param, tags, len); // make sure the value is null-terminated string } + + return param; } bool tSkipListNodeFilterCallback(const void* pNode, void* param) { @@ -799,8 +800,9 @@ bool tSkipListNodeFilterCallback(const void* pNode, void* param) { tQueryInfo* pInfo = (tQueryInfo*)param; STabObj* pMeter = (STabObj*)(((tSkipListNode*)pNode)->pData); - char name[TSDB_METER_NAME_LEN + 1] = {0}; - char* val = getTagValueFromMeter(pMeter, pInfo->offset, name); + char buf[TSDB_MAX_TAGS_LEN] = {0}; + + char* val = getTagValueFromMeter(pMeter, pInfo->offset, pInfo->sch.bytes, buf); int8_t type = pInfo->sch.type; int32_t ret = 0; From 8636bc95aaf7aac1596abfdc8d45c8b6da4b250d Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Wed, 11 Dec 2019 20:23:34 +0800 Subject: [PATCH 112/123] [tbase-1326] --- src/client/src/tscJoinProcess.c | 3 +-- src/system/detail/inc/vnode.h | 2 +- src/system/detail/src/vnodeQueryProcess.c | 6 ------ src/system/detail/src/vnodeRead.c | 8 +++++--- src/system/detail/src/vnodeShell.c | 2 +- 5 files changed, 8 insertions(+), 13 deletions(-) diff --git a/src/client/src/tscJoinProcess.c b/src/client/src/tscJoinProcess.c index 375e0066b1..a94b308e87 100644 --- a/src/client/src/tscJoinProcess.c +++ b/src/client/src/tscJoinProcess.c @@ -528,8 +528,7 @@ void tscFetchDatablockFromSubquery(SSqlObj* pSql) { numOfFetch++; } } else { - if ((pRes->row >= pRes->numOfRows && (!tscHasReachLimitation(pSql->pSubs[i])) && - tscProjectionQueryOnTable(&pSql->cmd)) || (pRes->numOfRows == 0)) { + if (pRes->row >= pRes->numOfRows && (!tscHasReachLimitation(pSql->pSubs[i]))) { numOfFetch++; } } diff --git a/src/system/detail/inc/vnode.h b/src/system/detail/inc/vnode.h index 4ac0848427..37e31803d0 100644 --- a/src/system/detail/inc/vnode.h +++ b/src/system/detail/inc/vnode.h @@ -339,7 +339,7 @@ extern void * vnodeTmrCtrl; // read API extern int (*vnodeSearchKeyFunc[])(char *pValue, int num, TSKEY key, int order); -void *vnodeQueryInTimeRange(SMeterObj **pMeterObj, SSqlGroupbyExpr *pGroupbyExpr, SSqlFunctionExpr *sqlExprs, +void *vnodeQueryOnSingleTable(SMeterObj **pMeterObj, SSqlGroupbyExpr *pGroupbyExpr, SSqlFunctionExpr *sqlExprs, SQueryMeterMsg *pQueryMsg, int *code); void *vnodeQueryOnMultiMeters(SMeterObj **pMeterObj, SSqlGroupbyExpr *pGroupbyExpr, SSqlFunctionExpr *pSqlExprs, diff --git a/src/system/detail/src/vnodeQueryProcess.c b/src/system/detail/src/vnodeQueryProcess.c index f39ca0dea1..231b47206e 100644 --- a/src/system/detail/src/vnodeQueryProcess.c +++ b/src/system/detail/src/vnodeQueryProcess.c @@ -890,8 +890,6 @@ static void vnodeMultiMeterQueryProcessor(SQInfo *pQInfo) { dTrace("QInfo:%p points returned:%d, totalRead:%d totalReturn:%d", pQInfo, pQuery->pointsRead, pQInfo->pointsRead, pQInfo->pointsReturned); - - vnodeDecRefCount(pQInfo); return; } @@ -899,7 +897,6 @@ static void vnodeMultiMeterQueryProcessor(SQInfo *pQInfo) { if (pSupporter->pMeterDataInfo == NULL) { dError("QInfo:%p failed to allocate memory, %s", pQInfo, strerror(errno)); pQInfo->code = -TSDB_CODE_SERV_OUT_OF_MEMORY; - vnodeDecRefCount(pQInfo); return; } @@ -916,7 +913,6 @@ static void vnodeMultiMeterQueryProcessor(SQInfo *pQInfo) { // failed to save all intermediate results into disk, abort further query processing if (doCloseAllOpenedResults(pSupporter) != TSDB_CODE_SUCCESS) { dError("QInfo:%p failed to save intermediate results, abort further query processing", pQInfo); - vnodeDecRefCount(pQInfo); return; } @@ -924,7 +920,6 @@ static void vnodeMultiMeterQueryProcessor(SQInfo *pQInfo) { if (isQueryKilled(pQuery)) { dTrace("QInfo:%p query killed, abort", pQInfo); - vnodeDecRefCount(pQInfo); return; } @@ -946,7 +941,6 @@ static void vnodeMultiMeterQueryProcessor(SQInfo *pQInfo) { pQInfo->pointsRead += pQuery->pointsRead; dTrace("QInfo:%p points returned:%d, totalRead:%d totalReturn:%d", pQInfo, pQuery->pointsRead, pQInfo->pointsRead, pQInfo->pointsReturned); - vnodeDecRefCount(pQInfo); } /* diff --git a/src/system/detail/src/vnodeRead.c b/src/system/detail/src/vnodeRead.c index 2ea41d9d58..5640c6bc0d 100644 --- a/src/system/detail/src/vnodeRead.c +++ b/src/system/detail/src/vnodeRead.c @@ -512,8 +512,9 @@ void vnodeDecRefCount(void *param) { assert(vnodeIsQInfoValid(pQInfo)); int32_t ref = atomic_sub_fetch_32(&pQInfo->refCount, 1); - dTrace("QInfo:%p decrease obj refcount, %d", pQInfo, ref); + assert(ref >= 0); + dTrace("QInfo:%p decrease obj refcount, %d", pQInfo, ref); if (ref == 0) { vnodeFreeQInfo(pQInfo, true); } @@ -616,7 +617,7 @@ void vnodeQueryData(SSchedMsg *pMsg) { vnodeDecRefCount(pQInfo); } -void *vnodeQueryInTimeRange(SMeterObj **pMetersObj, SSqlGroupbyExpr *pGroupbyExpr, SSqlFunctionExpr *pSqlExprs, +void *vnodeQueryOnSingleTable(SMeterObj **pMetersObj, SSqlGroupbyExpr *pGroupbyExpr, SSqlFunctionExpr *pSqlExprs, SQueryMeterMsg *pQueryMsg, int32_t *code) { SQInfo *pQInfo; SQuery *pQuery; @@ -687,6 +688,7 @@ void *vnodeQueryInTimeRange(SMeterObj **pMetersObj, SSqlGroupbyExpr *pGroupbyExp } if (pQInfo->over == 1) { + vnodeAddRefCount(pQInfo); // for retrieve procedure return pQInfo; } @@ -811,13 +813,13 @@ void *vnodeQueryOnMultiMeters(SMeterObj **pMetersObj, SSqlGroupbyExpr *pGroupbyE goto _error; } + vnodeAddRefCount(pQInfo); if (pQInfo->over == 1) { return pQInfo; } // pQInfo->signature = TSDB_QINFO_QUERY_FLAG; vnodeAddRefCount(pQInfo); - vnodeAddRefCount(pQInfo); schedMsg.msg = NULL; schedMsg.thandle = (void *)1; diff --git a/src/system/detail/src/vnodeShell.c b/src/system/detail/src/vnodeShell.c index 6ff3f4d27c..5efbb41014 100644 --- a/src/system/detail/src/vnodeShell.c +++ b/src/system/detail/src/vnodeShell.c @@ -376,7 +376,7 @@ int vnodeProcessQueryRequest(char *pMsg, int msgLen, SShellObj *pObj) { if (QUERY_IS_STABLE_QUERY(pQueryMsg->queryType)) { pObj->qhandle = vnodeQueryOnMultiMeters(pMeterObjList, pGroupbyExpr, pExprs, pQueryMsg, &code); } else { - pObj->qhandle = vnodeQueryInTimeRange(pMeterObjList, pGroupbyExpr, pExprs, pQueryMsg, &code); + pObj->qhandle = vnodeQueryOnSingleTable(pMeterObjList, pGroupbyExpr, pExprs, pQueryMsg, &code); } _query_over: From c060be25b937f946eabcfceaac6342e022398733 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Wed, 11 Dec 2019 21:02:32 +0800 Subject: [PATCH 113/123] [tbase-1327] --- src/client/src/tscServer.c | 6 ++++++ src/client/src/tscUtil.c | 8 +++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index e10865b642..40399d85b7 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1290,6 +1290,12 @@ static SSqlObj *tscCreateSqlObjForSubquery(SSqlObj *pSql, SRetrieveSupport *trsu SSqlObj *pNew = createSubqueryObj(pSql, 0, tscRetrieveDataRes, trsupport, prevSqlObj); if (pNew != NULL) { // the sub query of two-stage super table query pNew->cmd.type |= TSDB_QUERY_TYPE_STABLE_SUBQUERY; + assert(pNew->cmd.numOfTables == 1); + + //launch subquery for each vnode, so the subquery index equals to the vnodeIndex. + SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(&pNew->cmd, 0); + pMeterMetaInfo->vnodeIndex = trsupport->subqueryIndex; + pSql->pSubs[trsupport->subqueryIndex] = pNew; } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 9876dad906..4b0df76784 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1706,12 +1706,10 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void } pNew->fp = fp; - pNew->param = param; - SMeterMetaInfo* pMetermetaInfo = tscGetMeterMetaInfo(pCmd, tableIndex); - + char key[TSDB_MAX_TAGS_LEN + 1] = {0}; - tscGetMetricMetaCacheKey(pCmd, key, pMetermetaInfo->pMeterMeta->uid); + tscGetMetricMetaCacheKey(pCmd, key, uid); #ifdef _DEBUG_VIEW printf("the metricmeta key is:%s\n", key); @@ -1736,7 +1734,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void } assert(pFinalInfo->pMeterMeta != NULL); - if (UTIL_METER_IS_METRIC(pMetermetaInfo)) { + if (UTIL_METER_IS_METRIC(pMeterMetaInfo)) { assert(pFinalInfo->pMetricMeta != NULL); } From e03f3364b80e7b7ba7a4b1479390216c826df6d2 Mon Sep 17 00:00:00 2001 From: hjxilinx Date: Wed, 11 Dec 2019 21:03:52 +0800 Subject: [PATCH 114/123] [tbase-1326] --- src/system/detail/inc/vnode.h | 22 ------------------ src/system/detail/src/vnodeQueryProcess.c | 28 ++++++----------------- src/system/detail/src/vnodeRead.c | 25 ++------------------ 3 files changed, 9 insertions(+), 66 deletions(-) diff --git a/src/system/detail/inc/vnode.h b/src/system/detail/inc/vnode.h index 37e31803d0..3a5e7e7260 100644 --- a/src/system/detail/inc/vnode.h +++ b/src/system/detail/inc/vnode.h @@ -206,28 +206,6 @@ typedef struct { char cont[]; } SVMsgHeader; -/* - * The value of QInfo.signature is used to denote that a query is executing, it isn't safe to release QInfo yet. - * The release operations will be blocked in a busy-waiting until the query operation reach a safepoint. - * Then it will reset the signature in a atomic operation, followed by release operation. - * Only the QInfo.signature == QInfo, this structure can be released safely. - */ -#define TSDB_QINFO_QUERY_FLAG 0x1 -//#define TSDB_QINFO_RESET_SIG(x) atomic_store_64(&((x)->signature), (uint64_t)(x)) -#define TSDB_QINFO_RESET_SIG(x) -#define TSDB_QINFO_SET_QUERY_FLAG(x) -//#define TSDB_QINFO_SET_QUERY_FLAG(x) \ -// atomic_val_compare_exchange_64(&((x)->signature), (uint64_t)(x), TSDB_QINFO_QUERY_FLAG); - -// live lock: wait for query reaching a safe-point, release all resources -// belongs to this query -#define TSDB_WAIT_TO_SAFE_DROP_QINFO(x) \ -// { \ -// while (atomic_val_compare_exchange_64(&((x)->signature), (x), 0) == TSDB_QINFO_QUERY_FLAG) { \ -// taosMsleep(1); \ -// } \ -// } - struct tSQLBinaryExpr; typedef struct SColumnInfoEx { diff --git a/src/system/detail/src/vnodeQueryProcess.c b/src/system/detail/src/vnodeQueryProcess.c index 231b47206e..a545645a09 100644 --- a/src/system/detail/src/vnodeQueryProcess.c +++ b/src/system/detail/src/vnodeQueryProcess.c @@ -1173,15 +1173,13 @@ void vnodeSingleMeterQuery(SSchedMsg *pMsg) { } if (pQInfo->killed) { - TSDB_QINFO_RESET_SIG(pQInfo); - dTrace("QInfo:%p it is already killed, reset signature and abort", pQInfo); + dTrace("QInfo:%p it is already killed, abort", pQInfo); vnodeDecRefCount(pQInfo); return; } assert(pQInfo->refCount >= 1); -// assert(pQInfo->signature == TSDB_QINFO_QUERY_FLAG); SQuery * pQuery = &pQInfo->query; SMeterObj *pMeterObj = pQInfo->pObj; @@ -1215,10 +1213,7 @@ void vnodeSingleMeterQuery(SSchedMsg *pMsg) { pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->pointsRead, numOfInterpo, pQInfo->pointsRead, pQInfo->pointsInterpo, pQInfo->pointsReturned); - dTrace("QInfo:%p reset signature", pQInfo); - sem_post(&pQInfo->dataReady); - TSDB_QINFO_RESET_SIG(pQInfo); vnodeDecRefCount(pQInfo); return; @@ -1238,10 +1233,7 @@ void vnodeSingleMeterQuery(SSchedMsg *pMsg) { pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->pointsRead, pQInfo->pointsRead, pQInfo->pointsInterpo, pQInfo->pointsReturned); - dTrace("QInfo:%p reset signature", pQInfo); - sem_post(&pQInfo->dataReady); - TSDB_QINFO_RESET_SIG(pQInfo); vnodeDecRefCount(pQInfo); return; @@ -1250,12 +1242,12 @@ void vnodeSingleMeterQuery(SSchedMsg *pMsg) { } pQInfo->over = 1; - dTrace("QInfo:%p vid:%d sid:%d id:%s, query over, %d points are returned, reset signature", pQInfo, + dTrace("QInfo:%p vid:%d sid:%d id:%s, query over, %d points are returned", pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQInfo->pointsRead); vnodePrintQueryStatistics(pQInfo->pMeterQuerySupporter); sem_post(&pQInfo->dataReady); - TSDB_QINFO_RESET_SIG(pQInfo); + vnodeDecRefCount(pQInfo); return; } @@ -1284,16 +1276,14 @@ void vnodeSingleMeterQuery(SSchedMsg *pMsg) { /* check if query is killed or not */ if (isQueryKilled(pQuery)) { - dTrace("QInfo:%p query is killed, reset signature", pQInfo); + dTrace("QInfo:%p query is killed", pQInfo); pQInfo->over = 1; } else { - dTrace("QInfo:%p vid:%d sid:%d id:%s, meter query thread completed, %d points are returned, reset signature", + dTrace("QInfo:%p vid:%d sid:%d id:%s, meter query thread completed, %d points are returned", pQInfo, pMeterObj->vnode, pMeterObj->sid, pMeterObj->meterId, pQuery->pointsRead); } - TSDB_QINFO_RESET_SIG(pQInfo); sem_post(&pQInfo->dataReady); - vnodeDecRefCount(pQInfo); } @@ -1305,14 +1295,12 @@ void vnodeMultiMeterQuery(SSchedMsg *pMsg) { } if (pQInfo->killed) { - TSDB_QINFO_RESET_SIG(pQInfo); vnodeDecRefCount(pQInfo); - dTrace("QInfo:%p it is already killed, reset signature and abort", pQInfo); + dTrace("QInfo:%p it is already killed, abort", pQInfo); return; } assert(pQInfo->refCount >= 1); -// assert(pQInfo->signature == TSDB_QINFO_QUERY_FLAG); SQuery *pQuery = &pQInfo->query; pQuery->pointsRead = 0; @@ -1333,7 +1321,6 @@ void vnodeMultiMeterQuery(SSchedMsg *pMsg) { pQInfo->useconds += (taosGetTimestampUs() - st); pQInfo->over = isQueryKilled(pQuery) ? 1 : 0; - dTrace("QInfo:%p reset signature", pQInfo); taosInterpoSetStartInfo(&pQInfo->pMeterQuerySupporter->runtimeEnv.interpoInfo, pQuery->pointsRead, pQInfo->query.interpoType); @@ -1341,12 +1328,11 @@ void vnodeMultiMeterQuery(SSchedMsg *pMsg) { if (pQuery->pointsRead == 0) { pQInfo->over = 1; - dTrace("QInfo:%p over, %d meters queried, %d points are returned, reset signature", pQInfo, pSupporter->numOfMeters, + dTrace("QInfo:%p over, %d meters queried, %d points are returned", pQInfo, pSupporter->numOfMeters, pQInfo->pointsRead); vnodePrintQueryStatistics(pSupporter); } sem_post(&pQInfo->dataReady); - TSDB_QINFO_RESET_SIG(pQInfo); vnodeDecRefCount(pQInfo); } diff --git a/src/system/detail/src/vnodeRead.c b/src/system/detail/src/vnodeRead.c index 5640c6bc0d..ccd59f356b 100644 --- a/src/system/detail/src/vnodeRead.c +++ b/src/system/detail/src/vnodeRead.c @@ -422,8 +422,6 @@ void vnodeFreeQInfo(void *param, bool decQueryRef) { if (!vnodeIsQInfoValid(param)) return; pQInfo->killed = 1; - TSDB_WAIT_TO_SAFE_DROP_QINFO(pQInfo); - SMeterObj *pObj = pQInfo->pObj; dTrace("QInfo:%p start to free SQInfo", pQInfo); @@ -502,7 +500,6 @@ bool vnodeIsQInfoValid(void *param) { * into local variable, then compare by using local variable */ uint64_t sig = pQInfo->signature; -// return (sig == (uint64_t)pQInfo) || (sig == TSDB_QINFO_QUERY_FLAG); return (sig == (uint64_t)pQInfo); } @@ -536,13 +533,11 @@ void vnodeQueryData(SSchedMsg *pMsg) { pQInfo = (SQInfo *)pMsg->ahandle; if (pQInfo->killed) { - TSDB_QINFO_RESET_SIG(pQInfo); - dTrace("QInfo:%p it is already killed, reset signature and abort", pQInfo); + dTrace("QInfo:%p it is already killed, abort", pQInfo); vnodeDecRefCount(pQInfo); return; } -// assert(pQInfo->signature == TSDB_QINFO_QUERY_FLAG); pQuery = &(pQInfo->query); SMeterObj *pObj = pQInfo->pObj; @@ -610,9 +605,6 @@ void vnodeQueryData(SSchedMsg *pMsg) { tclose(pQInfo->query.lfd); } - /* reset QInfo signature */ - dTrace("QInfo:%p reset signature", pQInfo); - TSDB_QINFO_RESET_SIG(pQInfo); sem_post(&pQInfo->dataReady); vnodeDecRefCount(pQInfo); } @@ -697,9 +689,6 @@ void *vnodeQueryOnSingleTable(SMeterObj **pMetersObj, SSqlGroupbyExpr *pGroupbyE schedMsg.fp = vnodeQueryData; } - // set in query flag -// pQInfo->signature = TSDB_QINFO_QUERY_FLAG; - /* * The reference count, which is 2, is for both the current query thread and the future retrieve request, * which will always be issued by client to acquire data or free SQInfo struct. @@ -818,7 +807,6 @@ void *vnodeQueryOnMultiMeters(SMeterObj **pMetersObj, SSqlGroupbyExpr *pGroupbyE return pQInfo; } -// pQInfo->signature = TSDB_QINFO_QUERY_FLAG; vnodeAddRefCount(pQInfo); schedMsg.msg = NULL; @@ -900,18 +888,9 @@ int vnodeSaveQueryResult(void *handle, char *data, int32_t *size) { pQInfo->pointsRead); if (pQInfo->over == 0) { - //dTrace("QInfo:%p set query flag, oldSig:%p, func:%s", pQInfo, pQInfo->signature, __FUNCTION__); - dTrace("QInfo:%p set query flag, oldSig:%p", pQInfo, pQInfo->signature); -// uint64_t oldSignature = TSDB_QINFO_SET_QUERY_FLAG(pQInfo); + dTrace("QInfo:%p set query flag, sig:%p, func:%s", pQInfo, pQInfo->signature, __FUNCTION__); - /* - * If SQInfo has been released, the value of signature cannot be equalled to the address of pQInfo, - * since in release function, the original value has been destroyed. However, this memory area may be reused - * by another function. It may be 0 or any value, but it is rarely still be equalled to the address of SQInfo. - */ -// if (oldSignature == 0 || oldSignature != (uint64_t)pQInfo) { if (pQInfo->killed == 1) { -// dTrace("%p freed or killed, old sig:%p abort query", pQInfo, oldSignature); dTrace("%p freed or killed, abort query", pQInfo); } else { vnodeAddRefCount(pQInfo); From cbd3228f2d761916fd383e3eb770c8042c17afae Mon Sep 17 00:00:00 2001 From: slguan Date: Wed, 11 Dec 2019 22:08:17 +0800 Subject: [PATCH 115/123] remove some warnings --- src/client/src/sql.c | 2 ++ src/client/src/tscServer.c | 2 +- src/modules/monitor/src/monitorSystem.c | 38 +++++++++++++++---------- 3 files changed, 26 insertions(+), 16 deletions(-) diff --git a/src/client/src/sql.c b/src/client/src/sql.c index e0d96623e2..ffcdc4bc0e 100644 --- a/src/client/src/sql.c +++ b/src/client/src/sql.c @@ -22,6 +22,8 @@ ** The following is the concatenation of all %include directives from the ** input grammar file: */ +#pragma GCC diagnostic ignored "-Wunused-variable" + #include /************ Begin %include sections from the grammar ************************/ diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 40399d85b7..2b0b3c10da 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -211,7 +211,6 @@ void tscGetConnToMgmt(SSqlObj *pSql, uint8_t *pCode) { } void tscGetConnToVnode(SSqlObj *pSql, uint8_t *pCode) { - char ipstr[40] = {0}; SVPeerDesc *pVPeersDesc = NULL; static int vidIndex = 0; STscObj * pTscObj = pSql->pTscObj; @@ -244,6 +243,7 @@ void tscGetConnToVnode(SSqlObj *pSql, uint8_t *pCode) { while (pSql->retry < pSql->maxRetry) { (pSql->retry)++; #ifdef CLUSTER + char ipstr[40] = {0}; if (pVPeersDesc[pSql->index].ip == 0) { (pSql->index) = (pSql->index + 1) % TSDB_VNODES_SUPPORT; continue; diff --git a/src/modules/monitor/src/monitorSystem.c b/src/modules/monitor/src/monitorSystem.c index f89f259688..b4b84b6d29 100644 --- a/src/modules/monitor/src/monitorSystem.c +++ b/src/modules/monitor/src/monitorSystem.c @@ -403,21 +403,29 @@ void monitorSaveAcctLog(char *acctId, int64_t currentPointsPerSecond, int64_t ma char sql[1024] = {0}; sprintf(sql, "insert into %s.acct_%s using %s.acct tags('%s') values(now" - ", %" PRId64, "%" PRId64 - ", %" PRId64, "%" PRId64 - ", %" PRId64, "%" PRId64 - ", %" PRId64, "%" PRId64 - ", %" PRId64, "%" PRId64 - ", %" PRId64, "%" PRId64 - ", %" PRId64, "%" PRId64 - ", %" PRId64, "%" PRId64 - ", %" PRId64, "%" PRId64 - ", %" PRId64, "%" PRId64 + ", %" PRId64 ", %" PRId64 + ", %" PRId64 ", %" PRId64 + ", %" PRId64 ", %" PRId64 + ", %" PRId64 ", %" PRId64 + ", %" PRId64 ", %" PRId64 + ", %" PRId64 ", %" PRId64 + ", %" PRId64 ", %" PRId64 + ", %" PRId64 ", %" PRId64 + ", %" PRId64 ", %" PRId64 + ", %" PRId64 ", %" PRId64 ", %d)", - tsMonitorDbName, acctId, tsMonitorDbName, acctId, currentPointsPerSecond, maxPointsPerSecond, totalTimeSeries, - maxTimeSeries, totalStorage, maxStorage, totalQueryTime, maxQueryTime, totalInbound, maxInbound, - totalOutbound, maxOutbound, totalDbs, maxDbs, totalUsers, maxUsers, totalStreams, maxStreams, totalConns, - maxConns, accessState); + tsMonitorDbName, acctId, tsMonitorDbName, acctId, + currentPointsPerSecond, maxPointsPerSecond, + totalTimeSeries, maxTimeSeries, + totalStorage, maxStorage, + totalQueryTime, maxQueryTime, + totalInbound, maxInbound, + totalOutbound, maxOutbound, + totalDbs, maxDbs, + totalUsers, maxUsers, + totalStreams, maxStreams, + totalConns, maxConns, + accessState); monitorTrace("monitor:%p, save account info, sql %s", monitor->conn, sql); taos_query_a(monitor->conn, sql, dnodeMontiorInsertAcctCallback, "account"); @@ -432,7 +440,7 @@ void monitorSaveLog(int level, const char *const format, ...) { return; } - int len = snprintf(sql, (size_t)max_length, "import into %s.log values(%\" PRId64 \", %d,'", tsMonitorDbName, + int len = snprintf(sql, (size_t)max_length, "import into %s.log values(%" PRId64 ", %d,'", tsMonitorDbName, taosGetTimestampUs(), level); va_start(argpointer, format); From c6f9e04b67da5b8b5a5c5b0004c7ae1897b1e147 Mon Sep 17 00:00:00 2001 From: slguan Date: Thu, 12 Dec 2019 10:45:29 +0800 Subject: [PATCH 116/123] remove some warnings --- src/util/src/tutil.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/util/src/tutil.c b/src/util/src/tutil.c index 12932f0553..cdd017fb56 100644 --- a/src/util/src/tutil.c +++ b/src/util/src/tutil.c @@ -503,7 +503,6 @@ int taosCheckVersion(char *input_client_version, char *input_server_version, int if (!taosGetVersionNumber(server_version, serverVersionNumber)) { pError("invalid server version:%s", server_version); return TSDB_CODE_INVALID_CLIENT_VERSION; - return NULL; } for(int32_t i = 0; i < comparedSegments; ++i) { From 7b4830cf8b8a87395b1396076b8ca177e5b2a0d6 Mon Sep 17 00:00:00 2001 From: lihui Date: Thu, 12 Dec 2019 13:54:38 +0800 Subject: [PATCH 117/123] [uint64_t printf format] --- src/system/detail/src/vnodeRead.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/system/detail/src/vnodeRead.c b/src/system/detail/src/vnodeRead.c index ccd59f356b..abbcc7da2e 100644 --- a/src/system/detail/src/vnodeRead.c +++ b/src/system/detail/src/vnodeRead.c @@ -888,7 +888,7 @@ int vnodeSaveQueryResult(void *handle, char *data, int32_t *size) { pQInfo->pointsRead); if (pQInfo->over == 0) { - dTrace("QInfo:%p set query flag, sig:%p, func:%s", pQInfo, pQInfo->signature, __FUNCTION__); + dTrace("QInfo:%p set query flag, sig:%" PRIu64 ", func:%s", pQInfo, pQInfo->signature, __FUNCTION__); if (pQInfo->killed == 1) { dTrace("%p freed or killed, abort query", pQInfo); From 24f6170b6473061c161579dec7f10f84f70d3d5a Mon Sep 17 00:00:00 2001 From: lihui Date: Thu, 12 Dec 2019 15:41:50 +0800 Subject: [PATCH 118/123] [update] --- packaging/tools/make_install.sh | 92 ++++++++++++++++++++++++--------- 1 file changed, 69 insertions(+), 23 deletions(-) diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh index c43f918e8d..b461d5c46d 100755 --- a/packaging/tools/make_install.sh +++ b/packaging/tools/make_install.sh @@ -47,21 +47,54 @@ initd_mod=0 service_mod=2 if pidof systemd &> /dev/null; then service_mod=0 -elif $(which insserv &> /dev/null); then +elif $(which service &> /dev/null); then service_mod=1 - initd_mod=1 - service_config_dir="/etc/init.d" -elif $(which update-rc.d &> /dev/null); then - service_mod=1 - initd_mod=2 - service_config_dir="/etc/init.d" + service_config_dir="/etc/init.d" + if $(which chkconfig &> /dev/null); then + initd_mod=1 + elif $(which insserv &> /dev/null); then + initd_mod=2 + elif $(which update-rc.d &> /dev/null); then + initd_mod=3 + else + service_mod=2 + fi else service_mod=2 fi + +# get the operating system type for using the corresponding init file +# ubuntu/debian(deb), centos/fedora(rpm), others: opensuse, redhat, ..., no verification +#osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release) +osinfo=$(cat /etc/os-release | grep "NAME" | cut -d '"' -f2) +#echo "osinfo: ${osinfo}" +os_type=0 +if echo $osinfo | grep -qwi "ubuntu" ; then + echo "this is ubuntu system" + os_type=1 +elif echo $osinfo | grep -qwi "debian" ; then + echo "this is debian system" + os_type=1 +elif echo $osinfo | grep -qwi "Kylin" ; then + echo "this is Kylin system" + os_type=1 +elif echo $osinfo | grep -qwi "centos" ; then + echo "this is centos system" + os_type=2 +elif echo $osinfo | grep -qwi "fedora" ; then + echo "this is fedora system" + os_type=2 +else + echo "this is other linux system" + os_type=0 +fi + function kill_taosd() { pid=$(ps -ef | grep "taosd" | grep -v "grep" | awk '{print $2}') - ${csudo} kill -9 ${pid} || : + if [ -n "$pid" ]; then + ${csudo} kill -9 $pid || : + fi } function install_main_path() { @@ -153,20 +186,26 @@ function install_examples() { } function clean_service_on_sysvinit() { - restart_config_str="taos:2345:respawn:${service_config_dir}/taosd start" + #restart_config_str="taos:2345:respawn:${service_config_dir}/taosd start" + #${csudo} sed -i "\|${restart_config_str}|d" /etc/inittab || : + if pidof taosd &> /dev/null; then ${csudo} service taosd stop || : fi - ${csudo} sed -i "\|${restart_config_str}|d" /etc/inittab || : - ${csudo} rm -f ${service_config_dir}/taosd || : if ((${initd_mod}==1)); then - ${csudo} grep -q -F "taos" /etc/inittab && ${csudo} insserv -r taosd || : + ${csudo} chkconfig --del taosd || : elif ((${initd_mod}==2)); then - ${csudo} grep -q -F "taos" /etc/inittab && ${csudo} update-rc.d -f taosd remove || : + ${csudo} insserv -r taosd || : + elif ((${initd_mod}==3)); then + ${csudo} update-rc.d -f taosd remove || : + fi + + ${csudo} rm -f ${service_config_dir}/taosd || : + + if $(which init &> /dev/null); then + ${csudo} init q || : fi -# ${csudo} update-rc.d -f taosd remove || : - ${csudo} init q || : } function install_service_on_sysvinit() { @@ -175,19 +214,26 @@ function install_service_on_sysvinit() { sleep 1 # Install taosd service + if ((${os_type}==1)); then ${csudo} cp -f ${script_dir}/../deb/init.d/taosd ${install_main_dir}/init.d ${csudo} cp ${script_dir}/../deb/init.d/taosd ${service_config_dir} && ${csudo} chmod a+x ${service_config_dir}/taosd - restart_config_str="taos:2345:respawn:${service_config_dir}/taosd start" - - ${csudo} grep -q -F "$restart_config_str" /etc/inittab || ${csudo} bash -c "echo '${restart_config_str}' >> /etc/inittab" - # TODO: for centos, change here + elif ((${os_type}==2)); then + ${csudo} cp -f ${script_dir}/../rpm/init.d/taosd ${install_main_dir}/init.d + ${csudo} cp ${script_dir}/../rpm/init.d/taosd ${service_config_dir} && ${csudo} chmod a+x ${service_config_dir}/taosd + fi + + #restart_config_str="taos:2345:respawn:${service_config_dir}/taosd start" + #${csudo} grep -q -F "$restart_config_str" /etc/inittab || ${csudo} bash -c "echo '${restart_config_str}' >> /etc/inittab" + if ((${initd_mod}==1)); then - ${csudo} insserv taosd || : + ${csudo} chkconfig --add taosd || : + ${csudo} chkconfig --level 2345 taosd on || : elif ((${initd_mod}==2)); then + ${csudo} insserv taosd || : + ${csudo} insserv -d taosd || : + elif ((${initd_mod}==3)); then ${csudo} update-rc.d taosd defaults || : fi -# ${csudo} update-rc.d taosd defaults - # chkconfig mysqld on } function clean_service_on_systemd() { @@ -237,7 +283,7 @@ function install_service() { elif ((${service_mod}==1)); then install_service_on_sysvinit else - # must manual start taosd + # must manual stop taosd kill_taosd fi } From b473ba8eccc9bfe77cd3d4d7eff4f7a59cd61f7d Mon Sep 17 00:00:00 2001 From: lihui Date: Thu, 12 Dec 2019 16:00:27 +0800 Subject: [PATCH 119/123] [check return value] --- src/rpc/src/trpc.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/rpc/src/trpc.c b/src/rpc/src/trpc.c index 5071376bcd..db1ca33841 100755 --- a/src/rpc/src/trpc.c +++ b/src/rpc/src/trpc.c @@ -328,6 +328,10 @@ int taosSendSimpleRsp(void *thandle, char rsptype, char code) { } pStart = taosBuildRspMsgWithSize(thandle, rsptype, 32); + if (pStart == NULL) { + tError("build rsp msg error, return null prt"); + return -1; + } pMsg = pStart; *pMsg = code; From 409695fb394c90f188ce8f7ef9ec0829ac18bb3a Mon Sep 17 00:00:00 2001 From: lihui Date: Thu, 12 Dec 2019 17:20:45 +0800 Subject: [PATCH 120/123] [add compile parameter if is arm] --- README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index c82cda6f3c..fddd7d3132 100644 --- a/README.md +++ b/README.md @@ -39,12 +39,18 @@ sudo apt-get install maven ``` Build TDengine: -```cmd +``` mkdir build && cd build cmake .. && cmake --build . ``` +if compiling on an arm64 processor, you need add one parameter: + +```cmd +cmake .. -DARMVER=arm64 && cmake --build . +``` + # Quick Run To quickly start a TDengine server after building, run the command below in terminal: ```cmd From 115912544d1dedc1ebe6bede72b7576d26b6c9e2 Mon Sep 17 00:00:00 2001 From: lihui Date: Fri, 13 Dec 2019 15:12:35 +0800 Subject: [PATCH 121/123] [TBASE-1339] --- src/client/src/tscParseInsert.c | 103 ++++++++++++++++++++++++++++---- 1 file changed, 90 insertions(+), 13 deletions(-) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 592dd97f61..03e2241437 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -759,20 +759,84 @@ static int32_t tscParseSqlForCreateTableOnDemand(char **sqlstr, SSqlObj *pSql) { return tscInvalidSQLErrMsg(pCmd->payload, "create table only from super table is allowed", sToken.z); } - char * tagVal = pTag->data; SSchema *pTagSchema = tsGetTagSchema(pMeterMetaInfo->pMeterMeta); index = 0; sToken = tStrGetToken(sql, &index, false, 0, NULL); sql += index; + + SParsedDataColInfo spd = {0}; + uint8_t numOfTags = pMeterMetaInfo->pMeterMeta->numOfTags; + spd.numOfCols = numOfTags; + + // if specify some tags column + if (sToken.type != TK_LP) { + tscSetAssignedColumnInfo(&spd, pTagSchema, numOfTags); + } else { + /* insert into tablename (col1, col2,..., coln) using superTableName (tagName1, tagName2, ..., tagNamen) tags(tagVal1, tagVal2, ..., tagValn) values(v1, v2,... vn); */ + int16_t offset[TSDB_MAX_COLUMNS] = {0}; + for (int32_t t = 1; t < numOfTags; ++t) { + offset[t] = offset[t - 1] + pTagSchema[t - 1].bytes; + } + + while (1) { + index = 0; + sToken = tStrGetToken(sql, &index, false, 0, NULL); + sql += index; + + if (TK_STRING == sToken.type) { + sToken.n = strdequote(sToken.z); + strtrim(sToken.z); + sToken.n = (uint32_t)strlen(sToken.z); + } + + if (sToken.type == TK_RP) { + break; + } + + bool findColumnIndex = false; + + // todo speedup by using hash list + for (int32_t t = 0; t < numOfTags; ++t) { + if (strncmp(sToken.z, pTagSchema[t].name, sToken.n) == 0 && strlen(pTagSchema[t].name) == sToken.n) { + SParsedColElem *pElem = &spd.elems[spd.numOfAssignedCols++]; + pElem->offset = offset[t]; + pElem->colIndex = t; + + if (spd.hasVal[t] == true) { + return tscInvalidSQLErrMsg(pCmd->payload, "duplicated tag name", sToken.z); + } + + spd.hasVal[t] = true; + findColumnIndex = true; + break; + } + } + + if (!findColumnIndex) { + return tscInvalidSQLErrMsg(pCmd->payload, "invalid tag name", sToken.z); + } + } + + if (spd.numOfAssignedCols == 0 || spd.numOfAssignedCols > numOfTags) { + return tscInvalidSQLErrMsg(pCmd->payload, "tag name expected", sToken.z); + } + } + + index = 0; + sToken = tStrGetToken(sql, &index, false, 0, NULL); + sql += index; + if (sToken.type != TK_TAGS) { - return tscInvalidSQLErrMsg(pCmd->payload, "keyword TAGS expected", sql); + return tscInvalidSQLErrMsg(pCmd->payload, "keyword TAGS expected", sToken.z); } - int32_t numOfTagValues = 0; uint32_t ignoreTokenTypes = TK_LP; uint32_t numOfIgnoreToken = 1; - while (1) { + for (int i = 0; i < spd.numOfAssignedCols; ++i) { + char* tagVal = pTag->data + spd.elems[i].offset; + int16_t colIndex = spd.elems[i].colIndex; + index = 0; sToken = tStrGetToken(sql, &index, true, numOfIgnoreToken, &ignoreTokenTypes); sql += index; @@ -788,26 +852,39 @@ static int32_t tscParseSqlForCreateTableOnDemand(char **sqlstr, SSqlObj *pSql) { sToken.n -= 2; } - code = tsParseOneColumnData(&pTagSchema[numOfTagValues], &sToken, tagVal, pCmd->payload, &sql, false, - pMeterMetaInfo->pMeterMeta->precision); + code = tsParseOneColumnData(&pTagSchema[colIndex], &sToken, tagVal, pCmd->payload, &sql, false, pMeterMetaInfo->pMeterMeta->precision); if (code != TSDB_CODE_SUCCESS) { return code; } - if ((pTagSchema[numOfTagValues].type == TSDB_DATA_TYPE_BINARY || - pTagSchema[numOfTagValues].type == TSDB_DATA_TYPE_NCHAR) && sToken.n > pTagSchema[numOfTagValues].bytes) { + if ((pTagSchema[colIndex].type == TSDB_DATA_TYPE_BINARY || + pTagSchema[colIndex].type == TSDB_DATA_TYPE_NCHAR) && sToken.n > pTagSchema[colIndex].bytes) { return tscInvalidSQLErrMsg(pCmd->payload, "string too long", sToken.z); } - - tagVal += pTagSchema[numOfTagValues++].bytes; } - if (numOfTagValues != pMeterMetaInfo->pMeterMeta->numOfTags) { - return tscInvalidSQLErrMsg(pCmd->payload, "number of tags mismatch", sql); + index = 0; + sToken = tStrGetToken(sql, &index, false, 0, NULL); + sql += index; + if (sToken.n == 0 || sToken.type != TK_RP) { + return tscInvalidSQLErrMsg(pCmd->payload, ") expected", sToken.z); + } + + // 2. set the null value for the columns that do not assign values + if (spd.numOfAssignedCols < spd.numOfCols) { + char *ptr = pTag->data; + + for (int32_t i = 0; i < spd.numOfCols; ++i) { + if (!spd.hasVal[i]) { // current tag column do not have any value to insert, set it to null + setNull(ptr, pTagSchema[i].type, pTagSchema[i].bytes); + } + + ptr += pTagSchema[i].bytes; + } } if (tscValidateName(&tableToken) != TSDB_CODE_SUCCESS) { - return tscInvalidSQLErrMsg(pCmd->payload, "invalid table name", sql); + return tscInvalidSQLErrMsg(pCmd->payload, "invalid table name", *sqlstr); } int32_t ret = setMeterID(pSql, &tableToken, 0); From 227744a0b06fcb1b9783b1f070914ad17fd23fc3 Mon Sep 17 00:00:00 2001 From: fang Date: Sat, 14 Dec 2019 13:37:22 +0800 Subject: [PATCH 122/123] python connector user guide --- documentation/webdocs/markdowndocs/Connector.md | 2 +- documentation/webdocs/markdowndocs/connector-ch.md | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/documentation/webdocs/markdowndocs/Connector.md b/documentation/webdocs/markdowndocs/Connector.md index c4a08ca88e..46a2b04daa 100644 --- a/documentation/webdocs/markdowndocs/Connector.md +++ b/documentation/webdocs/markdowndocs/Connector.md @@ -510,7 +510,7 @@ Query OK, 1 row(s) in set (0.000141s) ## Python Connector ### Pre-requirement -* TDengine installed, TDengine-client installed if on Windows +* TDengine installed, TDengine-client installed if on Windows [(Windows TDengine client installation)](https://www.taosdata.com/cn/documentation/connector/#Windows客户端及程序接口) * python 2.7 or >= 3.4 * pip installed diff --git a/documentation/webdocs/markdowndocs/connector-ch.md b/documentation/webdocs/markdowndocs/connector-ch.md index 1e2e3b6989..e91c9d667a 100644 --- a/documentation/webdocs/markdowndocs/connector-ch.md +++ b/documentation/webdocs/markdowndocs/connector-ch.md @@ -502,7 +502,7 @@ Query OK, 1 row(s) in set (0.000141s) ## Python Connector ### 安装准备 -* 已安装TDengine, 如果客户端在Windows上，需要安装Windows 版本的TDengine客户端 +* 已安装TDengine, 如果客户端在Windows上，需要安装Windows 版本的TDengine客户端 [（Windows TDengine 客户端安装）](https://www.taosdata.com/cn/documentation/connector/#Windows客户端及程序接口) * 已安装python 2.7 or >= 3.4 * 已安装pip @@ -1114,6 +1114,13 @@ TDengine在Window系统上提供的API与Linux系统是相同的， 应用程序 + 将Windows开发包(taos.dll)放置到system32目录下。 +#### python接口注意事项 +在Windows系统上，应用程序可以通过导入taos这个模块来操纵数据库，使用python接口的注意事项如下： + ++ 确定在Windows上安装了TDengine客户端 + ++ 将Windows开发包(taos.dll)放置到system32目录下。 + [1]: https://search.maven.org/artifact/com.taosdata.jdbc/taos-jdbcdriver [2]: https://mvnrepository.com/artifact/com.taosdata.jdbc/taos-jdbcdriver From 04b511632bf290799ba663c71720d172bd706a7c Mon Sep 17 00:00:00 2001 From: lihui Date: Sat, 14 Dec 2019 16:54:06 +0800 Subject: [PATCH 123/123] [TBASE-1337] --- src/system/detail/inc/mgmt.h | 2 +- src/system/detail/src/mgmtDb.c | 3 ++- src/system/detail/src/mgmtShell.c | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/system/detail/inc/mgmt.h b/src/system/detail/inc/mgmt.h index f837f72424..39a94f320b 100644 --- a/src/system/detail/inc/mgmt.h +++ b/src/system/detail/inc/mgmt.h @@ -314,7 +314,7 @@ int mgmtUpdateDb(SDbObj *pDb); SDbObj *mgmtGetDb(char *db); SDbObj *mgmtGetDbByMeterId(char *db); int mgmtCreateDb(SAcctObj *pAcct, SCreateDbMsg *pCreate); -int mgmtDropDbByName(SAcctObj *pAcct, char *name); +int mgmtDropDbByName(SAcctObj *pAcct, char *name, short ignoreNotExists); int mgmtDropDb(SDbObj *pDb); /* void mgmtMonitorDbDrop(void *unused); */ void mgmtMonitorDbDrop(void *unused, void *unusedt); diff --git a/src/system/detail/src/mgmtDb.c b/src/system/detail/src/mgmtDb.c index 4bd6433068..900bbf91c6 100644 --- a/src/system/detail/src/mgmtDb.c +++ b/src/system/detail/src/mgmtDb.c @@ -310,10 +310,11 @@ int mgmtDropDb(SDbObj *pDb) { } } -int mgmtDropDbByName(SAcctObj *pAcct, char *name) { +int mgmtDropDbByName(SAcctObj *pAcct, char *name, short ignoreNotExists) { SDbObj *pDb; pDb = (SDbObj *)sdbGetRow(dbSdb, name); if (pDb == NULL) { + if (ignoreNotExists) return TSDB_CODE_SUCCESS; mWarn("db:%s is not there", name); return TSDB_CODE_INVALID_DB; } diff --git a/src/system/detail/src/mgmtShell.c b/src/system/detail/src/mgmtShell.c index f382abb602..8e94e7f401 100644 --- a/src/system/detail/src/mgmtShell.c +++ b/src/system/detail/src/mgmtShell.c @@ -763,7 +763,7 @@ int mgmtProcessDropDbMsg(char *pMsg, int msgLen, SConnObj *pConn) { if (!pConn->writeAuth) { code = TSDB_CODE_NO_RIGHTS; } else { - code = mgmtDropDbByName(pConn->pAcct, pDrop->db); + code = mgmtDropDbByName(pConn->pAcct, pDrop->db, pDrop->ignoreNotExists); if (code == 0) { mLPrint("DB:%s is dropped by %s", pDrop->db, pConn->pUser->user); }