diff --git a/Jenkinsfile b/Jenkinsfile index 433b46067a..bac0cce33b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -42,12 +42,12 @@ def pre_test(){ killall -9 taosd ||echo "no taosd running" killall -9 gdb || echo "no gdb running" cd ${WKC} - git checkout develop git reset --hard HEAD~10 >/dev/null + git checkout develop git pull >/dev/null git fetch origin +refs/pull/${CHANGE_ID}/merge git checkout -qf FETCH_HEAD - find ${WKC}/tests/pytest -name \'*\'.sql -exec rm -rf {} \\; + git clean -dfx cd ${WK} git reset --hard HEAD~10 git checkout develop @@ -55,7 +55,7 @@ def pre_test(){ cd ${WK} export TZ=Asia/Harbin date - rm -rf ${WK}/debug + git clean -dfx mkdir debug cd debug cmake .. > /dev/null diff --git a/cmake/install.inc b/cmake/install.inc index 01d3c8a4df..5823ef743e 100755 --- a/cmake/install.inc +++ b/cmake/install.inc @@ -32,7 +32,7 @@ ELSEIF (TD_WINDOWS) #INSTALL(TARGETS taos RUNTIME DESTINATION driver) #INSTALL(TARGETS shell RUNTIME DESTINATION .) IF (TD_MVN_INSTALLED) - INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.24-dist.jar DESTINATION connector/jdbc) + INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.25-dist.jar DESTINATION connector/jdbc) ENDIF () ELSEIF (TD_DARWIN) SET(TD_MAKE_INSTALL_SH "${TD_COMMUNITY_DIR}/packaging/tools/make_install.sh") diff --git a/documentation20/cn/02.getting-started/docs.md b/documentation20/cn/02.getting-started/docs.md index f86e616c42..db2885d302 100644 --- a/documentation20/cn/02.getting-started/docs.md +++ b/documentation20/cn/02.getting-started/docs.md @@ -101,7 +101,7 @@ $ taos -h 192.168.0.1 -s "use db; show tables;" ### 运行SQL命令脚本 -TDengine终端可以通过`source`命令来运行SQL命令脚本. +TDengine 终端可以通过 `source` 命令来运行 SQL 命令脚本. ```mysql taos> source ; @@ -109,10 +109,10 @@ taos> source ; ### Shell小技巧 -- 可以使用上下光标键查看已经历史输入的命令 -- 修改用户密码。在shell中使用alter user命令 +- 可以使用上下光标键查看历史输入的指令 +- 修改用户密码。在 shell 中使用 alter user 指令 - ctrl+c 中止正在进行中的查询 -- 执行`RESET QUERY CACHE`清空本地缓存的表的schema +- 执行 `RESET QUERY CACHE` 清空本地缓存的表 schema ## TDengine 极速体验 @@ -179,19 +179,20 @@ taos> select avg(f1), max(f2), min(f3) from test.t10 interval(10s); ### TDengine服务器支持的平台列表 -| | **CentOS 6/7/8** | **Ubuntu 16/18/20** | **Other Linux** | **统信 UOS** | **银河/中标麒麟** | **凝思 V60/V80** | -| -------------- | --------------------- | ------------------------ | --------------- | --------------- | ------------------------- | --------------------- | -| X64 | ● | ● | | ○ | ● | ● | -| 树莓派 ARM32 | | ● | ● | | | | -| 龙芯 MIPS64 | | | ● | | | | -| 鲲鹏 ARM64 | | ○ | ○ | | ● | | -| 申威 Alpha64 | | | ○ | ● | | | -| 飞腾 ARM64 | | ○ 优麒麟 | | | | | -| 海光 X64 | ● | ● | ● | ○ | ● | ● | -| 瑞芯微 ARM64/32 | | | ○ | | | | -| 全志 ARM64/32 | | | ○ | | | | -| 炬力 ARM64/32 | | | ○ | | | | -| TI ARM32 | | | ○ | | | | +| | **CentOS 6/7/8** | **Ubuntu 16/18/20** | **Other Linux** | **统信 UOS** | **银河/中标麒麟** | **凝思 V60/V80** | **华为 EulerOS** | +| -------------- | --------------------- | ------------------------ | --------------- | --------------- | ------------------------- | --------------------- | --------------------- | +| X64 | ● | ● | | ○ | ● | ● | ● | +| 树莓派 ARM32 | | ● | ● | | | | | +| 龙芯 MIPS64 | | | ● | | | | | +| 鲲鹏 ARM64 | | ○ | ○ | | ● | | | +| 申威 Alpha64 | | | ○ | ● | | | | +| 飞腾 ARM64 | | ○ 优麒麟 | | | | | | +| 海光 X64 | ● | ● | ● | ○ | ● | ● | | +| 瑞芯微 ARM64/32 | | | ○ | | | | | +| 全志 ARM64/32 | | | ○ | | | | | +| 炬力 ARM64/32 | | | ○ | | | | | +| TI ARM32 | | | ○ | | | | | +| 华为云 ARM64 | | | | | | | ● | 注： ● 表示经过官方测试验证， ○ 表示非官方测试验证。 diff --git a/documentation20/cn/07.advanced-features/docs.md b/documentation20/cn/07.advanced-features/docs.md index bdf93fdc3d..650a2ca96b 100644 --- a/documentation20/cn/07.advanced-features/docs.md +++ b/documentation20/cn/07.advanced-features/docs.md @@ -1,27 +1,16 @@ # 高级功能 -## 连续查询(Continuous Query) +## 连续查询（Continuous Query） -连续查询是TDengine定期自动执行的查询，采用滑动窗口的方式进行计算，是一种简化的时间驱动的流式计算。 -针对库中的表或超级表，TDengine可提供定期自动执行的连续查询， -用户可让TDengine推送查询的结果，也可以将结果再写回到TDengine中。 -每次执行的查询是一个时间窗口，时间窗口随着时间流动向前滑动。 -在定义连续查询的时候需要指定时间窗口（time window, 参数interval）大小和每次前向增量时间（forward sliding times, 参数sliding）。 +连续查询是TDengine定期自动执行的查询，采用滑动窗口的方式进行计算，是一种简化的时间驱动的流式计算。针对库中的表或超级表，TDengine可提供定期自动执行的连续查询，用户可让TDengine推送查询的结果，也可以将结果再写回到TDengine中。每次执行的查询是一个时间窗口，时间窗口随着时间流动向前滑动。在定义连续查询的时候需要指定时间窗口（time window, 参数interval）大小和每次前向增量时间（forward sliding times, 参数sliding）。 -TDengine的连续查询采用时间驱动模式，可以直接使用TAOS SQL进行定义，不需要额外的操作。 -使用连续查询，可以方便快捷地按照时间窗口生成结果，从而对原始采集数据进行降采样（down sampling）。 -用户通过TAOS SQL定义连续查询以后，TDengine自动在最后的一个完整的时间周期末端拉起查询， -并将计算获得的结果推送给用户或者写回TDengine。 +TDengine的连续查询采用时间驱动模式，可以直接使用TAOS SQL进行定义，不需要额外的操作。使用连续查询，可以方便快捷地按照时间窗口生成结果，从而对原始采集数据进行降采样（down sampling）。用户通过TAOS SQL定义连续查询以后，TDengine自动在最后的一个完整的时间周期末端拉起查询，并将计算获得的结果推送给用户或者写回TDengine。 TDengine提供的连续查询与普通流计算中的时间窗口计算具有以下区别： -- 不同于流计算的实时反馈计算结果，连续查询只在时间窗口关闭以后才开始计算。 -例如时间周期是1天，那么当天的结果只会在23:59:59以后才会生成。 -- 如果有历史记录写入到已经计算完成的时间区间，连续查询并不会重新进行计算， -也不会重新将结果推送给用户。对于写回TDengine的模式，也不会更新已经存在的计算结果。 -- 使用连续查询推送结果的模式，服务端并不缓存客户端计算状态，也不提供Exactly-Once的语意保证。 -如果用户的应用端崩溃，再次拉起的连续查询将只会从再次拉起的时间开始重新计算最近的一个完整的时间窗口。 -如果使用写回模式，TDengine可确保数据写回的有效性和连续性。 +- 不同于流计算的实时反馈计算结果，连续查询只在时间窗口关闭以后才开始计算。例如时间周期是1天，那么当天的结果只会在23:59:59以后才会生成。 +- 如果有历史记录写入到已经计算完成的时间区间，连续查询并不会重新进行计算，也不会重新将结果推送给用户。对于写回TDengine的模式，也不会更新已经存在的计算结果。 +- 使用连续查询推送结果的模式，服务端并不缓存客户端计算状态，也不提供Exactly-Once的语意保证。如果用户的应用端崩溃，再次拉起的连续查询将只会从再次拉起的时间开始重新计算最近的一个完整的时间窗口。如果使用写回模式，TDengine可确保数据写回的有效性和连续性。 ### 使用连续查询 @@ -40,23 +29,19 @@ create table D1002 using meters tags ("Beijing.Haidian", 2); select avg(voltage) from meters interval(1m) sliding(30s); ``` -每次执行这条语句，都会重新计算所有数据。 -如果需要每隔30秒执行一次来增量计算最近一分钟的数据， -可以把上面的语句改进成下面的样子，每次使用不同的 `startTime` 并定期执行： +每次执行这条语句，都会重新计算所有数据。 如果需要每隔30秒执行一次来增量计算最近一分钟的数据，可以把上面的语句改进成下面的样子，每次使用不同的 `startTime` 并定期执行： ```sql select avg(voltage) from meters where ts > {startTime} interval(1m) sliding(30s); ``` -这样做没有问题，但TDengine提供了更简单的方法， -只要在最初的查询语句前面加上 `create table {tableName} as ` 就可以了, 例如： +这样做没有问题，但TDengine提供了更简单的方法，只要在最初的查询语句前面加上 `create table {tableName} as ` 就可以了, 例如： ```sql create table avg_vol as select avg(voltage) from meters interval(1m) sliding(30s); ``` -会自动创建一个名为 `avg_vol` 的新表，然后每隔30秒，TDengine会增量执行 `as` 后面的 SQL 语句， -并将查询结果写入这个表中，用户程序后续只要从 `avg_vol` 中查询数据即可。 例如： +会自动创建一个名为 `avg_vol` 的新表，然后每隔30秒，TDengine会增量执行 `as` 后面的 SQL 语句，并将查询结果写入这个表中，用户程序后续只要从 `avg_vol` 中查询数据即可。 例如： ```mysql taos> select * from avg_vol; @@ -70,43 +55,27 @@ taos> select * from avg_vol; 需要注意，查询时间窗口的最小值是10毫秒，没有时间窗口范围的上限。 -此外，TDengine还支持用户指定连续查询的起止时间。 -如果不输入开始时间，连续查询将从第一条原始数据所在的时间窗口开始； -如果没有输入结束时间，连续查询将永久运行； -如果用户指定了结束时间，连续查询在系统时间达到指定的时间以后停止运行。 -比如使用下面的SQL创建的连续查询将运行一小时，之后会自动停止。 +此外，TDengine还支持用户指定连续查询的起止时间。如果不输入开始时间，连续查询将从第一条原始数据所在的时间窗口开始；如果没有输入结束时间，连续查询将永久运行；如果用户指定了结束时间，连续查询在系统时间达到指定的时间以后停止运行。比如使用下面的SQL创建的连续查询将运行一小时，之后会自动停止。 ```mysql create table avg_vol as select avg(voltage) from meters where ts > now and ts <= now + 1h interval(1m) sliding(30s); ``` -需要说明的是，上面例子中的 `now` 是指创建连续查询的时间，而不是查询执行的时间，否则，查询就无法自动停止了。 -另外，为了尽量避免原始数据延迟写入导致的问题，TDengine中连续查询的计算有一定的延迟。 -也就是说，一个时间窗口过去后，TDengine并不会立即计算这个窗口的数据， -所以要稍等一会（一般不会超过1分钟）才能查到计算结果。 +需要说明的是，上面例子中的 `now` 是指创建连续查询的时间，而不是查询执行的时间，否则，查询就无法自动停止了。另外，为了尽量避免原始数据延迟写入导致的问题，TDengine中连续查询的计算有一定的延迟。也就是说，一个时间窗口过去后，TDengine并不会立即计算这个窗口的数据，所以要稍等一会（一般不会超过1分钟）才能查到计算结果。 ### 管理连续查询 -用户可在控制台中通过 `show streams` 命令来查看系统中全部运行的连续查询， -并可以通过 `kill stream` 命令杀掉对应的连续查询。 -后续版本会提供更细粒度和便捷的连续查询管理命令。 +用户可在控制台中通过 `show streams` 命令来查看系统中全部运行的连续查询，并可以通过 `kill stream` 命令杀掉对应的连续查询。后续版本会提供更细粒度和便捷的连续查询管理命令。 -## 数据订阅(Publisher/Subscriber) +## 数据订阅（Publisher/Subscriber） -基于数据天然的时间序列特性，TDengine的数据写入（insert）与消息系统的数据发布（pub）逻辑上一致， -均可视为系统中插入一条带时间戳的新记录。 -同时，TDengine在内部严格按照数据时间序列单调递增的方式保存数据。 -本质上来说，TDengine中里每一张表均可视为一个标准的消息队列。 +基于数据天然的时间序列特性，TDengine的数据写入（insert）与消息系统的数据发布（pub）逻辑上一致，均可视为系统中插入一条带时间戳的新记录。同时，TDengine在内部严格按照数据时间序列单调递增的方式保存数据。本质上来说，TDengine中里每一张表均可视为一个标准的消息队列。 -TDengine内嵌支持轻量级的消息订阅与推送服务。 -使用系统提供的API，用户可使用普通查询语句订阅数据库中的一张或多张表。 -订阅的逻辑和操作状态的维护均是由客户端完成，客户端定时轮询服务器是否有新的记录到达， -有新的记录到达就会将结果反馈到客户。 +TDengine内嵌支持轻量级的消息订阅与推送服务。使用系统提供的API，用户可使用普通查询语句订阅数据库中的一张或多张表。订阅的逻辑和操作状态的维护均是由客户端完成，客户端定时轮询服务器是否有新的记录到达，有新的记录到达就会将结果反馈到客户。 -TDengine的订阅与推送服务的状态是客户端维持，TDengine服务器并不维持。 -因此如果应用重启，从哪个时间点开始获取最新数据，由应用决定。 +TDengine的订阅与推送服务的状态是客户端维持，TDengine服务器并不维持。因此如果应用重启，从哪个时间点开始获取最新数据，由应用决定。 TDengine的API中，与订阅相关的主要有以下三个： @@ -116,12 +85,9 @@ taos_consume taos_unsubscribe ``` -这些API的文档请见 [C/C++ Connector](https://www.taosdata.com/cn/documentation/connector/)， -下面仍以智能电表场景为例介绍一下它们的具体用法（超级表和子表结构请参考上一节“连续查询”）， -完整的示例代码可以在 [这里](https://github.com/taosdata/TDengine/blob/master/tests/examples/c/subscribe.c) 找到。 +这些API的文档请见 [C/C++ Connector](https://www.taosdata.com/cn/documentation/connector#c-cpp)，下面仍以智能电表场景为例介绍一下它们的具体用法（超级表和子表结构请参考上一节“连续查询”），完整的示例代码可以在 [这里](https://github.com/taosdata/TDengine/blob/master/tests/examples/c/subscribe.c) 找到。 -如果我们希望当某个电表的电流超过一定限制（比如10A）后能得到通知并进行一些处理， 有两种方法： -一是分别对每张子表进行查询，每次查询后记录最后一条数据的时间戳，后续只查询这个时间戳之后的数据： +如果我们希望当某个电表的电流超过一定限制（比如10A）后能得到通知并进行一些处理， 有两种方法：一是分别对每张子表进行查询，每次查询后记录最后一条数据的时间戳，后续只查询这个时间戳之后的数据： ```sql select * from D1001 where ts > {last_timestamp1} and current > 10; @@ -129,8 +95,7 @@ select * from D1002 where ts > {last_timestamp2} and current > 10; ... ``` -这确实可行，但随着电表数量的增加，查询数量也会增加，客户端和服务端的性能都会受到影响， -当电表数增长到一定的程度，系统就无法承受了。 +这确实可行，但随着电表数量的增加，查询数量也会增加，客户端和服务端的性能都会受到影响，当电表数增长到一定的程度，系统就无法承受了。 另一种方法是对超级表进行查询。这样，无论有多少电表，都只需一次查询： @@ -138,12 +103,7 @@ select * from D1002 where ts > {last_timestamp2} and current > 10; select * from meters where ts > {last_timestamp} and current > 10; ``` -但是，如何选择 `last_timestamp` 就成了一个新的问题。 -因为，一方面数据的产生时间（也就是数据时间戳）和数据入库的时间一般并不相同，有时偏差还很大； -另一方面，不同电表的数据到达TDengine的时间也会有差异。 -所以，如果我们在查询中使用最慢的那台电表的数据的时间戳作为 `last_timestamp`， -就可能重复读入其它电表的数据； -如果使用最快的电表的时间戳，其它电表的数据就可能被漏掉。 +但是，如何选择 `last_timestamp` 就成了一个新的问题。因为，一方面数据的产生时间（也就是数据时间戳）和数据入库的时间一般并不相同，有时偏差还很大；另一方面，不同电表的数据到达TDengine的时间也会有差异。所以，如果我们在查询中使用最慢的那台电表的数据的时间戳作为 `last_timestamp`，就可能重复读入其它电表的数据；如果使用最快的电表的时间戳，其它电表的数据就可能被漏掉。 TDengine的订阅功能为上面这个问题提供了一个彻底的解决方案。 @@ -160,47 +120,29 @@ if (async) { } ``` -TDengine中的订阅既可以是同步的，也可以是异步的， -上面的代码会根据从命令行获取的参数`async`的值来决定使用哪种方式。 -这里，同步的意思是用户程序要直接调用`taos_consume`来拉取数据， -而异步则由API在内部的另一个线程中调用`taos_consume`， -然后把拉取到的数据交给回调函数`subscribe_callback`去处理。 +TDengine中的订阅既可以是同步的，也可以是异步的，上面的代码会根据从命令行获取的参数`async`的值来决定使用哪种方式。这里，同步的意思是用户程序要直接调用`taos_consume`来拉取数据，而异步则由API在内部的另一个线程中调用`taos_consume`，然后把拉取到的数据交给回调函数`subscribe_callback`去处理。 -参数`taos`是一个已经建立好的数据库连接，在同步模式下无特殊要求。 -但在异步模式下，需要注意它不会被其它线程使用，否则可能导致不可预计的错误， -因为回调函数在API的内部线程中被调用，而TDengine的部分API不是线程安全的。 +参数`taos`是一个已经建立好的数据库连接，在同步模式下无特殊要求。但在异步模式下，需要注意它不会被其它线程使用，否则可能导致不可预计的错误，因为回调函数在API的内部线程中被调用，而TDengine的部分API不是线程安全的。 -参数`sql`是查询语句，可以在其中使用where子句指定过滤条件。 -在我们的例子中，如果只想订阅电流超过10A时的数据，可以这样写： +参数`sql`是查询语句，可以在其中使用where子句指定过滤条件。在我们的例子中，如果只想订阅电流超过10A时的数据，可以这样写： ```sql select * from meters where current > 10; ``` -注意，这里没有指定起始时间，所以会读到所有时间的数据。 -如果只想从一天前的数据开始订阅，而不需要更早的历史数据，可以再加上一个时间条件： +注意，这里没有指定起始时间，所以会读到所有时间的数据。如果只想从一天前的数据开始订阅，而不需要更早的历史数据，可以再加上一个时间条件： ```sql select * from meters where ts > now - 1d and current > 10; ``` -订阅的`topic`实际上是它的名字，因为订阅功能是在客户端API中实现的， -所以没必要保证它全局唯一，但需要它在一台客户端机器上唯一。 +订阅的`topic`实际上是它的名字，因为订阅功能是在客户端API中实现的，所以没必要保证它全局唯一，但需要它在一台客户端机器上唯一。 -如果名`topic`的订阅不存在，参数`restart`没有意义； -但如果用户程序创建这个订阅后退出，当它再次启动并重新使用这个`topic`时， -`restart`就会被用于决定是从头开始读取数据，还是接续上次的位置进行读取。 -本例中，如果`restart`是 **true**（非零值），用户程序肯定会读到所有数据。 -但如果这个订阅之前就存在了，并且已经读取了一部分数据， -且`restart`是 **false**（**0**），用户程序就不会读到之前已经读取的数据了。 +如果名`topic`的订阅不存在，参数`restart`没有意义；但如果用户程序创建这个订阅后退出，当它再次启动并重新使用这个`topic`时，`restart`就会被用于决定是从头开始读取数据，还是接续上次的位置进行读取。本例中，如果`restart`是 **true**（非零值），用户程序肯定会读到所有数据。但如果这个订阅之前就存在了，并且已经读取了一部分数据，且`restart`是 **false**（**0**），用户程序就不会读到之前已经读取的数据了。 -`taos_subscribe`的最后一个参数是以毫秒为单位的轮询周期。 -在同步模式下，如果前后两次调用`taos_consume`的时间间隔小于此时间， -`taos_consume`会阻塞，直到间隔超过此时间。 -异步模式下，这个时间是两次调用回调函数的最小时间间隔。 +`taos_subscribe`的最后一个参数是以毫秒为单位的轮询周期。在同步模式下，如果前后两次调用`taos_consume`的时间间隔小于此时间，`taos_consume`会阻塞，直到间隔超过此时间。异步模式下，这个时间是两次调用回调函数的最小时间间隔。 -`taos_subscribe`的倒数第二个参数用于用户程序向回调函数传递附加参数， -订阅API不对其做任何处理，只原样传递给回调函数。此参数在同步模式下无意义。 +`taos_subscribe`的倒数第二个参数用于用户程序向回调函数传递附加参数，订阅API不对其做任何处理，只原样传递给回调函数。此参数在同步模式下无意义。 订阅创建以后，就可以消费其数据了，同步模式下，示例代码是下面的 else 部分： @@ -219,9 +161,7 @@ if (async) { } ``` -这里是一个 **while** 循环，用户每按一次回车键就调用一次`taos_consume`， -而`taos_consume`的返回值是查询到的结果集，与`taos_use_result`完全相同， -例子中使用这个结果集的代码是函数`print_result`： +这里是一个 **while** 循环，用户每按一次回车键就调用一次`taos_consume`，而`taos_consume`的返回值是查询到的结果集，与`taos_use_result`完全相同，例子中使用这个结果集的代码是函数`print_result`： ```c void print_result(TAOS_RES* res, int blockFetch) { @@ -247,8 +187,7 @@ void print_result(TAOS_RES* res, int blockFetch) { } ``` -其中的 `taos_print_row` 用于处理订阅到数据，在我们的例子中，它会打印出所有符合条件的记录。 -而异步模式下，消费订阅到的数据则显得更为简单： +其中的 `taos_print_row` 用于处理订阅到数据，在我们的例子中，它会打印出所有符合条件的记录。而异步模式下，消费订阅到的数据则显得更为简单： ```c void subscribe_callback(TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) { @@ -262,11 +201,7 @@ void subscribe_callback(TAOS_SUB* tsub, TAOS_RES *res, void* param, int code) { taos_unsubscribe(tsub, keep); ``` -其第二个参数，用于决定是否在客户端保留订阅的进度信息。 -如果这个参数是**false**（**0**），那无论下次调用`taos_subscribe`的时的`restart`参数是什么， -订阅都只能重新开始。 -另外，进度信息的保存位置是 *{DataDir}/subscribe/* 这个目录下， -每个订阅有一个与其`topic`同名的文件，删掉某个文件，同样会导致下次创建其对应的订阅时只能重新开始。 +其第二个参数，用于决定是否在客户端保留订阅的进度信息。如果这个参数是**false**（**0**），那无论下次调用`taos_subscribe`时的`restart`参数是什么，订阅都只能重新开始。另外，进度信息的保存位置是 *{DataDir}/subscribe/* 这个目录下，每个订阅有一个与其`topic`同名的文件，删掉某个文件，同样会导致下次创建其对应的订阅时只能重新开始。 代码介绍完毕，我们来看一下实际的运行效果。假设： @@ -289,12 +224,11 @@ $ taos > insert into D1001 values(now, 12, 220, 1); ``` -这时，因为电流超过了10A，您应该可以看到示例程序将它输出到了屏幕上。 -您可以继续插入一些数据观察示例程序的输出。 +这时，因为电流超过了10A，您应该可以看到示例程序将它输出到了屏幕上。您可以继续插入一些数据观察示例程序的输出。 ### Java 使用数据订阅功能 -订阅功能也提供了 Java 开发接口，相关说明请见 [Java Connector](https://www.taosdata.com/cn/documentation/connector/)。需要注意的是，目前 Java 接口没有提供异步订阅模式，但用户程序可以通过创建 `TimerTask` 等方式达到同样的效果。 +订阅功能也提供了 Java 开发接口，相关说明请见 [Java Connector](https://www.taosdata.com/cn/documentation/connector/java#subscribe)。需要注意的是，目前 Java 接口没有提供异步订阅模式，但用户程序可以通过创建 `TimerTask` 等方式达到同样的效果。 下面以一个示例程序介绍其具体使用方法。它所完成的功能与前面介绍的 C 语言示例基本相同，也是订阅数据库中所有电流超过 10A 的记录。 @@ -404,7 +338,7 @@ ts: 1597466400000 current: 12.4 voltage: 220 phase: 1 location: Beijing.Chaoyang ``` -## 缓存(Cache) +## 缓存（Cache） TDengine采用时间驱动缓存管理策略（First-In-First-Out，FIFO），又称为写驱动的缓存管理机制。这种策略有别于读驱动的数据缓存模式（Least-Recent-Use，LRU），直接将最近写入的数据保存在系统的缓存中。当缓存达到临界值的时候，将最早的数据批量写入磁盘。一般意义上来说，对于物联网数据的使用，用户最为关心最近产生的数据，即当前状态。TDengine充分利用了这一特性，将最近到达的（当前状态）数据保存在缓存中。 @@ -423,7 +357,7 @@ select last_row(voltage) from meters where location='Beijing.Chaoyang'; 该SQL语句将获取所有位于北京朝阳区的电表最后记录的电压值。 -## 报警监测(Alert) +## 报警监测（Alert） 在 TDengine 的应用场景中，报警监测是一个常见需求，从概念上说，它要求程序从最近一段时间的数据中筛选出符合一定条件的数据，并基于这些数据根据定义好的公式计算出一个结果，当这个结果符合某个条件且持续一定时间后，以某种形式通知用户。 diff --git a/documentation20/cn/08.connector/01.java/docs.md b/documentation20/cn/08.connector/01.java/docs.md index 30b485f9b6..3442a2248c 100644 --- a/documentation20/cn/08.connector/01.java/docs.md +++ b/documentation20/cn/08.connector/01.java/docs.md @@ -285,7 +285,7 @@ JDBC连接器可能报错的错误码包括3种：JDBC driver本身的报错（ * https://github.com/taosdata/TDengine/blob/develop/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBErrorNumbers.java * https://github.com/taosdata/TDengine/blob/develop/src/inc/taoserror.h -### 订阅 +### 订阅 #### 创建 diff --git a/documentation20/cn/11.administrator/docs.md b/documentation20/cn/11.administrator/docs.md index c2c2927387..cc8689786d 100644 --- a/documentation20/cn/11.administrator/docs.md +++ b/documentation20/cn/11.administrator/docs.md @@ -120,17 +120,17 @@ taosd -C 不同应用场景的数据往往具有不同的数据特征，比如保留天数、副本数、采集频次、记录大小、采集点的数量、压缩等都可完全不同。为获得在存储上的最高效率，TDengine提供如下存储相关的系统配置参数： - days：一个数据文件存储数据的时间跨度，单位为天，默认值：10。 -- keep：数据库中数据保留的天数，单位为天，默认值：3650。 +- keep：数据库中数据保留的天数，单位为天，默认值：3650。（可通过 alter database 修改） - minRows：文件块中记录的最小条数，单位为条，默认值：100。 - maxRows：文件块中记录的最大条数，单位为条，默认值：4096。 -- comp：文件压缩标志位，0：关闭；1：一阶段压缩；2：两阶段压缩。默认值：2。 +- comp：文件压缩标志位，0：关闭；1：一阶段压缩；2：两阶段压缩。默认值：2。（可通过 alter database 修改） - walLevel：WAL级别。1：写wal，但不执行fsync；2：写wal, 而且执行fsync。默认值：1。 - fsync：当wal设置为2时，执行fsync的周期。设置为0，表示每次写入，立即执行fsync。单位为毫秒，默认值：3000。 - cache：内存块的大小，单位为兆字节（MB），默认值：16。 -- blocks：每个VNODE（TSDB）中有多少cache大小的内存块。因此一个VNODE的用的内存大小粗略为（cache * blocks）。单位为块，默认值：4。 -- replica：副本个数，取值范围：1-3。单位为个，默认值：1 -- precision：时间戳精度标识，ms表示毫秒，us表示微秒。默认值：ms -- cacheLast：是否在内存中缓存子表 last_row，0：关闭；1：开启。默认值：0。（从 2.0.11 版本开始支持此参数） +- blocks：每个VNODE（TSDB）中有多少cache大小的内存块。因此一个VNODE的用的内存大小粗略为（cache * blocks）。单位为块，默认值：4。（可通过 alter database 修改） +- replica：副本个数，取值范围：1-3。单位为个，默认值：1。（可通过 alter database 修改） +- precision：时间戳精度标识，ms表示毫秒，us表示微秒。默认值：ms。 +- cacheLast：是否在内存中缓存子表 last_row，0：关闭；1：开启。默认值：0。（可通过 alter database 修改）（从 2.0.11 版本开始支持此参数） 对于一个应用场景，可能有多种数据特征的数据并存，最佳的设计是将具有相同数据特征的表放在一个库里，这样一个应用有多个库，而每个库可以配置不同的存储参数，从而保证系统有最优的性能。TDengine允许应用在创建库时指定上述存储参数，如果指定，该参数就将覆盖对应的系统配置参数。举例，有下述SQL： diff --git a/documentation20/cn/12.taos-sql/docs.md b/documentation20/cn/12.taos-sql/docs.md index b4fa2b160a..58191e0bd8 100644 --- a/documentation20/cn/12.taos-sql/docs.md +++ b/documentation20/cn/12.taos-sql/docs.md @@ -29,21 +29,21 @@ taos> DESCRIBE meters; ## 支持的数据类型 -使用TDengine，最重要的是时间戳。创建并插入记录、查询历史记录的时候，均需要指定时间戳。时间戳有如下规则： +使用 TDengine，最重要的是时间戳。创建并插入记录、查询历史记录的时候，均需要指定时间戳。时间戳有如下规则： -- 时间格式为```YYYY-MM-DD HH:mm:ss.MS```, 默认时间分辨率为毫秒。比如：```2017-08-12 18:25:58.128``` -- 内部函数now是服务器的当前时间 -- 插入记录时，如果时间戳为now，插入数据时使用服务器当前时间 -- Epoch Time: 时间戳也可以是一个长整数，表示从1970-01-01 08:00:00.000开始的毫秒数 -- 时间可以加减，比如 now-2h，表明查询时刻向前推2个小时(最近2小时)。 数字后面的时间单位可以是 a(毫秒)、s(秒)、 m(分)、h(小时)、d(天)、w(周)。 比如select * from t1 where ts > now-2w and ts <= now-1w, 表示查询两周前整整一周的数据。 在指定降频操作(down sampling)的时间窗口(interval)时，时间单位还可以使用 n(自然月) 和 y(自然年)。 +- 时间格式为 ```YYYY-MM-DD HH:mm:ss.MS```，默认时间分辨率为毫秒。比如：```2017-08-12 18:25:58.128``` +- 内部函数 now 是客户端的当前时间 +- 插入记录时，如果时间戳为 now，插入数据时使用提交这条记录的客户端的当前时间 +- Epoch Time：时间戳也可以是一个长整数，表示从 1970-01-01 08:00:00.000 开始的毫秒数 +- 时间可以加减，比如 now-2h，表明查询时刻向前推 2 个小时（最近 2 小时）。数字后面的时间单位可以是 u(微秒)、a(毫秒)、s(秒)、m(分)、h(小时)、d(天)、w(周)。 比如 `select * from t1 where ts > now-2w and ts <= now-1w`，表示查询两周前整整一周的数据。在指定降频操作（down sampling）的时间窗口（interval）时，时间单位还可以使用 n(自然月) 和 y(自然年)。 -TDengine缺省的时间戳是毫秒精度，但通过修改配置参数enableMicrosecond就可支持微秒。 +TDengine 缺省的时间戳是毫秒精度，但通过修改配置参数 enableMicrosecond 就可以支持微秒。 -在TDengine中，普通表的数据模型中可使用以下10种数据类型。 +在TDengine中，普通表的数据模型中可使用以下 10 种数据类型。 | | 类型 | Bytes | 说明 | | ---- | :-------: | ------ | ------------------------------------------------------------ | -| 1 | TIMESTAMP | 8 | 时间戳。缺省精度毫秒，可支持微秒。从格林威治时间 1970-01-01 00:00:00.000 (UTC/GMT) 开始，计时不能早于该时间。 | +| 1 | TIMESTAMP | 8 | 时间戳。缺省精度毫秒，可支持微秒。从格林威治时间 1970-01-01 00:00:00.000 (UTC/GMT) 开始，计时不能早于该时间。（从 2.0.18 版本开始，已经去除了这一时间范围限制） | | 2 | INT | 4 | 整型，范围 [-2^31+1, 2^31-1], -2^31 用作 NULL | | 3 | BIGINT | 8 | 长整型，范围 [-2^63+1, 2^63-1], -2^63 用于 NULL | | 4 | FLOAT | 4 | 浮点型，有效位数 6-7，范围 [-3.4E38, 3.4E38] | diff --git a/documentation20/cn/13.faq/docs.md b/documentation20/cn/13.faq/docs.md index e2285b29e2..c01247d345 100644 --- a/documentation20/cn/13.faq/docs.md +++ b/documentation20/cn/13.faq/docs.md @@ -156,3 +156,13 @@ ALTER LOCAL RESETLOG; ``` 其含义是，清空本机所有由客户端生成的日志文件。 + +## 18. 时间戳的时区信息是怎样处理的？ + +TDengine 中时间戳的时区总是由客户端进行处理，而与服务端无关。具体来说，客户端会对 SQL 语句中的时间戳进行时区转换，转为 UTC 时区（即 Unix 时间戳——Unix Timestamp）再交由服务端进行写入和查询；在读取数据时，服务端也是采用 UTC 时区提供原始数据，客户端收到后再根据本地设置，把时间戳转换为本地系统所要求的时区进行显示。 + +客户端在处理时间戳字符串时，会采取如下逻辑： +1. 在未做特殊设置的情况下，客户端默认使用所在操作系统的时区设置。 +2. 如果在 taos.cfg 中设置了 timezone 参数，则客户端会以这个配置文件中的设置为准。 +3. 如果在 C/C++/Java/Python 等各种编程语言的 Connector Driver 中，在建立数据库连接时显式指定了 timezone，那么会以这个指定的时区设置为准。例如 Java Connector 的 JDBC URL 中就有 timezone 参数。 +4. 在书写 SQL 语句时，也可以直接使用 Unix 时间戳（例如 `1554984068000`）或带有时区的时间戳字符串，也即以 RFC 3339 格式（例如 `2013-04-12T15:52:01.123+08:00`）或 ISO-8601 格式（例如 `2013-04-12T15:52:01.123+0800`）来书写时间戳，此时这些时间戳的取值将不再受其他时区设置的影响。 diff --git a/packaging/tools/remove.sh b/packaging/tools/remove.sh index e63889aff1..9241f01efa 100755 --- a/packaging/tools/remove.sh +++ b/packaging/tools/remove.sh @@ -120,7 +120,7 @@ function clean_service_on_systemd() { if [ "$verMode" == "cluster" ]; then nginx_service_config="${service_config_dir}/${nginx_service_name}.service" - if [ -d ${bin_dir}/web ]; then + if [ -d ${install_nginxd_dir} ]; then if systemctl is-active --quiet ${nginx_service_name}; then echo "Nginx for TDengine is running, stopping it..." ${csudo} systemctl stop ${nginx_service_name} &> /dev/null || echo &> /dev/null diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index d0db130bf3..1e151584a1 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -36,19 +36,6 @@ extern "C" { #define UTIL_TABLE_IS_NORMAL_TABLE(metaInfo)\ (!(UTIL_TABLE_IS_SUPER_TABLE(metaInfo) || UTIL_TABLE_IS_CHILD_TABLE(metaInfo))) - -typedef struct SParsedColElem { - int16_t colIndex; - uint16_t offset; -} SParsedColElem; - -typedef struct SParsedDataColInfo { - int16_t numOfCols; - int16_t numOfAssignedCols; - SParsedColElem elems[TSDB_MAX_COLUMNS]; - bool hasVal[TSDB_MAX_COLUMNS]; -} SParsedDataColInfo; - #pragma pack(push,1) // this struct is transfered as binary, padding two bytes to avoid // an 'uid' whose low bytes is 0xff being recoginized as NULL, @@ -118,6 +105,8 @@ int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOff void tscDestroyDataBlock(STableDataBlocks* pDataBlock, bool removeMeta); void tscSortRemoveDataBlockDupRows(STableDataBlocks* dataBuf); +void tscDestroyBoundColumnInfo(SParsedDataColInfo* pColInfo); + SParamInfo* tscAddParamToDataBlock(STableDataBlocks* pDataBlock, char type, uint8_t timePrec, int16_t bytes, uint32_t offset); @@ -141,6 +130,8 @@ bool tscIsPointInterpQuery(SQueryInfo* pQueryInfo); bool tscIsTWAQuery(SQueryInfo* pQueryInfo); bool tscIsSecondStageQuery(SQueryInfo* pQueryInfo); bool tscGroupbyColumn(SQueryInfo* pQueryInfo); +bool tscIsTopbotQuery(SQueryInfo* pQueryInfo); +int32_t tscGetTopbotQueryParam(SQueryInfo* pQueryInfo); bool tscNonOrderedProjectionQueryOnSTable(SQueryInfo *pQueryInfo, int32_t tableIndex); bool tscOrderedProjectionQueryOnSTable(SQueryInfo* pQueryInfo, int32_t tableIndex); diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 36ea12170b..efbef284ae 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -175,6 +175,19 @@ typedef struct SParamInfo { uint32_t offset; } SParamInfo; + +typedef struct SBoundColumn { + bool hasVal; // denote if current column has bound or not + int32_t offset; // all column offset value +} SBoundColumn; + +typedef struct SParsedDataColInfo { + int16_t numOfCols; + int16_t numOfBound; + int32_t *boundedColumns; + SBoundColumn *cols; +} SParsedDataColInfo; + typedef struct STableDataBlocks { SName tableName; int8_t tsSource; // where does the UNIX timestamp come from, server or client @@ -189,6 +202,8 @@ typedef struct STableDataBlocks { STableMeta *pTableMeta; // the tableMeta of current table, the table meta will be used during submit, keep a ref to avoid to be removed from cache char *pData; + SParsedDataColInfo boundColumnInfo; + // for parameter ('?') binding uint32_t numOfAllocedParams; uint32_t numOfParams; @@ -426,6 +441,7 @@ void tscRestoreFuncForSTableQuery(SQueryInfo *pQueryInfo); int32_t tscCreateResPointerInfo(SSqlRes *pRes, SQueryInfo *pQueryInfo); void tscSetResRawPtr(SSqlRes* pRes, SQueryInfo* pQueryInfo); +void destroyTableNameList(SSqlCmd* pCmd); void tscResetSqlCmd(SSqlCmd *pCmd, bool removeMeta); @@ -440,6 +456,8 @@ void tscFreeSqlResult(SSqlObj *pSql); * @param pObj */ void tscFreeSqlObj(SSqlObj *pSql); +void tscFreeSubobj(SSqlObj* pSql); + void tscFreeRegisteredSqlObj(void *pSql); void tscCloseTscObj(void *pObj); @@ -461,6 +479,7 @@ char* tscGetSqlStr(SSqlObj* pSql); bool tscIsQueryWithLimit(SSqlObj* pSql); bool tscHasReachLimitation(SQueryInfo *pQueryInfo, SSqlRes *pRes); +void tscSetBoundColumnInfo(SParsedDataColInfo *pColInfo, SSchema *pSchema, int32_t numOfCols); char *tscGetErrorMsgPayload(SSqlCmd *pCmd); diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index 3249529b34..0cfcff3a98 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -301,7 +301,7 @@ static void tscAsyncResultCallback(SSchedMsg *pMsg) { taosReleaseRef(tscObjRef, pSql->self); } -void tscAsyncResultOnError(SSqlObj* pSql) { +void tscAsyncResultOnError(SSqlObj* pSql) { SSchedMsg schedMsg = {0}; schedMsg.fp = tscAsyncResultCallback; schedMsg.ahandle = (void *)pSql->self; @@ -505,10 +505,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { return; _error: - if (code != TSDB_CODE_SUCCESS) { - pSql->res.code = code; - tscAsyncResultOnError(pSql); - } - + pRes->code = code; + tscAsyncResultOnError(pSql); taosReleaseRef(tscObjRef, pSql->self); } diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c index baff0c46af..fd8d842871 100644 --- a/src/client/src/tscLocalMerge.c +++ b/src/client/src/tscLocalMerge.c @@ -338,11 +338,20 @@ void tscCreateLocalMerger(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrde pReducer->resColModel->capacity = pReducer->nResultBufSize; pReducer->finalModel = pFFModel; + int32_t expandFactor = 1; if (finalmodel->rowSize > 0) { - pReducer->resColModel->capacity /= finalmodel->rowSize; + bool topBotQuery = tscIsTopbotQuery(pQueryInfo); + if (topBotQuery) { + expandFactor = tscGetTopbotQueryParam(pQueryInfo); + pReducer->resColModel->capacity /= (finalmodel->rowSize * expandFactor); + pReducer->resColModel->capacity *= expandFactor; + } else { + pReducer->resColModel->capacity /= finalmodel->rowSize; + } } assert(finalmodel->rowSize > 0 && finalmodel->rowSize <= pReducer->rowSize); + pReducer->pFinalRes = calloc(1, pReducer->rowSize * pReducer->resColModel->capacity); if (pReducer->pTempBuffer == NULL || pReducer->discardData == NULL || pReducer->pResultBuf == NULL || @@ -1149,9 +1158,10 @@ static void fillMultiRowsOfTagsVal(SQueryInfo *pQueryInfo, int32_t numOfRes, SLo memset(buf, 0, (size_t)maxBufSize); memcpy(buf, pCtx->pOutput, (size_t)pCtx->outputBytes); + char* next = pCtx->pOutput; for (int32_t i = 0; i < inc; ++i) { - pCtx->pOutput += pCtx->outputBytes; - memcpy(pCtx->pOutput, buf, (size_t)pCtx->outputBytes); + next += pCtx->outputBytes; + memcpy(next, buf, (size_t)pCtx->outputBytes); } } @@ -1439,6 +1449,11 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); tFilePage *tmpBuffer = pLocalMerge->pTempBuffer; + int32_t remain = 1; + if (tscIsTopbotQuery(pQueryInfo)) { + remain = tscGetTopbotQueryParam(pQueryInfo); + } + if (doHandleLastRemainData(pSql)) { return TSDB_CODE_SUCCESS; } @@ -1527,7 +1542,7 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) { * if the previous group does NOT generate any result (pResBuf->num == 0), * continue to process results instead of return results. */ - if ((!sameGroup && pResBuf->num > 0) || (pResBuf->num == pLocalMerge->resColModel->capacity)) { + if ((!sameGroup && pResBuf->num > 0) || (pResBuf->num + remain >= pLocalMerge->resColModel->capacity)) { // does not belong to the same group bool notSkipped = genFinalResults(pSql, pLocalMerge, !sameGroup); diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 2b962333d5..926ee44b70 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -40,6 +40,7 @@ enum { }; static int32_t tscAllocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t * numOfRows); +static int32_t parseBoundColumns(SSqlCmd* pCmd, SParsedDataColInfo* pColInfo, SSchema* pSchema, char* str, char** end); static int32_t tscToDouble(SStrToken *pToken, double *value, char **endPtr) { errno = 0; @@ -94,12 +95,12 @@ int tsParseTime(SStrToken *pToken, int64_t *time, char **next, char *error, int1 */ SStrToken valueToken; index = 0; - sToken = tStrGetToken(pTokenEnd, &index, false, 0, NULL); + sToken = tStrGetToken(pTokenEnd, &index, false); pTokenEnd += index; if (sToken.type == TK_MINUS || sToken.type == TK_PLUS) { index = 0; - valueToken = tStrGetToken(pTokenEnd, &index, false, 0, NULL); + valueToken = tStrGetToken(pTokenEnd, &index, false); pTokenEnd += index; if (valueToken.n < 2) { @@ -117,7 +118,7 @@ int tsParseTime(SStrToken *pToken, int64_t *time, char **next, char *error, int1 if (sToken.type == TK_PLUS) { useconds += interval; } else { - useconds = (useconds >= interval) ? useconds - interval : 0; + useconds = useconds - interval; } *next = pTokenEnd; @@ -127,13 +128,12 @@ int tsParseTime(SStrToken *pToken, int64_t *time, char **next, char *error, int1 return TSDB_CODE_SUCCESS; } -// todo extract the null value check static bool isNullStr(SStrToken* pToken) { return (pToken->type == TK_NULL) || ((pToken->type == TK_STRING) && (pToken->n != 0) && (strncasecmp(TSDB_DATA_NULL_STR_L, pToken->z, pToken->n) == 0)); } -int32_t tsParseOneColumnData(SSchema *pSchema, SStrToken *pToken, char *payload, char *msg, char **str, bool primaryKey, - int16_t timePrec) { +int32_t tsParseOneColumn(SSchema *pSchema, SStrToken *pToken, char *payload, char *msg, char **str, bool primaryKey, + int16_t timePrec) { int64_t iv; int32_t ret; char *endptr = NULL; @@ -417,29 +417,32 @@ static int32_t tsCheckTimestamp(STableDataBlocks *pDataBlocks, const char *start return TSDB_CODE_SUCCESS; } -int tsParseOneRowData(char **str, STableDataBlocks *pDataBlocks, SSchema schema[], SParsedDataColInfo *spd, SSqlCmd* pCmd, - int16_t timePrec, int32_t *code, char *tmpTokenBuf) { - int32_t index = 0; - SStrToken sToken = {0}; - char * payload = pDataBlocks->pData + pDataBlocks->size; +int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, SSqlCmd *pCmd, int16_t timePrec, int32_t *len, + char *tmpTokenBuf) { + int32_t index = 0; + SStrToken sToken = {0}; + char *payload = pDataBlocks->pData + pDataBlocks->size; + + SParsedDataColInfo *spd = &pDataBlocks->boundColumnInfo; + SSchema *schema = tscGetTableSchema(pDataBlocks->pTableMeta); // 1. set the parsed value from sql string int32_t rowSize = 0; - for (int i = 0; i < spd->numOfAssignedCols; ++i) { + for (int i = 0; i < spd->numOfBound; ++i) { // the start position in data block buffer of current value in sql - char * start = payload + spd->elems[i].offset; - int16_t colIndex = spd->elems[i].colIndex; - SSchema *pSchema = schema + colIndex; + int32_t colIndex = spd->boundedColumns[i]; + + char *start = payload + spd->cols[colIndex].offset; + SSchema *pSchema = &schema[colIndex]; rowSize += pSchema->bytes; index = 0; - sToken = tStrGetToken(*str, &index, true, 0, NULL); + sToken = tStrGetToken(*str, &index, true); *str += index; if (sToken.type == TK_QUESTION) { if (pCmd->insertType != TSDB_QUERY_TYPE_STMT_INSERT) { - *code = tscSQLSyntaxErrMsg(pCmd->payload, "? only allowed in binding insertion", *str); - return -1; + return tscSQLSyntaxErrMsg(pCmd->payload, "? only allowed in binding insertion", *str); } uint32_t offset = (uint32_t)(start - pDataBlocks->pData); @@ -448,15 +451,13 @@ int tsParseOneRowData(char **str, STableDataBlocks *pDataBlocks, SSchema schema[ } strcpy(pCmd->payload, "client out of memory"); - *code = TSDB_CODE_TSC_OUT_OF_MEMORY; - return -1; + return TSDB_CODE_TSC_OUT_OF_MEMORY; } int16_t type = sToken.type; if ((type != TK_NOW && type != TK_INTEGER && type != TK_STRING && type != TK_FLOAT && type != TK_BOOL && type != TK_NULL && type != TK_HEX && type != TK_OCT && type != TK_BIN) || (sToken.n == 0) || (type == TK_RP)) { - *code = tscSQLSyntaxErrMsg(pCmd->payload, "invalid data or symbol", sToken.z); - return -1; + return tscSQLSyntaxErrMsg(pCmd->payload, "invalid data or symbol", sToken.z); } // Remove quotation marks @@ -485,26 +486,23 @@ int tsParseOneRowData(char **str, STableDataBlocks *pDataBlocks, SSchema schema[ } bool isPrimaryKey = (colIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX); - int32_t ret = tsParseOneColumnData(pSchema, &sToken, start, pCmd->payload, str, isPrimaryKey, timePrec); + int32_t ret = tsParseOneColumn(pSchema, &sToken, start, pCmd->payload, str, isPrimaryKey, timePrec); if (ret != TSDB_CODE_SUCCESS) { - *code = TSDB_CODE_TSC_SQL_SYNTAX_ERROR; - return -1; // NOTE: here 0 mean error! + return ret; } if (isPrimaryKey && tsCheckTimestamp(pDataBlocks, start) != TSDB_CODE_SUCCESS) { tscInvalidSQLErrMsg(pCmd->payload, "client time/server time can not be mixed up", sToken.z); - *code = TSDB_CODE_TSC_INVALID_TIME_STAMP; - return -1; + return TSDB_CODE_TSC_INVALID_TIME_STAMP; } } // 2. set the null value for the columns that do not assign values - if (spd->numOfAssignedCols < spd->numOfCols) { + if (spd->numOfBound < spd->numOfCols) { char *ptr = payload; for (int32_t i = 0; i < spd->numOfCols; ++i) { - - if (!spd->hasVal[i]) { // current column do not have any value to insert, set it to null + if (!spd->cols[i].hasVal) { // current column do not have any value to insert, set it to null if (schema[i].type == TSDB_DATA_TYPE_BINARY) { varDataSetLen(ptr, sizeof(int8_t)); *(uint8_t*) varDataVal(ptr) = TSDB_DATA_BINARY_NULL; @@ -522,7 +520,8 @@ int tsParseOneRowData(char **str, STableDataBlocks *pDataBlocks, SSchema schema[ rowSize = (int32_t)(ptr - payload); } - return rowSize; + *len = rowSize; + return TSDB_CODE_SUCCESS; } static int32_t rowDataCompar(const void *lhs, const void *rhs) { @@ -536,80 +535,79 @@ static int32_t rowDataCompar(const void *lhs, const void *rhs) { } } -int tsParseValues(char **str, STableDataBlocks *pDataBlock, STableMeta *pTableMeta, int maxRows, - SParsedDataColInfo *spd, SSqlCmd* pCmd, int32_t *code, char *tmpTokenBuf) { - int32_t index = 0; +int32_t tsParseValues(char **str, STableDataBlocks *pDataBlock, int maxRows, SSqlCmd* pCmd, int32_t* numOfRows, char *tmpTokenBuf) { + int32_t index = 0; + int32_t code = 0; + + (*numOfRows) = 0; + SStrToken sToken; - int32_t numOfRows = 0; - - SSchema *pSchema = tscGetTableSchema(pTableMeta); + STableMeta* pTableMeta = pDataBlock->pTableMeta; STableComInfo tinfo = tscGetTableInfo(pTableMeta); int32_t precision = tinfo.precision; - if (spd->hasVal[0] == false) { - *code = tscInvalidSQLErrMsg(pCmd->payload, "primary timestamp column can not be null", *str); - return -1; - } - while (1) { index = 0; - sToken = tStrGetToken(*str, &index, false, 0, NULL); + sToken = tStrGetToken(*str, &index, false); if (sToken.n == 0 || sToken.type != TK_LP) break; *str += index; - if (numOfRows >= maxRows || pDataBlock->size + tinfo.rowSize >= pDataBlock->nAllocSize) { + if ((*numOfRows) >= maxRows || pDataBlock->size + tinfo.rowSize >= pDataBlock->nAllocSize) { int32_t tSize; - *code = tscAllocateMemIfNeed(pDataBlock, tinfo.rowSize, &tSize); - if (*code != TSDB_CODE_SUCCESS) { //TODO pass the correct error code to client + code = tscAllocateMemIfNeed(pDataBlock, tinfo.rowSize, &tSize); + if (code != TSDB_CODE_SUCCESS) { //TODO pass the correct error code to client strcpy(pCmd->payload, "client out of memory"); - return -1; + return TSDB_CODE_TSC_OUT_OF_MEMORY; } ASSERT(tSize > maxRows); maxRows = tSize; } - int32_t len = tsParseOneRowData(str, pDataBlock, pSchema, spd, pCmd, precision, code, tmpTokenBuf); - if (len <= 0) { // error message has been set in tsParseOneRowData - return -1; + int32_t len = 0; + code = tsParseOneRow(str, pDataBlock, pCmd, precision, &len, tmpTokenBuf); + if (code != TSDB_CODE_SUCCESS) { // error message has been set in tsParseOneRow, return directly + return TSDB_CODE_TSC_SQL_SYNTAX_ERROR; } pDataBlock->size += len; index = 0; - sToken = tStrGetToken(*str, &index, false, 0, NULL); + sToken = tStrGetToken(*str, &index, false); *str += index; if (sToken.n == 0 || sToken.type != TK_RP) { tscSQLSyntaxErrMsg(pCmd->payload, ") expected", *str); - *code = TSDB_CODE_TSC_SQL_SYNTAX_ERROR; + code = TSDB_CODE_TSC_SQL_SYNTAX_ERROR; return -1; } - numOfRows++; + (*numOfRows)++; } - if (numOfRows <= 0) { + if ((*numOfRows) <= 0) { strcpy(pCmd->payload, "no any data points"); - *code = TSDB_CODE_TSC_SQL_SYNTAX_ERROR; - return -1; + return TSDB_CODE_TSC_SQL_SYNTAX_ERROR; } else { - return numOfRows; + return TSDB_CODE_SUCCESS; } } -static void tscSetAssignedColumnInfo(SParsedDataColInfo *spd, SSchema *pSchema, int32_t numOfCols) { - spd->numOfCols = numOfCols; - spd->numOfAssignedCols = numOfCols; +void tscSetBoundColumnInfo(SParsedDataColInfo *pColInfo, SSchema *pSchema, int32_t numOfCols) { + pColInfo->numOfCols = numOfCols; + pColInfo->numOfBound = numOfCols; - for (int32_t i = 0; i < numOfCols; ++i) { - spd->hasVal[i] = true; - spd->elems[i].colIndex = i; + pColInfo->boundedColumns = calloc(pColInfo->numOfCols, sizeof(int32_t)); + pColInfo->cols = calloc(pColInfo->numOfCols, sizeof(SBoundColumn)); + for (int32_t i = 0; i < pColInfo->numOfCols; ++i) { if (i > 0) { - spd->elems[i].offset = spd->elems[i - 1].offset + pSchema[i - 1].bytes; + pColInfo->cols[i].offset = pSchema[i - 1].bytes + pColInfo->cols[i - 1].offset; } + + pColInfo->cols[i].hasVal = true; + pColInfo->boundedColumns[i] = i; } } @@ -697,33 +695,26 @@ void tscSortRemoveDataBlockDupRows(STableDataBlocks *dataBuf) { } } -static int32_t doParseInsertStatement(SSqlCmd* pCmd, char **str, SParsedDataColInfo *spd, int32_t *totalNum) { - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); - STableMeta *pTableMeta = pTableMetaInfo->pTableMeta; - STableComInfo tinfo = tscGetTableInfo(pTableMeta); - - STableDataBlocks *dataBuf = NULL; - int32_t ret = tscGetDataBlockFromList(pCmd->pTableBlockHashList, pTableMeta->id.uid, TSDB_DEFAULT_PAYLOAD_SIZE, - sizeof(SSubmitBlk), tinfo.rowSize, &pTableMetaInfo->name, pTableMeta, &dataBuf, NULL); - if (ret != TSDB_CODE_SUCCESS) { - return ret; - } +static int32_t doParseInsertStatement(SSqlCmd* pCmd, char **str, STableDataBlocks* dataBuf, int32_t *totalNum) { + STableComInfo tinfo = tscGetTableInfo(dataBuf->pTableMeta); int32_t maxNumOfRows; - ret = tscAllocateMemIfNeed(dataBuf, tinfo.rowSize, &maxNumOfRows); - if (TSDB_CODE_SUCCESS != ret) { + int32_t code = tscAllocateMemIfNeed(dataBuf, tinfo.rowSize, &maxNumOfRows); + if (TSDB_CODE_SUCCESS != code) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } - int32_t code = TSDB_CODE_TSC_INVALID_SQL; - char * tmpTokenBuf = calloc(1, 16*1024); // used for deleting Escape character: \\, \', \" + code = TSDB_CODE_TSC_INVALID_SQL; + char *tmpTokenBuf = calloc(1, 16*1024); // used for deleting Escape character: \\, \', \" if (NULL == tmpTokenBuf) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } - int32_t numOfRows = tsParseValues(str, dataBuf, pTableMeta, maxNumOfRows, spd, pCmd, &code, tmpTokenBuf); + int32_t numOfRows = 0; + code = tsParseValues(str, dataBuf, maxNumOfRows, pCmd, &numOfRows, tmpTokenBuf); + free(tmpTokenBuf); - if (numOfRows <= 0) { + if (code != TSDB_CODE_SUCCESS) { return code; } @@ -736,25 +727,23 @@ static int32_t doParseInsertStatement(SSqlCmd* pCmd, char **str, SParsedDataColI } SSubmitBlk *pBlocks = (SSubmitBlk *)(dataBuf->pData); - code = tsSetBlockInfo(pBlocks, pTableMeta, numOfRows); + code = tsSetBlockInfo(pBlocks, dataBuf->pTableMeta, numOfRows); if (code != TSDB_CODE_SUCCESS) { tscInvalidSQLErrMsg(pCmd->payload, "too many rows in sql, total number of rows should be less than 32767", *str); return code; } - dataBuf->vgId = pTableMeta->vgId; dataBuf->numOfTables = 1; - *totalNum += numOfRows; return TSDB_CODE_SUCCESS; } -static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { +static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql, char** boundColumn) { int32_t index = 0; SStrToken sToken = {0}; SStrToken tableToken = {0}; int32_t code = TSDB_CODE_SUCCESS; - + const int32_t TABLE_INDEX = 0; const int32_t STABLE_INDEX = 1; @@ -767,38 +756,37 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { // get the token of specified table index = 0; - tableToken = tStrGetToken(sql, &index, false, 0, NULL); + tableToken = tStrGetToken(sql, &index, false); sql += index; - char *cstart = NULL; - char *cend = NULL; - // skip possibly exists column list index = 0; - sToken = tStrGetToken(sql, &index, false, 0, NULL); + sToken = tStrGetToken(sql, &index, false); sql += index; int32_t numOfColList = 0; - bool createTable = false; + // Bind table columns list in string, skip it and continue if (sToken.type == TK_LP) { - cstart = &sToken.z[0]; - index = 0; + *boundColumn = &sToken.z[0]; + while (1) { - sToken = tStrGetToken(sql, &index, false, 0, NULL); + index = 0; + sToken = tStrGetToken(sql, &index, false); + if (sToken.type == TK_RP) { - cend = &sToken.z[0]; break; } + sql += index; ++numOfColList; } - sToken = tStrGetToken(sql, &index, false, 0, NULL); + sToken = tStrGetToken(sql, &index, false); sql += index; } - if (numOfColList == 0 && cstart != NULL) { + if (numOfColList == 0 && (*boundColumn) != NULL) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -806,7 +794,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { if (sToken.type == TK_USING) { // create table if not exists according to the super table index = 0; - sToken = tStrGetToken(sql, &index, false, 0, NULL); + sToken = tStrGetToken(sql, &index, false); sql += index; //the source super table is moved to the secondary position of the pTableMetaInfo list @@ -835,82 +823,42 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { SSchema *pTagSchema = tscGetTableTagSchema(pSTableMetaInfo->pTableMeta); STableComInfo tinfo = tscGetTableInfo(pSTableMetaInfo->pTableMeta); - index = 0; - sToken = tStrGetToken(sql, &index, false, 0, NULL); - sql += index; - SParsedDataColInfo spd = {0}; - - uint8_t numOfTags = tscGetNumOfTags(pSTableMetaInfo->pTableMeta); - spd.numOfCols = numOfTags; + tscSetBoundColumnInfo(&spd, pTagSchema, tscGetNumOfTags(pSTableMetaInfo->pTableMeta)); - // if specify some tags column - if (sToken.type != TK_LP) { - tscSetAssignedColumnInfo(&spd, pTagSchema, numOfTags); - } else { - /* insert into tablename (col1, col2,..., coln) using superTableName (tagName1, tagName2, ..., tagNamen) - * tags(tagVal1, tagVal2, ..., tagValn) values(v1, v2,... vn); */ - int16_t offset[TSDB_MAX_COLUMNS] = {0}; - for (int32_t t = 1; t < numOfTags; ++t) { - offset[t] = offset[t - 1] + pTagSchema[t - 1].bytes; - } - - while (1) { - index = 0; - sToken = tStrGetToken(sql, &index, false, 0, NULL); - sql += index; - - if (TK_STRING == sToken.type) { - strdequote(sToken.z); - sToken.n = (uint32_t)strtrim(sToken.z); - } - - if (sToken.type == TK_RP) { - break; - } - - bool findColumnIndex = false; - - // todo speedup by using hash list - for (int32_t t = 0; t < numOfTags; ++t) { - if (strncmp(sToken.z, pTagSchema[t].name, sToken.n) == 0 && strlen(pTagSchema[t].name) == sToken.n) { - SParsedColElem *pElem = &spd.elems[spd.numOfAssignedCols++]; - pElem->offset = offset[t]; - pElem->colIndex = t; - - if (spd.hasVal[t] == true) { - return tscInvalidSQLErrMsg(pCmd->payload, "duplicated tag name", sToken.z); - } - - spd.hasVal[t] = true; - findColumnIndex = true; - break; - } - } - - if (!findColumnIndex) { - return tscInvalidSQLErrMsg(pCmd->payload, "invalid tag name", sToken.z); - } - } - - if (spd.numOfAssignedCols == 0 || spd.numOfAssignedCols > numOfTags) { - return tscInvalidSQLErrMsg(pCmd->payload, "tag name expected", sToken.z); - } - - index = 0; - sToken = tStrGetToken(sql, &index, false, 0, NULL); - sql += index; - } - - if (sToken.type != TK_TAGS) { + index = 0; + sToken = tStrGetToken(sql, &index, false); + if (sToken.type != TK_TAGS && sToken.type != TK_LP) { return tscInvalidSQLErrMsg(pCmd->payload, "keyword TAGS expected", sToken.z); } + // parse the bound tags column + if (sToken.type == TK_LP) { + /* + * insert into tablename (col1, col2,..., coln) using superTableName (tagName1, tagName2, ..., tagNamen) + * tags(tagVal1, tagVal2, ..., tagValn) values(v1, v2,... vn); + */ + char* end = NULL; + code = parseBoundColumns(pCmd, &spd, pTagSchema, sql, &end); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + sql = end; + + index = 0; // keywords of "TAGS" + sToken = tStrGetToken(sql, &index, false); + sql += index; + } else { + sql += index; + } + index = 0; - sToken = tStrGetToken(sql, &index, false, 0, NULL); + sToken = tStrGetToken(sql, &index, false); sql += index; + if (sToken.type != TK_LP) { - return tscInvalidSQLErrMsg(pCmd->payload, NULL, sToken.z); + return tscInvalidSQLErrMsg(pCmd->payload, "( is expected", sToken.z); } SKVRowBuilder kvRowBuilder = {0}; @@ -918,13 +866,11 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } - uint32_t ignoreTokenTypes = TK_LP; - uint32_t numOfIgnoreToken = 1; - for (int i = 0; i < spd.numOfAssignedCols; ++i) { - SSchema* pSchema = pTagSchema + spd.elems[i].colIndex; + for (int i = 0; i < spd.numOfBound; ++i) { + SSchema* pSchema = &pTagSchema[spd.boundedColumns[i]]; index = 0; - sToken = tStrGetToken(sql, &index, true, numOfIgnoreToken, &ignoreTokenTypes); + sToken = tStrGetToken(sql, &index, true); sql += index; if (TK_ILLEGAL == sToken.type) { @@ -943,7 +889,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { } char tagVal[TSDB_MAX_TAGS_LEN]; - code = tsParseOneColumnData(pSchema, &sToken, tagVal, pCmd->payload, &sql, false, tinfo.precision); + code = tsParseOneColumn(pSchema, &sToken, tagVal, pCmd->payload, &sql, false, tinfo.precision); if (code != TSDB_CODE_SUCCESS) { tdDestroyKVRowBuilder(&kvRowBuilder); return code; @@ -952,6 +898,8 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { tdAddColToKVRow(&kvRowBuilder, pSchema->colId, pSchema->type, tagVal); } + tscDestroyBoundColumnInfo(&spd); + SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder); tdDestroyKVRowBuilder(&kvRowBuilder); if (row == NULL) { @@ -974,7 +922,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { pCmd->tagData.data = pTag; index = 0; - sToken = tStrGetToken(sql, &index, false, 0, NULL); + sToken = tStrGetToken(sql, &index, false); sql += index; if (sToken.n == 0 || sToken.type != TK_RP) { return tscSQLSyntaxErrMsg(pCmd->payload, ") expected", sToken.z); @@ -989,33 +937,21 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { return ret; } - createTable = true; code = tscGetTableMetaEx(pSql, pTableMetaInfo, true); if (TSDB_CODE_TSC_ACTION_IN_PROGRESS == code) { return code; } } else { - if (cstart != NULL) { - sql = cstart; - } else { - sql = sToken.z; - } + sql = sToken.z; + code = tscGetTableMetaEx(pSql, pTableMetaInfo, false); - if (pCmd->curSql == NULL) { assert(code == TSDB_CODE_TSC_ACTION_IN_PROGRESS); } } - int32_t len = (int32_t)(cend - cstart + 1); - if (cstart != NULL && createTable == true) { - /* move the column list to start position of the next accessed points */ - memmove(sql - len, cstart, len); - *sqlstr = sql - len; - } else { - *sqlstr = sql; - } + *sqlstr = sql; if (*sqlstr == NULL) { code = TSDB_CODE_TSC_INVALID_SQL; @@ -1043,6 +979,76 @@ static int32_t validateDataSource(SSqlCmd *pCmd, int8_t type, const char *sql) { return TSDB_CODE_SUCCESS; } +static int32_t parseBoundColumns(SSqlCmd* pCmd, SParsedDataColInfo* pColInfo, SSchema* pSchema, + char* str, char **end) { + pColInfo->numOfBound = 0; + + memset(pColInfo->boundedColumns, 0, sizeof(int32_t) * pColInfo->numOfCols); + for(int32_t i = 0; i < pColInfo->numOfCols; ++i) { + pColInfo->cols[i].hasVal = false; + } + + int32_t code = TSDB_CODE_SUCCESS; + + int32_t index = 0; + SStrToken sToken = tStrGetToken(str, &index, false); + str += index; + + if (sToken.type != TK_LP) { + code = tscInvalidSQLErrMsg(pCmd->payload, "( is expected", sToken.z); + goto _clean; + } + + while (1) { + index = 0; + sToken = tStrGetToken(str, &index, false); + str += index; + + if (TK_STRING == sToken.type) { + tscDequoteAndTrimToken(&sToken); + } + + if (sToken.type == TK_RP) { + if (end != NULL) { // set the end position + *end = str; + } + + break; + } + + bool findColumnIndex = false; + + // todo speedup by using hash list + for (int32_t t = 0; t < pColInfo->numOfCols; ++t) { + if (strncmp(sToken.z, pSchema[t].name, sToken.n) == 0 && strlen(pSchema[t].name) == sToken.n) { + if (pColInfo->cols[t].hasVal == true) { + code = tscInvalidSQLErrMsg(pCmd->payload, "duplicated column name", sToken.z); + goto _clean; + } + + pColInfo->cols[t].hasVal = true; + pColInfo->boundedColumns[pColInfo->numOfBound] = t; + pColInfo->numOfBound += 1; + findColumnIndex = true; + break; + } + } + + if (!findColumnIndex) { + code = tscInvalidSQLErrMsg(pCmd->payload, "invalid column/tag name", sToken.z); + goto _clean; + } + } + + memset(&pColInfo->boundedColumns[pColInfo->numOfBound], 0 , sizeof(int32_t) * (pColInfo->numOfCols - pColInfo->numOfBound)); + return TSDB_CODE_SUCCESS; + + _clean: + pCmd->curSql = NULL; + pCmd->parseFinished = 1; + return code; +} + /** * parse insert sql * @param pSql @@ -1083,7 +1089,7 @@ int tsParseInsertSql(SSqlObj *pSql) { while (1) { int32_t index = 0; - SStrToken sToken = tStrGetToken(str, &index, false, 0, NULL); + SStrToken sToken = tStrGetToken(str, &index, false); // no data in the sql string anymore. if (sToken.n == 0) { @@ -1108,7 +1114,7 @@ int tsParseInsertSql(SSqlObj *pSql) { } pCmd->curSql = sToken.z; - char buf[TSDB_TABLE_FNAME_LEN]; + char buf[TSDB_TABLE_FNAME_LEN]; SStrToken sTblToken; sTblToken.z = buf; // Check if the table name available or not @@ -1121,7 +1127,8 @@ int tsParseInsertSql(SSqlObj *pSql) { goto _clean; } - if ((code = tscCheckIfCreateTable(&str, pSql)) != TSDB_CODE_SUCCESS) { + char *bindedColumns = NULL; + if ((code = tscCheckIfCreateTable(&str, pSql, &bindedColumns)) != TSDB_CODE_SUCCESS) { /* * After retrieving the table meta from server, the sql string will be parsed from the paused position. * And during the getTableMetaCallback function, the sql string will be parsed from the paused position. @@ -1129,7 +1136,7 @@ int tsParseInsertSql(SSqlObj *pSql) { if (TSDB_CODE_TSC_ACTION_IN_PROGRESS == code) { return code; } - + tscError("%p async insert parse error, code:%s", pSql, tstrerror(code)); pCmd->curSql = NULL; goto _clean; @@ -1141,41 +1148,22 @@ int tsParseInsertSql(SSqlObj *pSql) { } index = 0; - sToken = tStrGetToken(str, &index, false, 0, NULL); + sToken = tStrGetToken(str, &index, false); str += index; - if (sToken.n == 0) { + if (sToken.n == 0 || (sToken.type != TK_FILE && sToken.type != TK_VALUES)) { code = tscInvalidSQLErrMsg(pCmd->payload, "keyword VALUES or FILE required", sToken.z); goto _clean; } - + STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); - - if (sToken.type == TK_VALUES) { - SParsedDataColInfo spd = {.numOfCols = tinfo.numOfColumns}; - - SSchema *pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); - tscSetAssignedColumnInfo(&spd, pSchema, tinfo.numOfColumns); - - if (validateDataSource(pCmd, DATA_FROM_SQL_STRING, sToken.z) != TSDB_CODE_SUCCESS) { - goto _clean; - } - - /* - * app here insert data in different vnodes, so we need to set the following - * data in another submit procedure using async insert routines - */ - code = doParseInsertStatement(pCmd, &str, &spd, &totalNum); - if (code != TSDB_CODE_SUCCESS) { - goto _clean; - } - } else if (sToken.type == TK_FILE) { + if (sToken.type == TK_FILE) { if (validateDataSource(pCmd, DATA_FROM_DATA_FILE, sToken.z) != TSDB_CODE_SUCCESS) { goto _clean; } index = 0; - sToken = tStrGetToken(str, &index, false, 0, NULL); + sToken = tStrGetToken(str, &index, false); if (sToken.type != TK_STRING && sToken.type != TK_ID) { code = tscInvalidSQLErrMsg(pCmd->payload, "file path is required following keyword FILE", sToken.z); goto _clean; @@ -1199,83 +1187,63 @@ int tsParseInsertSql(SSqlObj *pSql) { tstrncpy(pCmd->payload, full_path.we_wordv[0], pCmd->allocSize); wordfree(&full_path); - } else if (sToken.type == TK_LP) { - /* insert into tablename(col1, col2,..., coln) values(v1, v2,... vn); */ - STableMeta *pTableMeta = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0)->pTableMeta; - SSchema * pSchema = tscGetTableSchema(pTableMeta); + } else { + if (bindedColumns == NULL) { + STableMeta *pTableMeta = pTableMetaInfo->pTableMeta; - if (validateDataSource(pCmd, DATA_FROM_SQL_STRING, sToken.z) != TSDB_CODE_SUCCESS) { - goto _clean; - } - - SParsedDataColInfo spd = {0}; - spd.numOfCols = tinfo.numOfColumns; - - int16_t offset[TSDB_MAX_COLUMNS] = {0}; - for (int32_t t = 1; t < tinfo.numOfColumns; ++t) { - offset[t] = offset[t - 1] + pSchema[t - 1].bytes; - } - - while (1) { - index = 0; - sToken = tStrGetToken(str, &index, false, 0, NULL); - str += index; - - if (TK_STRING == sToken.type) { - tscDequoteAndTrimToken(&sToken); + if (validateDataSource(pCmd, DATA_FROM_SQL_STRING, sToken.z) != TSDB_CODE_SUCCESS) { + goto _clean; } - if (sToken.type == TK_RP) { - break; + STableDataBlocks *dataBuf = NULL; + int32_t ret = tscGetDataBlockFromList(pCmd->pTableBlockHashList, pTableMeta->id.uid, TSDB_DEFAULT_PAYLOAD_SIZE, + sizeof(SSubmitBlk), tinfo.rowSize, &pTableMetaInfo->name, pTableMeta, + &dataBuf, NULL); + if (ret != TSDB_CODE_SUCCESS) { + goto _clean; } - bool findColumnIndex = false; + code = doParseInsertStatement(pCmd, &str, dataBuf, &totalNum); + if (code != TSDB_CODE_SUCCESS) { + goto _clean; + } + } else { // bindedColumns != NULL + // insert into tablename(col1, col2,..., coln) values(v1, v2,... vn); + STableMeta *pTableMeta = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0)->pTableMeta; - // todo speedup by using hash list - for (int32_t t = 0; t < tinfo.numOfColumns; ++t) { - if (strncmp(sToken.z, pSchema[t].name, sToken.n) == 0 && strlen(pSchema[t].name) == sToken.n) { - SParsedColElem *pElem = &spd.elems[spd.numOfAssignedCols++]; - pElem->offset = offset[t]; - pElem->colIndex = t; - - if (spd.hasVal[t] == true) { - code = tscInvalidSQLErrMsg(pCmd->payload, "duplicated column name", sToken.z); - goto _clean; - } - - spd.hasVal[t] = true; - findColumnIndex = true; - break; - } + if (validateDataSource(pCmd, DATA_FROM_SQL_STRING, sToken.z) != TSDB_CODE_SUCCESS) { + goto _clean; } - if (!findColumnIndex) { - code = tscInvalidSQLErrMsg(pCmd->payload, "invalid column name", sToken.z); + STableDataBlocks *dataBuf = NULL; + int32_t ret = tscGetDataBlockFromList(pCmd->pTableBlockHashList, pTableMeta->id.uid, TSDB_DEFAULT_PAYLOAD_SIZE, + sizeof(SSubmitBlk), tinfo.rowSize, &pTableMetaInfo->name, pTableMeta, + &dataBuf, NULL); + if (ret != TSDB_CODE_SUCCESS) { + goto _clean; + } + + SSchema *pSchema = tscGetTableSchema(pTableMeta); + code = parseBoundColumns(pCmd, &dataBuf->boundColumnInfo, pSchema, bindedColumns, NULL); + if (code != TSDB_CODE_SUCCESS) { + goto _clean; + } + + if (dataBuf->boundColumnInfo.cols[0].hasVal == false) { + code = tscInvalidSQLErrMsg(pCmd->payload, "primary timestamp column can not be null", NULL); + goto _clean; + } + + if (sToken.type != TK_VALUES) { + code = tscInvalidSQLErrMsg(pCmd->payload, "keyword VALUES is expected", sToken.z); + goto _clean; + } + + code = doParseInsertStatement(pCmd, &str, dataBuf, &totalNum); + if (code != TSDB_CODE_SUCCESS) { goto _clean; } } - - if (spd.numOfAssignedCols == 0 || spd.numOfAssignedCols > tinfo.numOfColumns) { - code = tscInvalidSQLErrMsg(pCmd->payload, "column name expected", sToken.z); - goto _clean; - } - - index = 0; - sToken = tStrGetToken(str, &index, false, 0, NULL); - str += index; - - if (sToken.type != TK_VALUES) { - code = tscInvalidSQLErrMsg(pCmd->payload, "keyword VALUES is expected", sToken.z); - goto _clean; - } - - code = doParseInsertStatement(pCmd, &str, &spd, &totalNum); - if (code != TSDB_CODE_SUCCESS) { - goto _clean; - } - } else { - code = tscInvalidSQLErrMsg(pCmd->payload, "keyword VALUES or FILE are required", sToken.z); - goto _clean; } } @@ -1294,7 +1262,7 @@ int tsParseInsertSql(SSqlObj *pSql) { goto _clean; _clean: - pCmd->curSql = NULL; + pCmd->curSql = NULL; pCmd->parseFinished = 1; return code; } @@ -1307,7 +1275,7 @@ int tsInsertInitialCheck(SSqlObj *pSql) { int32_t index = 0; SSqlCmd *pCmd = &pSql->cmd; - SStrToken sToken = tStrGetToken(pSql->sqlstr, &index, false, 0, NULL); + SStrToken sToken = tStrGetToken(pSql->sqlstr, &index, false); assert(sToken.type == TK_INSERT || sToken.type == TK_IMPORT); pCmd->count = 0; @@ -1317,7 +1285,7 @@ int tsInsertInitialCheck(SSqlObj *pSql) { TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_INSERT | pCmd->insertType); - sToken = tStrGetToken(pSql->sqlstr, &index, false, 0, NULL); + sToken = tStrGetToken(pSql->sqlstr, &index, false); if (sToken.type != TK_INTO) { return tscInvalidSQLErrMsg(pCmd->payload, "keyword INTO is expected", sToken.z); } @@ -1450,13 +1418,10 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int32_t numOfRow STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; - SSchema * pSchema = tscGetTableSchema(pTableMeta); STableComInfo tinfo = tscGetTableInfo(pTableMeta); - SParsedDataColInfo spd = {.numOfCols = tinfo.numOfColumns}; - tscSetAssignedColumnInfo(&spd, pSchema, tinfo.numOfColumns); + destroyTableNameList(pCmd); - tfree(pCmd->pTableNameList); pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); if (pCmd->pTableBlockHashList == NULL) { @@ -1495,8 +1460,9 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int32_t numOfRow char *lineptr = line; strtolower(line, line); - int32_t len = tsParseOneRowData(&lineptr, pTableDataBlock, pSchema, &spd, pCmd, tinfo.precision, &code, tokenBuf); - if (len <= 0 || pTableDataBlock->numOfParams > 0) { + int32_t len = 0; + code = tsParseOneRow(&lineptr, pTableDataBlock, pCmd, tinfo.precision, &len, tokenBuf); + if (code != TSDB_CODE_SUCCESS || pTableDataBlock->numOfParams > 0) { pSql->res.code = code; break; } diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 80a8cf3d6c..6cdd186c28 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -3567,7 +3567,8 @@ static int32_t extractColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SC } if (pSchema->type == TSDB_DATA_TYPE_BOOL) { - if (pExpr->tokenId != TK_EQ && pExpr->tokenId != TK_NE) { + int32_t t = pExpr->tokenId; + if (t != TK_EQ && t != TK_NE && t != TK_NOTNULL && t != TK_ISNULL) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } } @@ -3767,7 +3768,8 @@ static int32_t validateSQLExpr(SSqlCmd* pCmd, tSqlExpr* pExpr, SQueryInfo* pQuer } pList->ids[pList->num++] = index; - } else if (pExpr->tokenId == TK_FLOAT && (isnan(pExpr->value.dKey) || isinf(pExpr->value.dKey))) { + } else if ((pExpr->tokenId == TK_FLOAT && (isnan(pExpr->value.dKey) || isinf(pExpr->value.dKey))) || + pExpr->tokenId == TK_NULL) { return TSDB_CODE_TSC_INVALID_SQL; } else if (pExpr->type == SQL_NODE_SQLFUNCTION) { if (*type == NON_ARITHMEIC_EXPR) { @@ -3999,6 +4001,39 @@ static int32_t setExprToCond(tSqlExpr** parent, tSqlExpr* pExpr, const char* msg return TSDB_CODE_SUCCESS; } +static int32_t validateNullExpr(tSqlExpr* pExpr, char* msgBuf) { + const char* msg = "only support is [not] null"; + + tSqlExpr* pRight = pExpr->pRight; + if (pRight->tokenId == TK_NULL && (!(pExpr->tokenId == TK_ISNULL || pExpr->tokenId == TK_NOTNULL))) { + return invalidSqlErrMsg(msgBuf, msg); + } + + return TSDB_CODE_SUCCESS; +} + +// check for like expression +static int32_t validateLikeExpr(tSqlExpr* pExpr, STableMeta* pTableMeta, int32_t index, char* msgBuf) { + const char* msg1 = "wildcard string should be less than 20 characters"; + const char* msg2 = "illegal column name"; + + tSqlExpr* pLeft = pExpr->pLeft; + tSqlExpr* pRight = pExpr->pRight; + + if (pExpr->tokenId == TK_LIKE) { + if (pRight->value.nLen > TSDB_PATTERN_STRING_MAX_LEN) { + return invalidSqlErrMsg(msgBuf, msg1); + } + + SSchema* pSchema = tscGetTableSchema(pTableMeta); + if ((!isTablenameToken(&pLeft->colInfo)) && !IS_VAR_DATA_TYPE(pSchema[index].type)) { + return invalidSqlErrMsg(msgBuf, msg2); + } + } + + return TSDB_CODE_SUCCESS; +} + static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExpr** pExpr, SCondExpr* pCondExpr, int32_t* type, int32_t parentOptr) { const char* msg1 = "table query cannot use tags filter"; @@ -4008,8 +4043,7 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSql const char* msg5 = "not support ordinary column join"; const char* msg6 = "only one query condition on tbname allowed"; const char* msg7 = "only in/like allowed in filter table name"; - const char* msg8 = "wildcard string should be less than 20 characters"; - + tSqlExpr* pLeft = (*pExpr)->pLeft; tSqlExpr* pRight = (*pExpr)->pRight; @@ -4025,6 +4059,18 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSql STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; + // validate the null expression + int32_t code = validateNullExpr(*pExpr, tscGetErrorMsgPayload(pCmd)); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + // validate the like expression + code = validateLikeExpr(*pExpr, pTableMeta, index.columnIndex, tscGetErrorMsgPayload(pCmd)); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + if (index.columnIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX) { // query on time range if (!validateJoinExprNode(pCmd, pQueryInfo, *pExpr, &index)) { return TSDB_CODE_TSC_INVALID_SQL; @@ -4046,7 +4092,6 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSql int16_t leftIdx = index.tableIndex; - SColumnIndex index = COLUMN_INDEX_INITIALIZER; if (getColumnIndexByName(pCmd, &pRight->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } @@ -4093,20 +4138,6 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSql return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } - // check for like expression - if ((*pExpr)->tokenId == TK_LIKE) { - if (pRight->value.nLen > TSDB_PATTERN_STRING_MAX_LEN) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg8); - } - - SSchema* pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); - - if ((!isTablenameToken(&pLeft->colInfo)) && pSchema[index.columnIndex].type != TSDB_DATA_TYPE_BINARY && - pSchema[index.columnIndex].type != TSDB_DATA_TYPE_NCHAR) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - } - // in case of in operator, keep it in a seprate attribute if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { if (!validTableNameOptr(*pExpr)) { diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 3416530ab5..b94690f7fc 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -2826,7 +2826,6 @@ void tscInitMsgsFp() { tscProcessMsgRsp[TSDB_SQL_SHOW_CREATE_TABLE] = tscProcessShowCreateRsp; tscProcessMsgRsp[TSDB_SQL_SHOW_CREATE_DATABASE] = tscProcessShowCreateRsp; - tscKeepConn[TSDB_SQL_SHOW] = 1; tscKeepConn[TSDB_SQL_RETRIEVE] = 1; tscKeepConn[TSDB_SQL_SELECT] = 1; diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index a9cd1965e8..7699e6f459 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -299,6 +299,7 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf tfree(pTableMetaInfo->pTableMeta); tscFreeSqlResult(pSql); + tscFreeSubobj(pSql); tfree(pSql->pSubs); pSql->subState.numOfSub = 0; pTableMetaInfo->vgroupList = tscVgroupInfoClear(pTableMetaInfo->vgroupList); diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 76835552b1..f5d616fef1 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -271,6 +271,41 @@ bool tscIsTWAQuery(SQueryInfo* pQueryInfo) { return false; } +bool tscIsTopbotQuery(SQueryInfo* pQueryInfo) { + size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); + for (int32_t i = 0; i < numOfExprs; ++i) { + SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); + if (pExpr == NULL) { + continue; + } + + int32_t functionId = pExpr->functionId; + if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) { + return true; + } + } + + return false; +} + +int32_t tscGetTopbotQueryParam(SQueryInfo* pQueryInfo) { + size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); + for (int32_t i = 0; i < numOfExprs; ++i) { + SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, i); + if (pExpr == NULL) { + continue; + } + + int32_t functionId = pExpr->functionId; + if (functionId == TSDB_FUNC_TOP || functionId == TSDB_FUNC_BOTTOM) { + return (int32_t) pExpr->param[0].i64; + } + } + + return 0; +} + + void tscClearInterpInfo(SQueryInfo* pQueryInfo) { if (!tscIsPointInterpQuery(pQueryInfo)) { return; @@ -309,7 +344,7 @@ void tscSetResRawPtr(SSqlRes* pRes, SQueryInfo* pQueryInfo) { int32_t offset = 0; - for (int32_t i = 0; i < pRes->numOfCols; ++i) { + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { SInternalField* pInfo = (SInternalField*)TARRAY_GET_ELEM(pQueryInfo->fieldsInfo.internalField, i); pRes->urow[i] = pRes->data + offset * pRes->numOfRows; @@ -415,6 +450,20 @@ void tscFreeQueryInfo(SSqlCmd* pCmd, bool removeMeta) { tfree(pCmd->pQueryInfo); } +void destroyTableNameList(SSqlCmd* pCmd) { + if (pCmd->numOfTables == 0) { + assert(pCmd->pTableNameList == NULL); + return; + } + + for(int32_t i = 0; i < pCmd->numOfTables; ++i) { + tfree(pCmd->pTableNameList[i]); + } + + pCmd->numOfTables = 0; + tfree(pCmd->pTableNameList); +} + void tscResetSqlCmd(SSqlCmd* pCmd, bool removeMeta) { pCmd->command = 0; pCmd->numOfCols = 0; @@ -424,14 +473,7 @@ void tscResetSqlCmd(SSqlCmd* pCmd, bool removeMeta) { pCmd->parseFinished = 0; pCmd->autoCreated = 0; - for(int32_t i = 0; i < pCmd->numOfTables; ++i) { - if (pCmd->pTableNameList && pCmd->pTableNameList[i]) { - tfree(pCmd->pTableNameList[i]); - } - } - - pCmd->numOfTables = 0; - tfree(pCmd->pTableNameList); + destroyTableNameList(pCmd); pCmd->pTableBlockHashList = tscDestroyBlockHashTable(pCmd->pTableBlockHashList, removeMeta); pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); @@ -448,7 +490,7 @@ void tscFreeSqlResult(SSqlObj* pSql) { memset(&pSql->res, 0, sizeof(SSqlRes)); } -static void tscFreeSubobj(SSqlObj* pSql) { +void tscFreeSubobj(SSqlObj* pSql) { if (pSql->subState.numOfSub == 0) { return; } @@ -549,6 +591,11 @@ void tscFreeSqlObj(SSqlObj* pSql) { free(pSql); } +void tscDestroyBoundColumnInfo(SParsedDataColInfo* pColInfo) { + tfree(pColInfo->boundedColumns); + tfree(pColInfo->cols); +} + void tscDestroyDataBlock(STableDataBlocks* pDataBlock, bool removeMeta) { if (pDataBlock == NULL) { return; @@ -569,6 +616,7 @@ void tscDestroyDataBlock(STableDataBlocks* pDataBlock, bool removeMeta) { taosHashRemove(tscTableMetaInfo, name, strnlen(name, TSDB_TABLE_FNAME_LEN)); } + tscDestroyBoundColumnInfo(&pDataBlock->boundColumnInfo); tfree(pDataBlock); } @@ -699,7 +747,7 @@ int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock) { * @param dataBlocks * @return */ -int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOffset, SName* name, +int32_t tscCreateDataBlock(size_t defaultSize, int32_t rowSize, int32_t startOffset, SName* name, STableMeta* pTableMeta, STableDataBlocks** dataBlocks) { STableDataBlocks* dataBuf = (STableDataBlocks*)calloc(1, sizeof(STableDataBlocks)); if (dataBuf == NULL) { @@ -707,10 +755,12 @@ int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOff return TSDB_CODE_TSC_OUT_OF_MEMORY; } - dataBuf->nAllocSize = (uint32_t)initialSize; - dataBuf->headerSize = startOffset; // the header size will always be the startOffset value, reserved for the subumit block header + dataBuf->nAllocSize = (uint32_t)defaultSize; + dataBuf->headerSize = startOffset; + + // the header size will always be the startOffset value, reserved for the subumit block header if (dataBuf->nAllocSize <= dataBuf->headerSize) { - dataBuf->nAllocSize = dataBuf->headerSize*2; + dataBuf->nAllocSize = dataBuf->headerSize * 2; } dataBuf->pData = calloc(1, dataBuf->nAllocSize); @@ -720,25 +770,31 @@ int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOff return TSDB_CODE_TSC_OUT_OF_MEMORY; } - dataBuf->ordered = true; - dataBuf->prevTS = INT64_MIN; + //Here we keep the tableMeta to avoid it to be remove by other threads. + dataBuf->pTableMeta = tscTableMetaDup(pTableMeta); - dataBuf->rowSize = rowSize; - dataBuf->size = startOffset; + SParsedDataColInfo* pColInfo = &dataBuf->boundColumnInfo; + SSchema* pSchema = tscGetTableSchema(dataBuf->pTableMeta); + tscSetBoundColumnInfo(pColInfo, pSchema, dataBuf->pTableMeta->tableInfo.numOfColumns); + + dataBuf->ordered = true; + dataBuf->prevTS = INT64_MIN; + dataBuf->rowSize = rowSize; + dataBuf->size = startOffset; dataBuf->tsSource = -1; + dataBuf->vgId = dataBuf->pTableMeta->vgId; tNameAssign(&dataBuf->tableName, name); - //Here we keep the tableMeta to avoid it to be remove by other threads. - dataBuf->pTableMeta = tscTableMetaDup(pTableMeta); - assert(initialSize > 0 && pTableMeta != NULL && dataBuf->pTableMeta != NULL); + assert(defaultSize > 0 && pTableMeta != NULL && dataBuf->pTableMeta != NULL); *dataBlocks = dataBuf; return TSDB_CODE_SUCCESS; } int32_t tscGetDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize, - SName* name, STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList) { + SName* name, STableMeta* pTableMeta, STableDataBlocks** dataBlocks, + SArray* pBlockList) { *dataBlocks = NULL; STableDataBlocks** t1 = (STableDataBlocks**)taosHashGet(pHashList, (const char*)&id, sizeof(id)); if (t1 != NULL) { @@ -847,6 +903,8 @@ static void extractTableNameList(SSqlCmd* pCmd, bool freeBlockMap) { int32_t i = 0; while(p1) { STableDataBlocks* pBlocks = *p1; + tfree(pCmd->pTableNameList[i]); + pCmd->pTableNameList[i++] = tNameDup(&pBlocks->tableName); p1 = taosHashIterate(pCmd->pTableBlockHashList, p1); } @@ -963,7 +1021,7 @@ bool tscIsInsertData(char* sqlstr) { int32_t index = 0; do { - SStrToken t0 = tStrGetToken(sqlstr, &index, false, 0, NULL); + SStrToken t0 = tStrGetToken(sqlstr, &index, false); if (t0.type != TK_LP) { return t0.type == TK_INSERT || t0.type == TK_IMPORT; } diff --git a/src/connector/jdbc/CMakeLists.txt b/src/connector/jdbc/CMakeLists.txt index 540dc8eb58..eb158b1f76 100644 --- a/src/connector/jdbc/CMakeLists.txt +++ b/src/connector/jdbc/CMakeLists.txt @@ -8,7 +8,7 @@ IF (TD_MVN_INSTALLED) ADD_CUSTOM_COMMAND(OUTPUT ${JDBC_CMD_NAME} POST_BUILD COMMAND mvn -Dmaven.test.skip=true install -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml - COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.24-dist.jar ${LIBRARY_OUTPUT_PATH} + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.25-dist.jar ${LIBRARY_OUTPUT_PATH} COMMAND mvn -Dmaven.test.skip=true clean -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml COMMENT "build jdbc driver") ADD_CUSTOM_TARGET(${JDBC_TARGET_NAME} ALL WORKING_DIRECTORY ${EXECUTABLE_OUTPUT_PATH} DEPENDS ${JDBC_CMD_NAME}) diff --git a/src/connector/jdbc/deploy-pom.xml b/src/connector/jdbc/deploy-pom.xml index 935b3f7e4a..eb8c92575c 100755 --- a/src/connector/jdbc/deploy-pom.xml +++ b/src/connector/jdbc/deploy-pom.xml @@ -5,7 +5,7 @@ com.taosdata.jdbc taos-jdbcdriver - 2.0.24 + 2.0.25 jar JDBCDriver diff --git a/src/connector/jdbc/pom.xml b/src/connector/jdbc/pom.xml index 5d7c89e2d2..1f75754b0c 100755 --- a/src/connector/jdbc/pom.xml +++ b/src/connector/jdbc/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.taosdata.jdbc taos-jdbcdriver - 2.0.24 + 2.0.25 jar JDBCDriver https://github.com/taosdata/TDengine/tree/master/src/connector/jdbc @@ -62,6 +62,14 @@ + + + src/main/resources + + **/*.md + + + org.apache.maven.plugins diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractParameterMetaData.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractParameterMetaData.java index 999df06fc7..7df7252ae2 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractParameterMetaData.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractParameterMetaData.java @@ -15,7 +15,7 @@ public abstract class AbstractParameterMetaData extends WrapperImpl implements P @Override public int getParameterCount() throws SQLException { - return parameters.length; + return parameters == null ? 0 : parameters.length; } @Override diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java index 7d3741917c..5e3ffffa4f 100755 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java @@ -29,45 +29,25 @@ public class TSDBJNIConnector { private static volatile Boolean isInitialized = false; private TaosInfo taosInfo = TaosInfo.getInstance(); + // Connection pointer used in C + private long taos = TSDBConstants.JNI_NULL_POINTER; + // result set status in current connection + private boolean isResultsetClosed = true; + private int affectedRows = -1; static { System.loadLibrary("taos"); System.out.println("java.library.path:" + System.getProperty("java.library.path")); } - /** - * Connection pointer used in C - */ - private long taos = TSDBConstants.JNI_NULL_POINTER; - - /** - * Result set pointer for the current connection - */ -// private long taosResultSetPointer = TSDBConstants.JNI_NULL_POINTER; - - /** - * result set status in current connection - */ - private boolean isResultsetClosed = true; - private int affectedRows = -1; - - /** - * Whether the connection is closed - */ public boolean isClosed() { return this.taos == TSDBConstants.JNI_NULL_POINTER; } - /** - * Returns the status of last result set in current connection - */ public boolean isResultsetClosed() { return this.isResultsetClosed; } - /** - * Initialize static variables in JNI to optimize performance - */ public static void init(String configDir, String locale, String charset, String timezone) throws SQLWarning { synchronized (isInitialized) { if (!isInitialized) { @@ -93,11 +73,6 @@ public class TSDBJNIConnector { public static native String getTsCharset(); - /** - * Get connection pointer - * - * @throws SQLException - */ public boolean connect(String host, int port, String dbName, String user, String password) throws SQLException { if (this.taos != TSDBConstants.JNI_NULL_POINTER) { // this.closeConnectionImp(this.taos); @@ -185,13 +160,6 @@ public class TSDBJNIConnector { private native String getErrMsgImp(long pSql); - /** - * Get resultset pointer - * Each connection should have a single open result set at a time - */ -// public long getResultSet() { -// return taosResultSetPointer; -// } private native long getResultSetImp(long connection, long pSql); public boolean isUpdateQuery(long pSql) { @@ -231,6 +199,7 @@ public class TSDBJNIConnector { // } // return resCode; // } + private native int freeResultSetImp(long connection, long result); /** @@ -323,8 +292,7 @@ public class TSDBJNIConnector { * Validate if a create table sql statement is correct without actually creating that table */ public boolean validateCreateTableSql(String sql) { - long connection = taos; - int res = validateCreateTableSqlImp(connection, sql.getBytes()); + int res = validateCreateTableSqlImp(taos, sql.getBytes()); return res != 0 ? false : true; } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java index 856f5257bf..5c2d4c45b0 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulResultSet.java @@ -84,9 +84,9 @@ public class RestfulResultSet extends AbstractResultSet implements ResultSet { case TSDBConstants.TSDB_DATA_TYPE_TIMESTAMP: return new Timestamp(row.getDate(colIndex).getTime()); case TSDBConstants.TSDB_DATA_TYPE_BINARY: - return row.getString(colIndex).getBytes(); + return row.getString(colIndex) == null ? null : row.getString(colIndex).getBytes(); case TSDBConstants.TSDB_DATA_TYPE_NCHAR: - return row.getString(colIndex); + return row.getString(colIndex) == null ? null : row.getString(colIndex); default: return row.get(colIndex); } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/OSUtils.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/OSUtils.java new file mode 100644 index 0000000000..a67b4763f9 --- /dev/null +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/OSUtils.java @@ -0,0 +1,17 @@ +package com.taosdata.jdbc.utils; + +public class OSUtils { + private static final String OS = System.getProperty("os.name").toLowerCase(); + + public static boolean isWindows() { + return OS.indexOf("win") >= 0; + } + + public static boolean isMac() { + return OS.indexOf("mac") >= 0; + } + + public static boolean isLinux() { + return OS.indexOf("nux") >= 0; + } +} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DriverAutoloadTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DriverAutoloadTest.java index 580b2ac1b5..9826e6ed76 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DriverAutoloadTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/DriverAutoloadTest.java @@ -17,7 +17,6 @@ public class DriverAutoloadTest { @Test public void testRestful() throws SQLException { -// Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); final String url = "jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata"; Connection conn = DriverManager.getConnection(url, properties); Assert.assertNotNull(conn); diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NullValueInResultSetForJdbcRestfulTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NullValueInResultSetForJdbcRestfulTest.java new file mode 100644 index 0000000000..f2ac94adc1 --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/NullValueInResultSetForJdbcRestfulTest.java @@ -0,0 +1,64 @@ +package com.taosdata.jdbc.cases; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.sql.*; + +public class NullValueInResultSetForJdbcRestfulTest { + + private static final String host = "127.0.0.1"; + Connection conn; + + @Test + public void test() { + try (Statement stmt = conn.createStatement()) { + ResultSet rs = stmt.executeQuery("select * from weather"); + ResultSetMetaData meta = rs.getMetaData(); + while (rs.next()) { + for (int i = 1; i <= meta.getColumnCount(); i++) { + Object value = rs.getObject(i); + System.out.print(meta.getColumnLabel(i) + ": " + value + "\t"); + } + System.out.println(); + } + + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Before + public void before() throws SQLException { + final String url = "jdbc:TAOS-RS://" + host + ":6041/?user=root&password=taosdata"; + conn = DriverManager.getConnection(url); + try (Statement stmt = conn.createStatement()) { + stmt.execute("drop database if exists test_null"); + stmt.execute("create database if not exists test_null"); + stmt.execute("use test_null"); + stmt.execute("create table weather(ts timestamp, f1 int, f2 bigint, f3 float, f4 double, f5 smallint, f6 tinyint, f7 bool, f8 binary(64), f9 nchar(64))"); + stmt.executeUpdate("insert into weather(ts, f1) values(now+1s, 1)"); + stmt.executeUpdate("insert into weather(ts, f2) values(now+2s, 2)"); + stmt.executeUpdate("insert into weather(ts, f3) values(now+3s, 3.0)"); + stmt.executeUpdate("insert into weather(ts, f4) values(now+4s, 4.0)"); + stmt.executeUpdate("insert into weather(ts, f5) values(now+5s, 5)"); + stmt.executeUpdate("insert into weather(ts, f6) values(now+6s, 6)"); + stmt.executeUpdate("insert into weather(ts, f7) values(now+7s, true)"); + stmt.executeUpdate("insert into weather(ts, f8) values(now+8s, 'hello')"); + stmt.executeUpdate("insert into weather(ts, f9) values(now+9s, '涛思数据')"); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @After + public void after() { + try { + if (conn != null) + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } +} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/OSUtilsTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/OSUtilsTest.java new file mode 100644 index 0000000000..fd6c83ad1c --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/OSUtilsTest.java @@ -0,0 +1,30 @@ +package com.taosdata.jdbc.utils; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class OSUtilsTest { + + private String OS; + + @Test + public void inWindows() { + Assert.assertEquals(OS.indexOf("win") >= 0, OSUtils.isWindows()); + } + + @Test + public void isMac() { + Assert.assertEquals(OS.indexOf("mac") >= 0, OSUtils.isMac()); + } + + @Test + public void isLinux() { + Assert.assertEquals(OS.indexOf("nux") >= 0, OSUtils.isLinux()); + } + + @Before + public void before() { + OS = System.getProperty("os.name").toLowerCase(); + } +} \ No newline at end of file diff --git a/src/cq/src/cqMain.c b/src/cq/src/cqMain.c index fb0c1508cb..f620eb4dd5 100644 --- a/src/cq/src/cqMain.c +++ b/src/cq/src/cqMain.c @@ -294,7 +294,7 @@ void cqStop(void *handle) { pthread_mutex_unlock(&pContext->mutex); } -void *cqCreate(void *handle, uint64_t uid, int32_t sid, const char* dstTable, char *sqlStr, STSchema *pSchema) { +void *cqCreate(void *handle, uint64_t uid, int32_t sid, const char* dstTable, char *sqlStr, STSchema *pSchema, int start) { if (tsEnableStream == 0) { return NULL; } @@ -326,7 +326,11 @@ void *cqCreate(void *handle, uint64_t uid, int32_t sid, const char* dstTable, ch pObj->rid = taosAddRef(cqObjRef, pObj); - cqCreateStream(pContext, pObj); + if(start && pContext->master) { + cqCreateStream(pContext, pObj); + } else { + pObj->pContext = pContext; + } rid = pObj->rid; diff --git a/src/cq/test/cqtest.c b/src/cq/test/cqtest.c index f378835f0a..b1153397ba 100644 --- a/src/cq/test/cqtest.c +++ b/src/cq/test/cqtest.c @@ -70,7 +70,7 @@ int main(int argc, char *argv[]) { tdDestroyTSchemaBuilder(&schemaBuilder); for (int sid =1; sid<10; ++sid) { - cqCreate(pCq, sid, sid, NULL, "select avg(speed) from demo.t1 sliding(1s) interval(5s)", pSchema); + cqCreate(pCq, sid, sid, NULL, "select avg(speed) from demo.t1 sliding(1s) interval(5s)", pSchema, 1); } tdFreeSchema(pSchema); diff --git a/src/dnode/src/dnodeMain.c b/src/dnode/src/dnodeMain.c index 455092ad83..dd628ffe20 100644 --- a/src/dnode/src/dnodeMain.c +++ b/src/dnode/src/dnodeMain.c @@ -71,13 +71,16 @@ static SStep tsDnodeSteps[] = { {"dnode-vread", dnodeInitVRead, dnodeCleanupVRead}, {"dnode-vwrite", dnodeInitVWrite, dnodeCleanupVWrite}, {"dnode-vmgmt", dnodeInitVMgmt, dnodeCleanupVMgmt}, - {"dnode-mread", dnodeInitMRead, dnodeCleanupMRead}, - {"dnode-mwrite", dnodeInitMWrite, dnodeCleanupMWrite}, - {"dnode-mpeer", dnodeInitMPeer, dnodeCleanupMPeer}, + {"dnode-mread", dnodeInitMRead, NULL}, + {"dnode-mwrite", dnodeInitMWrite, NULL}, + {"dnode-mpeer", dnodeInitMPeer, NULL}, {"dnode-client", dnodeInitClient, dnodeCleanupClient}, {"dnode-server", dnodeInitServer, dnodeCleanupServer}, {"dnode-vnodes", dnodeInitVnodes, dnodeCleanupVnodes}, {"dnode-modules", dnodeInitModules, dnodeCleanupModules}, + {"dnode-mread", NULL, dnodeCleanupMRead}, + {"dnode-mwrite", NULL, dnodeCleanupMWrite}, + {"dnode-mpeer", NULL, dnodeCleanupMPeer}, {"dnode-shell", dnodeInitShell, dnodeCleanupShell}, {"dnode-statustmr", dnodeInitStatusTimer,dnodeCleanupStatusTimer}, {"dnode-telemetry", dnodeInitTelemetry, dnodeCleanupTelemetry}, @@ -236,6 +239,20 @@ static int32_t dnodeInitStorage() { return -1; } + TDIR *tdir = tfsOpendir("vnode_bak/.staging"); + bool stagingNotEmpty = tfsReaddir(tdir) != NULL; + tfsClosedir(tdir); + + if (stagingNotEmpty) { + dError("vnode_bak/.staging dir not empty, fix it first."); + return -1; + } + + if (tfsMkdir("vnode_bak/.staging") < 0) { + dError("failed to create vnode_bak/.staging dir since %s", tstrerror(terrno)); + return -1; + } + dnodeCheckDataDirOpenned(tsDnodeDir); dInfo("dnode storage is initialized at %s", tsDnodeDir); diff --git a/src/dnode/src/dnodeVWrite.c b/src/dnode/src/dnodeVWrite.c index 87b31e4604..26084a52eb 100644 --- a/src/dnode/src/dnodeVWrite.c +++ b/src/dnode/src/dnodeVWrite.c @@ -205,7 +205,7 @@ static void *dnodeProcessVWriteQueue(void *wparam) { pWrite->rpcMsg.ahandle, taosMsg[pWrite->pHead.msgType], qtypeStr[qtype], pWrite->pHead.version); pWrite->code = vnodeProcessWrite(pVnode, &pWrite->pHead, qtype, pWrite); - if (pWrite->code <= 0) pWrite->processedCount = 1; + if (pWrite->code <= 0) atomic_add_fetch_32(&pWrite->processedCount, 1); if (pWrite->code > 0) pWrite->code = 0; if (pWrite->code == 0 && pWrite->pHead.msgType != TSDB_MSG_TYPE_SUBMIT) forceFsync = true; @@ -222,7 +222,7 @@ static void *dnodeProcessVWriteQueue(void *wparam) { dnodeSendRpcVWriteRsp(pVnode, pWrite, pWrite->code); } else { if (qtype == TAOS_QTYPE_FWD) { - vnodeConfirmForward(pVnode, pWrite->pHead.version, 0, pWrite->pHead.msgType != TSDB_MSG_TYPE_SUBMIT); + vnodeConfirmForward(pVnode, pWrite->pHead.version, pWrite->code, pWrite->pHead.msgType != TSDB_MSG_TYPE_SUBMIT); } if (pWrite->rspRet.rsp) { rpcFreeCont(pWrite->rspRet.rsp); diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 85708e1382..eb11717a87 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -262,7 +262,7 @@ do { \ #define TSDB_MIN_TABLES 4 #define TSDB_MAX_TABLES 10000000 #define TSDB_DEFAULT_TABLES 1000000 -#define TSDB_TABLES_STEP 100 +#define TSDB_TABLES_STEP 1000 #define TSDB_MIN_DAYS_PER_FILE 1 #define TSDB_MAX_DAYS_PER_FILE 3650 diff --git a/src/inc/tcq.h b/src/inc/tcq.h index 1941649d0a..552a40665a 100644 --- a/src/inc/tcq.h +++ b/src/inc/tcq.h @@ -42,7 +42,7 @@ void cqStart(void *handle); void cqStop(void *handle); // cqCreate is called by TSDB to start an instance of CQ -void *cqCreate(void *handle, uint64_t uid, int32_t sid, const char* dstTable, char *sqlStr, STSchema *pSchema); +void *cqCreate(void *handle, uint64_t uid, int32_t sid, const char* dstTable, char *sqlStr, STSchema *pSchema, int start); // cqDrop is called by TSDB to stop an instance of CQ, handle is the return value of cqCreate void cqDrop(void *handle); diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index 495bfa2384..85ee9f0443 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -51,7 +51,7 @@ typedef struct { void *cqH; int (*notifyStatus)(void *, int status, int eno); int (*eventCallBack)(void *); - void *(*cqCreateFunc)(void *handle, uint64_t uid, int32_t sid, const char *dstTable, char *sqlStr, STSchema *pSchema); + void *(*cqCreateFunc)(void *handle, uint64_t uid, int32_t sid, const char *dstTable, char *sqlStr, STSchema *pSchema, int start); void (*cqDropFunc)(void *handle); } STsdbAppH; diff --git a/src/inc/tsync.h b/src/inc/tsync.h index 99dfd3a6a3..d1b68e3f5a 100644 --- a/src/inc/tsync.h +++ b/src/inc/tsync.h @@ -79,9 +79,6 @@ typedef void (*FStopSyncFile)(int32_t vgId, uint64_t fversion); // get file version typedef int32_t (*FGetVersion)(int32_t vgId, uint64_t *fver, uint64_t *vver); -// reset version -typedef int32_t (*FResetVersion)(int32_t vgId, uint64_t fver); - typedef int32_t (*FSendFile)(void *tsdb, SOCKET socketFd); typedef int32_t (*FRecvFile)(void *tsdb, SOCKET socketFd); @@ -99,7 +96,6 @@ typedef struct { FStartSyncFile startSyncFileFp; FStopSyncFile stopSyncFileFp; FGetVersion getVersionFp; - FResetVersion resetVersionFp; FSendFile sendFileFp; FRecvFile recvFileFp; } SSyncInfo; diff --git a/src/inc/ttokendef.h b/src/inc/ttokendef.h index 81be7e1984..20f81acc31 100644 --- a/src/inc/ttokendef.h +++ b/src/inc/ttokendef.h @@ -221,6 +221,7 @@ + #define TK_SPACE 300 #define TK_COMMENT 301 #define TK_ILLEGAL 302 diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index f4009459a7..87a08dee49 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -85,7 +85,7 @@ enum TEST_MODE { #define MAX_NUM_DATATYPE 10 #define MAX_DB_COUNT 8 -#define MAX_SUPER_TABLE_COUNT 8 +#define MAX_SUPER_TABLE_COUNT 200 #define MAX_COLUMN_COUNT 1024 #define MAX_TAG_COUNT 128 @@ -124,7 +124,7 @@ typedef enum enum_INSERT_MODE { typedef enum enumQUERY_TYPE { NO_INSERT_TYPE, - INSERT_TYPE, + INSERT_TYPE, QUERY_TYPE_BUT } QUERY_TYPE; @@ -211,8 +211,8 @@ typedef struct SArguments_S { int num_of_tables; int num_of_DPT; int abort; - int disorderRatio; - int disorderRange; + int disorderRatio; // 0: no disorder, >0: x% + int disorderRange; // ms or us by database precision int method_of_delete; char ** arg_list; int64_t totalInsertRows; @@ -229,25 +229,25 @@ typedef struct SColumn_S { typedef struct SSuperTable_S { char sTblName[MAX_TB_NAME_SIZE+1]; int childTblCount; - bool childTblExists; // 0: no, 1: yes - int batchCreateTableNum; // 0: no batch, > 0: batch table number in one sql - int8_t autoCreateTable; // 0: create sub table, 1: auto create sub table + bool childTblExists; // 0: no, 1: yes + int batchCreateTableNum; // 0: no batch, > 0: batch table number in one sql + int8_t autoCreateTable; // 0: create sub table, 1: auto create sub table char childTblPrefix[MAX_TB_NAME_SIZE]; char dataSource[MAX_TB_NAME_SIZE+1]; // rand_gen or sample - char insertMode[MAX_TB_NAME_SIZE]; // taosc, restful + char insertMode[MAX_TB_NAME_SIZE]; // taosc, restful int childTblLimit; int childTblOffset; - int multiThreadWriteOneTbl; // 0: no, 1: yes - int interlaceRows; // - int disorderRatio; // 0: no disorder, >0: x% - int disorderRange; // ms or us by database precision - int maxSqlLen; // + int multiThreadWriteOneTbl; // 0: no, 1: yes + int interlaceRows; // + int disorderRatio; // 0: no disorder, >0: x% + int disorderRange; // ms or us by database precision + int maxSqlLen; // int insertInterval; // insert interval, will override global insert interval - int64_t insertRows; // 0: no limit + int64_t insertRows; // 0: no limit int timeStampStep; - char startTimestamp[MAX_TB_NAME_SIZE]; // + char startTimestamp[MAX_TB_NAME_SIZE]; char sampleFormat[MAX_TB_NAME_SIZE]; // csv, json char sampleFile[MAX_FILE_NAME_LEN+1]; char tagsFile[MAX_FILE_NAME_LEN+1]; @@ -263,7 +263,6 @@ typedef struct SSuperTable_S { int lenOfTagOfOneRow; char* sampleDataBuf; - int sampleDataBufSize; //int sampleRowCount; //int sampleUsePos; @@ -488,7 +487,7 @@ static int taosRandom() return number; } -#else +#else // Not windows static void setupForAnsiEscape(void) {} static void resetAfterAnsiEscape(void) { @@ -500,13 +499,17 @@ static void resetAfterAnsiEscape(void) { static int taosRandom() { - srand(time(NULL)); + struct timeval tv; + + gettimeofday(&tv, NULL); + srand(tv.tv_usec); + return rand(); } -#endif +#endif // ifdef Windows -static int createDatabases(); +static int createDatabasesAndStables(); static void createChildTables(); static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet); @@ -539,7 +542,7 @@ SArguments g_args = { true, // insert_only false, // debug_print false, // verbose_print - false, // performance statistic print + false, // performance statistic print false, // answer_yes; "./output.txt", // output_file 0, // mode : sync or async @@ -641,7 +644,7 @@ static void printHelp() { "The password to use when connecting to the server. Default is 'taosdata'."); printf("%s%s%s%s\n", indent, "-c", indent, "Configuration directory. Default is '/etc/taos/'."); -#endif +#endif printf("%s%s%s%s\n", indent, "-h", indent, "The host to connect to TDengine. Default is localhost."); printf("%s%s%s%s\n", indent, "-p", indent, @@ -677,33 +680,26 @@ static void printHelp() { printf("%s%s%s%s\n", indent, "-x", indent, "Not insert only flag."); printf("%s%s%s%s\n", indent, "-y", indent, "Default input yes for prompt."); printf("%s%s%s%s\n", indent, "-O", indent, - "Insert mode--0: In order, > 0: disorder ratio. Default is in order."); + "Insert mode--0: In order, 1 ~ 50: disorder ratio. Default is in order."); printf("%s%s%s%s\n", indent, "-R", indent, "Out of order data's range, ms, default is 1000."); printf("%s%s%s%s\n", indent, "-g", indent, "Print debug info."); printf("%s%s%s%s\n", indent, "-V, --version", indent, "Print version info."); -/* printf("%s%s%s%s\n", indent, "-D", indent, +/* printf("%s%s%s%s\n", indent, "-D", indent, "if elete database if exists. 0: no, 1: yes, default is 1"); */ } static void parse_args(int argc, char *argv[], SArguments *arguments) { char **sptr; - wordexp_t full_path; for (int i = 1; i < argc; i++) { if (strcmp(argv[i], "-f") == 0) { arguments->metaFile = argv[++i]; } else if (strcmp(argv[i], "-c") == 0) { - char *configPath = argv[++i]; - if (wordexp(configPath, &full_path, 0) != 0) { - errorPrint( "Invalid path %s\n", configPath); - return; - } - taos_options(TSDB_OPTION_CONFIGDIR, full_path.we_wordv[0]); - wordfree(&full_path); + tstrncpy(configDir, argv[++i], MAX_FILE_NAME_LEN); } else if (strcmp(argv[i], "-h") == 0) { arguments->host = argv[++i]; @@ -749,7 +745,7 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { && strcasecmp(argv[i], "SMALLINT") && strcasecmp(argv[i], "BIGINT") && strcasecmp(argv[i], "DOUBLE") - && strcasecmp(argv[i], "BINARY") + && strcasecmp(argv[i], "BINARY") && strcasecmp(argv[i], "NCHAR")) { printHelp(); ERROR_EXIT( "Invalid data_type!\n"); @@ -798,23 +794,22 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { arguments->verbose_print = true; } else if (strcmp(argv[i], "-pp") == 0) { arguments->performance_print = true; - } else if (strcmp(argv[i], "-c") == 0) { - strcpy(configDir, argv[++i]); } else if (strcmp(argv[i], "-O") == 0) { + arguments->disorderRatio = atoi(argv[++i]); - if (arguments->disorderRatio > 1 - || arguments->disorderRatio < 0) { + + if (arguments->disorderRatio > 50) + arguments->disorderRatio = 50; + + if (arguments->disorderRatio < 0) arguments->disorderRatio = 0; - } else if (arguments->disorderRatio == 1) { - arguments->disorderRange = 10; - } + } else if (strcmp(argv[i], "-R") == 0) { + arguments->disorderRange = atoi(argv[++i]); - if (arguments->disorderRange == 1 - && (arguments->disorderRange > 50 - || arguments->disorderRange <= 0)) { - arguments->disorderRange = 10; - } + if (arguments->disorderRange < 0) + arguments->disorderRange = 1000; + } else if (strcmp(argv[i], "-a") == 0) { arguments->replica = atoi(argv[++i]); if (arguments->replica > 3 || arguments->replica < 1) { @@ -998,8 +993,9 @@ static void selectAndGetResult(TAOS *taos, char *command, char* resultFileName) taos_free_result(res); } -static double getCurrentTime() { +static double getCurrentTimeUs() { struct timeval tv; + if (gettimeofday(&tv, NULL) != 0) { perror("Failed to get current time in ms"); return 0.0; @@ -1107,6 +1103,7 @@ static int printfInsertMeta() { printf("host: \033[33m%s:%u\033[0m\n", g_Dbs.host, g_Dbs.port); printf("user: \033[33m%s\033[0m\n", g_Dbs.user); printf("password: \033[33m%s\033[0m\n", g_Dbs.password); + printf("configDir: \033[33m%s\033[0m\n", configDir); printf("resultFile: \033[33m%s\033[0m\n", g_Dbs.resultFile); printf("thread num of insert data: \033[33m%d\033[0m\n", g_Dbs.threadCount); printf("thread num of create table: \033[33m%d\033[0m\n", g_Dbs.threadCountByCreateTbl); @@ -1194,17 +1191,24 @@ static int printfInsertMeta() { printf(" childTblExists: \033[33m%s\033[0m\n", "error"); } - printf(" childTblCount: \033[33m%d\033[0m\n", g_Dbs.db[i].superTbls[j].childTblCount); - printf(" childTblPrefix: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].childTblPrefix); - printf(" dataSource: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].dataSource); - printf(" insertMode: \033[33m%s\033[0m\n", g_Dbs.db[i].superTbls[j].insertMode); + printf(" childTblCount: \033[33m%d\033[0m\n", + g_Dbs.db[i].superTbls[j].childTblCount); + printf(" childTblPrefix: \033[33m%s\033[0m\n", + g_Dbs.db[i].superTbls[j].childTblPrefix); + printf(" dataSource: \033[33m%s\033[0m\n", + g_Dbs.db[i].superTbls[j].dataSource); + printf(" insertMode: \033[33m%s\033[0m\n", + g_Dbs.db[i].superTbls[j].insertMode); if (g_Dbs.db[i].superTbls[j].childTblLimit > 0) { - printf(" childTblLimit: \033[33m%d\033[0m\n", g_Dbs.db[i].superTbls[j].childTblLimit); + printf(" childTblLimit: \033[33m%d\033[0m\n", + g_Dbs.db[i].superTbls[j].childTblLimit); } if (g_Dbs.db[i].superTbls[j].childTblOffset >= 0) { - printf(" childTblOffset: \033[33m%d\033[0m\n", g_Dbs.db[i].superTbls[j].childTblOffset); + printf(" childTblOffset: \033[33m%d\033[0m\n", + g_Dbs.db[i].superTbls[j].childTblOffset); } - printf(" insertRows: \033[33m%"PRId64"\033[0m\n", g_Dbs.db[i].superTbls[j].insertRows); + printf(" insertRows: \033[33m%"PRId64"\033[0m\n", + g_Dbs.db[i].superTbls[j].insertRows); if (0 == g_Dbs.db[i].superTbls[j].multiThreadWriteOneTbl) { printf(" multiThreadWriteOneTbl: \033[33mno\033[0m\n"); @@ -1257,8 +1261,10 @@ static int printfInsertMeta() { g_Dbs.db[i].superTbls[j].tagCount); for (int k = 0; k < g_Dbs.db[i].superTbls[j].tagCount; k++) { //printf("dataType:%s, dataLen:%d\t", g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen); - if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType, "binary", 6)) - || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType, "nchar", 5))) { + if ((0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType, + "binary", strlen("binary"))) + || (0 == strncasecmp(g_Dbs.db[i].superTbls[j].tags[k].dataType, + "nchar", strlen("nchar")))) { printf("tag[%d]:\033[33m%s(%d)\033[0m ", k, g_Dbs.db[i].superTbls[j].tags[k].dataType, g_Dbs.db[i].superTbls[j].tags[k].dataLen); @@ -1283,6 +1289,7 @@ static void printfInsertMetaToFile(FILE* fp) { fprintf(fp, "host: %s:%u\n", g_Dbs.host, g_Dbs.port); fprintf(fp, "user: %s\n", g_Dbs.user); + fprintf(fp, "configDir: %s\n", configDir); fprintf(fp, "resultFile: %s\n", g_Dbs.resultFile); fprintf(fp, "thread num of insert data: %d\n", g_Dbs.threadCount); fprintf(fp, "thread num of create table: %d\n", g_Dbs.threadCountByCreateTbl); @@ -1630,7 +1637,10 @@ static int getDbFromServer(TAOS * taos, SDbInfo** dbInfos) { while ((row = taos_fetch_row(res)) != NULL) { // sys database name : 'log' - if (strncasecmp(row[TSDB_SHOW_DB_NAME_INDEX], "log", fields[TSDB_SHOW_DB_NAME_INDEX].bytes) == 0) continue; + if (strncasecmp(row[TSDB_SHOW_DB_NAME_INDEX], "log", + fields[TSDB_SHOW_DB_NAME_INDEX].bytes) == 0) { + continue; + } dbInfos[count] = (SDbInfo *)calloc(1, sizeof(SDbInfo)); if (dbInfos[count] == NULL) { @@ -1677,7 +1687,8 @@ static int getDbFromServer(TAOS * taos, SDbInfo** dbInfos) { return count; } -static void printfDbInfoForQueryToFile(char* filename, SDbInfo* dbInfos, int index) { +static void printfDbInfoForQueryToFile( + char* filename, SDbInfo* dbInfos, int index) { if (filename[0] == 0) return; @@ -1749,7 +1760,7 @@ static void printfQuerySystemInfo(TAOS * taos) { } for (int i = 0; i < dbCount; i++) { - // printf database info + // printf database info printfDbInfoForQueryToFile(filename, dbInfos[i], i); // show db.vgroups @@ -1869,7 +1880,8 @@ static int postProceSql(char* host, uint16_t port, char* sqlstr) for (int l = 0; l < mod_table[userpass_buf_len % 3]; l++) base64_buf[encoded_len - 1 - l] = '='; - debugPrint("%s() LN%d: auth string base64 encoded: %s\n", __func__, __LINE__, base64_buf); + debugPrint("%s() LN%d: auth string base64 encoded: %s\n", + __func__, __LINE__, base64_buf); char *auth = base64_buf; int r = snprintf(request_buf, @@ -1948,7 +1960,7 @@ static char* getTagValueFromTagSample(SSuperTable* stbInfo, int tagUsePos) { return dataBuf; } -static char* generateTagVaulesForStb(SSuperTable* stbInfo) { +static char* generateTagVaulesForStb(SSuperTable* stbInfo, int32_t tableSeq) { char* dataBuf = (char*)calloc(TSDB_MAX_SQL_LEN+1, 1); if (NULL == dataBuf) { printf("calloc failed! size:%d\n", TSDB_MAX_SQL_LEN+1); @@ -1967,20 +1979,27 @@ static char* generateTagVaulesForStb(SSuperTable* stbInfo) { return NULL; } - char* buf = (char*)calloc(stbInfo->tags[i].dataLen+1, 1); + int tagBufLen = stbInfo->tags[i].dataLen + 1; + char* buf = (char*)calloc(tagBufLen, 1); if (NULL == buf) { printf("calloc failed! size:%d\n", stbInfo->tags[i].dataLen); tmfree(dataBuf); return NULL; } - rand_string(buf, stbInfo->tags[i].dataLen); + + if (tableSeq % 2) { + tstrncpy(buf, "beijing", tagBufLen); + } else { + tstrncpy(buf, "shanghai", tagBufLen); + } + //rand_string(buf, stbInfo->tags[i].dataLen); dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, "\'%s\', ", buf); tmfree(buf); } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "int", strlen("int"))) { dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, - "%d, ", rand_int()); + "%d, ", tableSeq); } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "bigint", strlen("bigint"))) { dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, @@ -2077,7 +2096,7 @@ static int calcRowLen(SSuperTable* superTbls) { lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6; } else if (strcasecmp(dataType, "FLOAT") == 0) { lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 22; - } else if (strcasecmp(dataType, "DOUBLE") == 0) { + } else if (strcasecmp(dataType, "DOUBLE") == 0) { lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 42; } else { printf("get error tag type : %s\n", dataType); @@ -2108,7 +2127,8 @@ static int getChildNameOfSuperTableWithLimitAndOffset(TAOS * taos, } //get all child table name use cmd: select tbname from superTblName; - snprintf(command, BUFFER_SIZE, "select tbname from %s.%s %s", dbName, sTblName, limitBuf); + snprintf(command, BUFFER_SIZE, "select tbname from %s.%s %s", + dbName, sTblName, limitBuf); res = taos_query(taos, command); int32_t code = taos_errno(res); @@ -2240,7 +2260,7 @@ static int getSuperTableFromServer(TAOS * taos, char* dbName, /* if (TBL_ALREADY_EXISTS == superTbls->childTblExists) { - //get all child table name use cmd: select tbname from superTblName; + //get all child table name use cmd: select tbname from superTblName; int childTblCount = 10000; superTbls->childTblName = (char*)calloc(1, childTblCount * TSDB_TABLE_NAME_LEN); if (superTbls->childTblName == NULL) { @@ -2256,7 +2276,8 @@ static int getSuperTableFromServer(TAOS * taos, char* dbName, return 0; } -static int createSuperTable(TAOS * taos, char* dbName, SSuperTable* superTbls, bool use_metric) { +static int createSuperTable(TAOS * taos, char* dbName, + SSuperTable* superTbl) { char command[BUFFER_SIZE] = "\0"; char cols[STRING_LEN] = "\0"; @@ -2264,19 +2285,26 @@ static int createSuperTable(TAOS * taos, char* dbName, SSuperTable* superTbls, int len = 0; int lenOfOneRow = 0; - for (colIndex = 0; colIndex < superTbls->columnCount; colIndex++) { - char* dataType = superTbls->columns[colIndex].dataType; - + + if (superTbl->columnCount == 0) { + errorPrint("%s() LN%d, super table column count is %d\n", + __func__, __LINE__, superTbl->columnCount); + return -1; + } + + for (colIndex = 0; colIndex < superTbl->columnCount; colIndex++) { + char* dataType = superTbl->columns[colIndex].dataType; + if (strcasecmp(dataType, "BINARY") == 0) { len += snprintf(cols + len, STRING_LEN - len, ", col%d %s(%d)", colIndex, "BINARY", - superTbls->columns[colIndex].dataLen); - lenOfOneRow += superTbls->columns[colIndex].dataLen + 3; + superTbl->columns[colIndex].dataLen); + lenOfOneRow += superTbl->columns[colIndex].dataLen + 3; } else if (strcasecmp(dataType, "NCHAR") == 0) { len += snprintf(cols + len, STRING_LEN - len, ", col%d %s(%d)", colIndex, "NCHAR", - superTbls->columns[colIndex].dataLen); - lenOfOneRow += superTbls->columns[colIndex].dataLen + 3; + superTbl->columns[colIndex].dataLen); + lenOfOneRow += superTbl->columns[colIndex].dataLen + 3; } else if (strcasecmp(dataType, "INT") == 0) { len += snprintf(cols + len, STRING_LEN - len, ", col%d %s", colIndex, "INT"); lenOfOneRow += 11; @@ -2308,92 +2336,99 @@ static int createSuperTable(TAOS * taos, char* dbName, SSuperTable* superTbls, } } - superTbls->lenOfOneRow = lenOfOneRow + 20; // timestamp - //printf("%s.%s column count:%d, column length:%d\n\n", g_Dbs.db[i].dbName, g_Dbs.db[i].superTbls[j].sTblName, g_Dbs.db[i].superTbls[j].columnCount, lenOfOneRow); + superTbl->lenOfOneRow = lenOfOneRow + 20; // timestamp + //printf("%s.%s column count:%d, column length:%d\n\n", g_Dbs.db[i].dbName, g_Dbs.db[i].superTbl[j].sTblName, g_Dbs.db[i].superTbl[j].columnCount, lenOfOneRow); // save for creating child table - superTbls->colsOfCreateChildTable = (char*)calloc(len+20, 1); - if (NULL == superTbls->colsOfCreateChildTable) { - printf("Failed when calloc, size:%d", len+1); + superTbl->colsOfCreateChildTable = (char*)calloc(len+20, 1); + if (NULL == superTbl->colsOfCreateChildTable) { + errorPrint("%s() LN%d, Failed when calloc, size:%d", + __func__, __LINE__, len+1); taos_close(taos); exit(-1); } - snprintf(superTbls->colsOfCreateChildTable, len+20, "(ts timestamp%s)", cols); - verbosePrint("%s() LN%d: %s\n", __func__, __LINE__, superTbls->colsOfCreateChildTable); - if (use_metric) { - char tags[STRING_LEN] = "\0"; - int tagIndex; - len = 0; + snprintf(superTbl->colsOfCreateChildTable, len+20, "(ts timestamp%s)", cols); + verbosePrint("%s() LN%d: %s\n", __func__, __LINE__, superTbl->colsOfCreateChildTable); - int lenOfTagOfOneRow = 0; - len += snprintf(tags + len, STRING_LEN - len, "("); - for (tagIndex = 0; tagIndex < superTbls->tagCount; tagIndex++) { - char* dataType = superTbls->tags[tagIndex].dataType; - - if (strcasecmp(dataType, "BINARY") == 0) { - len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", tagIndex, - "BINARY", superTbls->tags[tagIndex].dataLen); - lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3; - } else if (strcasecmp(dataType, "NCHAR") == 0) { - len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", tagIndex, - "NCHAR", superTbls->tags[tagIndex].dataLen); - lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 3; - } else if (strcasecmp(dataType, "INT") == 0) { - len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, - "INT"); - lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 11; - } else if (strcasecmp(dataType, "BIGINT") == 0) { - len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, - "BIGINT"); - lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 21; - } else if (strcasecmp(dataType, "SMALLINT") == 0) { - len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, - "SMALLINT"); - lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6; - } else if (strcasecmp(dataType, "TINYINT") == 0) { - len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, - "TINYINT"); - lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 4; - } else if (strcasecmp(dataType, "BOOL") == 0) { - len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, - "BOOL"); - lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 6; - } else if (strcasecmp(dataType, "FLOAT") == 0) { - len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, - "FLOAT"); - lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 22; - } else if (strcasecmp(dataType, "DOUBLE") == 0) { - len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, - "DOUBLE"); - lenOfTagOfOneRow += superTbls->tags[tagIndex].dataLen + 42; - } else { - taos_close(taos); - printf("config error tag type : %s\n", dataType); - exit(-1); - } - } - len -= 2; - len += snprintf(tags + len, STRING_LEN - len, ")"); - - superTbls->lenOfTagOfOneRow = lenOfTagOfOneRow; - - snprintf(command, BUFFER_SIZE, - "create table if not exists %s.%s (ts timestamp%s) tags %s", - dbName, superTbls->sTblName, cols, tags); - verbosePrint("%s() LN%d: %s\n", __func__, __LINE__, command); - - if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) { - errorPrint( "create supertable %s failed!\n\n", - superTbls->sTblName); - return -1; - } - debugPrint("create supertable %s success!\n\n", superTbls->sTblName); + if (superTbl->tagCount == 0) { + errorPrint("%s() LN%d, super table tag count is %d\n", + __func__, __LINE__, superTbl->tagCount); + return -1; } + + char tags[STRING_LEN] = "\0"; + int tagIndex; + len = 0; + + int lenOfTagOfOneRow = 0; + len += snprintf(tags + len, STRING_LEN - len, "("); + for (tagIndex = 0; tagIndex < superTbl->tagCount; tagIndex++) { + char* dataType = superTbl->tags[tagIndex].dataType; + + if (strcasecmp(dataType, "BINARY") == 0) { + len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", tagIndex, + "BINARY", superTbl->tags[tagIndex].dataLen); + lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 3; + } else if (strcasecmp(dataType, "NCHAR") == 0) { + len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", tagIndex, + "NCHAR", superTbl->tags[tagIndex].dataLen); + lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 3; + } else if (strcasecmp(dataType, "INT") == 0) { + len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, + "INT"); + lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 11; + } else if (strcasecmp(dataType, "BIGINT") == 0) { + len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, + "BIGINT"); + lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 21; + } else if (strcasecmp(dataType, "SMALLINT") == 0) { + len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, + "SMALLINT"); + lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 6; + } else if (strcasecmp(dataType, "TINYINT") == 0) { + len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, + "TINYINT"); + lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 4; + } else if (strcasecmp(dataType, "BOOL") == 0) { + len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, + "BOOL"); + lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 6; + } else if (strcasecmp(dataType, "FLOAT") == 0) { + len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, + "FLOAT"); + lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 22; + } else if (strcasecmp(dataType, "DOUBLE") == 0) { + len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, + "DOUBLE"); + lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 42; + } else { + taos_close(taos); + printf("config error tag type : %s\n", dataType); + exit(-1); + } + } + + len -= 2; + len += snprintf(tags + len, STRING_LEN - len, ")"); + + superTbl->lenOfTagOfOneRow = lenOfTagOfOneRow; + + snprintf(command, BUFFER_SIZE, + "create table if not exists %s.%s (ts timestamp%s) tags %s", + dbName, superTbl->sTblName, cols, tags); + verbosePrint("%s() LN%d: %s\n", __func__, __LINE__, command); + + if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) { + errorPrint( "create supertable %s failed!\n\n", + superTbl->sTblName); + return -1; + } + debugPrint("create supertable %s success!\n\n", superTbl->sTblName); return 0; } -static int createDatabases() { +static int createDatabasesAndStables() { TAOS * taos = NULL; int ret = 0; taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, NULL, g_Dbs.port); @@ -2411,85 +2446,88 @@ static int createDatabases() { taos_close(taos); return -1; } - } - int dataLen = 0; - dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, "create database if not exists %s", g_Dbs.db[i].dbName); + int dataLen = 0; + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, "create database if not exists %s", g_Dbs.db[i].dbName); - if (g_Dbs.db[i].dbCfg.blocks > 0) { - dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, " blocks %d", g_Dbs.db[i].dbCfg.blocks); - } - if (g_Dbs.db[i].dbCfg.cache > 0) { - dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, " cache %d", g_Dbs.db[i].dbCfg.cache); - } - if (g_Dbs.db[i].dbCfg.days > 0) { - dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, " days %d", g_Dbs.db[i].dbCfg.days); - } - if (g_Dbs.db[i].dbCfg.keep > 0) { - dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, " keep %d", g_Dbs.db[i].dbCfg.keep); - } - if (g_Dbs.db[i].dbCfg.quorum > 1) { - dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, " quorum %d", g_Dbs.db[i].dbCfg.quorum); - } - if (g_Dbs.db[i].dbCfg.replica > 0) { - dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, " replica %d", g_Dbs.db[i].dbCfg.replica); - } - if (g_Dbs.db[i].dbCfg.update > 0) { - dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, " update %d", g_Dbs.db[i].dbCfg.update); - } - //if (g_Dbs.db[i].dbCfg.maxtablesPerVnode > 0) { - // dataLen += snprintf(command + dataLen, - // BUFFER_SIZE - dataLen, "tables %d ", g_Dbs.db[i].dbCfg.maxtablesPerVnode); - //} - if (g_Dbs.db[i].dbCfg.minRows > 0) { - dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, " minrows %d", g_Dbs.db[i].dbCfg.minRows); - } - if (g_Dbs.db[i].dbCfg.maxRows > 0) { - dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, " maxrows %d", g_Dbs.db[i].dbCfg.maxRows); - } - if (g_Dbs.db[i].dbCfg.comp > 0) { - dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, " comp %d", g_Dbs.db[i].dbCfg.comp); - } - if (g_Dbs.db[i].dbCfg.walLevel > 0) { - dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, " wal %d", g_Dbs.db[i].dbCfg.walLevel); - } - if (g_Dbs.db[i].dbCfg.cacheLast > 0) { - dataLen += snprintf(command + dataLen, - BUFFER_SIZE - dataLen, " cachelast %d", g_Dbs.db[i].dbCfg.cacheLast); - } - if (g_Dbs.db[i].dbCfg.fsync > 0) { - dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, - " fsync %d", g_Dbs.db[i].dbCfg.fsync); - } - if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", strlen("ms"))) - || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, - "us", strlen("us")))) { - dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, - " precision \'%s\';", g_Dbs.db[i].dbCfg.precision); - } + if (g_Dbs.db[i].dbCfg.blocks > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " blocks %d", g_Dbs.db[i].dbCfg.blocks); + } + if (g_Dbs.db[i].dbCfg.cache > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " cache %d", g_Dbs.db[i].dbCfg.cache); + } + if (g_Dbs.db[i].dbCfg.days > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " days %d", g_Dbs.db[i].dbCfg.days); + } + if (g_Dbs.db[i].dbCfg.keep > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " keep %d", g_Dbs.db[i].dbCfg.keep); + } + if (g_Dbs.db[i].dbCfg.quorum > 1) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " quorum %d", g_Dbs.db[i].dbCfg.quorum); + } + if (g_Dbs.db[i].dbCfg.replica > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " replica %d", g_Dbs.db[i].dbCfg.replica); + } + if (g_Dbs.db[i].dbCfg.update > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " update %d", g_Dbs.db[i].dbCfg.update); + } + //if (g_Dbs.db[i].dbCfg.maxtablesPerVnode > 0) { + // dataLen += snprintf(command + dataLen, + // BUFFER_SIZE - dataLen, "tables %d ", g_Dbs.db[i].dbCfg.maxtablesPerVnode); + //} + if (g_Dbs.db[i].dbCfg.minRows > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " minrows %d", g_Dbs.db[i].dbCfg.minRows); + } + if (g_Dbs.db[i].dbCfg.maxRows > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " maxrows %d", g_Dbs.db[i].dbCfg.maxRows); + } + if (g_Dbs.db[i].dbCfg.comp > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " comp %d", g_Dbs.db[i].dbCfg.comp); + } + if (g_Dbs.db[i].dbCfg.walLevel > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " wal %d", g_Dbs.db[i].dbCfg.walLevel); + } + if (g_Dbs.db[i].dbCfg.cacheLast > 0) { + dataLen += snprintf(command + dataLen, + BUFFER_SIZE - dataLen, " cachelast %d", g_Dbs.db[i].dbCfg.cacheLast); + } + if (g_Dbs.db[i].dbCfg.fsync > 0) { + dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, + " fsync %d", g_Dbs.db[i].dbCfg.fsync); + } + if ((0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, "ms", strlen("ms"))) + || (0 == strncasecmp(g_Dbs.db[i].dbCfg.precision, + "us", strlen("us")))) { + dataLen += snprintf(command + dataLen, BUFFER_SIZE - dataLen, + " precision \'%s\';", g_Dbs.db[i].dbCfg.precision); + } - debugPrint("%s() %d command: %s\n", __func__, __LINE__, command); - if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) { - taos_close(taos); - errorPrint( "\ncreate database %s failed!\n\n", g_Dbs.db[i].dbName); - return -1; + debugPrint("%s() %d command: %s\n", __func__, __LINE__, command); + if (0 != queryDbExec(taos, command, NO_INSERT_TYPE, false)) { + taos_close(taos); + errorPrint( "\ncreate database %s failed!\n\n", g_Dbs.db[i].dbName); + return -1; + } + printf("\ncreate database %s success!\n\n", g_Dbs.db[i].dbName); } - printf("\ncreate database %s success!\n\n", g_Dbs.db[i].dbName); debugPrint("%s() %d supertbl count:%d\n", __func__, __LINE__, g_Dbs.db[i].superTblCount); + + int validStbCount = 0; + for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { sprintf(command, "describe %s.%s;", g_Dbs.db[i].dbName, g_Dbs.db[i].superTbls[j].sTblName); @@ -2499,12 +2537,11 @@ static int createDatabases() { if ((ret != 0) || (g_Dbs.db[i].drop)) { ret = createSuperTable(taos, g_Dbs.db[i].dbName, - &g_Dbs.db[i].superTbls[j], g_Dbs.use_metric); + &g_Dbs.db[i].superTbls[j]); if (0 != ret) { - errorPrint("\ncreate super table %d failed!\n\n", j); - taos_close(taos); - return -1; + errorPrint("create super table %d failed!\n\n", j); + continue; } } @@ -2513,10 +2550,13 @@ static int createDatabases() { if (0 != ret) { errorPrint("\nget super table %s.%s info failed!\n\n", g_Dbs.db[i].dbName, g_Dbs.db[i].superTbls[j].sTblName); - taos_close(taos); - return -1; + continue; } + + validStbCount ++; } + + g_Dbs.db[i].superTblCount = validStbCount; } taos_close(taos); @@ -2569,10 +2609,9 @@ static void* createTable(void *sarg) len += snprintf(buffer + len, buff_len - len, "create table "); } - char* tagsValBuf = NULL; if (0 == superTblInfo->tagSource) { - tagsValBuf = generateTagVaulesForStb(superTblInfo); + tagsValBuf = generateTagVaulesForStb(superTblInfo, i); } else { tagsValBuf = getTagValueFromTagSample( superTblInfo, @@ -2582,7 +2621,6 @@ static void* createTable(void *sarg) free(buffer); return NULL; } - len += snprintf(buffer + len, superTblInfo->maxSqlLen - len, "if not exists %s.%s%d using %s.%s tags %s ", @@ -2591,7 +2629,6 @@ static void* createTable(void *sarg) superTblInfo->sTblName, tagsValBuf); free(tagsValBuf); batchNum++; - if ((batchNum < superTblInfo->batchCreateTableNum) && ((superTblInfo->maxSqlLen - len) >= (superTblInfo->lenOfTagOfOneRow + 256))) { @@ -2615,7 +2652,7 @@ static void* createTable(void *sarg) lastPrintTime = currentPrintTime; } } - + if (0 != len) { verbosePrint("%s() %d buffer: %s\n", __func__, __LINE__, buffer); if (0 != queryDbExec(winfo->taos, buffer, NO_INSERT_TYPE, false)) { @@ -2680,7 +2717,7 @@ static int startMultiThreadCreateChildTable( t_info->minDelay = INT16_MAX; pthread_create(pids + i, NULL, createTable, t_info); } - + for (int i = 0; i < threads; i++) { pthread_join(pids[i], NULL); } @@ -2701,27 +2738,29 @@ static void createChildTables() { int len; for (int i = 0; i < g_Dbs.dbCount; i++) { - if (g_Dbs.db[i].superTblCount > 0) { - // with super table - for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { - if ((AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) - || (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists)) { - continue; + if (g_Dbs.use_metric) { + if (g_Dbs.db[i].superTblCount > 0) { + // with super table + for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { + if ((AUTO_CREATE_SUBTBL == g_Dbs.db[i].superTbls[j].autoCreateTable) + || (TBL_ALREADY_EXISTS == g_Dbs.db[i].superTbls[j].childTblExists)) { + continue; + } + + verbosePrint("%s() LN%d: %s\n", __func__, __LINE__, + g_Dbs.db[i].superTbls[j].colsOfCreateChildTable); + int startFrom = 0; + g_totalChildTables += g_Dbs.db[i].superTbls[j].childTblCount; + + verbosePrint("%s() LN%d: create %d child tables from %d\n", + __func__, __LINE__, g_totalChildTables, startFrom); + startMultiThreadCreateChildTable( + g_Dbs.db[i].superTbls[j].colsOfCreateChildTable, + g_Dbs.threadCountByCreateTbl, + startFrom, + g_Dbs.db[i].superTbls[j].childTblCount, + g_Dbs.db[i].dbName, &(g_Dbs.db[i].superTbls[j])); } - - verbosePrint("%s() LN%d: %s\n", __func__, __LINE__, - g_Dbs.db[i].superTbls[j].colsOfCreateChildTable); - int startFrom = 0; - g_totalChildTables += g_Dbs.db[i].superTbls[j].childTblCount; - - verbosePrint("%s() LN%d: create %d child tables from %d\n", - __func__, __LINE__, g_totalChildTables, startFrom); - startMultiThreadCreateChildTable( - g_Dbs.db[i].superTbls[j].colsOfCreateChildTable, - g_Dbs.threadCountByCreateTbl, - startFrom, - g_Dbs.db[i].superTbls[j].childTblCount, - g_Dbs.db[i].dbName, &(g_Dbs.db[i].superTbls[j])); } } else { // normal table @@ -2741,7 +2780,7 @@ static void createChildTables() { j++; } - len = snprintf(tblColsBuf + len, MAX_SQL_SIZE - len, ")"); + snprintf(tblColsBuf + len, MAX_SQL_SIZE - len, ")"); verbosePrint("%s() LN%d: dbName: %s num of tb: %d schema: %s\n", __func__, __LINE__, @@ -2897,7 +2936,7 @@ static bool getColumnAndTagTypeFromInsertJsonFile( cJSON* stbInfo, SSuperTable* superTbls) { bool ret = false; - // columns + // columns cJSON *columns = cJSON_GetObjectItem(stbInfo, "columns"); if (columns && columns->type != cJSON_Array) { printf("ERROR: failed to read json, columns not found\n"); @@ -2907,7 +2946,7 @@ static bool getColumnAndTagTypeFromInsertJsonFile( superTbls->tagCount = 0; return true; } - + int columnSize = cJSON_GetArraySize(columns); if (columnSize > MAX_COLUMN_COUNT) { errorPrint("%s() LN%d, failed to read json, column size overflow, max column size is %d\n", @@ -2935,7 +2974,7 @@ static bool getColumnAndTagTypeFromInsertJsonFile( count = 1; } - // column info + // column info memset(&columnCase, 0, sizeof(StrColumn)); cJSON *dataType = cJSON_GetObjectItem(column, "type"); if (!dataType || dataType->type != cJSON_String || dataType->valuestring == NULL) { @@ -2963,10 +3002,10 @@ static bool getColumnAndTagTypeFromInsertJsonFile( } } superTbls->columnCount = index; - + count = 1; index = 0; - // tags + // tags cJSON *tags = cJSON_GetObjectItem(stbInfo, "tags"); if (!tags || tags->type != cJSON_Array) { debugPrint("%s() LN%d, failed to read json, tags not found\n", __func__, __LINE__); @@ -2995,7 +3034,7 @@ static bool getColumnAndTagTypeFromInsertJsonFile( count = 1; } - // column info + // column info memset(&columnCase, 0, sizeof(StrColumn)); cJSON *dataType = cJSON_GetObjectItem(tag, "type"); if (!dataType || dataType->type != cJSON_String || dataType->valuestring == NULL) { @@ -3143,7 +3182,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { if (numRecPerReq && numRecPerReq->type == cJSON_Number) { g_args.num_of_RPR = numRecPerReq->valueint; } else if (!numRecPerReq) { - g_args.num_of_RPR = 100; + g_args.num_of_RPR = 0xffff; } else { errorPrint("%s() LN%d, failed to read json, num_of_records_per_req not found\n", __func__, __LINE__); goto PARSE_OVER; @@ -3186,7 +3225,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { cJSON* dbinfos = cJSON_GetArrayItem(dbs, i); if (dbinfos == NULL) continue; - // dbinfo + // dbinfo cJSON *dbinfo = cJSON_GetObjectItem(dbinfos, "dbinfo"); if (!dbinfo || dbinfo->type != cJSON_Object) { printf("ERROR: failed to read json, dbinfo not found\n"); @@ -3364,22 +3403,24 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } else if (!fsync) { g_Dbs.db[i].dbCfg.fsync = -1; } else { - printf("ERROR: failed to read json, fsync not found\n"); - goto PARSE_OVER; + errorPrint("%s() LN%d, failed to read json, fsync input mistake\n", + __func__, __LINE__); + goto PARSE_OVER; } // super_talbes cJSON *stables = cJSON_GetObjectItem(dbinfos, "super_tables"); if (!stables || stables->type != cJSON_Array) { - printf("ERROR: failed to read json, super_tables not found\n"); + errorPrint("%s() LN%d, failed to read json, super_tables not found\n", + __func__, __LINE__); goto PARSE_OVER; } int stbSize = cJSON_GetArraySize(stables); if (stbSize > MAX_SUPER_TABLE_COUNT) { errorPrint( - "ERROR: failed to read json, databases size overflow, max database is %d\n", - MAX_SUPER_TABLE_COUNT); + "%s() LN%d, failed to read json, supertable size overflow, max supertable is %d\n", + __func__, __LINE__, MAX_SUPER_TABLE_COUNT); goto PARSE_OVER; } @@ -3391,7 +3432,8 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { // dbinfo cJSON *stbName = cJSON_GetObjectItem(stbInfo, "name"); if (!stbName || stbName->type != cJSON_String || stbName->valuestring == NULL) { - printf("ERROR: failed to read json, stb name not found\n"); + errorPrint("%s() LN%d, failed to read json, stb name not found\n", + __func__, __LINE__); goto PARSE_OVER; } tstrncpy(g_Dbs.db[i].superTbls[j].sTblName, stbName->valuestring, MAX_TB_NAME_SIZE); @@ -3526,19 +3568,6 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { goto PARSE_OVER; } - cJSON* sampleDataBufSize = cJSON_GetObjectItem(stbInfo, "sample_buf_size"); - if (sampleDataBufSize && sampleDataBufSize->type == cJSON_Number) { - g_Dbs.db[i].superTbls[j].sampleDataBufSize = sampleDataBufSize->valueint; - if (g_Dbs.db[i].superTbls[j].sampleDataBufSize < 1024*1024) { - g_Dbs.db[i].superTbls[j].sampleDataBufSize = 1024*1024 + 1024; - } - } else if (!sampleDataBufSize) { - g_Dbs.db[i].superTbls[j].sampleDataBufSize = 1024*1024 + 1024; - } else { - printf("ERROR: failed to read json, sample_buf_size not found\n"); - goto PARSE_OVER; - } - cJSON *sampleFormat = cJSON_GetObjectItem(stbInfo, "sample_format"); if (sampleFormat && sampleFormat->type == cJSON_String && sampleFormat->valuestring != NULL) { @@ -3589,7 +3618,7 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { } g_Dbs.db[i].superTbls[j].maxSqlLen = len; } else if (!maxSqlLen) { - g_Dbs.db[i].superTbls[j].maxSqlLen = TSDB_MAX_SQL_LEN; + g_Dbs.db[i].superTbls[j].maxSqlLen = g_args.max_sql_len; } else { printf("ERROR: failed to read json, maxSqlLen not found\n"); goto PARSE_OVER; @@ -3636,6 +3665,12 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { cJSON* disorderRatio = cJSON_GetObjectItem(stbInfo, "disorder_ratio"); if (disorderRatio && disorderRatio->type == cJSON_Number) { + if (disorderRatio->valueint > 50) + disorderRatio->valueint = 50; + + if (disorderRatio->valueint < 0) + disorderRatio->valueint = 0; + g_Dbs.db[i].superTbls[j].disorderRatio = disorderRatio->valueint; } else if (!disorderRatio) { g_Dbs.db[i].superTbls[j].disorderRatio = 0; @@ -3722,7 +3757,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { cJSON* user = cJSON_GetObjectItem(root, "user"); if (user && user->type == cJSON_String && user->valuestring != NULL) { - tstrncpy(g_queryInfo.user, user->valuestring, MAX_USERNAME_SIZE); + tstrncpy(g_queryInfo.user, user->valuestring, MAX_USERNAME_SIZE); } else if (!user) { tstrncpy(g_queryInfo.user, "root", MAX_USERNAME_SIZE); ; } @@ -3779,7 +3814,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { goto PARSE_OVER; } - // super_table_query + // super_table_query cJSON *specifiedQuery = cJSON_GetObjectItem(root, "specified_table_query"); if (!specifiedQuery) { g_queryInfo.specifiedQueryInfo.concurrent = 0; @@ -3904,7 +3939,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { } } - // sub_table_query + // sub_table_query cJSON *superQuery = cJSON_GetObjectItem(root, "super_table_query"); if (!superQuery) { g_queryInfo.superQueryInfo.threadCnt = 0; @@ -3970,7 +4005,7 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { cJSON* subinterval = cJSON_GetObjectItem(superQuery, "interval"); if (subinterval && subinterval->type == cJSON_Number) { g_queryInfo.superQueryInfo.subscribeInterval = subinterval->valueint; - } else if (!subinterval) { + } else if (!subinterval) { //printf("failed to read json, subscribe interval no found\n"); //goto PARSE_OVER; g_queryInfo.superQueryInfo.subscribeInterval = 10000; @@ -4064,7 +4099,7 @@ static bool getInfoFromJsonFile(char* file) { } bool ret = false; - int maxLen = 64000; + int maxLen = 6400000; char *content = calloc(1, maxLen + 1); int len = fread(content, 1, maxLen, fp); if (len <= 0) { @@ -4174,71 +4209,77 @@ static int getRowDataFromSample(char* dataBuf, int maxLen, int64_t timestamp, dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, ")"); (*sampleUsePos)++; - + return dataLen; } -static int generateRowData(char* dataBuf, int maxLen, int64_t timestamp, SSuperTable* stbInfo) { - int dataLen = 0; - dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "(%" PRId64 ", ", timestamp); +static int generateRowData(char* recBuf, int64_t timestamp, SSuperTable* stbInfo) { + int dataLen = 0; + char *pstr = recBuf; + int maxLen = MAX_DATA_SIZE; + + dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "(%" PRId64 ", ", timestamp); + for (int i = 0; i < stbInfo->columnCount; i++) { if ((0 == strncasecmp(stbInfo->columns[i].dataType, "binary", 6)) || (0 == strncasecmp(stbInfo->columns[i].dataType, "nchar", 5))) { if (stbInfo->columns[i].dataLen > TSDB_MAX_BINARY_LEN) { errorPrint( "binary or nchar length overflow, max size:%u\n", (uint32_t)TSDB_MAX_BINARY_LEN); - return (-1); + return -1; } char* buf = (char*)calloc(stbInfo->columns[i].dataLen+1, 1); if (NULL == buf) { errorPrint( "calloc failed! size:%d\n", stbInfo->columns[i].dataLen); - return (-1); + return -1; } rand_string(buf, stbInfo->columns[i].dataLen); - dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "\'%s\', ", buf); + dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "\'%s\', ", buf); tmfree(buf); } else if (0 == strncasecmp(stbInfo->columns[i].dataType, "int", 3)) { - dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, + dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "%d, ", rand_int()); } else if (0 == strncasecmp(stbInfo->columns[i].dataType, "bigint", 6)) { - dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, + dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "%"PRId64", ", rand_bigint()); } else if (0 == strncasecmp(stbInfo->columns[i].dataType, "float", 5)) { - dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, + dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "%f, ", rand_float()); } else if (0 == strncasecmp(stbInfo->columns[i].dataType, "double", 6)) { - dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, + dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "%f, ", rand_double()); } else if (0 == strncasecmp(stbInfo->columns[i].dataType, "smallint", 8)) { - dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%d, ", rand_smallint()); + dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "%d, ", rand_smallint()); } else if (0 == strncasecmp(stbInfo->columns[i].dataType, "tinyint", 7)) { - dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%d, ", rand_tinyint()); + dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "%d, ", rand_tinyint()); } else if (0 == strncasecmp(stbInfo->columns[i].dataType, "bool", 4)) { - dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%d, ", rand_bool()); + dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "%d, ", rand_bool()); } else if (0 == strncasecmp(stbInfo->columns[i].dataType, "timestamp", 9)) { - dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, "%"PRId64", ", rand_bigint()); + dataLen += snprintf(pstr + dataLen, maxLen - dataLen, "%"PRId64", ", rand_bigint()); } else { errorPrint( "No support data type: %s\n", stbInfo->columns[i].dataType); - return (-1); + return -1; } } dataLen -= 2; - dataLen += snprintf(dataBuf + dataLen, maxLen - dataLen, ")"); + dataLen += snprintf(pstr + dataLen, maxLen - dataLen, ")"); - return dataLen; + verbosePrint("%s() LN%d, recBuf:\n\t%s\n", __func__, __LINE__, recBuf); + + return strlen(recBuf); } -static int32_t generateData(char *res, char **data_type, +static int32_t generateData(char *recBuf, char **data_type, int num_of_cols, int64_t timestamp, int lenOfBinary) { - memset(res, 0, MAX_DATA_SIZE); - char *pstr = res; + memset(recBuf, 0, MAX_DATA_SIZE); + char *pstr = recBuf; pstr += sprintf(pstr, "(%" PRId64, timestamp); int c = 0; @@ -4259,7 +4300,7 @@ static int32_t generateData(char *res, char **data_type, } else if (strcasecmp(data_type[i % c], "smallint") == 0) { pstr += sprintf(pstr, ", %d", rand_smallint()); } else if (strcasecmp(data_type[i % c], "int") == 0) { - pstr += sprintf(pstr, ", %d", rand_int()); + pstr += sprintf(pstr, ", %d", rand_int()); } else if (strcasecmp(data_type[i % c], "bigint") == 0) { pstr += sprintf(pstr, ", %" PRId64, rand_bigint()); } else if (strcasecmp(data_type[i % c], "float") == 0) { @@ -4282,7 +4323,7 @@ static int32_t generateData(char *res, char **data_type, free(s); } - if (pstr - res > MAX_DATA_SIZE) { + if (strlen(recBuf) > MAX_DATA_SIZE) { perror("column length too long, abort"); exit(-1); } @@ -4290,7 +4331,9 @@ static int32_t generateData(char *res, char **data_type, pstr += sprintf(pstr, ")"); - return (int32_t)(pstr - res); + verbosePrint("%s() LN%d, recBuf:\n\t%s\n", __func__, __LINE__, recBuf); + + return (int32_t)strlen(recBuf); } static int prepareSampleDataForSTable(SSuperTable *superTblInfo) { @@ -4299,9 +4342,9 @@ static int prepareSampleDataForSTable(SSuperTable *superTblInfo) { sampleDataBuf = calloc( superTblInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE, 1); if (sampleDataBuf == NULL) { - errorPrint("%s() LN%d, Failed to calloc %d Bytes, reason:%s\n", + errorPrint("%s() LN%d, Failed to calloc %d Bytes, reason:%s\n", __func__, __LINE__, - superTblInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE, + superTblInfo->lenOfOneRow * MAX_SAMPLES_ONCE_FROM_FILE, strerror(errno)); return -1; } @@ -4351,7 +4394,8 @@ static void getTableName(char *pTblName, threadInfo* pThreadInfo, int tableSeq) if ((superTblInfo->childTblOffset >= 0) && (superTblInfo->childTblLimit > 0)) { snprintf(pTblName, TSDB_TABLE_NAME_LEN, "%s", - superTblInfo->childTblName + (tableSeq - superTblInfo->childTblOffset) * TSDB_TABLE_NAME_LEN); + superTblInfo->childTblName + + (tableSeq - superTblInfo->childTblOffset) * TSDB_TABLE_NAME_LEN); } else { verbosePrint("[%d] %s() LN%d: from=%d count=%d seq=%d\n", @@ -4369,7 +4413,7 @@ static void getTableName(char *pTblName, threadInfo* pThreadInfo, int tableSeq) static int generateDataTail(char *tableName, int32_t tableSeq, threadInfo* pThreadInfo, SSuperTable* superTblInfo, - int batch, char* buffer, int64_t insertRows, + int batch, char* buffer, int remainderBufLen, int64_t insertRows, int64_t startFrom, uint64_t startTime, int *pSamplePos, int *dataLen) { int len = 0; int ncols_per_record = 1; // count first col ts @@ -4386,80 +4430,80 @@ static int generateDataTail(char *tableName, int32_t tableSeq, int k = 0; for (k = 0; k < batch;) { - if (superTblInfo) { - int retLen = 0; + char data[MAX_DATA_SIZE]; + memset(data, 0, MAX_DATA_SIZE); - if (0 == strncasecmp(superTblInfo->dataSource, + int retLen = 0; + + if (superTblInfo) { + if (0 == strncasecmp(superTblInfo->dataSource, "sample", strlen("sample"))) { retLen = getRowDataFromSample( - buffer + len, - superTblInfo->maxSqlLen - len, + data, + remainderBufLen, startTime + superTblInfo->timeStampStep * k, superTblInfo, pSamplePos); - } else if (0 == strncasecmp(superTblInfo->dataSource, + } else if (0 == strncasecmp(superTblInfo->dataSource, "rand", strlen("rand"))) { - int rand_num = rand_tinyint() % 100; + int rand_num = taosRandom() % 100; if (0 != superTblInfo->disorderRatio && rand_num < superTblInfo->disorderRatio) { int64_t d = startTime + superTblInfo->timeStampStep * k - taosRandom() % superTblInfo->disorderRange; retLen = generateRowData( - buffer + len, - superTblInfo->maxSqlLen - len, + data, d, superTblInfo); } else { retLen = generateRowData( - buffer + len, - superTblInfo->maxSqlLen - len, + data, startTime + superTblInfo->timeStampStep * k, superTblInfo); - } - } + } + } - if (retLen < 0) { - return -1; - } + if (retLen > remainderBufLen) { + break; + } - len += retLen; - - if (len >= (superTblInfo->maxSqlLen - 256)) { // reserve for overwrite - k++; - break; - } + buffer += snprintf(buffer, retLen + 1, "%s", data); + k++; + len += retLen; + remainderBufLen -= retLen; } else { + char **data_type = g_args.datatype; + int lenOfBinary = g_args.len_of_binary; + int rand_num = taosRandom() % 100; - char data[MAX_DATA_SIZE]; - char **data_type = g_args.datatype; - int lenOfBinary = g_args.len_of_binary; - if ((g_args.disorderRatio != 0) - && (rand_num < g_args.disorderRange)) { + && (rand_num < g_args.disorderRatio)) { - int64_t d = startTime + DEFAULT_TIMESTAMP_STEP * k - - taosRandom() % 1000000 + rand_num; - len = generateData(data, data_type, + int64_t d = startTime + DEFAULT_TIMESTAMP_STEP * k + - taosRandom() % g_args.disorderRange; + + retLen = generateData(data, data_type, ncols_per_record, d, lenOfBinary); } else { - len = generateData(data, data_type, + retLen = generateData(data, data_type, ncols_per_record, startTime + DEFAULT_TIMESTAMP_STEP * k, lenOfBinary); } + if (len > remainderBufLen) + break; + buffer += sprintf(buffer, " %s", data); - if (strlen(buffer) >= (g_args.max_sql_len - 256)) { // too long - k++; - break; - } + k++; + len += retLen; + remainderBufLen -= retLen; } verbosePrint("%s() LN%d len=%d k=%d \nbuffer=%s\n", __func__, __LINE__, len, k, buffer); - k++; startFrom ++; if (startFrom >= insertRows) { @@ -4479,7 +4523,7 @@ static int generateSQLHead(char *tableName, int32_t tableSeq, if (AUTO_CREATE_SUBTBL == superTblInfo->autoCreateTable) { char* tagsValBuf = NULL; if (0 == superTblInfo->tagSource) { - tagsValBuf = generateTagVaulesForStb(superTblInfo); + tagsValBuf = generateTagVaulesForStb(superTblInfo, tableSeq); } else { tagsValBuf = getTagValueFromTagSample( superTblInfo, @@ -4543,20 +4587,25 @@ static int generateProgressiveDataBuffer(char *pTblName, assert(buffer != NULL); - memset(buffer, 0, superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len); + int maxSqlLen = superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len; + int remainderBufLen = maxSqlLen; + + memset(buffer, 0, maxSqlLen); char *pstr = buffer; int headLen = generateSQLHead(pTblName, tableSeq, pThreadInfo, superTblInfo, buffer); pstr += headLen; + remainderBufLen -= headLen; int k; int dataLen; k = generateDataTail(pTblName, tableSeq, pThreadInfo, superTblInfo, - g_args.num_of_RPR, pstr, insertRows, startFrom, + g_args.num_of_RPR, pstr, remainderBufLen, insertRows, startFrom, startTime, pSamplePos, &dataLen); + return k; } @@ -4586,12 +4635,13 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { return NULL; } - char tableName[TSDB_TABLE_NAME_LEN]; pThreadInfo->totalInsertRows = 0; pThreadInfo->totalAffectedRows = 0; + int nTimeStampStep = superTblInfo?superTblInfo->timeStampStep:DEFAULT_TIMESTAMP_STEP; + int64_t insertRows = (superTblInfo)?superTblInfo->insertRows:g_args.num_of_DPT; int insert_interval = superTblInfo?superTblInfo->insertInterval:g_args.insert_interval; @@ -4629,19 +4679,30 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { int generatedRecPerTbl = 0; bool flagSleep = true; int sleepTimeTotal = 0; + + int maxSqlLen = superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len; + int remainderBufLen; + while(pThreadInfo->totalInsertRows < pThreadInfo->ntables * insertRows) { if ((flagSleep) && (insert_interval)) { st = taosGetTimestampUs(); flagSleep = false; } // generate data - memset(buffer, 0, superTblInfo?superTblInfo->maxSqlLen:g_args.max_sql_len); + memset(buffer, 0, maxSqlLen); + remainderBufLen = maxSqlLen; char *pstr = buffer; int recOfBatch = 0; for (int i = 0; i < batchPerTblTimes; i ++) { getTableName(tableName, pThreadInfo, tableSeq); + if (0 == strlen(tableName)) { + errorPrint("[%d] %s() LN%d, getTableName return null\n", + pThreadInfo->threadID, __func__, __LINE__); + free(buffer); + return NULL; + } int headLen; if (i == 0) { @@ -4658,6 +4719,8 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { pThreadInfo->threadID, __func__, __LINE__, i, buffer); pstr += headLen; + remainderBufLen -= headLen; + int dataLen = 0; verbosePrint("[%d] %s() LN%d i=%d batchPerTblTimes=%d batchPerTbl = %d\n", @@ -4671,15 +4734,22 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { } else { startTime = 1500000000000; } - generateDataTail( + int generated = generateDataTail( tableName, tableSeq, pThreadInfo, superTblInfo, - batchPerTbl, pstr, insertRows, 0, + batchPerTbl, pstr, remainderBufLen, insertRows, 0, startTime, &(pThreadInfo->samplePos), &dataLen); + if (generated < 0) { + debugPrint("[%d] %s() LN%d, generated data is %d\n", + pThreadInfo->threadID, __func__, __LINE__, generated); + goto free_and_statistics_interlace; + } pstr += dataLen; + remainderBufLen -= dataLen; + recOfBatch += batchPerTbl; - startTime += batchPerTbl * superTblInfo->timeStampStep; +// startTime += batchPerTbl * superTblInfo->timeStampStep; pThreadInfo->totalInsertRows += batchPerTbl; verbosePrint("[%d] %s() LN%d batchPerTbl=%d recOfBatch=%d\n", pThreadInfo->threadID, __func__, __LINE__, @@ -4689,9 +4759,12 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { if (insertMode == INTERLACE_INSERT_MODE) { if (tableSeq == pThreadInfo->start_table_from + pThreadInfo->ntables) { // turn to first table - startTime += batchPerTbl * superTblInfo->timeStampStep; tableSeq = pThreadInfo->start_table_from; generatedRecPerTbl += batchPerTbl; + + startTime = pThreadInfo->start_time + + generatedRecPerTbl * nTimeStampStep; + flagSleep = true; if (generatedRecPerTbl >= insertRows) break; @@ -4769,9 +4842,9 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { free_and_statistics_interlace: tmfree(buffer); - printf("====thread[%d] completed total inserted rows: %"PRId64 ", total affected rows: %"PRId64 "====\n", - pThreadInfo->threadID, - pThreadInfo->totalInsertRows, + printf("====thread[%d] completed total inserted rows: %"PRId64 ", total affected rows: %"PRId64 "====\n", + pThreadInfo->threadID, + pThreadInfo->totalInsertRows, pThreadInfo->totalAffectedRows); return NULL; } @@ -4902,16 +4975,16 @@ static void* syncWriteProgressive(threadInfo *pThreadInfo) { free_and_statistics_2: tmfree(buffer); - printf("====thread[%d] completed total inserted rows: %"PRId64 ", total affected rows: %"PRId64 "====\n", - pThreadInfo->threadID, - pThreadInfo->totalInsertRows, + printf("====thread[%d] completed total inserted rows: %"PRId64 ", total affected rows: %"PRId64 "====\n", + pThreadInfo->threadID, + pThreadInfo->totalInsertRows, pThreadInfo->totalAffectedRows); return NULL; } static void* syncWrite(void *sarg) { - threadInfo *winfo = (threadInfo *)sarg; + threadInfo *winfo = (threadInfo *)sarg; SSuperTable* superTblInfo = winfo->superTblInfo; int interlaceRows = superTblInfo?superTblInfo->interlaceRows:g_args.interlace_rows; @@ -4926,7 +4999,7 @@ static void* syncWrite(void *sarg) { } static void callBack(void *param, TAOS_RES *res, int code) { - threadInfo* winfo = (threadInfo*)param; + threadInfo* winfo = (threadInfo*)param; SSuperTable* superTblInfo = winfo->superTblInfo; int insert_interval = @@ -4939,7 +5012,7 @@ static void callBack(void *param, TAOS_RES *res, int code) { } char *buffer = calloc(1, winfo->superTblInfo->maxSqlLen); - char *data = calloc(1, MAX_DATA_SIZE); + char data[MAX_DATA_SIZE]; char *pstr = buffer; pstr += sprintf(pstr, "insert into %s.%s%d values", winfo->db_name, winfo->tb_prefix, winfo->start_table_from); @@ -4951,7 +5024,6 @@ static void callBack(void *param, TAOS_RES *res, int code) { if (winfo->start_table_from > winfo->end_table_to) { tsem_post(&winfo->lock_sem); free(buffer); - free(data); taos_free_result(res); return; } @@ -4960,12 +5032,10 @@ static void callBack(void *param, TAOS_RES *res, int code) { int rand_num = taosRandom() % 100; if (0 != winfo->superTblInfo->disorderRatio && rand_num < winfo->superTblInfo->disorderRatio) { - int64_t d = winfo->lastTs - taosRandom() % 1000000 + rand_num; - //generateData(data, datatype, ncols_per_record, d, len_of_binary); - generateRowData(data, MAX_DATA_SIZE, d, winfo->superTblInfo); + int64_t d = winfo->lastTs - taosRandom() % winfo->superTblInfo->disorderRange; + generateRowData(data, d, winfo->superTblInfo); } else { - //generateData(data, datatype, ncols_per_record, start_time += 1000, len_of_binary); - generateRowData(data, MAX_DATA_SIZE, winfo->lastTs += 1000, winfo->superTblInfo); + generateRowData(data, winfo->lastTs += 1000, winfo->superTblInfo); } pstr += sprintf(pstr, "%s", data); winfo->counter++; @@ -4980,7 +5050,6 @@ static void callBack(void *param, TAOS_RES *res, int code) { } taos_query_a(winfo->taos, buffer, callBack, winfo); free(buffer); - free(data); taos_free_result(res); } @@ -5055,7 +5124,7 @@ static void startMultiThreadInsertData(int threads, char* db_name, start_time = 1500000000000; } - double start = getCurrentTime(); + double start = getCurrentTimeUs(); // read sample data from file first if ((superTblInfo) && (0 == strncasecmp(superTblInfo->dataSource, @@ -5092,16 +5161,14 @@ static void startMultiThreadInsertData(int threads, char* db_name, if (superTblInfo) { int limit, offset; - if (superTblInfo->childTblOffset >= superTblInfo->childTblCount) { - printf("WARNING: specified offset >= child table count! \n"); - if (!g_args.answer_yes) { - printf(" Press enter key to continue or Ctrl-C to stop\n\n"); - (void)getchar(); - } + if ((superTblInfo->childTblExists == TBL_NO_EXISTS) && + ((superTblInfo->childTblOffset != 0) || (superTblInfo->childTblLimit != 0))) { + printf("WARNING: offset and limit will not be used since the child tables are not exists!\n"); } - if (superTblInfo->childTblOffset >= 0) { - if (superTblInfo->childTblLimit <= 0) { + if ((superTblInfo->childTblExists == TBL_ALREADY_EXISTS) + && (superTblInfo->childTblOffset >= 0)) { + if (superTblInfo->childTblLimit < 0) { superTblInfo->childTblLimit = superTblInfo->childTblCount - superTblInfo->childTblOffset; } @@ -5109,13 +5176,32 @@ static void startMultiThreadInsertData(int threads, char* db_name, offset = superTblInfo->childTblOffset; limit = superTblInfo->childTblLimit; } else { - limit = superTblInfo->childTblCount; - offset = 0; + limit = superTblInfo->childTblCount; + offset = 0; } ntables = limit; startFrom = offset; + if ((superTblInfo->childTblExists != TBL_NO_EXISTS) + && ((superTblInfo->childTblOffset + superTblInfo->childTblLimit ) + > superTblInfo->childTblCount)) { + printf("WARNING: specified offset + limit > child table count!\n"); + if (!g_args.answer_yes) { + printf(" Press enter key to continue or Ctrl-C to stop\n\n"); + (void)getchar(); + } + } + + if ((superTblInfo->childTblExists != TBL_NO_EXISTS) + && (0 == superTblInfo->childTblLimit)) { + printf("WARNING: specified limit = 0, which cannot find table name to insert or query! \n"); + if (!g_args.answer_yes) { + printf(" Press enter key to continue or Ctrl-C to stop\n\n"); + (void)getchar(); + } + } + superTblInfo->childTblName = (char*)calloc(1, limit * TSDB_TABLE_NAME_LEN); if (superTblInfo->childTblName == NULL) { @@ -5233,7 +5319,7 @@ static void startMultiThreadInsertData(int threads, char* db_name, if (cntDelay == 0) cntDelay = 1; avgDelay = (double)totalDelay / cntDelay; - double end = getCurrentTime(); + double end = getCurrentTimeUs(); double t = end - start; if (superTblInfo) { @@ -5312,7 +5398,7 @@ static void *readTable(void *sarg) { sprintf(command, "select %s from %s%d where ts>= %" PRId64, aggreFunc[j], tb_prefix, i, sTime); - double t = getCurrentTime(); + double t = getCurrentTimeUs(); TAOS_RES *pSql = taos_query(taos, command); int32_t code = taos_errno(pSql); @@ -5328,7 +5414,7 @@ static void *readTable(void *sarg) { count++; } - t = getCurrentTime() - t; + t = getCurrentTimeUs() - t; totalT += t; taos_free_result(pSql); @@ -5346,7 +5432,7 @@ static void *readTable(void *sarg) { } static void *readMetric(void *sarg) { -#if 1 +#if 1 threadInfo *rinfo = (threadInfo *)sarg; TAOS *taos = rinfo->taos; char command[BUFFER_SIZE] = "\0"; @@ -5387,7 +5473,7 @@ static void *readMetric(void *sarg) { printf("Where condition: %s\n", condition); fprintf(fp, "%s\n", command); - double t = getCurrentTime(); + double t = getCurrentTimeUs(); TAOS_RES *pSql = taos_query(taos, command); int32_t code = taos_errno(pSql); @@ -5403,7 +5489,7 @@ static void *readMetric(void *sarg) { while (taos_fetch_row(pSql) != NULL) { count++; } - t = getCurrentTime() - t; + t = getCurrentTimeUs() - t; fprintf(fp, "| Speed: %12.2f(per s) | Latency: %.4f(ms) |\n", num_of_tables * num_of_DPT / t, t * 1000); @@ -5445,7 +5531,7 @@ static int insertTestProcess() { init_rand_data(); // create database and super tables - if(createDatabases() != 0) { + if(createDatabasesAndStables() != 0) { fclose(g_fpOfInsertResult); return -1; } @@ -5457,9 +5543,9 @@ static int insertTestProcess() { double end; // create child tables - start = getCurrentTime(); + start = getCurrentTimeUs(); createChildTables(); - end = getCurrentTime(); + end = getCurrentTimeUs(); if (g_totalChildTables > 0) { printf("Spent %.4f seconds to create %d tables with %d thread(s)\n\n", @@ -5471,20 +5557,23 @@ static int insertTestProcess() { taosMsleep(1000); // create sub threads for inserting data - //start = getCurrentTime(); + //start = getCurrentTimeUs(); for (int i = 0; i < g_Dbs.dbCount; i++) { - if (g_Dbs.db[i].superTblCount > 0) { - for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { - SSuperTable* superTblInfo = &g_Dbs.db[i].superTbls[j]; - if (0 == g_Dbs.db[i].superTbls[j].insertRows) { - continue; - } - startMultiThreadInsertData( - g_Dbs.threadCount, - g_Dbs.db[i].dbName, - g_Dbs.db[i].dbCfg.precision, - superTblInfo); + if (g_Dbs.use_metric) { + if (g_Dbs.db[i].superTblCount > 0) { + for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { + + SSuperTable* superTblInfo = &g_Dbs.db[i].superTbls[j]; + + if (superTblInfo && (superTblInfo->insertRows > 0)) { + startMultiThreadInsertData( + g_Dbs.threadCount, + g_Dbs.db[i].dbName, + g_Dbs.db[i].dbCfg.precision, + superTblInfo); + } } + } } else { startMultiThreadInsertData( g_Dbs.threadCount, @@ -5493,11 +5582,11 @@ static int insertTestProcess() { NULL); } } - //end = getCurrentTime(); + //end = getCurrentTimeUs(); //int64_t totalInsertRows = 0; //int64_t totalAffectedRows = 0; - //for (int i = 0; i < g_Dbs.dbCount; i++) { + //for (int i = 0; i < g_Dbs.dbCount; i++) { // for (int j = 0; j < g_Dbs.db[i].superTblCount; j++) { // totalInsertRows+= g_Dbs.db[i].superTbls[j].totalInsertRows; // totalAffectedRows += g_Dbs.db[i].superTbls[j].totalAffectedRows; @@ -5848,6 +5937,7 @@ static TAOS_SUB* subscribeImpl(TAOS *taos, char *sql, char* topic, char* resultF static void *subSubscribeProcess(void *sarg) { threadInfo *winfo = (threadInfo *)sarg; char subSqlstr[1024]; + TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT] = {0}; if (winfo->taos == NULL) { TAOS * taos = NULL; @@ -5893,9 +5983,8 @@ static void *subSubscribeProcess(void *sarg) { sprintf(tmpFile, "%s-%d", g_queryInfo.superQueryInfo.result[i], winfo->threadID); } - g_queryInfo.superQueryInfo.tsub[i] = subscribeImpl( - winfo->taos, subSqlstr, topic, tmpFile); - if (NULL == g_queryInfo.superQueryInfo.tsub[i]) { + tsub[i] = subscribeImpl(winfo->taos, subSqlstr, topic, tmpFile); + if (NULL == tsub[i]) { taos_close(winfo->taos); return NULL; } @@ -5912,7 +6001,7 @@ static void *subSubscribeProcess(void *sarg) { continue; } - res = taos_consume(g_queryInfo.superQueryInfo.tsub[i]); + res = taos_consume(tsub[i]); if (res) { char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; if (g_queryInfo.superQueryInfo.result[i][0] != 0) { @@ -5927,8 +6016,7 @@ static void *subSubscribeProcess(void *sarg) { taos_free_result(res); for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { - taos_unsubscribe(g_queryInfo.superQueryInfo.tsub[i], - g_queryInfo.superQueryInfo.subscribeKeepProgress); + taos_unsubscribe(tsub[i], g_queryInfo.superQueryInfo.subscribeKeepProgress); } taos_close(winfo->taos); @@ -5937,6 +6025,7 @@ static void *subSubscribeProcess(void *sarg) { static void *superSubscribeProcess(void *sarg) { threadInfo *winfo = (threadInfo *)sarg; + TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT] = {0}; if (winfo->taos == NULL) { TAOS * taos = NULL; @@ -5979,10 +6068,7 @@ static void *superSubscribeProcess(void *sarg) { sprintf(tmpFile, "%s-%d", g_queryInfo.specifiedQueryInfo.result[i], winfo->threadID); } - g_queryInfo.specifiedQueryInfo.tsub[i] = - subscribeImpl(winfo->taos, - g_queryInfo.specifiedQueryInfo.sql[i], - topic, tmpFile); + tsub[i] = subscribeImpl(winfo->taos, g_queryInfo.specifiedQueryInfo.sql[i], topic, tmpFile); if (NULL == g_queryInfo.specifiedQueryInfo.tsub[i]) { taos_close(winfo->taos); return NULL; @@ -6000,7 +6086,7 @@ static void *superSubscribeProcess(void *sarg) { continue; } - res = taos_consume(g_queryInfo.specifiedQueryInfo.tsub[i]); + res = taos_consume(tsub[i]); if (res) { char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; if (g_queryInfo.specifiedQueryInfo.result[i][0] != 0) { @@ -6014,8 +6100,7 @@ static void *superSubscribeProcess(void *sarg) { taos_free_result(res); for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) { - taos_unsubscribe(g_queryInfo.specifiedQueryInfo.tsub[i], - g_queryInfo.specifiedQueryInfo.subscribeKeepProgress); + taos_unsubscribe(tsub[i], g_queryInfo.specifiedQueryInfo.subscribeKeepProgress); } taos_close(winfo->taos); @@ -6086,7 +6171,7 @@ static int subscribeTestProcess() { && (g_queryInfo.superQueryInfo.threadCnt > 0)) { pidsOfSub = malloc(g_queryInfo.superQueryInfo.threadCnt * sizeof(pthread_t)); - infosOfSub = malloc(g_queryInfo.superQueryInfo.threadCnt * + infosOfSub = malloc(g_queryInfo.superQueryInfo.threadCnt * sizeof(threadInfo)); if ((NULL == pidsOfSub) || (NULL == infosOfSub)) { errorPrint("%s() LN%d, malloc failed for create threads\n", @@ -6233,7 +6318,7 @@ static void setParaFromArg(){ g_Dbs.db[0].superTbls[0].timeStampStep = DEFAULT_TIMESTAMP_STEP; g_Dbs.db[0].superTbls[0].insertRows = g_args.num_of_DPT; - g_Dbs.db[0].superTbls[0].maxSqlLen = TSDB_PAYLOAD_SIZE; + g_Dbs.db[0].superTbls[0].maxSqlLen = g_args.max_sql_len; g_Dbs.db[0].superTbls[0].columnCount = 0; for (int i = 0; i < MAX_NUM_DATATYPE; i++) { @@ -6318,7 +6403,7 @@ static void querySqlFile(TAOS* taos, char* sqlFile) char * line = NULL; size_t line_len = 0; - double t = getCurrentTime(); + double t = getCurrentTimeUs(); while ((read_len = tgetline(&line, &line_len, fp)) != -1) { if (read_len >= MAX_SQL_SIZE) continue; @@ -6349,7 +6434,7 @@ static void querySqlFile(TAOS* taos, char* sqlFile) cmd_len = 0; } - t = getCurrentTime() - t; + t = getCurrentTimeUs() - t; printf("run %s took %.6f second(s)\n\n", sqlFile, t); tmfree(cmd); @@ -6433,6 +6518,16 @@ static void queryResult() { static void testCmdLine() { + if (strlen(configDir)) { + wordexp_t full_path; + if (wordexp(configDir, &full_path, 0) != 0) { + errorPrint( "Invalid path %s\n", configDir); + return; + } + taos_options(TSDB_OPTION_CONFIGDIR, full_path.we_wordv[0]); + wordfree(&full_path); + } + g_args.test_mode = INSERT_TEST; insertTestProcess(); diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index 3cee5f1b1d..9f176904fe 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -39,6 +39,22 @@ typedef struct { int8_t type; } SOColInfo; +#define debugPrint(fmt, ...) \ + do { if (g_args.debug_print || g_args.verbose_print) \ + fprintf(stderr, "DEBG: "fmt, __VA_ARGS__); } while(0) + +#define verbosePrint(fmt, ...) \ + do { if (g_args.verbose_print) \ + fprintf(stderr, "VERB: "fmt, __VA_ARGS__); } while(0) + +#define performancePrint(fmt, ...) \ + do { if (g_args.performance_print) \ + fprintf(stderr, "VERB: "fmt, __VA_ARGS__); } while(0) + +#define errorPrint(fmt, ...) \ + do { fprintf(stderr, "ERROR: "fmt, __VA_ARGS__); } while(0) + + // -------------------------- SHOW DATABASE INTERFACE----------------------- enum _show_db_index { TSDB_SHOW_DB_NAME_INDEX, @@ -46,7 +62,7 @@ enum _show_db_index { TSDB_SHOW_DB_NTABLES_INDEX, TSDB_SHOW_DB_VGROUPS_INDEX, TSDB_SHOW_DB_REPLICA_INDEX, - TSDB_SHOW_DB_QUORUM_INDEX, + TSDB_SHOW_DB_QUORUM_INDEX, TSDB_SHOW_DB_DAYS_INDEX, TSDB_SHOW_DB_KEEP_INDEX, TSDB_SHOW_DB_CACHE_INDEX, @@ -101,10 +117,10 @@ typedef struct { char name[TSDB_DB_NAME_LEN + 1]; char create_time[32]; int32_t ntables; - int32_t vgroups; + int32_t vgroups; int16_t replica; int16_t quorum; - int16_t days; + int16_t days; char keeplist[32]; //int16_t daysToKeep; //int16_t daysToKeep1; @@ -172,48 +188,50 @@ static char args_doc[] = "dbname [tbname ...]\n--databases dbname ...\n--all-dat /* The options we understand. */ static struct argp_option options[] = { // connection option - {"host", 'h', "HOST", 0, "Server host dumping data from. Default is localhost.", 0}, - {"user", 'u', "USER", 0, "User name used to connect to server. Default is root.", 0}, + {"host", 'h', "HOST", 0, "Server host dumping data from. Default is localhost.", 0}, + {"user", 'u', "USER", 0, "User name used to connect to server. Default is root.", 0}, #ifdef _TD_POWER_ - {"password", 'p', "PASSWORD", 0, "User password to connect to server. Default is powerdb.", 0}, + {"password", 'p', "PASSWORD", 0, "User password to connect to server. Default is powerdb.", 0}, #else - {"password", 'p', "PASSWORD", 0, "User password to connect to server. Default is taosdata.", 0}, + {"password", 'p', "PASSWORD", 0, "User password to connect to server. Default is taosdata.", 0}, #endif - {"port", 'P', "PORT", 0, "Port to connect", 0}, - {"cversion", 'v', "CVERION", 0, "client version", 0}, - {"mysqlFlag", 'q', "MYSQLFLAG", 0, "mysqlFlag, Default is 0", 0}, + {"port", 'P', "PORT", 0, "Port to connect", 0}, + {"cversion", 'v', "CVERION", 0, "client version", 0}, + {"mysqlFlag", 'q', "MYSQLFLAG", 0, "mysqlFlag, Default is 0", 0}, // input/output file - {"outpath", 'o', "OUTPATH", 0, "Output file path.", 1}, - {"inpath", 'i', "INPATH", 0, "Input file path.", 1}, - {"resultFile", 'r', "RESULTFILE", 0, "DumpOut/In Result file path and name.", 1}, + {"outpath", 'o', "OUTPATH", 0, "Output file path.", 1}, + {"inpath", 'i', "INPATH", 0, "Input file path.", 1}, + {"resultFile", 'r', "RESULTFILE", 0, "DumpOut/In Result file path and name.", 1}, #ifdef _TD_POWER_ - {"config", 'c', "CONFIG_DIR", 0, "Configure directory. Default is /etc/power/taos.cfg.", 1}, + {"config", 'c', "CONFIG_DIR", 0, "Configure directory. Default is /etc/power/taos.cfg.", 1}, #else - {"config", 'c', "CONFIG_DIR", 0, "Configure directory. Default is /etc/taos/taos.cfg.", 1}, + {"config", 'c', "CONFIG_DIR", 0, "Configure directory. Default is /etc/taos/taos.cfg.", 1}, #endif - {"encode", 'e', "ENCODE", 0, "Input file encoding.", 1}, + {"encode", 'e', "ENCODE", 0, "Input file encoding.", 1}, // dump unit options - {"all-databases", 'A', 0, 0, "Dump all databases.", 2}, - {"databases", 'B', 0, 0, "Dump assigned databases", 2}, + {"all-databases", 'A', 0, 0, "Dump all databases.", 2}, + {"databases", 'B', 0, 0, "Dump assigned databases", 2}, // dump format options - {"schemaonly", 's', 0, 0, "Only dump schema.", 3}, - {"with-property", 'M', 0, 0, "Dump schema with properties.", 3}, - {"start-time", 'S', "START_TIME", 0, "Start time to dump.", 3}, - {"end-time", 'E', "END_TIME", 0, "End time to dump.", 3}, - {"data-batch", 'N', "DATA_BATCH", 0, "Number of data point per insert statement. Default is 1.", 3}, - {"max-sql-len", 'L', "SQL_LEN", 0, "Max length of one sql. Default is 65480.", 3}, - {"table-batch", 't', "TABLE_BATCH", 0, "Number of table dumpout into one output file. Default is 1.", 3}, - {"thread_num", 'T', "THREAD_NUM", 0, "Number of thread for dump in file. Default is 5.", 3}, - {"allow-sys", 'a', 0, 0, "Allow to dump sys database", 3}, + {"schemaonly", 's', 0, 0, "Only dump schema.", 3}, + {"with-property", 'M', 0, 0, "Dump schema with properties.", 3}, + {"start-time", 'S', "START_TIME", 0, "Start time to dump.", 3}, + {"end-time", 'E', "END_TIME", 0, "End time to dump. Epoch or ISO8601/RFC3339 format is acceptable. For example: 2017-10-01T18:00:00+0800", 3}, + {"data-batch", 'N', "DATA_BATCH", 0, "Number of data point per insert statement. Default is 1.", 3}, + {"max-sql-len", 'L', "SQL_LEN", 0, "Max length of one sql. Default is 65480.", 3}, + {"table-batch", 't', "TABLE_BATCH", 0, "Number of table dumpout into one output file. Default is 1.", 3}, + {"thread_num", 'T', "THREAD_NUM", 0, "Number of thread for dump in file. Default is 5.", 3}, + {"allow-sys", 'a', 0, 0, "Allow to dump sys database", 3}, + {"debug", 'g', 0, 0, "Print debug info.", 1}, + {"verbose", 'v', 0, 0, "Print verbose debug info.", 1}, {0}}; /* Used by main to communicate with parse_opt. */ -struct arguments { +typedef struct arguments { // connection option char *host; char *user; char *password; - uint16_t port; + uint16_t port; char cversion[12]; uint16_t mysqlFlag; // output file @@ -238,9 +256,12 @@ struct arguments { int32_t thread_num; int abort; char **arg_list; - int arg_list_len; - bool isDumpIn; -}; + int arg_list_len; + bool isDumpIn; + bool debug_print; + bool verbose_print; + bool performance_print; +} SArguments; /* Parse a single option. */ static error_t parse_opt(int key, char *arg, struct argp_state *state) { @@ -286,6 +307,9 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { tstrncpy(arguments->outpath, full_path.we_wordv[0], TSDB_FILENAME_LEN); wordfree(&full_path); break; + case 'g': + arguments->debug_print = true; + break; case 'i': arguments->isDumpIn = true; if (wordexp(arg, &full_path, 0) != 0) { @@ -387,7 +411,7 @@ int taosCheckParam(struct arguments *arguments); void taosFreeDbInfos(); static void taosStartDumpOutWorkThreads(void* taosCon, struct arguments* args, int32_t numOfThread, char *dbName); -struct arguments tsArguments = { +struct arguments g_args = { // connection option NULL, "root", @@ -400,18 +424,18 @@ struct arguments tsArguments = { "", 0, // outpath and inpath - "", + "", "", "./dump_result.txt", NULL, // dump unit option - false, + false, false, // dump format option - false, - false, - 0, - INT64_MAX, + false, + false, + 0, + INT64_MAX, 1, TSDB_MAX_SQL_LEN, 1, @@ -419,11 +443,14 @@ struct arguments tsArguments = { // other options 5, 0, - NULL, - 0, - false + NULL, + 0, + false, + false, // debug_print + false, // verbose_print + false // performance_print }; - + static int queryDbImpl(TAOS *taos, char *command) { int i; TAOS_RES *res = NULL; @@ -434,7 +461,7 @@ static int queryDbImpl(TAOS *taos, char *command) { taos_free_result(res); res = NULL; } - + res = taos_query(taos, command); code = taos_errno(res); if (0 == code) { @@ -453,13 +480,44 @@ static int queryDbImpl(TAOS *taos, char *command) { return 0; } +static void parse_args(int argc, char *argv[], SArguments *arguments) { + for (int i = 1; i < argc; i++) { + if (strcmp(argv[i], "-E") == 0) { + if (argv[i+1]) { + char *tmp = argv[++i]; + int64_t tmpEpoch; + if (strchr(tmp, ':') && strchr(tmp, '-')) { + if (TSDB_CODE_SUCCESS != taosParseTime( + tmp, &tmpEpoch, strlen(tmp), TSDB_TIME_PRECISION_MILLI, 0)) { + fprintf(stderr, "Input end time error!\n"); + return; + } + } else { + tmpEpoch = atoll(tmp); + } + + sprintf(argv[i], "%"PRId64"", tmpEpoch); + debugPrint("%s() LN%d, tmp is: %s, argv[%d]: %s\n", + __func__, __LINE__, tmp, i, argv[i]); + } else { + fprintf(stderr, "Input end time error!\n"); + return; + } + } else if (strcmp(argv[i], "-g") == 0) { + arguments->debug_print = true; + } + } +} + int main(int argc, char *argv[]) { /* Parse our arguments; every option seen by parse_opt will be reflected in arguments. */ - argp_parse(&argp, argc, argv, 0, 0, &tsArguments); + parse_args(argc, argv, &g_args); - if (tsArguments.abort) { + argp_parse(&argp, argc, argv, 0, 0, &g_args); + + if (g_args.abort) { #ifndef _ALPINE error(10, 0, "ABORTED"); #else @@ -469,81 +527,82 @@ int main(int argc, char *argv[]) { printf("====== arguments config ======\n"); { - printf("host: %s\n", tsArguments.host); - printf("user: %s\n", tsArguments.user); - printf("password: %s\n", tsArguments.password); - printf("port: %u\n", tsArguments.port); - printf("cversion: %s\n", tsArguments.cversion); - printf("mysqlFlag: %d\n", tsArguments.mysqlFlag); - printf("outpath: %s\n", tsArguments.outpath); - printf("inpath: %s\n", tsArguments.inpath); - printf("resultFile: %s\n", tsArguments.resultFile); - printf("encode: %s\n", tsArguments.encode); - printf("all_databases: %d\n", tsArguments.all_databases); - printf("databases: %d\n", tsArguments.databases); - printf("schemaonly: %d\n", tsArguments.schemaonly); - printf("with_property: %d\n", tsArguments.with_property); - printf("start_time: %" PRId64 "\n", tsArguments.start_time); - printf("end_time: %" PRId64 "\n", tsArguments.end_time); - printf("data_batch: %d\n", tsArguments.data_batch); - printf("max_sql_len: %d\n", tsArguments.max_sql_len); - printf("table_batch: %d\n", tsArguments.table_batch); - printf("thread_num: %d\n", tsArguments.thread_num); - printf("allow_sys: %d\n", tsArguments.allow_sys); - printf("abort: %d\n", tsArguments.abort); - printf("isDumpIn: %d\n", tsArguments.isDumpIn); - printf("arg_list_len: %d\n", tsArguments.arg_list_len); + printf("host: %s\n", g_args.host); + printf("user: %s\n", g_args.user); + printf("password: %s\n", g_args.password); + printf("port: %u\n", g_args.port); + printf("cversion: %s\n", g_args.cversion); + printf("mysqlFlag: %d\n", g_args.mysqlFlag); + printf("outpath: %s\n", g_args.outpath); + printf("inpath: %s\n", g_args.inpath); + printf("resultFile: %s\n", g_args.resultFile); + printf("encode: %s\n", g_args.encode); + printf("all_databases: %d\n", g_args.all_databases); + printf("databases: %d\n", g_args.databases); + printf("schemaonly: %d\n", g_args.schemaonly); + printf("with_property: %d\n", g_args.with_property); + printf("start_time: %" PRId64 "\n", g_args.start_time); + printf("end_time: %" PRId64 "\n", g_args.end_time); + printf("data_batch: %d\n", g_args.data_batch); + printf("max_sql_len: %d\n", g_args.max_sql_len); + printf("table_batch: %d\n", g_args.table_batch); + printf("thread_num: %d\n", g_args.thread_num); + printf("allow_sys: %d\n", g_args.allow_sys); + printf("abort: %d\n", g_args.abort); + printf("isDumpIn: %d\n", g_args.isDumpIn); + printf("arg_list_len: %d\n", g_args.arg_list_len); + printf("debug_print: %d\n", g_args.debug_print); - for (int32_t i = 0; i < tsArguments.arg_list_len; i++) { - printf("arg_list[%d]: %s\n", i, tsArguments.arg_list[i]); + for (int32_t i = 0; i < g_args.arg_list_len; i++) { + printf("arg_list[%d]: %s\n", i, g_args.arg_list[i]); } - } + } printf("==============================\n"); - if (tsArguments.cversion[0] != 0){ - tstrncpy(version, tsArguments.cversion, 11); + if (g_args.cversion[0] != 0){ + tstrncpy(version, g_args.cversion, 11); } - if (taosCheckParam(&tsArguments) < 0) { + if (taosCheckParam(&g_args) < 0) { exit(EXIT_FAILURE); } - - g_fpOfResult = fopen(tsArguments.resultFile, "a"); + + g_fpOfResult = fopen(g_args.resultFile, "a"); if (NULL == g_fpOfResult) { - fprintf(stderr, "Failed to open %s for save result\n", tsArguments.resultFile); + fprintf(stderr, "Failed to open %s for save result\n", g_args.resultFile); return 1; }; fprintf(g_fpOfResult, "#############################################################################\n"); fprintf(g_fpOfResult, "============================== arguments config =============================\n"); { - fprintf(g_fpOfResult, "host: %s\n", tsArguments.host); - fprintf(g_fpOfResult, "user: %s\n", tsArguments.user); - fprintf(g_fpOfResult, "password: %s\n", tsArguments.password); - fprintf(g_fpOfResult, "port: %u\n", tsArguments.port); - fprintf(g_fpOfResult, "cversion: %s\n", tsArguments.cversion); - fprintf(g_fpOfResult, "mysqlFlag: %d\n", tsArguments.mysqlFlag); - fprintf(g_fpOfResult, "outpath: %s\n", tsArguments.outpath); - fprintf(g_fpOfResult, "inpath: %s\n", tsArguments.inpath); - fprintf(g_fpOfResult, "resultFile: %s\n", tsArguments.resultFile); - fprintf(g_fpOfResult, "encode: %s\n", tsArguments.encode); - fprintf(g_fpOfResult, "all_databases: %d\n", tsArguments.all_databases); - fprintf(g_fpOfResult, "databases: %d\n", tsArguments.databases); - fprintf(g_fpOfResult, "schemaonly: %d\n", tsArguments.schemaonly); - fprintf(g_fpOfResult, "with_property: %d\n", tsArguments.with_property); - fprintf(g_fpOfResult, "start_time: %" PRId64 "\n", tsArguments.start_time); - fprintf(g_fpOfResult, "end_time: %" PRId64 "\n", tsArguments.end_time); - fprintf(g_fpOfResult, "data_batch: %d\n", tsArguments.data_batch); - fprintf(g_fpOfResult, "max_sql_len: %d\n", tsArguments.max_sql_len); - fprintf(g_fpOfResult, "table_batch: %d\n", tsArguments.table_batch); - fprintf(g_fpOfResult, "thread_num: %d\n", tsArguments.thread_num); - fprintf(g_fpOfResult, "allow_sys: %d\n", tsArguments.allow_sys); - fprintf(g_fpOfResult, "abort: %d\n", tsArguments.abort); - fprintf(g_fpOfResult, "isDumpIn: %d\n", tsArguments.isDumpIn); - fprintf(g_fpOfResult, "arg_list_len: %d\n", tsArguments.arg_list_len); + fprintf(g_fpOfResult, "host: %s\n", g_args.host); + fprintf(g_fpOfResult, "user: %s\n", g_args.user); + fprintf(g_fpOfResult, "password: %s\n", g_args.password); + fprintf(g_fpOfResult, "port: %u\n", g_args.port); + fprintf(g_fpOfResult, "cversion: %s\n", g_args.cversion); + fprintf(g_fpOfResult, "mysqlFlag: %d\n", g_args.mysqlFlag); + fprintf(g_fpOfResult, "outpath: %s\n", g_args.outpath); + fprintf(g_fpOfResult, "inpath: %s\n", g_args.inpath); + fprintf(g_fpOfResult, "resultFile: %s\n", g_args.resultFile); + fprintf(g_fpOfResult, "encode: %s\n", g_args.encode); + fprintf(g_fpOfResult, "all_databases: %d\n", g_args.all_databases); + fprintf(g_fpOfResult, "databases: %d\n", g_args.databases); + fprintf(g_fpOfResult, "schemaonly: %d\n", g_args.schemaonly); + fprintf(g_fpOfResult, "with_property: %d\n", g_args.with_property); + fprintf(g_fpOfResult, "start_time: %" PRId64 "\n", g_args.start_time); + fprintf(g_fpOfResult, "end_time: %" PRId64 "\n", g_args.end_time); + fprintf(g_fpOfResult, "data_batch: %d\n", g_args.data_batch); + fprintf(g_fpOfResult, "max_sql_len: %d\n", g_args.max_sql_len); + fprintf(g_fpOfResult, "table_batch: %d\n", g_args.table_batch); + fprintf(g_fpOfResult, "thread_num: %d\n", g_args.thread_num); + fprintf(g_fpOfResult, "allow_sys: %d\n", g_args.allow_sys); + fprintf(g_fpOfResult, "abort: %d\n", g_args.abort); + fprintf(g_fpOfResult, "isDumpIn: %d\n", g_args.isDumpIn); + fprintf(g_fpOfResult, "arg_list_len: %d\n", g_args.arg_list_len); - for (int32_t i = 0; i < tsArguments.arg_list_len; i++) { - fprintf(g_fpOfResult, "arg_list[%d]: %s\n", i, tsArguments.arg_list[i]); + for (int32_t i = 0; i < g_args.arg_list_len; i++) { + fprintf(g_fpOfResult, "arg_list[%d]: %s\n", i, g_args.arg_list[i]); } } @@ -552,11 +611,11 @@ int main(int argc, char *argv[]) { time_t tTime = time(NULL); struct tm tm = *localtime(&tTime); - if (tsArguments.isDumpIn) { + if (g_args.isDumpIn) { fprintf(g_fpOfResult, "============================== DUMP IN ============================== \n"); fprintf(g_fpOfResult, "# DumpIn start time: %d-%02d-%02d %02d:%02d:%02d\n", tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); - if (taosDumpIn(&tsArguments) < 0) { + if (taosDumpIn(&g_args) < 0) { fprintf(g_fpOfResult, "\n"); fclose(g_fpOfResult); return -1; @@ -565,7 +624,7 @@ int main(int argc, char *argv[]) { fprintf(g_fpOfResult, "============================== DUMP OUT ============================== \n"); fprintf(g_fpOfResult, "# DumpOut start time: %d-%02d-%02d %02d:%02d:%02d\n", tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); - if (taosDumpOut(&tsArguments) < 0) { + if (taosDumpOut(&g_args) < 0) { fprintf(g_fpOfResult, "\n"); fclose(g_fpOfResult); return -1; @@ -573,9 +632,9 @@ int main(int argc, char *argv[]) { fprintf(g_fpOfResult, "\n============================== TOTAL STATISTICS ============================== \n"); fprintf(g_fpOfResult, "# total database count: %d\n", g_resultStatistics.totalDatabasesOfDumpOut); - fprintf(g_fpOfResult, "# total super table count: %d\n", g_resultStatistics.totalSuperTblsOfDumpOut); - fprintf(g_fpOfResult, "# total child table count: %"PRId64"\n", g_resultStatistics.totalChildTblsOfDumpOut); - fprintf(g_fpOfResult, "# total row count: %"PRId64"\n", g_resultStatistics.totalRowsOfDumpOut); + fprintf(g_fpOfResult, "# total super table count: %d\n", g_resultStatistics.totalSuperTblsOfDumpOut); + fprintf(g_fpOfResult, "# total child table count: %"PRId64"\n", g_resultStatistics.totalChildTblsOfDumpOut); + fprintf(g_fpOfResult, "# total row count: %"PRId64"\n", g_resultStatistics.totalRowsOfDumpOut); } fprintf(g_fpOfResult, "\n"); @@ -1236,8 +1295,8 @@ void* taosDumpOutWorkThreadFp(void *arg) FILE *fp = NULL; memset(tmpBuf, 0, TSDB_FILENAME_LEN + 128); - if (tsArguments.outpath[0] != 0) { - sprintf(tmpBuf, "%s/%s.tables.%d.sql", tsArguments.outpath, pThread->dbName, pThread->threadIndex); + if (g_args.outpath[0] != 0) { + sprintf(tmpBuf, "%s/%s.tables.%d.sql", g_args.outpath, pThread->dbName, pThread->threadIndex); } else { sprintf(tmpBuf, "%s.tables.%d.sql", pThread->dbName, pThread->threadIndex); } @@ -1270,7 +1329,7 @@ void* taosDumpOutWorkThreadFp(void *arg) ssize_t readLen = read(fd, &tableRecord, sizeof(STableRecord)); if (readLen <= 0) break; - int ret = taosDumpTable(tableRecord.name, tableRecord.metric, &tsArguments, fp, pThread->taosCon, pThread->dbName); + int ret = taosDumpTable(tableRecord.name, tableRecord.metric, &g_args, fp, pThread->taosCon, pThread->dbName); if (ret >= 0) { // TODO: sum table count and table rows by self pThread->tablesOfDumpOut++; @@ -1282,13 +1341,13 @@ void* taosDumpOutWorkThreadFp(void *arg) } tablesInOneFile++; - if (tablesInOneFile >= tsArguments.table_batch) { + if (tablesInOneFile >= g_args.table_batch) { fclose(fp); tablesInOneFile = 0; memset(tmpBuf, 0, TSDB_FILENAME_LEN + 128); - if (tsArguments.outpath[0] != 0) { - sprintf(tmpBuf, "%s/%s.tables.%d-%d.sql", tsArguments.outpath, pThread->dbName, pThread->threadIndex, fileNameIndex); + if (g_args.outpath[0] != 0) { + sprintf(tmpBuf, "%s/%s.tables.%d-%d.sql", g_args.outpath, pThread->dbName, pThread->threadIndex, fileNameIndex); } else { sprintf(tmpBuf, "%s.tables.%d-%d.sql", pThread->dbName, pThread->threadIndex, fileNameIndex); } @@ -1491,14 +1550,14 @@ int taosDumpDb(SDbInfo *dbInfo, struct arguments *arguments, FILE *fp, TAOS *tao taos_free_result(res); lseek(fd, 0, SEEK_SET); - int maxThreads = tsArguments.thread_num; + int maxThreads = g_args.thread_num; int tableOfPerFile ; - if (numOfTable <= tsArguments.thread_num) { + if (numOfTable <= g_args.thread_num) { tableOfPerFile = 1; maxThreads = numOfTable; } else { - tableOfPerFile = numOfTable / tsArguments.thread_num; - if (0 != numOfTable % tsArguments.thread_num) { + tableOfPerFile = numOfTable / g_args.thread_num; + if (0 != numOfTable % g_args.thread_num) { tableOfPerFile += 1; } } @@ -1806,9 +1865,9 @@ int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS* //} } - fprintf(fp, "\n"); + fprintf(fp, "\n"); atomic_add_fetch_64(&totalDumpOutRows, totalRows); - + taos_free_result(tmpResult); free(tmpBuffer); return totalRows; @@ -1824,7 +1883,7 @@ int taosCheckParam(struct arguments *arguments) { fprintf(stderr, "start time is larger than end time\n"); return -1; } - + if (arguments->arg_list_len == 0) { if ((!arguments->all_databases) && (!arguments->isDumpIn)) { fprintf(stderr, "taosdump requires parameters\n"); @@ -2214,7 +2273,7 @@ void* taosDumpInWorkThreadFp(void *arg) continue; } fprintf(stderr, "Success Open input file: %s\n", SQLFileName); - taosDumpInOneFile(pThread->taosCon, fp, tsfCharset, tsArguments.encode, SQLFileName); + taosDumpInOneFile(pThread->taosCon, fp, tsfCharset, g_args.encode, SQLFileName); } } diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index 304096f3ae..80473ba5ae 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -628,6 +628,11 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) { bnNotify(); } + if (!tsEnableBalance) { + int32_t numOfMnodes = mnodeGetMnodesNum(); + if (numOfMnodes < tsNumOfMnodes) bnNotify(); + } + if (openVnodes != pDnode->openVnodes) { mnodeCheckUnCreatedVgroup(pDnode, pStatus->load, openVnodes); } @@ -722,6 +727,10 @@ int32_t mnodeDropDnode(SDnodeObj *pDnode, void *pMsg) { static int32_t mnodeDropDnodeByEp(char *ep, SMnodeMsg *pMsg) { SDnodeObj *pDnode = mnodeGetDnodeByEp(ep); if (pDnode == NULL) { + if (strspn(ep, "0123456789 ;") != strlen(ep)) { + return TSDB_CODE_MND_DNODE_NOT_EXIST; + } + int32_t dnodeId = (int32_t)strtol(ep, NULL, 10); pDnode = mnodeGetDnode(dnodeId); if (pDnode == NULL) { diff --git a/src/mnode/src/mnodeMnode.c b/src/mnode/src/mnodeMnode.c index 49473d3e06..ca6d6400ae 100644 --- a/src/mnode/src/mnodeMnode.c +++ b/src/mnode/src/mnodeMnode.c @@ -381,6 +381,8 @@ static bool mnodeAllOnline() { void *pIter = NULL; bool allOnline = true; + sdbUpdateMnodeRoles(); + while (1) { SMnodeObj *pMnode = NULL; pIter = mnodeGetNextMnode(pIter, &pMnode); diff --git a/src/os/inc/osArm32.h b/src/os/inc/osArm32.h index 56ed1e48c0..e7c307dd9d 100644 --- a/src/os/inc/osArm32.h +++ b/src/os/inc/osArm32.h @@ -78,6 +78,7 @@ extern "C" { #include #include #include +#include #define TAOS_OS_FUNC_LZ4 #define BUILDIN_CLZL(val) __builtin_clzll(val) diff --git a/src/os/inc/osArm64.h b/src/os/inc/osArm64.h index f6c14d69e2..494dba751a 100644 --- a/src/os/inc/osArm64.h +++ b/src/os/inc/osArm64.h @@ -79,6 +79,7 @@ extern "C" { #include #include #include +#include #ifdef __cplusplus } diff --git a/src/os/inc/osLinux32.h b/src/os/inc/osLinux32.h index b55c85db88..fb101afc09 100644 --- a/src/os/inc/osLinux32.h +++ b/src/os/inc/osLinux32.h @@ -78,6 +78,7 @@ extern "C" { #include #include #include +#include #define TAOS_OS_FUNC_LZ4 #define BUILDIN_CLZL(val) __builtin_clzll(val) diff --git a/src/os/inc/osLinux64.h b/src/os/inc/osLinux64.h index a5296689c7..4ceb266b81 100644 --- a/src/os/inc/osLinux64.h +++ b/src/os/inc/osLinux64.h @@ -75,6 +75,7 @@ extern "C" { #include #include #include +#include #ifndef _ALPINE #include #endif diff --git a/src/os/inc/osSocket.h b/src/os/inc/osSocket.h index 111fca712c..cf75b74a9a 100644 --- a/src/os/inc/osSocket.h +++ b/src/os/inc/osSocket.h @@ -68,6 +68,8 @@ void taosSetMaskSIGPIPE(); // TAOS_OS_FUNC_SOCKET_SETSOCKETOPT int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen); +int32_t taosGetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t* optlen); + // TAOS_OS_FUNC_SOCKET_INET uint32_t taosInetAddr(char *ipAddr); const char *taosInetNtoa(struct in_addr ipInt); diff --git a/src/os/src/detail/osSocket.c b/src/os/src/detail/osSocket.c index 1186a6dd0a..db976acccd 100644 --- a/src/os/src/detail/osSocket.c +++ b/src/os/src/detail/osSocket.c @@ -71,6 +71,9 @@ int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *op return setsockopt(socketfd, level, optname, optval, (socklen_t)optlen); } +int32_t taosGetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t* optlen) { + return getsockopt(socketfd, level, optname, optval, (socklen_t *)optlen); +} #endif #ifndef TAOS_OS_FUNC_SOCKET_INET diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index f955155445..a02df1a10e 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -87,7 +87,8 @@ typedef struct SResultRow { bool closed; // this result status: closed or opened uint32_t numOfRows; // number of rows of current time window SResultRowCellInfo* pCellInfo; // For each result column, there is a resultInfo - union {STimeWindow win; char* key;}; // start key of current result row + STimeWindow win; + char* key; // start key of current result row } SResultRow; typedef struct SGroupResInfo { diff --git a/src/query/inc/sql.y b/src/query/inc/sql.y index 7991a50255..d39c938854 100644 --- a/src/query/inc/sql.y +++ b/src/query/inc/sql.y @@ -679,6 +679,7 @@ expr(A) ::= STRING(X). { A = tSqlExprCreateIdValue(&X, TK_STRING);} expr(A) ::= NOW(X). { A = tSqlExprCreateIdValue(&X, TK_NOW); } expr(A) ::= VARIABLE(X). { A = tSqlExprCreateIdValue(&X, TK_VARIABLE);} expr(A) ::= BOOL(X). { A = tSqlExprCreateIdValue(&X, TK_BOOL);} +expr(A) ::= NULL(X). { A = tSqlExprCreateIdValue(&X, TK_NULL);} // ordinary functions: min(x), max(x), top(k, 20) expr(A) ::= ID(X) LP exprlist(Y) RP(E). { A = tSqlExprCreateFunction(Y, &X, &E, X.type); } diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 0e196cadeb..005cf5ac4b 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -194,6 +194,7 @@ static bool isPointInterpoQuery(SQuery *pQuery); static void setResultBufSize(SQuery* pQuery, SRspResultInfo* pResultInfo); static void setCtxTagForJoin(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, SExprInfo* pExprInfo, void* pTable); static void setParamForStableStddev(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr); +static void setParamForStableStddevByColData(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr, char* val, int16_t bytes); static void doSetTableGroupOutputBuf(SQueryRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo, SQLFunctionCtx* pCtx, int32_t* rowCellInfoOffset, int32_t numOfOutput, int32_t groupIndex); @@ -1363,6 +1364,7 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SGroupbyOperatorInfo *pIn SColumnInfoData* pColInfoData = taosArrayGet(pSDataBlock->pDataBlock, pInfo->colIndex); int16_t bytes = pColInfoData->info.bytes; int16_t type = pColInfoData->info.type; + SQuery *pQuery = pRuntimeEnv->pQuery; if (type == TSDB_DATA_TYPE_FLOAT || type == TSDB_DATA_TYPE_DOUBLE) { qError("QInfo:%"PRIu64" group by not supported on double/float columns, abort", GET_QID(pRuntimeEnv)); @@ -1383,6 +1385,10 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SGroupbyOperatorInfo *pIn memcpy(pInfo->prevData, val, bytes); + if (pQuery->stableQuery && pQuery->stabledev && (pRuntimeEnv->prevResult != NULL)) { + setParamForStableStddevByColData(pRuntimeEnv, pInfo->binfo.pCtx, pOperator->numOfOutput, pOperator->pExpr, val, bytes); + } + int32_t ret = setGroupResultOutputBuf(pRuntimeEnv, pInfo, pOperator->numOfOutput, val, type, bytes, item->groupIndex); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code @@ -1912,14 +1918,15 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { taosHashCleanup(pRuntimeEnv->pResultRowHashTable); pRuntimeEnv->pResultRowHashTable = NULL; - pRuntimeEnv->pool = destroyResultRowPool(pRuntimeEnv->pool); - taosArrayDestroyEx(pRuntimeEnv->prevResult, freeInterResult); - pRuntimeEnv->prevResult = NULL; - taosHashCleanup(pRuntimeEnv->pTableRetrieveTsMap); pRuntimeEnv->pTableRetrieveTsMap = NULL; destroyOperatorInfo(pRuntimeEnv->proot); + + pRuntimeEnv->pool = destroyResultRowPool(pRuntimeEnv->pool); + taosArrayDestroyEx(pRuntimeEnv->prevResult, freeInterResult); + pRuntimeEnv->prevResult = NULL; + } static bool needBuildResAfterQueryComplete(SQInfo* pQInfo) { @@ -2677,6 +2684,21 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, STableScanInfo* pTa tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->pQueryHandle, &pBlock->pBlockStatis); if (pQuery->topBotQuery && pBlock->pBlockStatis != NULL) { + { // set previous window + if (QUERY_IS_INTERVAL_QUERY(pQuery)) { + SResultRow* pResult = NULL; + + bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); + TSKEY k = ascQuery? pBlock->info.window.skey : pBlock->info.window.ekey; + + STimeWindow win = getActiveTimeWindow(pTableScanInfo->pResultRowInfo, k, pQuery); + if (setWindowOutputBufByKey(pRuntimeEnv, pTableScanInfo->pResultRowInfo, &win, masterScan, &pResult, groupId, + pTableScanInfo->pCtx, pTableScanInfo->numOfOutput, + pTableScanInfo->rowCellInfoOffset) != TSDB_CODE_SUCCESS) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + } + } bool load = false; for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t functionId = pTableScanInfo->pCtx[i].functionId; @@ -3487,6 +3509,42 @@ void setParamForStableStddev(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx } +void setParamForStableStddevByColData(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr, char* val, int16_t bytes) { + SQuery* pQuery = pRuntimeEnv->pQuery; + + int32_t numOfExprs = pQuery->numOfOutput; + for(int32_t i = 0; i < numOfExprs; ++i) { + SExprInfo* pExprInfo = &(pExpr[i]); + if (pExprInfo->base.functionId != TSDB_FUNC_STDDEV_DST) { + continue; + } + + SSqlFuncMsg* pFuncMsg = &pExprInfo->base; + + pCtx[i].param[0].arr = NULL; + pCtx[i].param[0].nType = TSDB_DATA_TYPE_INT; // avoid freeing the memory by setting the type to be int + + // TODO use hash to speedup this loop + int32_t numOfGroup = (int32_t)taosArrayGetSize(pRuntimeEnv->prevResult); + for (int32_t j = 0; j < numOfGroup; ++j) { + SInterResult* p = taosArrayGet(pRuntimeEnv->prevResult, j); + if (bytes == 0 || memcmp(p->tags, val, bytes) == 0) { + int32_t numOfCols = (int32_t)taosArrayGetSize(p->pResult); + for (int32_t k = 0; k < numOfCols; ++k) { + SStddevInterResult* pres = taosArrayGet(p->pResult, k); + if (pres->colId == pFuncMsg->colInfo.colId) { + pCtx[i].param[0].arr = pres->pResult; + break; + } + } + } + } + } + +} + + + /* * There are two cases to handle: * @@ -6672,6 +6730,9 @@ void freeQInfo(SQInfo *pQInfo) { SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; releaseQueryBuf(pRuntimeEnv->tableqinfoGroupInfo.numOfTables); + + doDestroyTableQueryInfo(&pRuntimeEnv->tableqinfoGroupInfo); + teardownQueryRuntimeEnv(&pQInfo->runtimeEnv); SQuery *pQuery = pQInfo->runtimeEnv.pQuery; @@ -6707,7 +6768,6 @@ void freeQInfo(SQInfo *pQInfo) { } } - doDestroyTableQueryInfo(&pRuntimeEnv->tableqinfoGroupInfo); tfree(pQInfo->pBuf); tfree(pQInfo->sql); diff --git a/src/query/src/qSqlParser.c b/src/query/src/qSqlParser.c index 6211116194..438aa89c46 100644 --- a/src/query/src/qSqlParser.c +++ b/src/query/src/qSqlParser.c @@ -127,7 +127,12 @@ tSqlExpr *tSqlExprCreateIdValue(SStrToken *pToken, int32_t optrType) { pSqlExpr->token = *pToken; } - if (optrType == TK_INTEGER || optrType == TK_STRING || optrType == TK_FLOAT || optrType == TK_BOOL) { + if (optrType == TK_NULL) { + pToken->type = TSDB_DATA_TYPE_NULL; + tVariantCreate(&pSqlExpr->value, pToken); + pSqlExpr->tokenId = optrType; + pSqlExpr->type = SQL_NODE_VALUE; + } else if (optrType == TK_INTEGER || optrType == TK_STRING || optrType == TK_FLOAT || optrType == TK_BOOL) { toTSDBType(pToken->type); tVariantCreate(&pSqlExpr->value, pToken); @@ -356,7 +361,11 @@ void tSqlExprCompact(tSqlExpr** pExpr) { bool tSqlExprIsLeaf(tSqlExpr* pExpr) { return (pExpr->pRight == NULL && pExpr->pLeft == NULL) && - (pExpr->tokenId == 0 || pExpr->tokenId == TK_ID || (pExpr->tokenId >= TK_BOOL && pExpr->tokenId <= TK_NCHAR) || pExpr->tokenId == TK_SET); + (pExpr->tokenId == 0 || + (pExpr->tokenId == TK_ID) || + (pExpr->tokenId >= TK_BOOL && pExpr->tokenId <= TK_NCHAR) || + (pExpr->tokenId == TK_NULL) || + (pExpr->tokenId == TK_SET)); } bool tSqlExprIsParentOfLeaf(tSqlExpr* pExpr) { diff --git a/src/query/src/qTokenizer.c b/src/query/src/qTokenizer.c index d5d1cdc8d6..6bcf57b1a8 100644 --- a/src/query/src/qTokenizer.c +++ b/src/query/src/qTokenizer.c @@ -564,7 +564,7 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenId) { return 0; } -SStrToken tStrGetToken(char* str, int32_t* i, bool isPrevOptr, uint32_t numOfIgnoreToken, uint32_t* ignoreTokenTypes) { +SStrToken tStrGetToken(char* str, int32_t* i, bool isPrevOptr) { SStrToken t0 = {0}; // here we reach the end of sql string, null-terminated string @@ -589,7 +589,10 @@ SStrToken tStrGetToken(char* str, int32_t* i, bool isPrevOptr, uint32_t numOfIgn } t0.n = tSQLGetToken(&str[*i], &t0.type); + break; + // not support user specfied ignored symbol list +#if 0 bool ignore = false; for (uint32_t k = 0; k < numOfIgnoreToken; k++) { if (t0.type == ignoreTokenTypes[k]) { @@ -601,6 +604,7 @@ SStrToken tStrGetToken(char* str, int32_t* i, bool isPrevOptr, uint32_t numOfIgn if (!ignore) { break; } +#endif } if (t0.type == TK_SEMI) { diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index da7dbcd501..9b0046fda0 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -66,8 +66,8 @@ void cleanupResultRowInfo(SResultRowInfo *pResultRowInfo) { return; } - if (pResultRowInfo->type == TSDB_DATA_TYPE_BINARY || pResultRowInfo->type == TSDB_DATA_TYPE_NCHAR) { - for(int32_t i = 0; i < pResultRowInfo->size; ++i) { + for(int32_t i = 0; i < pResultRowInfo->size; ++i) { + if (pResultRowInfo->pResult[i]) { tfree(pResultRowInfo->pResult[i]->key); } } @@ -153,11 +153,8 @@ void clearResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResultRow, int16 pResultRow->offset = -1; pResultRow->closed = false; - if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - tfree(pResultRow->key); - } else { - pResultRow->win = TSWINDOW_INITIALIZER; - } + tfree(pResultRow->key); + pResultRow->win = TSWINDOW_INITIALIZER; } // TODO refactor: use macro diff --git a/src/query/src/queryMain.c b/src/query/src/queryMain.c index 06070ca6d1..7eb80e200d 100644 --- a/src/query/src/queryMain.c +++ b/src/query/src/queryMain.c @@ -375,6 +375,7 @@ int32_t qKillQuery(qinfo_t qinfo) { return TSDB_CODE_QRY_INVALID_QHANDLE; } + qDebug("QInfo:%"PRIu64" query killed", pQInfo->qId); setQueryKilled(pQInfo); // Wait for the query executing thread being stopped/ diff --git a/src/query/src/sql.c b/src/query/src/sql.c index 6f78272809..7dca72b937 100644 --- a/src/query/src/sql.c +++ b/src/query/src/sql.c @@ -138,17 +138,17 @@ typedef union { #define ParseCTX_STORE #define YYFALLBACK 1 #define YYNSTATE 327 -#define YYNRULE 270 -#define YYNRULE_WITH_ACTION 270 +#define YYNRULE 271 +#define YYNRULE_WITH_ACTION 271 #define YYNTOKEN 191 #define YY_MAX_SHIFT 326 -#define YY_MIN_SHIFTREDUCE 521 -#define YY_MAX_SHIFTREDUCE 790 -#define YY_ERROR_ACTION 791 -#define YY_ACCEPT_ACTION 792 -#define YY_NO_ACTION 793 -#define YY_MIN_REDUCE 794 -#define YY_MAX_REDUCE 1063 +#define YY_MIN_SHIFTREDUCE 522 +#define YY_MAX_SHIFTREDUCE 792 +#define YY_ERROR_ACTION 793 +#define YY_ACCEPT_ACTION 794 +#define YY_NO_ACTION 795 +#define YY_MIN_REDUCE 796 +#define YY_MAX_REDUCE 1066 /************* End control #defines *******************************************/ #define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) @@ -215,78 +215,77 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (696) +#define YY_ACTTAB_COUNT (688) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 965, 570, 188, 570, 209, 324, 930, 17, 71, 571, - /* 10 */ 84, 571, 1044, 49, 50, 147, 53, 54, 140, 186, + /* 0 */ 967, 571, 188, 571, 209, 324, 932, 17, 71, 572, + /* 10 */ 84, 572, 1047, 49, 50, 147, 53, 54, 140, 186, /* 20 */ 221, 43, 188, 52, 269, 57, 55, 59, 56, 192, - /* 30 */ 32, 216, 1045, 48, 47, 1041, 188, 46, 45, 44, - /* 40 */ 929, 927, 928, 29, 931, 215, 1045, 522, 523, 524, - /* 50 */ 525, 526, 527, 528, 529, 530, 531, 532, 533, 534, - /* 60 */ 535, 325, 962, 213, 238, 49, 50, 32, 53, 54, - /* 70 */ 147, 210, 221, 43, 941, 52, 269, 57, 55, 59, - /* 80 */ 56, 254, 997, 72, 264, 48, 47, 290, 944, 46, - /* 90 */ 45, 44, 49, 50, 147, 53, 54, 32, 290, 221, - /* 100 */ 43, 570, 52, 269, 57, 55, 59, 56, 224, 571, - /* 110 */ 32, 941, 48, 47, 76, 233, 46, 45, 44, 49, - /* 120 */ 51, 38, 53, 54, 854, 226, 221, 43, 570, 52, - /* 130 */ 269, 57, 55, 59, 56, 266, 571, 80, 225, 48, - /* 140 */ 47, 941, 83, 46, 45, 44, 50, 310, 53, 54, - /* 150 */ 944, 293, 221, 43, 941, 52, 269, 57, 55, 59, - /* 160 */ 56, 996, 737, 32, 228, 48, 47, 1040, 12, 46, - /* 170 */ 45, 44, 86, 23, 288, 319, 318, 287, 286, 285, - /* 180 */ 317, 284, 316, 315, 314, 283, 313, 312, 904, 944, - /* 190 */ 892, 893, 894, 895, 896, 897, 898, 899, 900, 901, - /* 200 */ 902, 903, 905, 906, 53, 54, 147, 940, 221, 43, - /* 210 */ 18, 52, 269, 57, 55, 59, 56, 232, 651, 87, - /* 220 */ 1039, 48, 47, 241, 230, 46, 45, 44, 196, 220, - /* 230 */ 750, 245, 244, 741, 198, 744, 25, 747, 46, 45, - /* 240 */ 44, 124, 123, 197, 220, 750, 300, 299, 741, 320, - /* 250 */ 744, 956, 747, 932, 32, 57, 55, 59, 56, 938, - /* 260 */ 1055, 217, 218, 48, 47, 268, 211, 46, 45, 44, - /* 270 */ 323, 322, 133, 81, 944, 76, 217, 218, 23, 690, - /* 280 */ 319, 318, 38, 1, 161, 317, 205, 316, 315, 314, - /* 290 */ 234, 313, 312, 297, 296, 294, 233, 231, 941, 912, - /* 300 */ 292, 227, 910, 911, 248, 853, 70, 913, 82, 915, - /* 310 */ 916, 914, 204, 917, 918, 841, 5, 163, 32, 233, - /* 320 */ 173, 73, 35, 162, 93, 98, 89, 97, 942, 270, - /* 330 */ 3, 174, 206, 109, 114, 792, 326, 58, 280, 103, - /* 340 */ 113, 850, 119, 122, 112, 687, 173, 111, 190, 749, - /* 350 */ 116, 24, 58, 675, 32, 310, 672, 943, 673, 298, - /* 360 */ 674, 694, 941, 842, 749, 748, 181, 177, 173, 718, - /* 370 */ 719, 1007, 179, 176, 128, 127, 126, 125, 250, 219, - /* 380 */ 748, 956, 33, 739, 235, 236, 48, 47, 63, 191, - /* 390 */ 46, 45, 44, 697, 28, 302, 249, 275, 941, 703, - /* 400 */ 252, 709, 710, 142, 770, 62, 20, 751, 19, 64, - /* 410 */ 743, 19, 746, 742, 66, 745, 661, 272, 663, 740, - /* 420 */ 33, 33, 62, 102, 101, 193, 274, 6, 187, 753, - /* 430 */ 85, 662, 69, 67, 650, 62, 194, 14, 13, 195, - /* 440 */ 108, 107, 201, 16, 15, 202, 679, 677, 680, 678, - /* 450 */ 200, 121, 120, 185, 138, 136, 1006, 199, 189, 222, - /* 460 */ 246, 139, 1003, 1002, 223, 301, 676, 964, 41, 972, - /* 470 */ 974, 141, 145, 957, 253, 137, 939, 989, 988, 157, - /* 480 */ 158, 937, 159, 255, 160, 702, 311, 855, 908, 110, - /* 490 */ 277, 954, 153, 278, 150, 148, 279, 149, 281, 282, - /* 500 */ 68, 65, 212, 39, 183, 36, 291, 849, 257, 1060, - /* 510 */ 262, 99, 60, 1059, 1057, 267, 164, 265, 295, 1054, - /* 520 */ 105, 1053, 1051, 165, 873, 37, 34, 40, 184, 838, - /* 530 */ 115, 836, 117, 118, 834, 833, 237, 175, 831, 830, - /* 540 */ 829, 828, 827, 826, 825, 263, 178, 261, 180, 822, - /* 550 */ 820, 818, 816, 814, 182, 259, 251, 74, 77, 256, - /* 560 */ 258, 990, 42, 303, 304, 305, 306, 307, 308, 207, - /* 570 */ 229, 309, 276, 321, 790, 240, 239, 789, 208, 94, - /* 580 */ 95, 203, 242, 243, 788, 776, 247, 775, 252, 8, - /* 590 */ 271, 75, 682, 832, 824, 167, 874, 170, 168, 166, - /* 600 */ 129, 172, 169, 171, 2, 130, 131, 823, 4, 132, - /* 610 */ 815, 78, 704, 143, 154, 155, 151, 152, 156, 707, - /* 620 */ 144, 214, 26, 920, 79, 260, 9, 711, 146, 7, - /* 630 */ 10, 752, 27, 11, 21, 273, 22, 754, 88, 86, - /* 640 */ 30, 90, 91, 31, 92, 614, 610, 608, 607, 606, - /* 650 */ 603, 33, 289, 96, 100, 574, 61, 104, 653, 652, - /* 660 */ 649, 598, 596, 588, 594, 590, 592, 586, 584, 617, - /* 670 */ 616, 106, 615, 613, 612, 611, 609, 605, 604, 62, - /* 680 */ 572, 794, 539, 537, 793, 793, 793, 134, 793, 793, - /* 690 */ 793, 793, 793, 793, 793, 135, + /* 30 */ 32, 216, 1048, 48, 47, 1044, 188, 46, 45, 44, + /* 40 */ 931, 929, 930, 29, 933, 215, 1048, 523, 524, 525, + /* 50 */ 526, 527, 528, 529, 530, 531, 532, 533, 534, 535, + /* 60 */ 536, 325, 964, 213, 238, 49, 50, 32, 53, 54, + /* 70 */ 147, 210, 221, 43, 943, 52, 269, 57, 55, 59, + /* 80 */ 56, 254, 999, 72, 264, 48, 47, 290, 946, 46, + /* 90 */ 45, 44, 49, 50, 290, 53, 54, 32, 83, 221, + /* 100 */ 43, 571, 52, 269, 57, 55, 59, 56, 224, 572, + /* 110 */ 32, 943, 48, 47, 76, 310, 46, 45, 44, 49, + /* 120 */ 51, 38, 53, 54, 12, 226, 221, 43, 86, 52, + /* 130 */ 269, 57, 55, 59, 56, 266, 228, 80, 225, 48, + /* 140 */ 47, 943, 320, 46, 45, 44, 50, 1043, 53, 54, + /* 150 */ 946, 293, 221, 43, 943, 52, 269, 57, 55, 59, + /* 160 */ 56, 946, 738, 233, 147, 48, 47, 946, 940, 46, + /* 170 */ 45, 44, 856, 23, 288, 319, 318, 287, 286, 285, + /* 180 */ 317, 284, 316, 315, 314, 283, 313, 312, 906, 32, + /* 190 */ 894, 895, 896, 897, 898, 899, 900, 901, 902, 903, + /* 200 */ 904, 905, 907, 908, 53, 54, 147, 1042, 221, 43, + /* 210 */ 227, 52, 269, 57, 55, 59, 56, 205, 32, 18, + /* 220 */ 958, 48, 47, 1, 161, 46, 45, 44, 220, 751, + /* 230 */ 294, 998, 742, 943, 745, 211, 748, 196, 220, 751, + /* 240 */ 206, 571, 742, 198, 745, 32, 748, 5, 163, 572, + /* 250 */ 124, 123, 197, 35, 162, 93, 98, 89, 97, 298, + /* 260 */ 217, 218, 943, 28, 268, 23, 275, 319, 318, 280, + /* 270 */ 217, 218, 317, 81, 316, 315, 314, 1058, 313, 312, + /* 280 */ 232, 691, 914, 843, 76, 912, 913, 32, 173, 942, + /* 290 */ 915, 38, 917, 918, 916, 82, 919, 920, 57, 55, + /* 300 */ 59, 56, 190, 87, 794, 326, 48, 47, 73, 852, + /* 310 */ 46, 45, 44, 248, 173, 70, 109, 114, 46, 45, + /* 320 */ 44, 204, 103, 113, 230, 119, 122, 112, 302, 719, + /* 330 */ 720, 943, 233, 116, 48, 47, 58, 934, 46, 45, + /* 340 */ 44, 855, 752, 111, 3, 174, 58, 219, 750, 25, + /* 350 */ 958, 310, 752, 234, 652, 250, 297, 296, 750, 33, + /* 360 */ 181, 177, 270, 695, 749, 249, 179, 176, 128, 127, + /* 370 */ 126, 125, 676, 698, 749, 673, 233, 674, 241, 675, + /* 380 */ 252, 191, 300, 299, 740, 944, 245, 244, 323, 322, + /* 390 */ 133, 844, 193, 688, 704, 6, 173, 231, 142, 24, + /* 400 */ 292, 710, 711, 235, 236, 62, 20, 63, 772, 753, + /* 410 */ 66, 187, 19, 19, 744, 743, 747, 746, 662, 272, + /* 420 */ 741, 664, 33, 33, 194, 62, 274, 663, 64, 67, + /* 430 */ 85, 62, 102, 101, 121, 120, 14, 13, 69, 680, + /* 440 */ 651, 681, 108, 107, 16, 15, 678, 945, 679, 138, + /* 450 */ 136, 195, 201, 202, 200, 185, 1009, 199, 189, 1008, + /* 460 */ 222, 1005, 755, 1004, 223, 301, 246, 41, 139, 966, + /* 470 */ 974, 976, 141, 959, 145, 253, 991, 990, 137, 941, + /* 480 */ 251, 158, 157, 939, 910, 677, 255, 110, 703, 159, + /* 490 */ 212, 303, 156, 154, 956, 151, 148, 160, 857, 277, + /* 500 */ 278, 279, 267, 68, 281, 257, 282, 39, 183, 36, + /* 510 */ 262, 65, 291, 851, 1063, 99, 1062, 149, 1060, 164, + /* 520 */ 295, 1057, 105, 60, 1056, 1054, 165, 875, 37, 34, + /* 530 */ 40, 184, 840, 115, 838, 117, 118, 836, 835, 237, + /* 540 */ 175, 265, 833, 263, 832, 831, 830, 829, 828, 827, + /* 550 */ 261, 178, 180, 824, 822, 820, 818, 816, 182, 259, + /* 560 */ 74, 77, 256, 258, 992, 42, 311, 304, 305, 306, + /* 570 */ 207, 308, 307, 309, 229, 276, 321, 792, 240, 208, + /* 580 */ 203, 94, 95, 791, 239, 243, 834, 242, 790, 778, + /* 590 */ 777, 247, 252, 271, 8, 129, 168, 130, 167, 876, + /* 600 */ 166, 170, 169, 172, 171, 2, 75, 826, 683, 131, + /* 610 */ 825, 4, 132, 78, 817, 705, 143, 155, 708, 152, + /* 620 */ 150, 79, 214, 153, 144, 712, 922, 260, 146, 9, + /* 630 */ 10, 88, 754, 7, 26, 27, 11, 21, 756, 22, + /* 640 */ 30, 273, 90, 92, 86, 91, 31, 615, 611, 609, + /* 650 */ 608, 607, 604, 575, 289, 96, 100, 33, 61, 654, + /* 660 */ 653, 650, 599, 597, 589, 595, 591, 593, 104, 106, + /* 670 */ 587, 585, 618, 617, 616, 614, 613, 612, 610, 606, + /* 680 */ 605, 62, 573, 540, 134, 135, 538, 796, }; static const YYCODETYPE yy_lookahead[] = { /* 0 */ 194, 1, 255, 1, 193, 194, 0, 255, 200, 9, @@ -298,67 +297,67 @@ static const YYCODETYPE yy_lookahead[] = { /* 60 */ 58, 59, 256, 214, 62, 13, 14, 194, 16, 17, /* 70 */ 194, 235, 20, 21, 238, 23, 24, 25, 26, 27, /* 80 */ 28, 257, 261, 83, 263, 33, 34, 81, 239, 37, - /* 90 */ 38, 39, 13, 14, 194, 16, 17, 194, 81, 20, + /* 90 */ 38, 39, 13, 14, 81, 16, 17, 194, 83, 20, /* 100 */ 21, 1, 23, 24, 25, 26, 27, 28, 235, 9, - /* 110 */ 194, 238, 33, 34, 109, 194, 37, 38, 39, 13, - /* 120 */ 14, 116, 16, 17, 203, 214, 20, 21, 1, 23, - /* 130 */ 24, 25, 26, 27, 28, 259, 9, 261, 235, 33, - /* 140 */ 34, 238, 83, 37, 38, 39, 14, 86, 16, 17, + /* 110 */ 194, 238, 33, 34, 109, 86, 37, 38, 39, 13, + /* 120 */ 14, 116, 16, 17, 109, 214, 20, 21, 113, 23, + /* 130 */ 24, 25, 26, 27, 28, 259, 214, 261, 235, 33, + /* 140 */ 34, 238, 214, 37, 38, 39, 14, 255, 16, 17, /* 150 */ 239, 235, 20, 21, 238, 23, 24, 25, 26, 27, - /* 160 */ 28, 261, 110, 194, 214, 33, 34, 255, 109, 37, - /* 170 */ 38, 39, 113, 93, 94, 95, 96, 97, 98, 99, - /* 180 */ 100, 101, 102, 103, 104, 105, 106, 107, 213, 239, + /* 160 */ 28, 239, 110, 194, 194, 33, 34, 239, 194, 37, + /* 170 */ 38, 39, 203, 93, 94, 95, 96, 97, 98, 99, + /* 180 */ 100, 101, 102, 103, 104, 105, 106, 107, 213, 194, /* 190 */ 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, - /* 200 */ 225, 226, 227, 228, 16, 17, 194, 238, 20, 21, - /* 210 */ 44, 23, 24, 25, 26, 27, 28, 68, 5, 200, - /* 220 */ 255, 33, 34, 139, 68, 37, 38, 39, 62, 1, - /* 230 */ 2, 147, 148, 5, 68, 7, 109, 9, 37, 38, - /* 240 */ 39, 75, 76, 77, 1, 2, 33, 34, 5, 214, - /* 250 */ 7, 237, 9, 234, 194, 25, 26, 27, 28, 194, - /* 260 */ 239, 33, 34, 33, 34, 37, 252, 37, 38, 39, - /* 270 */ 65, 66, 67, 261, 239, 109, 33, 34, 93, 37, - /* 280 */ 95, 96, 116, 201, 202, 100, 255, 102, 103, 104, - /* 290 */ 141, 106, 107, 144, 145, 235, 194, 141, 238, 213, - /* 300 */ 144, 236, 216, 217, 138, 203, 140, 221, 240, 223, - /* 310 */ 224, 225, 146, 227, 228, 199, 63, 64, 194, 194, - /* 320 */ 204, 253, 69, 70, 71, 72, 73, 74, 203, 15, - /* 330 */ 197, 198, 255, 63, 64, 191, 192, 109, 85, 69, - /* 340 */ 70, 199, 72, 73, 74, 114, 204, 78, 255, 121, - /* 350 */ 80, 120, 109, 2, 194, 86, 5, 239, 7, 235, - /* 360 */ 9, 119, 238, 199, 121, 137, 63, 64, 204, 128, - /* 370 */ 129, 230, 69, 70, 71, 72, 73, 74, 110, 61, - /* 380 */ 137, 237, 114, 1, 33, 34, 33, 34, 114, 255, - /* 390 */ 37, 38, 39, 110, 109, 235, 252, 112, 238, 110, - /* 400 */ 117, 110, 110, 114, 110, 114, 114, 110, 114, 135, - /* 410 */ 5, 114, 7, 5, 114, 7, 110, 110, 110, 37, - /* 420 */ 114, 114, 114, 142, 143, 255, 110, 109, 255, 115, - /* 430 */ 114, 110, 109, 133, 111, 114, 255, 142, 143, 255, - /* 440 */ 142, 143, 255, 142, 143, 255, 5, 5, 7, 7, - /* 450 */ 255, 78, 79, 255, 63, 64, 230, 255, 255, 230, - /* 460 */ 194, 194, 230, 230, 230, 230, 115, 194, 254, 194, - /* 470 */ 194, 194, 194, 237, 237, 61, 237, 262, 262, 241, - /* 480 */ 194, 194, 194, 258, 194, 121, 108, 194, 229, 92, - /* 490 */ 194, 251, 245, 194, 248, 250, 194, 249, 194, 194, - /* 500 */ 132, 134, 258, 194, 194, 194, 194, 194, 258, 194, - /* 510 */ 258, 194, 131, 194, 194, 126, 194, 130, 194, 194, - /* 520 */ 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, + /* 200 */ 225, 226, 227, 228, 16, 17, 194, 255, 20, 21, + /* 210 */ 236, 23, 24, 25, 26, 27, 28, 255, 194, 44, + /* 220 */ 237, 33, 34, 201, 202, 37, 38, 39, 1, 2, + /* 230 */ 235, 261, 5, 238, 7, 252, 9, 62, 1, 2, + /* 240 */ 255, 1, 5, 68, 7, 194, 9, 63, 64, 9, + /* 250 */ 75, 76, 77, 69, 70, 71, 72, 73, 74, 235, + /* 260 */ 33, 34, 238, 109, 37, 93, 112, 95, 96, 85, + /* 270 */ 33, 34, 100, 261, 102, 103, 104, 239, 106, 107, + /* 280 */ 68, 37, 213, 199, 109, 216, 217, 194, 204, 238, + /* 290 */ 221, 116, 223, 224, 225, 240, 227, 228, 25, 26, + /* 300 */ 27, 28, 255, 200, 191, 192, 33, 34, 253, 199, + /* 310 */ 37, 38, 39, 138, 204, 140, 63, 64, 37, 38, + /* 320 */ 39, 146, 69, 70, 68, 72, 73, 74, 235, 128, + /* 330 */ 129, 238, 194, 80, 33, 34, 109, 234, 37, 38, + /* 340 */ 39, 203, 115, 78, 197, 198, 109, 61, 121, 109, + /* 350 */ 237, 86, 115, 141, 5, 110, 144, 145, 121, 114, + /* 360 */ 63, 64, 15, 119, 137, 252, 69, 70, 71, 72, + /* 370 */ 73, 74, 2, 110, 137, 5, 194, 7, 139, 9, + /* 380 */ 117, 255, 33, 34, 1, 203, 147, 148, 65, 66, + /* 390 */ 67, 199, 255, 114, 110, 109, 204, 141, 114, 120, + /* 400 */ 144, 110, 110, 33, 34, 114, 114, 114, 110, 110, + /* 410 */ 114, 255, 114, 114, 5, 5, 7, 7, 110, 110, + /* 420 */ 37, 110, 114, 114, 255, 114, 110, 110, 135, 133, + /* 430 */ 114, 114, 142, 143, 78, 79, 142, 143, 109, 5, + /* 440 */ 111, 7, 142, 143, 142, 143, 5, 239, 7, 63, + /* 450 */ 64, 255, 255, 255, 255, 255, 230, 255, 255, 230, + /* 460 */ 230, 230, 115, 230, 230, 230, 194, 254, 194, 194, + /* 470 */ 194, 194, 194, 237, 194, 237, 262, 262, 61, 237, + /* 480 */ 195, 194, 241, 194, 229, 115, 258, 92, 121, 194, + /* 490 */ 258, 91, 242, 244, 251, 247, 250, 194, 194, 194, + /* 500 */ 194, 194, 126, 132, 194, 258, 194, 194, 194, 194, + /* 510 */ 258, 134, 194, 194, 194, 194, 194, 249, 194, 194, + /* 520 */ 194, 194, 194, 131, 194, 194, 194, 194, 194, 194, /* 530 */ 194, 194, 194, 194, 194, 194, 194, 194, 194, 194, - /* 540 */ 194, 194, 194, 194, 194, 125, 194, 124, 194, 194, - /* 550 */ 194, 194, 194, 194, 194, 123, 195, 195, 195, 122, - /* 560 */ 195, 195, 136, 91, 51, 88, 90, 55, 89, 195, - /* 570 */ 195, 87, 195, 81, 5, 5, 149, 5, 195, 200, - /* 580 */ 200, 195, 149, 5, 5, 95, 139, 94, 117, 109, - /* 590 */ 112, 118, 110, 195, 195, 210, 212, 207, 206, 211, - /* 600 */ 196, 205, 209, 208, 201, 196, 196, 195, 197, 196, - /* 610 */ 195, 114, 110, 109, 244, 243, 247, 246, 242, 110, - /* 620 */ 114, 1, 114, 229, 109, 109, 127, 110, 109, 109, - /* 630 */ 127, 110, 114, 109, 109, 112, 109, 115, 78, 113, - /* 640 */ 84, 83, 71, 84, 83, 9, 5, 5, 5, 5, - /* 650 */ 5, 114, 15, 78, 143, 82, 16, 143, 5, 5, - /* 660 */ 110, 5, 5, 5, 5, 5, 5, 5, 5, 5, - /* 670 */ 5, 143, 5, 5, 5, 5, 5, 5, 5, 114, - /* 680 */ 82, 0, 61, 60, 266, 266, 266, 21, 266, 266, - /* 690 */ 266, 266, 266, 266, 266, 21, 266, 266, 266, 266, + /* 540 */ 194, 130, 194, 125, 194, 194, 194, 194, 194, 194, + /* 550 */ 124, 194, 194, 194, 194, 194, 194, 194, 194, 123, + /* 560 */ 195, 195, 122, 195, 195, 136, 108, 51, 88, 90, + /* 570 */ 195, 89, 55, 87, 195, 195, 81, 5, 5, 195, + /* 580 */ 195, 200, 200, 5, 149, 5, 195, 149, 5, 95, + /* 590 */ 94, 139, 117, 112, 109, 196, 206, 196, 210, 212, + /* 600 */ 211, 207, 209, 205, 208, 201, 118, 195, 110, 196, + /* 610 */ 195, 197, 196, 114, 195, 110, 109, 243, 110, 246, + /* 620 */ 248, 109, 1, 245, 114, 110, 229, 109, 109, 127, + /* 630 */ 127, 78, 110, 109, 114, 114, 109, 109, 115, 109, + /* 640 */ 84, 112, 83, 83, 113, 71, 84, 9, 5, 5, + /* 650 */ 5, 5, 5, 82, 15, 78, 143, 114, 16, 5, + /* 660 */ 5, 110, 5, 5, 5, 5, 5, 5, 143, 143, + /* 670 */ 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + /* 680 */ 5, 114, 82, 61, 21, 21, 60, 0, 266, 266, + /* 690 */ 266, 266, 266, 266, 266, 266, 266, 266, 266, 266, /* 700 */ 266, 266, 266, 266, 266, 266, 266, 266, 266, 266, /* 710 */ 266, 266, 266, 266, 266, 266, 266, 266, 266, 266, /* 720 */ 266, 266, 266, 266, 266, 266, 266, 266, 266, 266, @@ -376,105 +375,104 @@ static const YYCODETYPE yy_lookahead[] = { /* 840 */ 266, 266, 266, 266, 266, 266, 266, 266, 266, 266, /* 850 */ 266, 266, 266, 266, 266, 266, 266, 266, 266, 266, /* 860 */ 266, 266, 266, 266, 266, 266, 266, 266, 266, 266, - /* 870 */ 266, 266, 266, 266, 266, 266, 266, 266, 266, 266, - /* 880 */ 266, 266, 266, 266, 266, 266, 266, + /* 870 */ 266, 266, 266, 266, 266, 266, 266, 266, 266, }; #define YY_SHIFT_COUNT (326) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (681) +#define YY_SHIFT_MAX (687) static const unsigned short int yy_shift_ofst[] = { - /* 0 */ 166, 80, 80, 185, 185, 17, 228, 243, 100, 100, - /* 10 */ 100, 100, 100, 100, 100, 100, 100, 0, 2, 243, - /* 20 */ 351, 351, 351, 351, 127, 5, 100, 100, 100, 6, - /* 30 */ 100, 100, 100, 100, 269, 17, 61, 61, 696, 696, - /* 40 */ 696, 243, 243, 243, 243, 243, 243, 243, 243, 243, - /* 50 */ 243, 243, 243, 243, 243, 243, 243, 243, 243, 243, - /* 60 */ 243, 351, 351, 213, 213, 213, 213, 213, 213, 213, - /* 70 */ 100, 100, 100, 242, 100, 5, 5, 100, 100, 100, - /* 80 */ 241, 241, 231, 5, 100, 100, 100, 100, 100, 100, + /* 0 */ 175, 80, 80, 172, 172, 13, 227, 237, 100, 100, + /* 10 */ 100, 100, 100, 100, 100, 100, 100, 0, 2, 237, + /* 20 */ 370, 370, 370, 370, 240, 5, 100, 100, 100, 6, + /* 30 */ 100, 100, 100, 100, 265, 13, 29, 29, 688, 688, + /* 40 */ 688, 237, 237, 237, 237, 237, 237, 237, 237, 237, + /* 50 */ 237, 237, 237, 237, 237, 237, 237, 237, 237, 237, + /* 60 */ 237, 370, 370, 349, 349, 349, 349, 349, 349, 349, + /* 70 */ 100, 100, 100, 244, 100, 5, 5, 100, 100, 100, + /* 80 */ 201, 201, 279, 5, 100, 100, 100, 100, 100, 100, /* 90 */ 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, /* 100 */ 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, /* 110 */ 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, /* 120 */ 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, - /* 130 */ 100, 100, 100, 100, 100, 100, 100, 100, 100, 414, - /* 140 */ 414, 414, 364, 364, 364, 414, 364, 414, 368, 367, - /* 150 */ 381, 389, 387, 420, 423, 432, 437, 426, 414, 414, - /* 160 */ 414, 378, 17, 17, 414, 414, 397, 472, 513, 477, - /* 170 */ 476, 512, 479, 484, 378, 414, 492, 492, 414, 492, - /* 180 */ 414, 492, 414, 696, 696, 52, 79, 106, 79, 79, - /* 190 */ 132, 188, 230, 230, 230, 230, 253, 270, 303, 353, - /* 200 */ 353, 353, 353, 149, 84, 201, 201, 59, 156, 205, - /* 210 */ 268, 283, 289, 291, 292, 294, 297, 405, 408, 382, - /* 220 */ 318, 314, 274, 300, 306, 307, 308, 316, 321, 285, - /* 230 */ 281, 295, 298, 323, 301, 441, 442, 373, 391, 569, - /* 240 */ 427, 570, 572, 433, 578, 579, 490, 493, 447, 471, - /* 250 */ 478, 480, 473, 482, 497, 502, 504, 509, 506, 515, - /* 260 */ 620, 516, 517, 519, 508, 499, 518, 503, 521, 520, - /* 270 */ 522, 524, 478, 525, 523, 527, 526, 560, 556, 558, - /* 280 */ 571, 559, 561, 636, 641, 642, 643, 644, 645, 573, - /* 290 */ 637, 575, 511, 537, 537, 640, 514, 528, 537, 653, - /* 300 */ 654, 550, 537, 656, 657, 658, 659, 660, 661, 662, - /* 310 */ 663, 664, 665, 667, 668, 669, 670, 671, 672, 673, - /* 320 */ 565, 598, 666, 674, 621, 623, 681, + /* 130 */ 100, 100, 100, 100, 100, 100, 100, 100, 100, 417, + /* 140 */ 417, 417, 367, 367, 367, 417, 367, 417, 371, 377, + /* 150 */ 392, 376, 411, 418, 426, 436, 440, 429, 417, 417, + /* 160 */ 417, 458, 13, 13, 417, 417, 395, 400, 516, 480, + /* 170 */ 479, 517, 482, 486, 458, 417, 495, 495, 417, 495, + /* 180 */ 417, 495, 417, 688, 688, 52, 79, 106, 79, 79, + /* 190 */ 132, 188, 273, 273, 273, 273, 184, 253, 297, 301, + /* 200 */ 301, 301, 301, 212, 239, 281, 281, 15, 256, 323, + /* 210 */ 245, 263, 284, 291, 292, 298, 299, 409, 410, 383, + /* 220 */ 286, 347, 293, 296, 308, 309, 311, 316, 317, 154, + /* 230 */ 290, 294, 300, 329, 302, 434, 441, 356, 386, 572, + /* 240 */ 435, 573, 578, 438, 580, 583, 494, 496, 452, 475, + /* 250 */ 481, 485, 488, 498, 499, 505, 507, 508, 510, 512, + /* 260 */ 621, 518, 515, 519, 520, 502, 521, 503, 522, 524, + /* 270 */ 523, 527, 481, 528, 529, 530, 531, 553, 556, 559, + /* 280 */ 574, 562, 560, 638, 643, 644, 645, 646, 647, 571, + /* 290 */ 639, 577, 513, 543, 543, 642, 525, 526, 543, 654, + /* 300 */ 655, 551, 543, 657, 658, 659, 660, 661, 662, 665, + /* 310 */ 666, 667, 668, 669, 670, 671, 672, 673, 674, 675, + /* 320 */ 567, 600, 663, 664, 622, 626, 687, }; #define YY_REDUCE_COUNT (184) #define YY_REDUCE_MIN (-253) -#define YY_REDUCE_MAX (415) +#define YY_REDUCE_MAX (419) static const short yy_reduce_ofst[] = { - /* 0 */ 144, -25, -25, 86, 86, -190, -233, -219, -164, -179, - /* 10 */ -124, -127, -97, -84, 60, 124, 160, -194, -189, -253, - /* 20 */ -151, -89, -50, 35, -176, 14, -100, 12, 65, 19, - /* 30 */ -79, 102, 125, -31, 116, -192, 142, 164, 68, 82, - /* 40 */ 133, -248, -236, -226, -220, -88, -35, 31, 77, 93, - /* 50 */ 134, 170, 173, 181, 184, 187, 190, 195, 198, 202, - /* 60 */ 203, 21, 118, 141, 226, 229, 232, 233, 234, 235, - /* 70 */ 266, 267, 273, 214, 275, 236, 237, 276, 277, 278, - /* 80 */ 215, 216, 238, 239, 286, 287, 288, 290, 293, 296, - /* 90 */ 299, 302, 304, 305, 309, 310, 311, 312, 313, 315, - /* 100 */ 317, 319, 320, 322, 324, 325, 326, 327, 328, 329, - /* 110 */ 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, - /* 120 */ 340, 341, 342, 343, 344, 345, 346, 347, 348, 349, - /* 130 */ 350, 352, 354, 355, 356, 357, 358, 359, 360, 361, - /* 140 */ 362, 363, 225, 244, 250, 365, 252, 366, 240, 245, - /* 150 */ 248, 246, 369, 371, 247, 370, 372, 376, 374, 375, - /* 160 */ 377, 259, 379, 380, 383, 386, 384, 388, 385, 392, - /* 170 */ 393, 390, 395, 396, 394, 398, 404, 409, 399, 410, - /* 180 */ 412, 413, 415, 403, 411, + /* 0 */ 113, -25, -25, 69, 69, -190, -233, -219, -164, -179, + /* 10 */ -124, -127, -97, -84, -5, 24, 93, -194, -189, -253, + /* 20 */ -151, -89, -78, -72, -176, -17, -30, 12, -26, 103, + /* 30 */ -31, 138, 182, 51, 84, -192, 110, 192, 55, 22, + /* 40 */ 147, -248, -236, -226, -220, -108, -48, -38, -15, 47, + /* 50 */ 126, 137, 156, 169, 196, 197, 198, 199, 200, 202, + /* 60 */ 203, 38, 208, 226, 229, 230, 231, 233, 234, 235, + /* 70 */ 272, 274, 275, 213, 276, 236, 238, 277, 278, 280, + /* 80 */ 214, 215, 241, 242, 287, 289, 295, 303, 304, 305, + /* 90 */ 306, 307, 310, 312, 313, 314, 315, 318, 319, 320, + /* 100 */ 321, 322, 324, 325, 326, 327, 328, 330, 331, 332, + /* 110 */ 333, 334, 335, 336, 337, 338, 339, 340, 341, 342, + /* 120 */ 343, 344, 345, 346, 348, 350, 351, 352, 353, 354, + /* 130 */ 355, 357, 358, 359, 360, 361, 362, 363, 364, 285, + /* 140 */ 365, 366, 228, 232, 247, 368, 252, 369, 243, 246, + /* 150 */ 268, 372, 248, 373, 378, 249, 374, 250, 375, 379, + /* 160 */ 380, 255, 381, 382, 384, 385, 387, 389, 388, 390, + /* 170 */ 393, 394, 396, 398, 397, 391, 399, 401, 412, 413, + /* 180 */ 415, 416, 419, 404, 414, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 791, 907, 851, 919, 839, 848, 1047, 1047, 791, 791, - /* 10 */ 791, 791, 791, 791, 791, 791, 791, 966, 811, 1047, - /* 20 */ 791, 791, 791, 791, 791, 791, 791, 791, 791, 848, - /* 30 */ 791, 791, 791, 791, 856, 848, 856, 856, 961, 891, - /* 40 */ 909, 791, 791, 791, 791, 791, 791, 791, 791, 791, - /* 50 */ 791, 791, 791, 791, 791, 791, 791, 791, 791, 791, - /* 60 */ 791, 791, 791, 791, 791, 791, 791, 791, 791, 791, - /* 70 */ 791, 791, 791, 968, 971, 791, 791, 973, 791, 791, - /* 80 */ 993, 993, 959, 791, 791, 791, 791, 791, 791, 791, - /* 90 */ 791, 791, 791, 791, 791, 791, 791, 791, 791, 791, - /* 100 */ 791, 791, 791, 791, 791, 791, 791, 791, 791, 791, - /* 110 */ 791, 791, 791, 791, 791, 837, 791, 835, 791, 791, - /* 120 */ 791, 791, 791, 791, 791, 791, 791, 791, 791, 791, - /* 130 */ 791, 791, 791, 821, 791, 791, 791, 791, 791, 813, - /* 140 */ 813, 813, 791, 791, 791, 813, 791, 813, 1000, 1004, - /* 150 */ 998, 986, 994, 985, 981, 979, 978, 1008, 813, 813, - /* 160 */ 813, 852, 848, 848, 813, 813, 872, 870, 868, 860, - /* 170 */ 866, 862, 864, 858, 840, 813, 846, 846, 813, 846, - /* 180 */ 813, 846, 813, 891, 909, 791, 1009, 791, 1046, 999, - /* 190 */ 1036, 1035, 1042, 1034, 1033, 1032, 791, 791, 791, 1028, - /* 200 */ 1029, 1031, 1030, 791, 791, 1038, 1037, 791, 791, 791, - /* 210 */ 791, 791, 791, 791, 791, 791, 791, 791, 791, 791, - /* 220 */ 1011, 791, 1005, 1001, 791, 791, 791, 791, 791, 791, - /* 230 */ 791, 791, 791, 921, 791, 791, 791, 791, 791, 791, - /* 240 */ 791, 791, 791, 791, 791, 791, 791, 791, 791, 958, - /* 250 */ 791, 791, 791, 791, 969, 791, 791, 791, 791, 791, - /* 260 */ 791, 791, 791, 791, 995, 791, 987, 791, 791, 791, - /* 270 */ 791, 791, 933, 791, 791, 791, 791, 791, 791, 791, - /* 280 */ 791, 791, 791, 791, 791, 791, 791, 791, 791, 791, - /* 290 */ 791, 791, 791, 1058, 1056, 791, 791, 791, 1052, 791, - /* 300 */ 791, 791, 1050, 791, 791, 791, 791, 791, 791, 791, - /* 310 */ 791, 791, 791, 791, 791, 791, 791, 791, 791, 791, - /* 320 */ 875, 791, 819, 817, 791, 809, 791, + /* 0 */ 793, 909, 853, 921, 841, 850, 1050, 1050, 793, 793, + /* 10 */ 793, 793, 793, 793, 793, 793, 793, 968, 813, 1050, + /* 20 */ 793, 793, 793, 793, 793, 793, 793, 793, 793, 850, + /* 30 */ 793, 793, 793, 793, 858, 850, 858, 858, 963, 893, + /* 40 */ 911, 793, 793, 793, 793, 793, 793, 793, 793, 793, + /* 50 */ 793, 793, 793, 793, 793, 793, 793, 793, 793, 793, + /* 60 */ 793, 793, 793, 793, 793, 793, 793, 793, 793, 793, + /* 70 */ 793, 793, 793, 970, 973, 793, 793, 975, 793, 793, + /* 80 */ 995, 995, 961, 793, 793, 793, 793, 793, 793, 793, + /* 90 */ 793, 793, 793, 793, 793, 793, 793, 793, 793, 793, + /* 100 */ 793, 793, 793, 793, 793, 793, 793, 793, 793, 793, + /* 110 */ 793, 793, 793, 793, 793, 839, 793, 837, 793, 793, + /* 120 */ 793, 793, 793, 793, 793, 793, 793, 793, 793, 793, + /* 130 */ 793, 793, 793, 823, 793, 793, 793, 793, 793, 815, + /* 140 */ 815, 815, 793, 793, 793, 815, 793, 815, 1002, 1006, + /* 150 */ 1000, 988, 996, 987, 983, 981, 980, 1010, 815, 815, + /* 160 */ 815, 854, 850, 850, 815, 815, 874, 872, 870, 862, + /* 170 */ 868, 864, 866, 860, 842, 815, 848, 848, 815, 848, + /* 180 */ 815, 848, 815, 893, 911, 793, 1011, 793, 1049, 1001, + /* 190 */ 1039, 1038, 1045, 1037, 1036, 1035, 793, 793, 793, 1031, + /* 200 */ 1032, 1034, 1033, 793, 793, 1041, 1040, 793, 793, 793, + /* 210 */ 793, 793, 793, 793, 793, 793, 793, 793, 793, 793, + /* 220 */ 1013, 793, 1007, 1003, 793, 793, 793, 793, 793, 793, + /* 230 */ 793, 793, 793, 923, 793, 793, 793, 793, 793, 793, + /* 240 */ 793, 793, 793, 793, 793, 793, 793, 793, 793, 960, + /* 250 */ 793, 793, 793, 793, 971, 793, 793, 793, 793, 793, + /* 260 */ 793, 793, 793, 793, 997, 793, 989, 793, 793, 793, + /* 270 */ 793, 793, 935, 793, 793, 793, 793, 793, 793, 793, + /* 280 */ 793, 793, 793, 793, 793, 793, 793, 793, 793, 793, + /* 290 */ 793, 793, 793, 1061, 1059, 793, 793, 793, 1055, 793, + /* 300 */ 793, 793, 1053, 793, 793, 793, 793, 793, 793, 793, + /* 310 */ 793, 793, 793, 793, 793, 793, 793, 793, 793, 793, + /* 320 */ 877, 793, 821, 819, 793, 811, 793, }; /********** End of lemon-generated parsing tables *****************************/ @@ -1275,46 +1273,47 @@ static const char *const yyRuleName[] = { /* 227 */ "expr ::= NOW", /* 228 */ "expr ::= VARIABLE", /* 229 */ "expr ::= BOOL", - /* 230 */ "expr ::= ID LP exprlist RP", - /* 231 */ "expr ::= ID LP STAR RP", - /* 232 */ "expr ::= expr IS NULL", - /* 233 */ "expr ::= expr IS NOT NULL", - /* 234 */ "expr ::= expr LT expr", - /* 235 */ "expr ::= expr GT expr", - /* 236 */ "expr ::= expr LE expr", - /* 237 */ "expr ::= expr GE expr", - /* 238 */ "expr ::= expr NE expr", - /* 239 */ "expr ::= expr EQ expr", - /* 240 */ "expr ::= expr BETWEEN expr AND expr", - /* 241 */ "expr ::= expr AND expr", - /* 242 */ "expr ::= expr OR expr", - /* 243 */ "expr ::= expr PLUS expr", - /* 244 */ "expr ::= expr MINUS expr", - /* 245 */ "expr ::= expr STAR expr", - /* 246 */ "expr ::= expr SLASH expr", - /* 247 */ "expr ::= expr REM expr", - /* 248 */ "expr ::= expr LIKE expr", - /* 249 */ "expr ::= expr IN LP exprlist RP", - /* 250 */ "exprlist ::= exprlist COMMA expritem", - /* 251 */ "exprlist ::= expritem", - /* 252 */ "expritem ::= expr", - /* 253 */ "expritem ::=", - /* 254 */ "cmd ::= RESET QUERY CACHE", - /* 255 */ "cmd ::= SYNCDB ids REPLICA", - /* 256 */ "cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist", - /* 257 */ "cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids", - /* 258 */ "cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist", - /* 259 */ "cmd ::= ALTER TABLE ids cpxName DROP TAG ids", - /* 260 */ "cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids", - /* 261 */ "cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem", - /* 262 */ "cmd ::= ALTER STABLE ids cpxName ADD COLUMN columnlist", - /* 263 */ "cmd ::= ALTER STABLE ids cpxName DROP COLUMN ids", - /* 264 */ "cmd ::= ALTER STABLE ids cpxName ADD TAG columnlist", - /* 265 */ "cmd ::= ALTER STABLE ids cpxName DROP TAG ids", - /* 266 */ "cmd ::= ALTER STABLE ids cpxName CHANGE TAG ids ids", - /* 267 */ "cmd ::= KILL CONNECTION INTEGER", - /* 268 */ "cmd ::= KILL STREAM INTEGER COLON INTEGER", - /* 269 */ "cmd ::= KILL QUERY INTEGER COLON INTEGER", + /* 230 */ "expr ::= NULL", + /* 231 */ "expr ::= ID LP exprlist RP", + /* 232 */ "expr ::= ID LP STAR RP", + /* 233 */ "expr ::= expr IS NULL", + /* 234 */ "expr ::= expr IS NOT NULL", + /* 235 */ "expr ::= expr LT expr", + /* 236 */ "expr ::= expr GT expr", + /* 237 */ "expr ::= expr LE expr", + /* 238 */ "expr ::= expr GE expr", + /* 239 */ "expr ::= expr NE expr", + /* 240 */ "expr ::= expr EQ expr", + /* 241 */ "expr ::= expr BETWEEN expr AND expr", + /* 242 */ "expr ::= expr AND expr", + /* 243 */ "expr ::= expr OR expr", + /* 244 */ "expr ::= expr PLUS expr", + /* 245 */ "expr ::= expr MINUS expr", + /* 246 */ "expr ::= expr STAR expr", + /* 247 */ "expr ::= expr SLASH expr", + /* 248 */ "expr ::= expr REM expr", + /* 249 */ "expr ::= expr LIKE expr", + /* 250 */ "expr ::= expr IN LP exprlist RP", + /* 251 */ "exprlist ::= exprlist COMMA expritem", + /* 252 */ "exprlist ::= expritem", + /* 253 */ "expritem ::= expr", + /* 254 */ "expritem ::=", + /* 255 */ "cmd ::= RESET QUERY CACHE", + /* 256 */ "cmd ::= SYNCDB ids REPLICA", + /* 257 */ "cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist", + /* 258 */ "cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids", + /* 259 */ "cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist", + /* 260 */ "cmd ::= ALTER TABLE ids cpxName DROP TAG ids", + /* 261 */ "cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids", + /* 262 */ "cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem", + /* 263 */ "cmd ::= ALTER STABLE ids cpxName ADD COLUMN columnlist", + /* 264 */ "cmd ::= ALTER STABLE ids cpxName DROP COLUMN ids", + /* 265 */ "cmd ::= ALTER STABLE ids cpxName ADD TAG columnlist", + /* 266 */ "cmd ::= ALTER STABLE ids cpxName DROP TAG ids", + /* 267 */ "cmd ::= ALTER STABLE ids cpxName CHANGE TAG ids ids", + /* 268 */ "cmd ::= KILL CONNECTION INTEGER", + /* 269 */ "cmd ::= KILL STREAM INTEGER COLON INTEGER", + /* 270 */ "cmd ::= KILL QUERY INTEGER COLON INTEGER", }; #endif /* NDEBUG */ @@ -2004,46 +2003,47 @@ static const YYCODETYPE yyRuleInfoLhs[] = { 255, /* (227) expr ::= NOW */ 255, /* (228) expr ::= VARIABLE */ 255, /* (229) expr ::= BOOL */ - 255, /* (230) expr ::= ID LP exprlist RP */ - 255, /* (231) expr ::= ID LP STAR RP */ - 255, /* (232) expr ::= expr IS NULL */ - 255, /* (233) expr ::= expr IS NOT NULL */ - 255, /* (234) expr ::= expr LT expr */ - 255, /* (235) expr ::= expr GT expr */ - 255, /* (236) expr ::= expr LE expr */ - 255, /* (237) expr ::= expr GE expr */ - 255, /* (238) expr ::= expr NE expr */ - 255, /* (239) expr ::= expr EQ expr */ - 255, /* (240) expr ::= expr BETWEEN expr AND expr */ - 255, /* (241) expr ::= expr AND expr */ - 255, /* (242) expr ::= expr OR expr */ - 255, /* (243) expr ::= expr PLUS expr */ - 255, /* (244) expr ::= expr MINUS expr */ - 255, /* (245) expr ::= expr STAR expr */ - 255, /* (246) expr ::= expr SLASH expr */ - 255, /* (247) expr ::= expr REM expr */ - 255, /* (248) expr ::= expr LIKE expr */ - 255, /* (249) expr ::= expr IN LP exprlist RP */ - 264, /* (250) exprlist ::= exprlist COMMA expritem */ - 264, /* (251) exprlist ::= expritem */ - 265, /* (252) expritem ::= expr */ - 265, /* (253) expritem ::= */ - 192, /* (254) cmd ::= RESET QUERY CACHE */ - 192, /* (255) cmd ::= SYNCDB ids REPLICA */ - 192, /* (256) cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ - 192, /* (257) cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ - 192, /* (258) cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ - 192, /* (259) cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ - 192, /* (260) cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ - 192, /* (261) cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ - 192, /* (262) cmd ::= ALTER STABLE ids cpxName ADD COLUMN columnlist */ - 192, /* (263) cmd ::= ALTER STABLE ids cpxName DROP COLUMN ids */ - 192, /* (264) cmd ::= ALTER STABLE ids cpxName ADD TAG columnlist */ - 192, /* (265) cmd ::= ALTER STABLE ids cpxName DROP TAG ids */ - 192, /* (266) cmd ::= ALTER STABLE ids cpxName CHANGE TAG ids ids */ - 192, /* (267) cmd ::= KILL CONNECTION INTEGER */ - 192, /* (268) cmd ::= KILL STREAM INTEGER COLON INTEGER */ - 192, /* (269) cmd ::= KILL QUERY INTEGER COLON INTEGER */ + 255, /* (230) expr ::= NULL */ + 255, /* (231) expr ::= ID LP exprlist RP */ + 255, /* (232) expr ::= ID LP STAR RP */ + 255, /* (233) expr ::= expr IS NULL */ + 255, /* (234) expr ::= expr IS NOT NULL */ + 255, /* (235) expr ::= expr LT expr */ + 255, /* (236) expr ::= expr GT expr */ + 255, /* (237) expr ::= expr LE expr */ + 255, /* (238) expr ::= expr GE expr */ + 255, /* (239) expr ::= expr NE expr */ + 255, /* (240) expr ::= expr EQ expr */ + 255, /* (241) expr ::= expr BETWEEN expr AND expr */ + 255, /* (242) expr ::= expr AND expr */ + 255, /* (243) expr ::= expr OR expr */ + 255, /* (244) expr ::= expr PLUS expr */ + 255, /* (245) expr ::= expr MINUS expr */ + 255, /* (246) expr ::= expr STAR expr */ + 255, /* (247) expr ::= expr SLASH expr */ + 255, /* (248) expr ::= expr REM expr */ + 255, /* (249) expr ::= expr LIKE expr */ + 255, /* (250) expr ::= expr IN LP exprlist RP */ + 264, /* (251) exprlist ::= exprlist COMMA expritem */ + 264, /* (252) exprlist ::= expritem */ + 265, /* (253) expritem ::= expr */ + 265, /* (254) expritem ::= */ + 192, /* (255) cmd ::= RESET QUERY CACHE */ + 192, /* (256) cmd ::= SYNCDB ids REPLICA */ + 192, /* (257) cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ + 192, /* (258) cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ + 192, /* (259) cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ + 192, /* (260) cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ + 192, /* (261) cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ + 192, /* (262) cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ + 192, /* (263) cmd ::= ALTER STABLE ids cpxName ADD COLUMN columnlist */ + 192, /* (264) cmd ::= ALTER STABLE ids cpxName DROP COLUMN ids */ + 192, /* (265) cmd ::= ALTER STABLE ids cpxName ADD TAG columnlist */ + 192, /* (266) cmd ::= ALTER STABLE ids cpxName DROP TAG ids */ + 192, /* (267) cmd ::= ALTER STABLE ids cpxName CHANGE TAG ids ids */ + 192, /* (268) cmd ::= KILL CONNECTION INTEGER */ + 192, /* (269) cmd ::= KILL STREAM INTEGER COLON INTEGER */ + 192, /* (270) cmd ::= KILL QUERY INTEGER COLON INTEGER */ }; /* For rule J, yyRuleInfoNRhs[J] contains the negative of the number @@ -2279,46 +2279,47 @@ static const signed char yyRuleInfoNRhs[] = { -1, /* (227) expr ::= NOW */ -1, /* (228) expr ::= VARIABLE */ -1, /* (229) expr ::= BOOL */ - -4, /* (230) expr ::= ID LP exprlist RP */ - -4, /* (231) expr ::= ID LP STAR RP */ - -3, /* (232) expr ::= expr IS NULL */ - -4, /* (233) expr ::= expr IS NOT NULL */ - -3, /* (234) expr ::= expr LT expr */ - -3, /* (235) expr ::= expr GT expr */ - -3, /* (236) expr ::= expr LE expr */ - -3, /* (237) expr ::= expr GE expr */ - -3, /* (238) expr ::= expr NE expr */ - -3, /* (239) expr ::= expr EQ expr */ - -5, /* (240) expr ::= expr BETWEEN expr AND expr */ - -3, /* (241) expr ::= expr AND expr */ - -3, /* (242) expr ::= expr OR expr */ - -3, /* (243) expr ::= expr PLUS expr */ - -3, /* (244) expr ::= expr MINUS expr */ - -3, /* (245) expr ::= expr STAR expr */ - -3, /* (246) expr ::= expr SLASH expr */ - -3, /* (247) expr ::= expr REM expr */ - -3, /* (248) expr ::= expr LIKE expr */ - -5, /* (249) expr ::= expr IN LP exprlist RP */ - -3, /* (250) exprlist ::= exprlist COMMA expritem */ - -1, /* (251) exprlist ::= expritem */ - -1, /* (252) expritem ::= expr */ - 0, /* (253) expritem ::= */ - -3, /* (254) cmd ::= RESET QUERY CACHE */ - -3, /* (255) cmd ::= SYNCDB ids REPLICA */ - -7, /* (256) cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ - -7, /* (257) cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ - -7, /* (258) cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ - -7, /* (259) cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ - -8, /* (260) cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ - -9, /* (261) cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ - -7, /* (262) cmd ::= ALTER STABLE ids cpxName ADD COLUMN columnlist */ - -7, /* (263) cmd ::= ALTER STABLE ids cpxName DROP COLUMN ids */ - -7, /* (264) cmd ::= ALTER STABLE ids cpxName ADD TAG columnlist */ - -7, /* (265) cmd ::= ALTER STABLE ids cpxName DROP TAG ids */ - -8, /* (266) cmd ::= ALTER STABLE ids cpxName CHANGE TAG ids ids */ - -3, /* (267) cmd ::= KILL CONNECTION INTEGER */ - -5, /* (268) cmd ::= KILL STREAM INTEGER COLON INTEGER */ - -5, /* (269) cmd ::= KILL QUERY INTEGER COLON INTEGER */ + -1, /* (230) expr ::= NULL */ + -4, /* (231) expr ::= ID LP exprlist RP */ + -4, /* (232) expr ::= ID LP STAR RP */ + -3, /* (233) expr ::= expr IS NULL */ + -4, /* (234) expr ::= expr IS NOT NULL */ + -3, /* (235) expr ::= expr LT expr */ + -3, /* (236) expr ::= expr GT expr */ + -3, /* (237) expr ::= expr LE expr */ + -3, /* (238) expr ::= expr GE expr */ + -3, /* (239) expr ::= expr NE expr */ + -3, /* (240) expr ::= expr EQ expr */ + -5, /* (241) expr ::= expr BETWEEN expr AND expr */ + -3, /* (242) expr ::= expr AND expr */ + -3, /* (243) expr ::= expr OR expr */ + -3, /* (244) expr ::= expr PLUS expr */ + -3, /* (245) expr ::= expr MINUS expr */ + -3, /* (246) expr ::= expr STAR expr */ + -3, /* (247) expr ::= expr SLASH expr */ + -3, /* (248) expr ::= expr REM expr */ + -3, /* (249) expr ::= expr LIKE expr */ + -5, /* (250) expr ::= expr IN LP exprlist RP */ + -3, /* (251) exprlist ::= exprlist COMMA expritem */ + -1, /* (252) exprlist ::= expritem */ + -1, /* (253) expritem ::= expr */ + 0, /* (254) expritem ::= */ + -3, /* (255) cmd ::= RESET QUERY CACHE */ + -3, /* (256) cmd ::= SYNCDB ids REPLICA */ + -7, /* (257) cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ + -7, /* (258) cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ + -7, /* (259) cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ + -7, /* (260) cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ + -8, /* (261) cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ + -9, /* (262) cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ + -7, /* (263) cmd ::= ALTER STABLE ids cpxName ADD COLUMN columnlist */ + -7, /* (264) cmd ::= ALTER STABLE ids cpxName DROP COLUMN ids */ + -7, /* (265) cmd ::= ALTER STABLE ids cpxName ADD TAG columnlist */ + -7, /* (266) cmd ::= ALTER STABLE ids cpxName DROP TAG ids */ + -8, /* (267) cmd ::= ALTER STABLE ids cpxName CHANGE TAG ids ids */ + -3, /* (268) cmd ::= KILL CONNECTION INTEGER */ + -5, /* (269) cmd ::= KILL STREAM INTEGER COLON INTEGER */ + -5, /* (270) cmd ::= KILL QUERY INTEGER COLON INTEGER */ }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -3125,7 +3126,7 @@ static YYACTIONTYPE yy_reduce( break; case 204: /* having_opt ::= */ case 214: /* where_opt ::= */ yytestcase(yyruleno==214); - case 253: /* expritem ::= */ yytestcase(yyruleno==253); + case 254: /* expritem ::= */ yytestcase(yyruleno==254); {yymsp[1].minor.yy454 = 0;} break; case 205: /* having_opt ::= HAVING expr */ @@ -3202,112 +3203,116 @@ static YYACTIONTYPE yy_reduce( { yylhsminor.yy454 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_BOOL);} yymsp[0].minor.yy454 = yylhsminor.yy454; break; - case 230: /* expr ::= ID LP exprlist RP */ + case 230: /* expr ::= NULL */ +{ yylhsminor.yy454 = tSqlExprCreateIdValue(&yymsp[0].minor.yy0, TK_NULL);} + yymsp[0].minor.yy454 = yylhsminor.yy454; + break; + case 231: /* expr ::= ID LP exprlist RP */ { yylhsminor.yy454 = tSqlExprCreateFunction(yymsp[-1].minor.yy193, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } yymsp[-3].minor.yy454 = yylhsminor.yy454; break; - case 231: /* expr ::= ID LP STAR RP */ + case 232: /* expr ::= ID LP STAR RP */ { yylhsminor.yy454 = tSqlExprCreateFunction(NULL, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } yymsp[-3].minor.yy454 = yylhsminor.yy454; break; - case 232: /* expr ::= expr IS NULL */ + case 233: /* expr ::= expr IS NULL */ {yylhsminor.yy454 = tSqlExprCreate(yymsp[-2].minor.yy454, NULL, TK_ISNULL);} yymsp[-2].minor.yy454 = yylhsminor.yy454; break; - case 233: /* expr ::= expr IS NOT NULL */ + case 234: /* expr ::= expr IS NOT NULL */ {yylhsminor.yy454 = tSqlExprCreate(yymsp[-3].minor.yy454, NULL, TK_NOTNULL);} yymsp[-3].minor.yy454 = yylhsminor.yy454; break; - case 234: /* expr ::= expr LT expr */ + case 235: /* expr ::= expr LT expr */ {yylhsminor.yy454 = tSqlExprCreate(yymsp[-2].minor.yy454, yymsp[0].minor.yy454, TK_LT);} yymsp[-2].minor.yy454 = yylhsminor.yy454; break; - case 235: /* expr ::= expr GT expr */ + case 236: /* expr ::= expr GT expr */ {yylhsminor.yy454 = tSqlExprCreate(yymsp[-2].minor.yy454, yymsp[0].minor.yy454, TK_GT);} yymsp[-2].minor.yy454 = yylhsminor.yy454; break; - case 236: /* expr ::= expr LE expr */ + case 237: /* expr ::= expr LE expr */ {yylhsminor.yy454 = tSqlExprCreate(yymsp[-2].minor.yy454, yymsp[0].minor.yy454, TK_LE);} yymsp[-2].minor.yy454 = yylhsminor.yy454; break; - case 237: /* expr ::= expr GE expr */ + case 238: /* expr ::= expr GE expr */ {yylhsminor.yy454 = tSqlExprCreate(yymsp[-2].minor.yy454, yymsp[0].minor.yy454, TK_GE);} yymsp[-2].minor.yy454 = yylhsminor.yy454; break; - case 238: /* expr ::= expr NE expr */ + case 239: /* expr ::= expr NE expr */ {yylhsminor.yy454 = tSqlExprCreate(yymsp[-2].minor.yy454, yymsp[0].minor.yy454, TK_NE);} yymsp[-2].minor.yy454 = yylhsminor.yy454; break; - case 239: /* expr ::= expr EQ expr */ + case 240: /* expr ::= expr EQ expr */ {yylhsminor.yy454 = tSqlExprCreate(yymsp[-2].minor.yy454, yymsp[0].minor.yy454, TK_EQ);} yymsp[-2].minor.yy454 = yylhsminor.yy454; break; - case 240: /* expr ::= expr BETWEEN expr AND expr */ + case 241: /* expr ::= expr BETWEEN expr AND expr */ { tSqlExpr* X2 = tSqlExprClone(yymsp[-4].minor.yy454); yylhsminor.yy454 = tSqlExprCreate(tSqlExprCreate(yymsp[-4].minor.yy454, yymsp[-2].minor.yy454, TK_GE), tSqlExprCreate(X2, yymsp[0].minor.yy454, TK_LE), TK_AND);} yymsp[-4].minor.yy454 = yylhsminor.yy454; break; - case 241: /* expr ::= expr AND expr */ + case 242: /* expr ::= expr AND expr */ {yylhsminor.yy454 = tSqlExprCreate(yymsp[-2].minor.yy454, yymsp[0].minor.yy454, TK_AND);} yymsp[-2].minor.yy454 = yylhsminor.yy454; break; - case 242: /* expr ::= expr OR expr */ + case 243: /* expr ::= expr OR expr */ {yylhsminor.yy454 = tSqlExprCreate(yymsp[-2].minor.yy454, yymsp[0].minor.yy454, TK_OR); } yymsp[-2].minor.yy454 = yylhsminor.yy454; break; - case 243: /* expr ::= expr PLUS expr */ + case 244: /* expr ::= expr PLUS expr */ {yylhsminor.yy454 = tSqlExprCreate(yymsp[-2].minor.yy454, yymsp[0].minor.yy454, TK_PLUS); } yymsp[-2].minor.yy454 = yylhsminor.yy454; break; - case 244: /* expr ::= expr MINUS expr */ + case 245: /* expr ::= expr MINUS expr */ {yylhsminor.yy454 = tSqlExprCreate(yymsp[-2].minor.yy454, yymsp[0].minor.yy454, TK_MINUS); } yymsp[-2].minor.yy454 = yylhsminor.yy454; break; - case 245: /* expr ::= expr STAR expr */ + case 246: /* expr ::= expr STAR expr */ {yylhsminor.yy454 = tSqlExprCreate(yymsp[-2].minor.yy454, yymsp[0].minor.yy454, TK_STAR); } yymsp[-2].minor.yy454 = yylhsminor.yy454; break; - case 246: /* expr ::= expr SLASH expr */ + case 247: /* expr ::= expr SLASH expr */ {yylhsminor.yy454 = tSqlExprCreate(yymsp[-2].minor.yy454, yymsp[0].minor.yy454, TK_DIVIDE);} yymsp[-2].minor.yy454 = yylhsminor.yy454; break; - case 247: /* expr ::= expr REM expr */ + case 248: /* expr ::= expr REM expr */ {yylhsminor.yy454 = tSqlExprCreate(yymsp[-2].minor.yy454, yymsp[0].minor.yy454, TK_REM); } yymsp[-2].minor.yy454 = yylhsminor.yy454; break; - case 248: /* expr ::= expr LIKE expr */ + case 249: /* expr ::= expr LIKE expr */ {yylhsminor.yy454 = tSqlExprCreate(yymsp[-2].minor.yy454, yymsp[0].minor.yy454, TK_LIKE); } yymsp[-2].minor.yy454 = yylhsminor.yy454; break; - case 249: /* expr ::= expr IN LP exprlist RP */ + case 250: /* expr ::= expr IN LP exprlist RP */ {yylhsminor.yy454 = tSqlExprCreate(yymsp[-4].minor.yy454, (tSqlExpr*)yymsp[-1].minor.yy193, TK_IN); } yymsp[-4].minor.yy454 = yylhsminor.yy454; break; - case 250: /* exprlist ::= exprlist COMMA expritem */ + case 251: /* exprlist ::= exprlist COMMA expritem */ {yylhsminor.yy193 = tSqlExprListAppend(yymsp[-2].minor.yy193,yymsp[0].minor.yy454,0, 0);} yymsp[-2].minor.yy193 = yylhsminor.yy193; break; - case 251: /* exprlist ::= expritem */ + case 252: /* exprlist ::= expritem */ {yylhsminor.yy193 = tSqlExprListAppend(0,yymsp[0].minor.yy454,0, 0);} yymsp[0].minor.yy193 = yylhsminor.yy193; break; - case 252: /* expritem ::= expr */ + case 253: /* expritem ::= expr */ {yylhsminor.yy454 = yymsp[0].minor.yy454;} yymsp[0].minor.yy454 = yylhsminor.yy454; break; - case 254: /* cmd ::= RESET QUERY CACHE */ + case 255: /* cmd ::= RESET QUERY CACHE */ { setDCLSqlElems(pInfo, TSDB_SQL_RESET_CACHE, 0);} break; - case 255: /* cmd ::= SYNCDB ids REPLICA */ + case 256: /* cmd ::= SYNCDB ids REPLICA */ { setDCLSqlElems(pInfo, TSDB_SQL_SYNC_DB_REPLICA, 1, &yymsp[-1].minor.yy0);} break; - case 256: /* cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ + case 257: /* cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy193, NULL, TSDB_ALTER_TABLE_ADD_COLUMN, -1); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 257: /* cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ + case 258: /* cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; @@ -3318,14 +3323,14 @@ static YYACTIONTYPE yy_reduce( setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 258: /* cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ + case 259: /* cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy193, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN, -1); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 259: /* cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ + case 260: /* cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; @@ -3336,7 +3341,7 @@ static YYACTIONTYPE yy_reduce( setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 260: /* cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ + case 261: /* cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ { yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n; @@ -3350,7 +3355,7 @@ static YYACTIONTYPE yy_reduce( setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 261: /* cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ + case 262: /* cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ { yymsp[-6].minor.yy0.n += yymsp[-5].minor.yy0.n; @@ -3362,14 +3367,14 @@ static YYACTIONTYPE yy_reduce( setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 262: /* cmd ::= ALTER STABLE ids cpxName ADD COLUMN columnlist */ + case 263: /* cmd ::= ALTER STABLE ids cpxName ADD COLUMN columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy193, NULL, TSDB_ALTER_TABLE_ADD_COLUMN, TSDB_SUPER_TABLE); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 263: /* cmd ::= ALTER STABLE ids cpxName DROP COLUMN ids */ + case 264: /* cmd ::= ALTER STABLE ids cpxName DROP COLUMN ids */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; @@ -3380,14 +3385,14 @@ static YYACTIONTYPE yy_reduce( setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 264: /* cmd ::= ALTER STABLE ids cpxName ADD TAG columnlist */ + case 265: /* cmd ::= ALTER STABLE ids cpxName ADD TAG columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; SAlterTableInfo* pAlterTable = tSetAlterTableInfo(&yymsp[-4].minor.yy0, yymsp[0].minor.yy193, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN, TSDB_SUPER_TABLE); setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 265: /* cmd ::= ALTER STABLE ids cpxName DROP TAG ids */ + case 266: /* cmd ::= ALTER STABLE ids cpxName DROP TAG ids */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; @@ -3398,7 +3403,7 @@ static YYACTIONTYPE yy_reduce( setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 266: /* cmd ::= ALTER STABLE ids cpxName CHANGE TAG ids ids */ + case 267: /* cmd ::= ALTER STABLE ids cpxName CHANGE TAG ids ids */ { yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n; @@ -3412,13 +3417,13 @@ static YYACTIONTYPE yy_reduce( setSqlInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 267: /* cmd ::= KILL CONNECTION INTEGER */ + case 268: /* cmd ::= KILL CONNECTION INTEGER */ {setKillSql(pInfo, TSDB_SQL_KILL_CONNECTION, &yymsp[0].minor.yy0);} break; - case 268: /* cmd ::= KILL STREAM INTEGER COLON INTEGER */ + case 269: /* cmd ::= KILL STREAM INTEGER COLON INTEGER */ {yymsp[-2].minor.yy0.n += (yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n); setKillSql(pInfo, TSDB_SQL_KILL_STREAM, &yymsp[-2].minor.yy0);} break; - case 269: /* cmd ::= KILL QUERY INTEGER COLON INTEGER */ + case 270: /* cmd ::= KILL QUERY INTEGER COLON INTEGER */ {yymsp[-2].minor.yy0.n += (yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n); setKillSql(pInfo, TSDB_SQL_KILL_QUERY, &yymsp[-2].minor.yy0);} break; default: diff --git a/src/query/tests/resultBufferTest.cpp b/src/query/tests/resultBufferTest.cpp index 19e0117742..491d75ccb9 100644 --- a/src/query/tests/resultBufferTest.cpp +++ b/src/query/tests/resultBufferTest.cpp @@ -10,7 +10,7 @@ namespace { // simple test void simpleTest() { SDiskbasedResultBuf* pResultBuf = NULL; - int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 1024, 4096, NULL); + int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 1024, 4096, 1); int32_t pageId = 0; int32_t groupId = 0; @@ -52,7 +52,7 @@ void simpleTest() { void writeDownTest() { SDiskbasedResultBuf* pResultBuf = NULL; - int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 1024, 4*1024, NULL); + int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 1024, 4*1024, 1); int32_t pageId = 0; int32_t writePageId = 0; @@ -99,7 +99,7 @@ void writeDownTest() { void recyclePageTest() { SDiskbasedResultBuf* pResultBuf = NULL; - int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 1024, 4*1024, NULL); + int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 1024, 4*1024, 1); int32_t pageId = 0; int32_t writePageId = 0; diff --git a/src/sync/inc/syncInt.h b/src/sync/inc/syncInt.h index 2b87938474..b4d9315a8e 100644 --- a/src/sync/inc/syncInt.h +++ b/src/sync/inc/syncInt.h @@ -35,7 +35,7 @@ extern "C" { #define SYNC_MAX_SIZE (TSDB_MAX_WAL_SIZE + sizeof(SWalHead) + sizeof(SSyncHead) + 16) #define SYNC_RECV_BUFFER_SIZE (5*1024*1024) -#define SYNC_MAX_FWDS 512 +#define SYNC_MAX_FWDS 1024 #define SYNC_FWD_TIMER 300 #define SYNC_ROLE_TIMER 15000 // ms #define SYNC_CHECK_INTERVAL 1000 // ms @@ -117,7 +117,6 @@ typedef struct SSyncNode { FStartSyncFile startSyncFileFp; FStopSyncFile stopSyncFileFp; FGetVersion getVersionFp; - FResetVersion resetVersionFp; FSendFile sendFileFp; FRecvFile recvFileFp; pthread_mutex_t mutex; diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index 76d4379c5f..72442eee6c 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -182,7 +182,6 @@ int64_t syncStart(const SSyncInfo *pInfo) { pNode->startSyncFileFp = pInfo->startSyncFileFp; pNode->stopSyncFileFp = pInfo->stopSyncFileFp; pNode->getVersionFp = pInfo->getVersionFp; - pNode->resetVersionFp = pInfo->resetVersionFp; pNode->sendFileFp = pInfo->sendFileFp; pNode->recvFileFp = pInfo->recvFileFp; @@ -1373,7 +1372,7 @@ static void syncMonitorNodeRole(void *param, void *tmrId) { if (/*pPeer->role > TAOS_SYNC_ROLE_UNSYNCED && */ nodeRole > TAOS_SYNC_ROLE_UNSYNCED) continue; if (/*pPeer->sstatus > TAOS_SYNC_STATUS_INIT || */ nodeSStatus > TAOS_SYNC_STATUS_INIT) continue; - sDebug("%s, check roles since self:%s sstatus:%s, peer:%s sstatus:%s", pPeer->id, syncRole[pPeer->role], + sDebug("%s, check roles since peer:%s sstatus:%s, self:%s sstatus:%s", pPeer->id, syncRole[pPeer->role], syncStatus[pPeer->sstatus], syncRole[nodeRole], syncStatus[nodeSStatus]); syncSendPeersStatusMsgToPeer(pPeer, 1, SYNC_STATUS_CHECK_ROLE, syncGenTranId()); break; @@ -1460,7 +1459,12 @@ static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle if ((pNode->quorum > 1 || force) && code == 0) { code = syncSaveFwdInfo(pNode, pWalHead->version, mhandle); - if (code >= 0) code = 1; + if (code >= 0) { + code = 1; + } else { + pthread_mutex_unlock(&pNode->mutex); + return code; + } } int32_t retLen = taosWriteMsg(pPeer->peerFd, pSyncHead, fwdLen); diff --git a/src/sync/src/syncRestore.c b/src/sync/src/syncRestore.c index 22d0a27581..c0d66316cd 100644 --- a/src/sync/src/syncRestore.c +++ b/src/sync/src/syncRestore.c @@ -238,7 +238,6 @@ static int32_t syncRestoreDataStepByStep(SSyncPeer *pPeer) { (*pNode->stopSyncFileFp)(pNode->vgId, fversion); nodeVersion = fversion; - if (pNode->resetVersionFp) (*pNode->resetVersionFp)(pNode->vgId, fversion); sInfo("%s, start to restore wal, fver:%" PRIu64, pPeer->id, nodeVersion); uint64_t wver = 0; diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c index dd0fd18019..4351cce51c 100644 --- a/src/tsdb/src/tsdbCommit.c +++ b/src/tsdb/src/tsdbCommit.c @@ -17,7 +17,7 @@ #define TSDB_MAX_SUBBLOCKS 8 static FORCE_INLINE int TSDB_KEY_FID(TSKEY key, int32_t days, int8_t precision) { if (key < 0) { - return (int)((key + 1) / tsMsPerDay[precision] / days + 1); + return (int)((key + 1) / tsMsPerDay[precision] / days - 1); } else { return (int)((key / tsMsPerDay[precision] / days)); } diff --git a/src/tsdb/src/tsdbFS.c b/src/tsdb/src/tsdbFS.c index a9e74cb229..c9f087a5cf 100644 --- a/src/tsdb/src/tsdbFS.c +++ b/src/tsdb/src/tsdbFS.c @@ -33,6 +33,7 @@ static int tsdbScanDataDir(STsdbRepo *pRepo); static bool tsdbIsTFileInFS(STsdbFS *pfs, const TFILE *pf); static int tsdbRestoreCurrent(STsdbRepo *pRepo); static int tsdbComparTFILE(const void *arg1, const void *arg2); +static void tsdbScanAndTryFixDFilesHeader(STsdbRepo *pRepo); // ================== CURRENT file header info static int tsdbEncodeFSHeader(void **buf, SFSHeader *pHeader) { @@ -246,6 +247,8 @@ int tsdbOpenFS(STsdbRepo *pRepo) { tsdbError("vgId:%d failed to open FS since %s", REPO_ID(pRepo), tstrerror(terrno)); return -1; } + + tsdbScanAndTryFixDFilesHeader(pRepo); } else { if (tsdbRestoreCurrent(pRepo) < 0) { tsdbError("vgId:%d failed to restore current file since %s", REPO_ID(pRepo), tstrerror(terrno)); @@ -1201,4 +1204,41 @@ static int tsdbComparTFILE(const void *arg1, const void *arg2) { return 0; } } +} + +static void tsdbScanAndTryFixDFilesHeader(STsdbRepo *pRepo) { + STsdbFS * pfs = REPO_FS(pRepo); + SFSStatus *pStatus = pfs->cstatus; + SDFInfo info; + + for (size_t i = 0; i < taosArrayGetSize(pStatus->df); i++) { + SDFileSet fset; + tsdbInitDFileSetEx(&fset, (SDFileSet *)taosArrayGet(pStatus->df, i)); + + tsdbDebug("vgId:%d scan DFileSet %d header", REPO_ID(pRepo), fset.fid); + + if (tsdbOpenDFileSet(&fset, O_RDWR) < 0) { + tsdbError("vgId:%d failed to open DFileSet %d since %s, continue", REPO_ID(pRepo), fset.fid, tstrerror(terrno)); + continue; + } + + for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { + SDFile *pDFile = TSDB_DFILE_IN_SET(&fset, ftype); + + if ((tsdbLoadDFileHeader(pDFile, &info) < 0) || pDFile->info.size != info.size || + pDFile->info.magic != info.magic) { + if (tsdbUpdateDFileHeader(pDFile) < 0) { + tsdbError("vgId:%d failed to update DFile header of %s since %s, continue", REPO_ID(pRepo), + TSDB_FILE_FULL_NAME(pDFile), tstrerror(terrno)); + } else { + tsdbInfo("vgId:%d DFile header of %s is updated", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile)); + TSDB_FILE_FSYNC(pDFile); + } + } else { + tsdbDebug("vgId:%d DFile header of %s is correct", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile)); + } + } + + tsdbCloseDFileSet(&fset); + } } \ No newline at end of file diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index 304ba2f073..5db993e463 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -567,6 +567,7 @@ void tsdbInitDFileSet(SDFileSet *pSet, SDiskID did, int vid, int fid, uint32_t v } void tsdbInitDFileSetEx(SDFileSet *pSet, SDFileSet *pOSet) { + pSet->fid = pOSet->fid; for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { tsdbInitDFileEx(TSDB_DFILE_IN_SET(pSet, ftype), TSDB_DFILE_IN_SET(pOSet, ftype)); } diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 8969f61596..99929f3542 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -526,7 +526,7 @@ static void tsdbStartStream(STsdbRepo *pRepo) { STable *pTable = pMeta->tables[i]; if (pTable && pTable->type == TSDB_STREAM_TABLE) { pTable->cqhandle = (*pRepo->appH.cqCreateFunc)(pRepo->appH.cqH, TABLE_UID(pTable), TABLE_TID(pTable), TABLE_NAME(pTable)->data, pTable->sql, - tsdbGetTableSchemaImpl(pTable, false, false, -1)); + tsdbGetTableSchemaImpl(pTable, false, false, -1), 0); } } } @@ -619,4 +619,4 @@ int tsdbRestoreInfo(STsdbRepo *pRepo) { tsdbDestroyReadH(&readh); return 0; -} \ No newline at end of file +} diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 824f69608e..3e6263b9d3 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -840,7 +840,7 @@ static int tsdbAddTableToMeta(STsdbRepo *pRepo, STable *pTable, bool addIdx, boo if (lock && tsdbUnlockRepoMeta(pRepo) < 0) return -1; if (TABLE_TYPE(pTable) == TSDB_STREAM_TABLE && addIdx) { pTable->cqhandle = (*pRepo->appH.cqCreateFunc)(pRepo->appH.cqH, TABLE_UID(pTable), TABLE_TID(pTable), TABLE_NAME(pTable)->data, pTable->sql, - tsdbGetTableSchemaImpl(pTable, false, false, -1)); + tsdbGetTableSchemaImpl(pTable, false, false, -1), 1); } tsdbDebug("vgId:%d table %s tid %d uid %" PRIu64 " is added to meta", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), @@ -1322,4 +1322,4 @@ static int tsdbCheckTableTagVal(SKVRow *pKVRow, STSchema *pSchema) { } return 0; -} \ No newline at end of file +} diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 6c0137abf5..ea72760568 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -917,7 +917,9 @@ static int32_t loadBlockInfo(STsdbQueryHandle * pQueryHandle, int32_t index, int pCheckInfo->compSize = compIndex->len; } - tsdbLoadBlockInfo(&(pQueryHandle->rhelper), (void*)(pCheckInfo->pCompInfo)); + if (tsdbLoadBlockInfo(&(pQueryHandle->rhelper), (void*)(pCheckInfo->pCompInfo)) < 0) { + return terrno; + } SBlockInfo* pCompInfo = pCheckInfo->pCompInfo; TSKEY s = TSKEY_INITIAL_VAL, e = TSKEY_INITIAL_VAL; @@ -2832,7 +2834,9 @@ int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT* pQueryHandle, SDataSta } int64_t stime = taosGetTimestampUs(); - tsdbLoadBlockStatis(&pHandle->rhelper, pBlockInfo->compBlock); + if (tsdbLoadBlockStatis(&pHandle->rhelper, pBlockInfo->compBlock) < 0) { + return terrno; + } int16_t* colIds = pHandle->defaultLoadColumn->pData; diff --git a/src/tsdb/src/tsdbSync.c b/src/tsdb/src/tsdbSync.c index cef561aebe..5a2756537e 100644 --- a/src/tsdb/src/tsdbSync.c +++ b/src/tsdb/src/tsdbSync.c @@ -194,8 +194,8 @@ static int32_t tsdbSyncRecvMeta(SSyncH *pSynch) { return 0; } - if (pLMFile == NULL || memcmp(&(pSynch->pmf->info), &(pLMFile->info), sizeof(SMFInfo)) != 0 || - TSDB_FILE_IS_BAD(pLMFile)) { + if (pLMFile == NULL || pSynch->pmf->info.size != pLMFile->info.size || + pSynch->pmf->info.magic != pLMFile->info.magic || TSDB_FILE_IS_BAD(pLMFile)) { // Local has no meta file or has a different meta file, need to copy from remote pSynch->mfChanged = true; @@ -536,7 +536,7 @@ static bool tsdbIsTowFSetSame(SDFileSet *pSet1, SDFileSet *pSet2) { SDFile *pDFile1 = TSDB_DFILE_IN_SET(pSet1, ftype); SDFile *pDFile2 = TSDB_DFILE_IN_SET(pSet2, ftype); - if (memcmp((void *)(TSDB_FILE_INFO(pDFile1)), (void *)(TSDB_FILE_INFO(pDFile2)), sizeof(SDFInfo)) != 0) { + if (pDFile1->info.size != pDFile2->info.size || pDFile1->info.magic != pDFile2->info.magic) { return false; } } diff --git a/src/util/inc/tstoken.h b/src/util/inc/tstoken.h index 7af03d96af..ab1ef7b279 100644 --- a/src/util/inc/tstoken.h +++ b/src/util/inc/tstoken.h @@ -51,11 +51,9 @@ uint32_t tSQLGetToken(char *z, uint32_t *tokenType); * @param str * @param i * @param isPrevOptr - * @param numOfIgnoreToken - * @param ignoreTokenTypes * @return */ -SStrToken tStrGetToken(char *str, int32_t *i, bool isPrevOptr, uint32_t numOfIgnoreToken, uint32_t *ignoreTokenTypes); +SStrToken tStrGetToken(char *str, int32_t *i, bool isPrevOptr); /** * check if it is a keyword or not diff --git a/src/util/src/tsocket.c b/src/util/src/tsocket.c index b33cdb8b80..77941cba82 100644 --- a/src/util/src/tsocket.c +++ b/src/util/src/tsocket.c @@ -22,6 +22,8 @@ #define SIGPIPE EPIPE #endif +#define TCP_CONN_TIMEOUT 3000 // conn timeout + int32_t taosGetFqdn(char *fqdn) { char hostname[1024]; hostname[1023] = '\0'; @@ -346,10 +348,47 @@ SOCKET taosOpenTcpClientSocket(uint32_t destIp, uint16_t destPort, uint32_t clie serverAddr.sin_addr.s_addr = destIp; serverAddr.sin_port = (uint16_t)htons((uint16_t)destPort); - ret = connect(sockFd, (struct sockaddr *)&serverAddr, sizeof(serverAddr)); +#ifdef _TD_LINUX + taosSetNonblocking(sockFd, 1); + ret = connect(sockFd, (struct sockaddr *)&serverAddr, sizeof(serverAddr)); + if (ret == -1) { + if (errno == EHOSTUNREACH) { + uError("failed to connect socket, ip:0x%x, port:%hu(%s)", destIp, destPort, strerror(errno)); + taosCloseSocket(sockFd); + return -1; + } else if (errno == EINPROGRESS || errno == EAGAIN || errno == EWOULDBLOCK) { + struct pollfd wfd[1]; + + wfd[0].fd = sockFd; + wfd[0].events = POLLOUT; + + int res = poll(wfd, 1, TCP_CONN_TIMEOUT); + if (res == -1 || res == 0) { + uError("failed to connect socket, ip:0x%x, port:%hu(poll error/conn timeout)", destIp, destPort); + taosCloseSocket(sockFd); // + return -1; + } + int optVal = -1, optLen = sizeof(int); + if ((0 != taosGetSockOpt(sockFd, SOL_SOCKET, SO_ERROR, &optVal, &optLen)) || (optVal != 0)) { + uError("failed to connect socket, ip:0x%x, port:%hu(connect host error)", destIp, destPort); + taosCloseSocket(sockFd); // + return -1; + } + ret = 0; + } else { // Other error + uError("failed to connect socket, ip:0x%x, port:%hu(target host cannot be reached)", destIp, destPort); + taosCloseSocket(sockFd); // + return -1; + } + } + taosSetNonblocking(sockFd, 0); + +#else + ret = connect(sockFd, (struct sockaddr *)&serverAddr, sizeof(serverAddr)); +#endif if (ret != 0) { - // uError("failed to connect socket, ip:0x%x, port:%hu(%s)", destIp, destPort, strerror(errno)); + uError("failed to connect socket, ip:0x%x, port:%hu(%s)", destIp, destPort, strerror(errno)); taosCloseSocket(sockFd); sockFd = -1; } else { diff --git a/src/vnode/inc/vnodeBackup.h b/src/vnode/inc/vnodeBackup.h new file mode 100644 index 0000000000..0a6b26546c --- /dev/null +++ b/src/vnode/inc/vnodeBackup.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_VNODE_BACKUP_H +#define TDENGINE_VNODE_BACKUP_H + +#ifdef __cplusplus +extern "C" { +#endif +#include "vnodeInt.h" + +int32_t vnodeInitBackup(); +void vnodeCleanupBackup(); +int32_t vnodeBackup(int32_t vgId); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/vnode/inc/vnodeSync.h b/src/vnode/inc/vnodeSync.h index 75d7ffbabd..28fb63dd6a 100644 --- a/src/vnode/inc/vnodeSync.h +++ b/src/vnode/inc/vnodeSync.h @@ -30,7 +30,6 @@ void vnodeStopSyncFile(int32_t vgId, uint64_t fversion); void vnodeConfirmForard(int32_t vgId, void *wparam, int32_t code); int32_t vnodeWriteToCache(int32_t vgId, void *wparam, int32_t qtype, void *rparam); int32_t vnodeGetVersion(int32_t vgId, uint64_t *fver, uint64_t *wver); -int32_t vnodeResetVersion(int32_t vgId, uint64_t fver); void vnodeConfirmForward(void *pVnode, uint64_t version, int32_t code, bool force); diff --git a/src/vnode/src/vnodeBackup.c b/src/vnode/src/vnodeBackup.c new file mode 100644 index 0000000000..a0a975be2b --- /dev/null +++ b/src/vnode/src/vnodeBackup.c @@ -0,0 +1,172 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "taoserror.h" +#include "taosmsg.h" +#include "tutil.h" +#include "tqueue.h" +#include "tglobal.h" +#include "tfs.h" +#include "vnodeBackup.h" +#include "vnodeMain.h" + +typedef struct { + int32_t vgId; +} SVBackupMsg; + +typedef struct { + pthread_t thread; + int32_t workerId; +} SVBackupWorker; + +typedef struct { + int32_t num; + SVBackupWorker *worker; +} SVBackupWorkerPool; + +static SVBackupWorkerPool tsVBackupPool; +static taos_qset tsVBackupQset; +static taos_queue tsVBackupQueue; + +static void vnodeProcessBackupMsg(SVBackupMsg *pMsg) { + int32_t vgId = pMsg->vgId; + char newDir[TSDB_FILENAME_LEN] = {0}; + char stagingDir[TSDB_FILENAME_LEN] = {0}; + + sprintf(newDir, "%s/vnode%d", "vnode_bak", vgId); + sprintf(stagingDir, "%s/.staging/vnode%d", "vnode_bak", vgId); + + if (tsEnableVnodeBak) { + tfsRmdir(newDir); + tfsRename(stagingDir, newDir); + } else { + vInfo("vgId:%d, vnode backup not enabled", vgId); + + tfsRmdir(stagingDir); + } +} + +static void *vnodeBackupFunc(void *param) { + while (1) { + SVBackupMsg *pMsg = NULL; + if (taosReadQitemFromQset(tsVBackupQset, NULL, (void **)&pMsg, NULL) == 0) { + vDebug("qset:%p, vbackup got no message from qset, exiting", tsVBackupQset); + break; + } + + vTrace("vgId:%d, will be processed in vbackup queue", pMsg->vgId); + vnodeProcessBackupMsg(pMsg); + + vTrace("vgId:%d, disposed in vbackup worker", pMsg->vgId); + taosFreeQitem(pMsg); + } + + return NULL; +} + +static int32_t vnodeStartBackup() { + tsVBackupQueue = taosOpenQueue(); + if (tsVBackupQueue == NULL) return TSDB_CODE_DND_OUT_OF_MEMORY; + + taosAddIntoQset(tsVBackupQset, tsVBackupQueue, NULL); + + for (int32_t i = 0; i < tsVBackupPool.num; ++i) { + SVBackupWorker *pWorker = tsVBackupPool.worker + i; + pWorker->workerId = i; + + pthread_attr_t thAttr; + pthread_attr_init(&thAttr); + pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); + + if (pthread_create(&pWorker->thread, &thAttr, vnodeBackupFunc, pWorker) != 0) { + vError("failed to create thread to process vbackup queue, reason:%s", strerror(errno)); + } + + pthread_attr_destroy(&thAttr); + + vDebug("vbackup:%d is launched, total:%d", pWorker->workerId, tsVBackupPool.num); + } + + vDebug("vbackup queue:%p is allocated", tsVBackupQueue); + + return TSDB_CODE_SUCCESS; +} + +static int32_t vnodeWriteIntoBackupWorker(int32_t vgId) { + SVBackupMsg *pMsg = taosAllocateQitem(sizeof(SVBackupMsg)); + if (pMsg == NULL) return TSDB_CODE_VND_OUT_OF_MEMORY; + + pMsg->vgId = vgId; + + int32_t code = taosWriteQitem(tsVBackupQueue, TAOS_QTYPE_RPC, pMsg); + if (code == 0) code = TSDB_CODE_DND_ACTION_IN_PROGRESS; + + return code; +} + +int32_t vnodeBackup(int32_t vgId) { + vTrace("vgId:%d, will backup", vgId); + return vnodeWriteIntoBackupWorker(vgId); +} + +int32_t vnodeInitBackup() { + tsVBackupQset = taosOpenQset(); + + tsVBackupPool.num = 1; + tsVBackupPool.worker = calloc(sizeof(SVBackupWorker), tsVBackupPool.num); + + if (tsVBackupPool.worker == NULL) return -1; + for (int32_t i = 0; i < tsVBackupPool.num; ++i) { + SVBackupWorker *pWorker = tsVBackupPool.worker + i; + pWorker->workerId = i; + vDebug("vbackup:%d is created", i); + } + + vDebug("vbackup is initialized, num:%d qset:%p", tsVBackupPool.num, tsVBackupQset); + + return vnodeStartBackup(); +} + +void vnodeCleanupBackup() { + for (int32_t i = 0; i < tsVBackupPool.num; ++i) { + SVBackupWorker *pWorker = tsVBackupPool.worker + i; + if (taosCheckPthreadValid(pWorker->thread)) { + taosQsetThreadResume(tsVBackupQset); + } + vDebug("vbackup:%d is closed", i); + } + + for (int32_t i = 0; i < tsVBackupPool.num; ++i) { + SVBackupWorker *pWorker = tsVBackupPool.worker + i; + vDebug("vbackup:%d start to join", i); + if (taosCheckPthreadValid(pWorker->thread)) { + pthread_join(pWorker->thread, NULL); + } + vDebug("vbackup:%d join success", i); + } + + vDebug("vbackup is closed, qset:%p", tsVBackupQset); + + taosCloseQset(tsVBackupQset); + tsVBackupQset = NULL; + + tfree(tsVBackupPool.worker); + + vDebug("vbackup queue:%p is freed", tsVBackupQueue); + taosCloseQueue(tsVBackupQueue); + tsVBackupQueue = NULL; +} diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 441e951250..69d94aff87 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -27,6 +27,7 @@ #include "vnodeVersion.h" #include "vnodeMgmt.h" #include "vnodeWorker.h" +#include "vnodeBackup.h" #include "vnodeMain.h" static int32_t vnodeProcessTsdbStatus(void *arg, int32_t status, int32_t eno); @@ -364,7 +365,6 @@ int32_t vnodeOpen(int32_t vgId) { syncInfo.startSyncFileFp = vnodeStartSyncFile; syncInfo.stopSyncFileFp = vnodeStopSyncFile; syncInfo.getVersionFp = vnodeGetVersion; - syncInfo.resetVersionFp = vnodeResetVersion; syncInfo.sendFileFp = tsdbSyncSend; syncInfo.recvFileFp = tsdbSyncRecv; syncInfo.pTsdb = pVnode->tsdb; @@ -448,18 +448,14 @@ void vnodeDestroy(SVnodeObj *pVnode) { if (pVnode->dropped) { char rootDir[TSDB_FILENAME_LEN] = {0}; - char newDir[TSDB_FILENAME_LEN] = {0}; + char stagingDir[TSDB_FILENAME_LEN] = {0}; sprintf(rootDir, "%s/vnode%d", "vnode", vgId); - sprintf(newDir, "%s/vnode%d", "vnode_bak", vgId); + sprintf(stagingDir, "%s/.staging/vnode%d", "vnode_bak", vgId); - if (0 == tsEnableVnodeBak) { - vInfo("vgId:%d, vnode backup not enabled", pVnode->vgId); - } else { - tfsRmdir(newDir); - tfsRename(rootDir, newDir); - } + tfsRename(rootDir, stagingDir); + + vnodeBackup(vgId); - tfsRmdir(rootDir); dnodeSendStatusMsgToMnode(); } diff --git a/src/vnode/src/vnodeMgmt.c b/src/vnode/src/vnodeMgmt.c index 71d9bc07f5..32f9532138 100644 --- a/src/vnode/src/vnodeMgmt.c +++ b/src/vnode/src/vnodeMgmt.c @@ -17,6 +17,7 @@ #include "os.h" #include "dnode.h" #include "vnodeStatus.h" +#include "vnodeBackup.h" #include "vnodeWorker.h" #include "vnodeRead.h" #include "vnodeWrite.h" @@ -29,6 +30,7 @@ static void vnodeCleanupHash(void); static void vnodeIncRef(void *ptNode); static SStep tsVnodeSteps[] = { + {"vnode-backup", vnodeInitBackup, vnodeCleanupBackup}, {"vnode-worker", vnodeInitMWorker, vnodeCleanupMWorker}, {"vnode-write", vnodeInitWrite, vnodeCleanupWrite}, {"vnode-read", vnodeInitRead, vnodeCleanupRead}, diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index 8233c45632..0836ade77f 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -364,7 +364,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pRead) { // register the qhandle to connect to quit query immediate if connection is broken if (vnodeNotifyCurrentQhandle(pRead->rpcHandle, pRetrieve->qId, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) { - vError("vgId:%d, QInfo:%"PRIu64 "-%p, retrieve discarded since link is broken, %p", pVnode->vgId, pRetrieve->qhandle, *handle, pRead->rpcHandle); + vError("vgId:%d, QInfo:%"PRIu64 "-%p, retrieve discarded since link is broken, conn:%p", pVnode->vgId, pRetrieve->qhandle, *handle, pRead->rpcHandle); code = TSDB_CODE_RPC_NETWORK_UNAVAIL; qKillQuery(*handle); qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true); @@ -409,7 +409,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pRead) { // client is broken, the query needs to be killed immediately. int32_t vnodeNotifyCurrentQhandle(void *handle, uint64_t qId, void *qhandle, int32_t vgId) { SRetrieveTableMsg *pMsg = rpcMallocCont(sizeof(SRetrieveTableMsg)); - pMsg->qhandle = htobe64((uint64_t)qhandle); + pMsg->qId = htobe64(qId); pMsg->header.vgId = htonl(vgId); pMsg->header.contLen = htonl(sizeof(SRetrieveTableMsg)); diff --git a/src/vnode/src/vnodeSync.c b/src/vnode/src/vnodeSync.c index 929dd15926..aa4cf0fc15 100644 --- a/src/vnode/src/vnodeSync.c +++ b/src/vnode/src/vnodeSync.c @@ -107,8 +107,9 @@ void vnodeStopSyncFile(int32_t vgId, uint64_t fversion) { pVnode->fversion = fversion; pVnode->version = fversion; vnodeSaveVersion(pVnode); + walResetVersion(pVnode->wal, fversion); - vDebug("vgId:%d, datafile is synced, fver:%" PRIu64 " vver:%" PRIu64, vgId, fversion, fversion); + vInfo("vgId:%d, datafile is synced, fver:%" PRIu64 " vver:%" PRIu64, vgId, fversion, fversion); vnodeSetReadyStatus(pVnode); vnodeRelease(pVnode); @@ -158,22 +159,6 @@ int32_t vnodeGetVersion(int32_t vgId, uint64_t *fver, uint64_t *wver) { return code; } -int32_t vnodeResetVersion(int32_t vgId, uint64_t fver) { - SVnodeObj *pVnode = vnodeAcquire(vgId); - if (pVnode == NULL) { - vError("vgId:%d, vnode not found while reset version", vgId); - return -1; - } - - pVnode->fversion = fver; - pVnode->version = fver; - walResetVersion(pVnode->wal, fver); - vInfo("vgId:%d, version reset to %" PRIu64, vgId, fver); - - vnodeRelease(pVnode); - return 0; -} - void vnodeConfirmForward(void *vparam, uint64_t version, int32_t code, bool force) { SVnodeObj *pVnode = vparam; syncConfirmForward(pVnode->sync, version, code, force); diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index e318978a11..a0be52db7a 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -91,12 +91,17 @@ int32_t vnodeProcessWrite(void *vparam, void *wparam, int32_t qtype, void *rpara int32_t syncCode = 0; bool force = (pWrite == NULL ? false : pWrite->pHead.msgType != TSDB_MSG_TYPE_SUBMIT); syncCode = syncForwardToPeer(pVnode->sync, pHead, pWrite, qtype, force); - if (syncCode < 0) return syncCode; + if (syncCode < 0) { + pHead->version = 0; + return syncCode; + } // write into WAL code = walWrite(pVnode->wal, pHead); if (code < 0) { + if (syncCode > 0) atomic_sub_fetch_32(&pWrite->processedCount, 1); vError("vgId:%d, hver:%" PRIu64 " vver:%" PRIu64 " code:0x%x", pVnode->vgId, pHead->version, pVnode->version, code); + pHead->version = 0; return code; } @@ -104,7 +109,10 @@ int32_t vnodeProcessWrite(void *vparam, void *wparam, int32_t qtype, void *rpara // write data locally code = (*vnodeProcessWriteMsgFp[pHead->msgType])(pVnode, pHead->cont, pRspRet); - if (code < 0) return code; + if (code < 0) { + if (syncCode > 0) atomic_sub_fetch_32(&pWrite->processedCount, 1); + return code; + } return syncCode; } diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JDBCDemo.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JDBCDemo.java index 07a21e79d1..da865b3ffd 100644 --- a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JDBCDemo.java +++ b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/JDBCDemo.java @@ -68,12 +68,12 @@ public class JDBCDemo { } private void insert() { - final String sql = "insert into test.weather (ts, temperature, humidity) values(now, 20.5, 34)"; + final String sql = "insert into " + dbName + "." + tbName + " (ts, temperature, humidity) values(now, 20.5, 34)"; exuete(sql); } private void select() { - final String sql = "select * from test.weather"; + final String sql = "select * from "+ dbName + "." + tbName; executeQuery(sql); } diff --git a/tests/examples/go/taosdemo.go b/tests/examples/go/taosdemo.go index 2c3a7d09b6..003f5aeddc 100644 --- a/tests/examples/go/taosdemo.go +++ b/tests/examples/go/taosdemo.go @@ -16,45 +16,47 @@ package main import ( "database/sql" + "flag" "fmt" - _ "github.com/taosdata/driver-go/taosSql" + "math/rand" "os" - "sync" "runtime" "strconv" + "sync" "time" - "flag" - "math/rand" + + _ "github.com/taosdata/driver-go/taosSql" + //"golang.org/x/sys/unix" ) const ( - maxLocationSize = 32 - maxSqlBufSize = 65480 + maxLocationSize = 32 + //maxSqlBufSize = 65480 ) -var locations = [maxLocationSize]string { - "Beijing", "Shanghai", "Guangzhou", "Shenzhen", - "HangZhou", "Tianjin", "Wuhan", "Changsha", - "Nanjing", "Xian"} +var locations = [maxLocationSize]string{ + "Beijing", "Shanghai", "Guangzhou", "Shenzhen", + "HangZhou", "Tianjin", "Wuhan", "Changsha", + "Nanjing", "Xian"} type config struct { - hostName string - serverPort int - user string - password string - dbName string - supTblName string - tablePrefix string - numOftables int - numOfRecordsPerTable int - numOfRecordsPerReq int - numOfThreads int - startTimestamp string - startTs int64 + hostName string + serverPort int + user string + password string + dbName string + supTblName string + tablePrefix string + numOftables int + numOfRecordsPerTable int + numOfRecordsPerReq int + numOfThreads int + startTimestamp string + startTs int64 - keep int - days int + keep int + days int } var configPara config @@ -62,7 +64,7 @@ var taosDriverName = "taosSql" var url string func init() { - flag.StringVar(&configPara.hostName, "h", "127.0.0.1","The host to connect to TDengine server.") + flag.StringVar(&configPara.hostName, "h", "127.0.0.1", "The host to connect to TDengine server.") flag.IntVar(&configPara.serverPort, "p", 6030, "The TCP/IP port number to use for the connection to TDengine server.") flag.StringVar(&configPara.user, "u", "root", "The TDengine user name to use when connecting to the server.") flag.StringVar(&configPara.password, "P", "taosdata", "The password to use when connecting to the server.") @@ -80,14 +82,14 @@ func init() { configPara.supTblName = "meters" startTs, err := time.ParseInLocation("2006-01-02 15:04:05", configPara.startTimestamp, time.Local) - if err==nil { - configPara.startTs = startTs.UnixNano() / 1e6 + if err == nil { + configPara.startTs = startTs.UnixNano() / 1e6 } } func printAllArgs() { fmt.Printf("\n============= args parse result: =============\n") - fmt.Printf("hostName: %v\n", configPara.hostName) + fmt.Printf("hostName: %v\n", configPara.hostName) fmt.Printf("serverPort: %v\n", configPara.serverPort) fmt.Printf("usr: %v\n", configPara.user) fmt.Printf("password: %v\n", configPara.password) @@ -104,10 +106,10 @@ func printAllArgs() { func main() { printAllArgs() fmt.Printf("Please press enter key to continue....\n") - fmt.Scanln() + _, _ = fmt.Scanln() url = "root:taosdata@/tcp(" + configPara.hostName + ":" + strconv.Itoa(configPara.serverPort) + ")/" - //url = fmt.Sprintf("%s:%s@/tcp(%s:%d)/%s?interpolateParams=true", configPara.user, configPara.password, configPara.hostName, configPara.serverPort, configPara.dbName) + //url = fmt.Sprintf("%s:%s@/tcp(%s:%d)/%s?interpolateParams=true", configPara.user, configPara.password, configPara.hostName, configPara.serverPort, configPara.dbName) // open connect to taos server //db, err := sql.Open(taosDriverName, url) //if err != nil { @@ -115,7 +117,7 @@ func main() { // os.Exit(1) //} //defer db.Close() - rand.Seed(time.Now().Unix()) + rand.Seed(time.Now().Unix()) createDatabase(configPara.dbName, configPara.supTblName) fmt.Printf("======== create database success! ========\n\n") @@ -138,7 +140,7 @@ func main() { func createDatabase(dbName string, supTblName string) { db, err := sql.Open(taosDriverName, url) if err != nil { - fmt.Println("Open database error: %s\n", err) + fmt.Printf("Open database error: %s\n", err) os.Exit(1) } defer db.Close() @@ -165,27 +167,27 @@ func createDatabase(dbName string, supTblName string) { checkErr(err, sqlStr) } -func multiThreadCreateTable(threads int, ntables int, dbName string, tablePrefix string) { +func multiThreadCreateTable(threads int, nTables int, dbName string, tablePrefix string) { st := time.Now().UnixNano() - if (threads < 1) { - threads = 1; + if threads < 1 { + threads = 1 } - a := ntables / threads; - if (a < 1) { - threads = ntables; - a = 1; + a := nTables / threads + if a < 1 { + threads = nTables + a = 1 } - b := ntables % threads; + b := nTables % threads - last := 0; + last := 0 endTblId := 0 wg := sync.WaitGroup{} for i := 0; i < threads; i++ { startTblId := last - if (i < b ) { + if i < b { endTblId = last + a } else { endTblId = last + a - 1 @@ -206,42 +208,43 @@ func createTable(dbName string, childTblPrefix string, startTblId int, endTblId db, err := sql.Open(taosDriverName, url) if err != nil { - fmt.Println("Open database error: %s\n", err) + fmt.Printf("Open database error: %s\n", err) os.Exit(1) } defer db.Close() - for i := startTblId; i <= endTblId; i++ { - sqlStr := "create table if not exists " + dbName + "." + childTblPrefix + strconv.Itoa(i) + " using " + dbName + ".meters tags('" + locations[i%maxLocationSize] + "', " + strconv.Itoa(i) + ");" - //fmt.Printf("sqlStr: %v\n", sqlStr) - _, err = db.Exec(sqlStr) - checkErr(err, sqlStr) - } - wg.Done() - runtime.Goexit() + for i := startTblId; i <= endTblId; i++ { + sqlStr := "create table if not exists " + dbName + "." + childTblPrefix + strconv.Itoa(i) + " using " + dbName + ".meters tags('" + locations[i%maxLocationSize] + "', " + strconv.Itoa(i) + ");" + //fmt.Printf("sqlStr: %v\n", sqlStr) + _, err = db.Exec(sqlStr) + checkErr(err, sqlStr) + } + wg.Done() + runtime.Goexit() } func generateRowData(ts int64) string { - voltage := rand.Int() % 1000 - current := 200 + rand.Float32() - phase := rand.Float32() - values := "( " + strconv.FormatInt(ts, 10) + ", " + strconv.FormatFloat(float64(current), 'f', 6, 64) + ", " + strconv.Itoa(voltage) + ", " + strconv.FormatFloat(float64(phase), 'f', 6, 64) + " ) " - return values + voltage := rand.Int() % 1000 + current := 200 + rand.Float32() + phase := rand.Float32() + values := "( " + strconv.FormatInt(ts, 10) + ", " + strconv.FormatFloat(float64(current), 'f', 6, 64) + ", " + strconv.Itoa(voltage) + ", " + strconv.FormatFloat(float64(phase), 'f', 6, 64) + " ) " + return values } + func insertData(dbName string, childTblPrefix string, startTblId int, endTblId int, wg *sync.WaitGroup) { //fmt.Printf("subThread[%d]: insert data to table from %d to %d \n", unix.Gettid(), startTblId, endTblId) // windows.GetCurrentThreadId() db, err := sql.Open(taosDriverName, url) if err != nil { - fmt.Println("Open database error: %s\n", err) + fmt.Printf("Open database error: %s\n", err) os.Exit(1) } defer db.Close() - tmpTs := configPara.startTs; + tmpTs := configPara.startTs //rand.New(rand.NewSource(time.Now().UnixNano())) - for tID := startTblId; tID <= endTblId; tID++{ + for tID := startTblId; tID <= endTblId; tID++ { totalNum := 0 for { sqlStr := "insert into " + dbName + "." + childTblPrefix + strconv.Itoa(tID) + " values " @@ -249,13 +252,13 @@ func insertData(dbName string, childTblPrefix string, startTblId int, endTblId i for { tmpTs += 1000 valuesOfRow := generateRowData(tmpTs) - currRowNum += 1 - totalNum += 1 + currRowNum += 1 + totalNum += 1 sqlStr = fmt.Sprintf("%s %s", sqlStr, valuesOfRow) - if (currRowNum >= configPara.numOfRecordsPerReq || totalNum >= configPara.numOfRecordsPerTable) { - break + if currRowNum >= configPara.numOfRecordsPerReq || totalNum >= configPara.numOfRecordsPerTable { + break } } @@ -265,12 +268,12 @@ func insertData(dbName string, childTblPrefix string, startTblId int, endTblId i count, err := res.RowsAffected() checkErr(err, "rows affected") - if (count != int64(currRowNum)) { - fmt.Printf("insert data, expect affected:%d, actual:%d\n", currRowNum, count) + if count != int64(currRowNum) { + fmt.Printf("insert data, expect affected:%d, actual:%d\n", currRowNum, count) os.Exit(1) } - if (totalNum >= configPara.numOfRecordsPerTable) { + if totalNum >= configPara.numOfRecordsPerTable { break } } @@ -279,44 +282,46 @@ func insertData(dbName string, childTblPrefix string, startTblId int, endTblId i wg.Done() runtime.Goexit() } -func multiThreadInsertData(threads int, ntables int, dbName string, tablePrefix string) { + +func multiThreadInsertData(threads int, nTables int, dbName string, tablePrefix string) { st := time.Now().UnixNano() - if (threads < 1) { - threads = 1; + if threads < 1 { + threads = 1 } - a := ntables / threads; - if (a < 1) { - threads = ntables; - a = 1; + a := nTables / threads + if a < 1 { + threads = nTables + a = 1 } - b := ntables % threads; + b := nTables % threads - last := 0; + last := 0 endTblId := 0 wg := sync.WaitGroup{} for i := 0; i < threads; i++ { startTblId := last - if (i < b ) { + if i < b { endTblId = last + a } else { endTblId = last + a - 1 } last = endTblId + 1 wg.Add(1) - go insertData(dbName, tablePrefix, startTblId , endTblId, &wg) + go insertData(dbName, tablePrefix, startTblId, endTblId, &wg) } wg.Wait() et := time.Now().UnixNano() fmt.Printf("insert data spent duration: %6.6fs\n", (float32(et-st))/1e9) } -func selectTest(dbName string, tbPrefix string, supTblName string){ + +func selectTest(dbName string, tbPrefix string, supTblName string) { db, err := sql.Open(taosDriverName, url) if err != nil { - fmt.Println("Open database error: %s\n", err) + fmt.Printf("Open database error: %s\n", err) os.Exit(1) } defer db.Close() @@ -332,12 +337,12 @@ func selectTest(dbName string, tbPrefix string, supTblName string){ fmt.Printf("query sql: %s\n", sqlStr) for rows.Next() { var ( - ts string - current float32 - voltage int - phase float32 - location string - groupid int + ts string + current float32 + voltage int + phase float32 + location string + groupid int ) err := rows.Scan(&ts, ¤t, &voltage, &phase, &location, &groupid) if err != nil { @@ -352,7 +357,7 @@ func selectTest(dbName string, tbPrefix string, supTblName string){ } // select sql 2 - sqlStr = "select avg(voltage), min(voltage), max(voltage) from " + dbName + "." + tbPrefix + strconv.Itoa( rand.Int() % configPara.numOftables) + sqlStr = "select avg(voltage), min(voltage), max(voltage) from " + dbName + "." + tbPrefix + strconv.Itoa(rand.Int()%configPara.numOftables) rows, err = db.Query(sqlStr) checkErr(err, sqlStr) @@ -360,9 +365,9 @@ func selectTest(dbName string, tbPrefix string, supTblName string){ fmt.Printf("\nquery sql: %s\n", sqlStr) for rows.Next() { var ( - voltageAvg float32 - voltageMin int - voltageMax int + voltageAvg float32 + voltageMin int + voltageMax int ) err := rows.Scan(&voltageAvg, &voltageMin, &voltageMax) if err != nil { @@ -385,10 +390,10 @@ func selectTest(dbName string, tbPrefix string, supTblName string){ fmt.Printf("\nquery sql: %s\n", sqlStr) for rows.Next() { var ( - lastTs string - lastCurrent float32 - lastVoltage int - lastPhase float32 + lastTs string + lastCurrent float32 + lastVoltage int + lastPhase float32 ) err := rows.Scan(&lastTs, &lastCurrent, &lastVoltage, &lastPhase) if err != nil { diff --git a/tests/perftest-scripts/perftest-query.sh b/tests/perftest-scripts/perftest-query.sh index e43a45ed5e..8b9fa1a546 100755 --- a/tests/perftest-scripts/perftest-query.sh +++ b/tests/perftest-scripts/perftest-query.sh @@ -50,7 +50,7 @@ function buildTDengine { echo "repo up-to-date" else echo "repo need to pull" - git pull > /dev/null + git pull > /dev/null 2>&1 LOCAL_COMMIT=`git rev-parse --short @` cd debug @@ -71,27 +71,14 @@ function runQueryPerfTest { python3 query/queryPerformance.py -c $LOCAL_COMMIT | tee -a $PERFORMANCE_TEST_REPORT python3 insert/insertFromCSVPerformance.py -c $LOCAL_COMMIT | tee -a $PERFORMANCE_TEST_REPORT - - yes | taosdemo -c /etc/taosperf/ -d taosdemo_insert_test -x > taosdemoperf.txt - - CREATETABLETIME=`grep 'Spent' taosdemoperf.txt | awk 'NR==1{print $2}'` - INSERTRECORDSTIME=`grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $2}'` - REQUESTSPERSECOND=`grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $13}'` - delay=`grep 'delay' taosdemoperf.txt | awk '{print $4}'` - AVGDELAY=`echo ${delay:0:${#delay}-3}` - delay=`grep 'delay' taosdemoperf.txt | awk '{print $6}'` - MAXDELAY=`echo ${delay:0:${#delay}-3}` - delay=`grep 'delay' taosdemoperf.txt | awk '{print $8}'` - MINDELAY=`echo ${delay:0:${#delay}-2}` - python3 tools/taosdemoPerformance.py -c $LOCAL_COMMIT -t $CREATETABLETIME -i $INSERTRECORDSTIME -r $REQUESTSPERSECOND -avg $AVGDELAY -max $MAXDELAY -min $MINDELAY | tee -a $PERFORMANCE_TEST_REPORT - [ -f taosdemoperf.txt ] && rm taosdemoperf.txt + python3 tools/taosdemoPerformance.py -c $LOCAL_COMMIT | tee -a $PERFORMANCE_TEST_REPORT } function sendReport { echo "send report" - receiver="pxiao@taosdata.com" + receiver="develop@taosdata.com" mimebody="MIME-Version: 1.0\nContent-Type: text/html; charset=utf-8\n" cd $TDENGINE_DIR diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index a631149c90..7ff41b13a6 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -139,6 +139,18 @@ python3 ./test.py -f import_merge/importInsertThenImport.py python3 ./test.py -f import_merge/importCSV.py #======================p1-end=============== #======================p2-start=============== +# tools +python3 test.py -f tools/taosdumpTest.py + +python3 test.py -f tools/taosdemoTest.py +python3 test.py -f tools/taosdemoTestWithoutMetric.py +python3 test.py -f tools/taosdemoTestWithJson.py +python3 test.py -f tools/taosdemoTestLimitOffset.py +python3 test.py -f tools/taosdemoTestTblAlt.py +python3 test.py -f tools/taosdemoTestSampleData.py +python3 test.py -f tools/taosdemoTestInterlace.py +python3 test.py -f tools/taosdemoTestQuery.py + # update python3 ./test.py -f update/allow_update.py python3 ./test.py -f update/allow_update-0.py @@ -166,6 +178,7 @@ python3 ./test.py -f stable/query_after_reset.py # perfbenchmark python3 ./test.py -f perfbenchmark/bug3433.py +#python3 ./test.py -f perfbenchmark/bug3589.py #query @@ -200,6 +213,8 @@ python3 ./test.py -f query/bug2119.py python3 ./test.py -f query/isNullTest.py python3 ./test.py -f query/queryWithTaosdKilled.py python3 ./test.py -f query/floatCompare.py +python3 ./test.py -f query/query1970YearsAf.py +python3 ./test.py -f query/bug3351.py python3 ./test.py -f query/bug3375.py @@ -246,18 +261,6 @@ python3 test.py -f subscribe/supertable.py #======================p3-end=============== #======================p4-start=============== -# tools -python3 test.py -f tools/taosdumpTest.py - -python3 test.py -f tools/taosdemoTest.py -python3 test.py -f tools/taosdemoTestWithoutMetric.py -python3 test.py -f tools/taosdemoTestWithJson.py -python3 test.py -f tools/taosdemoTestLimitOffset.py -python3 test.py -f tools/taosdemoTest2.py -python3 test.py -f tools/taosdemoTestSampleData.py -python3 test.py -f tools/taosdemoTestInterlace.py -python3 test.py -f tools/taosdemoTestQuery.py - python3 ./test.py -f update/merge_commit_data-0.py # wal python3 ./test.py -f wal/addOldWalTest.py diff --git a/tests/pytest/insert/metadataUpdate.py b/tests/pytest/insert/metadataUpdate.py index 4c32b0e39a..1a960a20e6 100644 --- a/tests/pytest/insert/metadataUpdate.py +++ b/tests/pytest/insert/metadataUpdate.py @@ -11,13 +11,13 @@ # -*- coding: utf-8 -*- -import sys import taos from util.log import tdLog from util.cases import tdCases from util.sql import tdSql from util.dnodes import tdDnodes -from multiprocessing import Process +from multiprocessing import Process +import subprocess class TDTestCase: def init(self, conn, logSql): @@ -40,27 +40,22 @@ class TDTestCase: print("alter table done") def deleteTableAndRecreate(self): - self.host = "127.0.0.1" - self.user = "root" - self.password = "taosdata" self.config = tdDnodes.getSimCfgPath() - self.conn = taos.connect(host = self.host, user = self.user, password = self.password, config = self.config) - self.cursor = self.conn.cursor() - - self.cursor.execute("use test") - print("drop table stb") - self.cursor.execute("drop table stb") - - print("create table stb") - self.cursor.execute("create table if not exists stb (ts timestamp, col1 int) tags(areaid int, city nchar(20))") - print("insert data") + sqlCmds = "use test; drop table stb;" + sqlCmds += "create table if not exists stb (ts timestamp, col1 int) tags(areaid int, city nchar(20));" for i in range(self.tables): city = "beijing" if i % 2 == 0 else "shanghai" - self.cursor.execute("create table tb%d using stb tags(%d, '%s')" % (i, i, city)) - for j in range(self.rows): - self.cursor.execute("insert into tb%d values(%d, %d)" % (i, self.ts + j, j * 100000)) - + sqlCmds += "create table tb%d using stb tags(%d, '%s');" % (i, i, city) + for j in range(5): + sqlCmds += "insert into tb%d values(%d, %d);" % (i, self.ts + j, j * 100000) + command = ["taos", "-c", self.config, "-s", sqlCmds] + print("drop stb, recreate stb and insert data ") + result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8") + if result.returncode == 0: + print("success:", result) + else: + print("error:", result) def run(self): tdSql.prepare() @@ -100,19 +95,17 @@ class TDTestCase: tdSql.query("select count(*) from stb") tdSql.checkData(0, 0, 10000) - tdSql.query("select count(*) from tb1") + tdSql.query("select count(*) from tb0") tdSql.checkData(0, 0, 1000) - p = Process(target=self.deleteTableAndRecreate, args=()) - p.start() - p.join() - p.terminate() + # drop stable in subprocess + self.deleteTableAndRecreate() tdSql.query("select count(*) from stb") - tdSql.checkData(0, 0, 10000) + tdSql.checkData(0, 0, 5 * self.tables) - tdSql.query("select count(*) from tb1") - tdSql.checkData(0, 0, 1000) + tdSql.query("select count(*) from tb0") + tdSql.checkData(0, 0, 5) def stop(self): tdSql.close() diff --git a/tests/pytest/perfbenchmark/bug3589.py b/tests/pytest/perfbenchmark/bug3589.py new file mode 100644 index 0000000000..c54ef8595d --- /dev/null +++ b/tests/pytest/perfbenchmark/bug3589.py @@ -0,0 +1,156 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import json + +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def __init__(self): + self.path = "" + + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/debug/build/bin")] + break + return buildPath + + def getCfgDir(self): + return self.getBuildPath() + "/sim/psim/cfg" + + def querycfg(self): + cfgdir = self.getCfgDir() + querycfg={ + "filetype": "query", + "cfgdir": cfgdir, + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "confirm_parameter_prompt": "yes", + "databases": "db", + "specified_table_query": { + "query_interval": 0, + "concurrent": 1, + "sqls": [ + { + "sql": "select * from t10, t11 where t10.ts=t11.ts" + } + ] + } + } + + return querycfg + + def querycfgfile(self): + querycfg = self.querycfg() + date = datetime.datetime.now().strftime("%Y%m%d%H%M") + querycfg.get("specified_table_query").get("sqls")[0]["result"] = f"/tmp/query_{date}.log" + file_query_table = f"/tmp/query_{date}.json" + with open(file_query_table, "w") as f: + json.dump(querycfg, f) + + return [file_query_table, querycfg] + + def querytable(self, filepath): + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info(f"taosd found in {buildPath}") + binPath = buildPath + "/debug/build/bin/" + + query_table_cmd = f"yes | {binPath}taosdemo -f {filepath}" + _ = subprocess.check_output(query_table_cmd, shell=True).decode("utf-8") + + def checkqueryresult(self, expectrows): + querycfg = self.querycfgfile()[1] + result_file = querycfg.get("specified_table_query").get("sqls")[0].get("result") + "-0" + if result_file: + check_cmd = f"wc -l {result_file}" + check_data_init = subprocess.check_output(check_cmd, shell=True).decode("utf-8") + check_data = int(check_data_init[0]) + if check_data == expectrows: + tdLog.info(f"queryResultRows:{check_data} == expect:{expectrows}") + else: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, check_data, expectrows) + tdLog.exit(f"{args[0]}({args[1]}) failed: result:{args[2]} != expect:{args[3]}") + + def droptmpfile(self): + drop_file_cmd = "rm -f /tmp/query_* " + _ = subprocess.check_output(drop_file_cmd, shell=True).decode("utf-8") + drop_file_cmd = "rm -f querySystemInfo-*" + _ = subprocess.check_output(drop_file_cmd, shell=True).decode("utf-8") + + def run(self): + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table && insert data") + tdSql.execute("alter database db keep 36500") + tdSql.execute( + "create table stb1 (ts timestamp, c1 int) TAGS(t1 int)" + ) + tdSql.execute("create table t10 using stb1 tags(1)") + tdSql.execute("create table t11 using stb1 tags(2)") + + tdSql.execute("insert into t10 values (-865000000, 1)") + tdSql.execute("insert into t11 values (-865000000, 2)") + tdSql.execute("insert into t10 values ('1969-12-31 23:59:59.000', 2)") + tdSql.execute("insert into t11 values ('1969-12-31 23:59:59.000', 3)") + tdSql.execute("insert into t10 values ('1970-01-01 00:00:00.000', 3)") + tdSql.execute("insert into t11 values ('1970-01-01 00:00:00.000', 4)") + tdSql.execute("insert into t10 values (-15230000, 4)") + tdSql.execute("insert into t11 values (-15230000, 5)") + tdSql.execute("insert into t10 values (-15220000, 5)") + tdSql.execute("insert into t11 values (-15220000, 6)") + tdSql.execute("insert into t10 values (-15210000, 6)") + tdSql.execute("insert into t11 values (-15210000, 7)") + tdSql.execute("insert into t10 values (0, 7)") + tdSql.execute("insert into t11 values (0, 8)") + tdSql.execute("insert into t10 values ('2020-10-01 00:00:00.000', 8)") + tdSql.execute("insert into t11 values ('2020-10-01 00:00:00.000', 9)") + + tdLog.printNoPrefix("==========step2:query") + query_file = self.querycfgfile()[0] + self.querytable(query_file) + self.checkqueryresult(8) + + self.droptmpfile() + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/pytest/query/bug3351.py b/tests/pytest/query/bug3351.py new file mode 100644 index 0000000000..288d071a69 --- /dev/null +++ b/tests/pytest/query/bug3351.py @@ -0,0 +1,74 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + tdSql.execute("drop database if exists db") + tdSql.execute("create database if not exists db keep 36500") + tdSql.execute("use db") + tdLog.printNoPrefix("==========step1:create table && insert data") + + tdSql.execute( + "create table stb1 (ts timestamp, c1 int) TAGS(t1 int)" + ) + tdSql.execute("create table t0 using stb1 tags(1)") + tdSql.execute("insert into t0 values (-865000000, 1)") + tdSql.execute("insert into t0 values (-864000000, 2)") + tdSql.execute("insert into t0 values (-863000000, 3)") + tdSql.execute("insert into t0 values (-15230000, 4)") + tdSql.execute("insert into t0 values (-15220000, 5)") + tdSql.execute("insert into t0 values (-15210000, 6)") + + tdLog.printNoPrefix("==========step2:query") + # bug1:when ts > -864000000, return 0 rows; + # bug2:when ts = -15220000, return 0 rows. + tdSql.query('select * from t0 where ts < -864000000') + tdSql.checkRows(1) + tdSql.query('select * from t0 where ts <= -864000000') + tdSql.checkRows(2) + tdSql.query('select * from t0 where ts = -864000000') + tdSql.checkRows(1) + tdSql.query('select * from t0 where ts > -864000000') + tdSql.checkRows(4) + tdSql.query('select * from t0 where ts >= -864000000') + tdSql.checkRows(5) + tdSql.query('select * from t0 where ts < -15220000') + tdSql.checkRows(4) + tdSql.query('select * from t0 where ts <= -15220000') + tdSql.checkRows(5) + tdSql.query('select * from t0 where ts = -15220000') + tdSql.checkRows(1) + tdSql.query('select * from t0 where ts > -15220000') + tdSql.checkRows(1) + tdSql.query('select * from t0 where ts >= -15220000') + tdSql.checkRows(2) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/pytest/query/query1970YearsAf.py b/tests/pytest/query/query1970YearsAf.py index 9794973325..e66a79f5e6 100644 --- a/tests/pytest/query/query1970YearsAf.py +++ b/tests/pytest/query/query1970YearsAf.py @@ -22,25 +22,34 @@ import datetime from util.log import * from util.sql import * from util.cases import * +from util.dnodes import * +from util.dnodes import TDDnode class TDTestCase: + def __init__(self): + self.path = "" + def init(self, conn, logSql): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor()) - def getCfgDir(self, path): + def getcfgPath(self, path): binPath = os.path.dirname(os.path.realpath(__file__)) binPath = binPath + "/../../../debug/" - tdLog.debug("binPath %s" % (binPath)) + tdLog.debug(f"binPath {binPath}") binPath = os.path.realpath(binPath) - tdLog.debug("binPath real path %s" % (binPath)) + tdLog.debug(f"binPath real path {binPath}") + if path == "": self.path = os.path.abspath(binPath + "../../") else: self.path = os.path.realpath(path) + return self.path - self.cfgDir = "%s/sim/psim/cfg" % (self.path) + def getCfgDir(self): + self.getcfgPath(self.path) + self.cfgDir = f"{self.path}/sim/psim/cfg" return self.cfgDir def creatcfg(self): @@ -63,7 +72,7 @@ class TDTestCase: "update": 0 } - # 设置创建的超级表格式 + # set stable schema stable1 = { "name": "stb2", "child_table_exists": "no", @@ -75,35 +84,49 @@ class TDTestCase: "insert_rows": 5000, "multi_thread_write_one_tbl": "no", "number_of_tbl_in_one_sql": 0, - "rows_per_tbl": 1000, + "rows_per_tbl": 1, "max_sql_len": 65480, "disorder_ratio": 0, "disorder_range": 1000, - "timestamp_step": 19000000, - "start_timestamp": "1969-01-01 00:00:00.000", + "timestamp_step": 20000, + "start_timestamp": "1969-12-31 00:00:00.000", "sample_format": "csv", "sample_file": "./sample.csv", "tags_file": "", "columns": [ - {"type": "INT"}, - {"type": "DOUBLE", "count": 10}, - {"type": "BINARY", "len": 16, "count": 3}, - {"type": "BINARY", "len": 32, "count": 6} + {"type": "INT", "count": 2}, + {"type": "DOUBLE", "count": 2}, + {"type": "BIGINT", "count": 2}, + {"type": "FLOAT", "count": 2}, + {"type": "SMALLINT", "count": 2}, + {"type": "TINYINT", "count": 2}, + {"type": "BOOL", "count": 2}, + {"type": "NCHAR", "len": 3, "count": 1}, + {"type": "BINARY", "len": 8, "count": 1} + ], "tags": [ + {"type": "INT", "count": 2}, + {"type": "DOUBLE", "count": 2}, + {"type": "BIGINT", "count": 2}, + {"type": "FLOAT", "count": 2}, + {"type": "SMALLINT", "count": 2}, {"type": "TINYINT", "count": 2}, - {"type": "BINARY", "len": 16, "count": 5} + {"type": "BOOL", "count": 2}, + {"type": "NCHAR", "len": 3, "count": 1}, + {"type": "BINARY", "len": 8, "count": 1} ] } - # 需要创建多个超级表时，只需创建不同的超级表格式并添加至super_tables - super_tables = [stable1] + # create different stables like stable1 and add to list super_tables + super_tables = [] + super_tables.append(stable1) database = { "dbinfo": dbinfo, "super_tables": super_tables } - cfgdir = self.getCfgDir("") + cfgdir = self.getCfgDir() create_table = { "filetype": "insert", "cfgdir": cfgdir, @@ -121,17 +144,82 @@ class TDTestCase: } return create_table - def inserttable(self): + def createinsertfile(self): create_table = self.creatcfg() date = datetime.datetime.now().strftime("%Y%m%d%H%M") file_create_table = f"/tmp/insert_{date}.json" with open(file_create_table, 'w') as f: json.dump(create_table, f) + return file_create_table - create_table_cmd = f"taosdemo -f {file_create_table}" + def inserttable(self, filepath): + create_table_cmd = f"taosdemo -f {filepath} > /dev/null 2>&1" _ = subprocess.check_output(create_table_cmd, shell=True).decode("utf-8") + def sqlsquery(self): + # stable query + tdSql.query( + "select * from stb2 where stb2.ts < '1970-01-01 00:00:00.000' " + ) + tdSql.checkRows(43200) + + tdSql.query( + "select * from stb2 where stb2.ts >= '1970-01-01 00:00:00.000' " + ) + tdSql.checkRows(6800) + + tdSql.query( + "select * from stb2 where stb2.ts > '1969-12-31 23:00:00.000' and stb2.ts <'1970-01-01 01:00:00.000' " + ) + tdSql.checkRows(3590) + + # child-tables query + tdSql.query( + "select * from t0 where t0.ts < '1970-01-01 00:00:00.000' " + ) + tdSql.checkRows(4320) + + tdSql.query( + "select * from t1 where t1.ts >= '1970-01-01 00:00:00.000' " + ) + tdSql.checkRows(680) + + tdSql.query( + "select * from t9 where t9.ts > '1969-12-31 22:00:00.000' and t9.ts <'1970-01-01 02:00:00.000' " + ) + tdSql.checkRows(719) + + tdSql.query( + "select * from t0,t1 where t0.ts=t1.ts and t1.ts >= '1970-01-01 00:00:00.000' " + ) + tdSql.checkRows(680) + + tdSql.query( + "select diff(col1) from t0 where t0.ts >= '1970-01-01 00:00:00.000' " + ) + tdSql.checkRows(679) + + tdSql.query( + "select t0,col1 from stb2 where stb2.ts < '1970-01-01 00:00:00.000' order by ts" + ) + tdSql.checkRows(43200) + + # query with timestamp in 'where ...' + tdSql.query( + "select * from stb2 where stb2.ts > -28800000 " + ) + tdSql.checkRows(6790) + + tdSql.query( + "select * from stb2 where stb2.ts > -28800000 and stb2.ts < '1970-01-01 08:00:00.000' " + ) + tdSql.checkRows(6790) + + tdSql.query( + "select * from stb2 where stb2.ts < -28800000 and stb2.ts > '1969-12-31 22:00:00.000' " + ) + tdSql.checkRows(3590) def run(self): s = 'reset query cache' @@ -142,84 +230,24 @@ class TDTestCase: tdSql.execute(s) tdLog.info("==========step1:create table stable and child table,then insert data automatically") - self.inserttable() - # tdSql.execute( - # '''create table if not exists supt - # (ts timestamp, c1 int, c2 float, c3 bigint, c4 double, c5 smallint, c6 tinyint) - # tags(location binary(64), type int, isused bool , family nchar(64))''' - # ) - # tdSql.execute("create table t1 using supt tags('beijing', 1, 1, '自行车')") - # tdSql.execute("create table t2 using supt tags('shanghai', 2, 0, '拖拉机')") - # tdSql.execute( - # f"insert into t1 values (-31564800000, 6, 5, 4, 3, 2, 1)" - # ) - - + insertfile = self.createinsertfile() + self.inserttable(insertfile) tdLog.info("==========step2:query join") + self.sqlsquery() - # stable query - tdSql.query( - "select * from stb2 where stb2.ts < '1970-01-01 00:00:00.000' " - ) - tdSql.checkRows(16600) + # after wal and sync, check again + tdSql.query("show dnodes") + index = tdSql.getData(0, 0) + tdDnodes.stop(index) + tdDnodes.start(index) - tdSql.query( - "select * from stb2 where stb2.ts >= '1970-01-01 00:00:00.000' " - ) - tdSql.checkRows(33400) - - tdSql.query( - "select * from stb2 where stb2.ts > '1969-12-01 00:00:00.000' and stb2.ts <'1970-01-31 00:00:00.000' " - ) - tdSql.checkRows(2780) - - # child-table query - tdSql.query( - "select * from t0 where t0.ts < '1970-01-01 00:00:00.000' " - ) - tdSql.checkRows(1660) - - tdSql.query( - "select * from t1 where t1.ts >= '1970-01-01 00:00:00.000' " - ) - tdSql.checkRows(3340) - - tdSql.query( - "select * from t9 where t9.ts > '1969-12-01 00:00:00.000' and t9.ts <'1970-01-31 00:00:00.000' " - ) - tdSql.checkRows(278) - - tdSql.query( - "select * from t0,t1 where t0.ts=t1.ts and t1.ts >= '1970-01-01 00:00:00.000' " - ) - tdSql.checkRows(3340) - - tdSql.query( - "select diff(col1) from t0 where t0.ts >= '1970-01-01 00:00:00.000' " - ) - tdSql.checkRows(3339) - - tdSql.query( - "select t0,col1 from stb2 where stb2.ts < '1970-01-01 00:00:00.000' order by ts" - ) - tdSql.checkRows(16600) - - # query with timestamp in 'where ...' - tdSql.query( - "select * from stb2 where stb2.ts > -28800000 " - ) - tdSql.checkRows(33400) - - tdSql.query( - "select * from stb2 where stb2.ts > -28800000 and stb2.ts < '1970-01-01 08:00:00.000' " - ) - tdSql.checkRows(20) - - tdSql.query( - "select * from stb2 where stb2.ts < -28800000 and stb2.ts > '1969-12-31 16:00:00.000' " - ) + tdLog.info("==========step3: query join again") + self.sqlsquery() + # delete temporary file + rm_cmd = f"rm -f /tmp/insert* > /dev/null 2>&1" + _ = subprocess.check_output(rm_cmd, shell=True).decode("utf-8") def stop(self): tdSql.close() diff --git a/tests/pytest/tools/insert-tblimit-tboffset-createdb.json b/tests/pytest/tools/insert-tblimit-tboffset-createdb.json new file mode 100644 index 0000000000..8c3b39fb0b --- /dev/null +++ b/tests/pytest/tools/insert-tblimit-tboffset-createdb.json @@ -0,0 +1,57 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "num_of_records_per_req": 100, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "db", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 16, + "blocks": 8, + "precision": "ms", + "keep": 365, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb", + "child_table_exists":"no", + "childtable_count": 100, + "childtable_prefix": "stb_", + "auto_create_table": "no", + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 0, + "multi_thread_write_one_tbl": "no", + "number_of_tbl_in_one_sql": 0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":10}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }] + }] +} diff --git a/tests/pytest/tools/insert-tblimit-tboffset-insertrec.json b/tests/pytest/tools/insert-tblimit-tboffset-insertrec.json new file mode 100644 index 0000000000..a9efa7c31c --- /dev/null +++ b/tests/pytest/tools/insert-tblimit-tboffset-insertrec.json @@ -0,0 +1,59 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "num_of_records_per_req": 100, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "db", + "drop": "no", + "replica": 1, + "days": 10, + "cache": 16, + "blocks": 8, + "precision": "ms", + "keep": 365, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb", + "child_table_exists":"yes", + "childtable_count": 100, + "childtable_prefix": "stb_", + "auto_create_table": "no", + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 1000, + "childtable_limit": 33, + "childtable_offset": 33, + "multi_thread_write_one_tbl": "no", + "number_of_tbl_in_one_sql": 0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":10}, {"type": "BINARY", "len": 16, "count":3}, {"type": "BINARY", "len": 32, "count":6}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":5}] + }] + }] +} diff --git a/tests/pytest/tools/insert-tblimit-tboffset0.json b/tests/pytest/tools/insert-tblimit-tboffset0.json index 7dcb2e0527..6bf3783cb2 100644 --- a/tests/pytest/tools/insert-tblimit-tboffset0.json +++ b/tests/pytest/tools/insert-tblimit-tboffset0.json @@ -15,7 +15,7 @@ "databases": [{ "dbinfo": { "name": "db", - "drop": "yes", + "drop": "no", "replica": 1, "days": 10, "cache": 16, @@ -33,7 +33,7 @@ }, "super_tables": [{ "name": "stb", - "child_table_exists":"no", + "child_table_exists":"yes", "childtable_count": 100, "childtable_prefix": "stb_", "auto_create_table": "no", diff --git a/tests/pytest/tools/insert-tblimit1-tboffset.json b/tests/pytest/tools/insert-tblimit1-tboffset.json index a33dc22d5d..292902093d 100644 --- a/tests/pytest/tools/insert-tblimit1-tboffset.json +++ b/tests/pytest/tools/insert-tblimit1-tboffset.json @@ -15,7 +15,7 @@ "databases": [{ "dbinfo": { "name": "db", - "drop": "yes", + "drop": "no", "replica": 1, "days": 10, "cache": 16, @@ -33,7 +33,7 @@ }, "super_tables": [{ "name": "stb", - "child_table_exists":"no", + "child_table_exists":"yes", "childtable_count": 100, "childtable_prefix": "stb_", "auto_create_table": "no", diff --git a/tests/pytest/tools/taosdemo-sampledata.json b/tests/pytest/tools/taosdemo-sampledata.json index 473c977773..d543b7e086 100644 --- a/tests/pytest/tools/taosdemo-sampledata.json +++ b/tests/pytest/tools/taosdemo-sampledata.json @@ -16,8 +16,6 @@ "name": "stb", "child_table_exists":"no", "childtable_count": 20, - "childtable_limit": 10, - "childtable_offset": 0, "childtable_prefix": "t_", "auto_create_table": "no", "data_source": "sample", diff --git a/tests/pytest/tools/taosdemoPerformance.py b/tests/pytest/tools/taosdemoPerformance.py index 28f451b6a0..1156cc2484 100644 --- a/tests/pytest/tools/taosdemoPerformance.py +++ b/tests/pytest/tools/taosdemoPerformance.py @@ -11,26 +11,16 @@ # -*- coding: utf-8 -*- -import sys import taos -import time -import datetime -import csv -import random import pandas as pd import argparse import os.path +import json class taosdemoPerformace: - def __init__(self, commitID, dbName, createTableTime, insertRecordsTime, recordsPerSecond, avgDelay, maxDelay, minDelay): + def __init__(self, commitID, dbName): self.commitID = commitID - self.dbName = dbName - self.createTableTime = createTableTime - self.insertRecordsTime = insertRecordsTime - self.recordsPerSecond = recordsPerSecond - self.avgDelay = avgDelay - self.maxDelay = maxDelay - self.minDelay = minDelay + self.dbName = dbName self.host = "127.0.0.1" self.user = "root" self.password = "taosdata" @@ -39,8 +29,97 @@ class taosdemoPerformace: self.host, self.user, self.password, - self.config) + self.config) + self.insertDB = "insertDB"; + def generateJson(self): + db = { + "name": "%s" % self.insertDB, + "drop": "yes", + "replica": 1 + } + + stb = { + "name": "meters", + "child_table_exists":"no", + "childtable_count": 10000, + "childtable_prefix": "stb_", + "auto_create_table": "no", + "data_source": "rand", + "batch_create_tbl_num": 10, + "insert_mode": "taosc", + "insert_rows": 100000, + "multi_thread_write_one_tbl": "no", + "number_of_tbl_in_one_sql": 0, + "rows_per_tbl": 100, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [ + {"type": "INT", "count": 4} + ], + "tags": [ + {"type": "INT", "count":1}, + {"type": "BINARY", "len": 16} + ] + } + + stables = [] + stables.append(stb) + + db = { + "dbinfo": db, + "super_tables": stables + } + + insert_data = { + "filetype": "insert", + "cfgdir": "/etc/taosperf", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 10, + "thread_count_create_tbl": 10, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "num_of_records_per_req": 30000, + "databases": [db] + } + + insert_json_file = f"/tmp/insert.json" + + with open(insert_json_file, 'w') as f: + json.dump(insert_data, f) + return insert_json_file + + def getCMDOutput(self, cmd): + cmd = os.popen(cmd) + output = cmd.read() + cmd.close() + return output + + def insertData(self): + os.system("taosdemo -f %s > taosdemoperf.txt" % self.generateJson()) + self.createTableTime = self.getCMDOutput("grep 'Spent' taosdemoperf.txt | awk 'NR==1{print $2}'") + self.insertRecordsTime = self.getCMDOutput("grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $2}'") + self.recordsPerSecond = self.getCMDOutput("grep 'Spent' taosdemoperf.txt | awk 'NR==2{print $16}'") + self.commitID = self.getCMDOutput("git rev-parse --short HEAD") + delay = self.getCMDOutput("grep 'delay' taosdemoperf.txt | awk '{print $4}'") + self.avgDelay = delay[:-4] + delay = self.getCMDOutput("grep 'delay' taosdemoperf.txt | awk '{print $6}'") + self.maxDelay = delay[:-4] + delay = self.getCMDOutput("grep 'delay' taosdemoperf.txt | awk '{print $8}'") + self.minDelay = delay[:-3] + + os.system("[ -f taosdemoperf.txt ] && rm taosdemoperf.txt") + def createTablesAndStoreData(self): cursor = self.conn.cursor() @@ -48,14 +127,15 @@ class taosdemoPerformace: cursor.execute("use %s" % self.dbName) cursor.execute("create table if not exists taosdemo_perf (ts timestamp, create_table_time float, insert_records_time float, records_per_second float, commit_id binary(50), avg_delay float, max_delay float, min_delay float)") print("==================== taosdemo performance ====================") - print("create tables time: %f" % self.createTableTime) - print("insert records time: %f" % self.insertRecordsTime) - print("records per second: %f" % self.recordsPerSecond) - print("avg delay: %f" % self.avgDelay) - print("max delay: %f" % self.maxDelay) - print("min delay: %f" % self.minDelay) - cursor.execute("insert into taosdemo_perf values(now, %f, %f, %f, '%s', %f, %f, %f)" % (self.createTableTime, self.insertRecordsTime, self.recordsPerSecond, self.commitID, self.avgDelay, self.maxDelay, self.minDelay)) - cursor.execute("drop database if exists taosdemo_insert_test") + print("create tables time: %f" % float(self.createTableTime)) + print("insert records time: %f" % float(self.insertRecordsTime)) + print("records per second: %f" % float(self.recordsPerSecond)) + print("avg delay: %f" % float(self.avgDelay)) + print("max delay: %f" % float(self.maxDelay)) + print("min delay: %f" % float(self.minDelay)) + cursor.execute("insert into taosdemo_perf values(now, %f, %f, %f, '%s', %f, %f, %f)" % + (float(self.createTableTime), float(self.insertRecordsTime), float(self.recordsPerSecond), self.commitID, float(self.avgDelay), float(self.maxDelay), float(self.minDelay))) + cursor.execute("drop database if exists %s" % self.insertDB) cursor.close() @@ -64,7 +144,7 @@ if __name__ == '__main__': parser.add_argument( '-c', '--commit-id', - action='store', + action='store', type=str, help='git commit id (default: null)') parser.add_argument( @@ -74,46 +154,9 @@ if __name__ == '__main__': default='perf', type=str, help='Database name to be created (default: perf)') - parser.add_argument( - '-t', - '--create-table', - action='store', - type=float, - help='create table time') - parser.add_argument( - '-i', - '--insert-records', - action='store', - type=float, - help='insert records time') - parser.add_argument( - '-r', - '---records-per-second', - action='store', - type=float, - help='records per request') - parser.add_argument( - '-avg', - '---avg-delay', - action='store', - type=float, - help='avg delay') - parser.add_argument( - '-max', - '---max-delay', - action='store', - type=float, - help='max delay') - parser.add_argument( - '-min', - '---min-delay', - action='store', - type=float, - help='min delay') - args = parser.parse_args() - perftest = taosdemoPerformace(args.commit_id, args.database_name, args.create_table, args.insert_records, args.records_per_second, - args.avg_delay, args.max_delay, args.min_delay) + perftest = taosdemoPerformace(args.commit_id, args.database_name) + perftest.insertData() perftest.createTablesAndStoreData() \ No newline at end of file diff --git a/tests/pytest/tools/taosdemoTestLimitOffset.py b/tests/pytest/tools/taosdemoTestLimitOffset.py index bce41e1c75..dd8a1bee70 100644 --- a/tests/pytest/tools/taosdemoTestLimitOffset.py +++ b/tests/pytest/tools/taosdemoTestLimitOffset.py @@ -51,7 +51,8 @@ class TDTestCase: else: tdLog.info("taosd found in %s" % buildPath) binPath = buildPath+ "/build/bin/" - os.system("%staosdemo -f tools/insert-tblimit-tboffset.json" % binPath) + os.system("%staosdemo -f tools/insert-tblimit-tboffset-createdb.json" % binPath) + os.system("%staosdemo -f tools/insert-tblimit-tboffset-insertrec.json" % binPath) tdSql.execute("use db") tdSql.query("select count(tbname) from db.stb") @@ -59,6 +60,7 @@ class TDTestCase: tdSql.query("select count(*) from db.stb") tdSql.checkData(0, 0, 33000) + os.system("%staosdemo -f tools/insert-tblimit-tboffset-createdb.json" % binPath) os.system("%staosdemo -f tools/insert-tblimit-tboffset0.json" % binPath) tdSql.execute("reset query cache") @@ -68,6 +70,7 @@ class TDTestCase: tdSql.query("select count(*) from db.stb") tdSql.checkData(0, 0, 20000) + os.system("%staosdemo -f tools/insert-tblimit-tboffset-createdb.json" % binPath) os.system("%staosdemo -f tools/insert-tblimit1-tboffset.json" % binPath) tdSql.execute("reset query cache") diff --git a/tests/pytest/tools/taosdemoTestSampleData.py b/tests/pytest/tools/taosdemoTestSampleData.py index 893c53984d..a8710a9df3 100644 --- a/tests/pytest/tools/taosdemoTestSampleData.py +++ b/tests/pytest/tools/taosdemoTestSampleData.py @@ -57,7 +57,7 @@ class TDTestCase: tdSql.query("select count(tbname) from db.stb") tdSql.checkData(0, 0, 20) tdSql.query("select count(*) from db.stb") - tdSql.checkData(0, 0, 200) + tdSql.checkData(0, 0, 400) def stop(self): tdSql.close() diff --git a/tests/pytest/tools/taosdemoTest2.py b/tests/pytest/tools/taosdemoTestTblAlt.py similarity index 63% rename from tests/pytest/tools/taosdemoTest2.py rename to tests/pytest/tools/taosdemoTestTblAlt.py index 74b05faf8b..bb367817cf 100644 --- a/tests/pytest/tools/taosdemoTest2.py +++ b/tests/pytest/tools/taosdemoTestTblAlt.py @@ -29,10 +29,33 @@ class TDTestCase: self.numberOfTables = 10 self.numberOfRecords = 1000000 + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + def insertDataAndAlterTable(self, threadID): + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + binPath = buildPath + "/build/bin/" + if(threadID == 0): - os.system("taosdemo -y -t %d -n %d" % - (self.numberOfTables, self.numberOfRecords)) + os.system("%staosdemo -y -t %d -n %d" % + (binPath, self.numberOfTables, self.numberOfRecords)) if(threadID == 1): time.sleep(2) print("use test") @@ -47,7 +70,13 @@ class TDTestCase: # check if all the tables have heen created while True: - tdSql.query("show tables") + try: + tdSql.query("show tables") + except Exception as e: + tdLog.info("show tables test failed") + time.sleep(1) + continue + rows = tdSql.queryRows print("number of tables: %d" % rows) if(rows == self.numberOfTables): @@ -56,16 +85,23 @@ class TDTestCase: # check if there are any records in the last created table while True: print("query started") - tdSql.query("select * from test.t9") + try: + tdSql.query("select * from test.t9") + except Exception as e: + tdLog.info("select * test failed") + time.sleep(2) + continue + rows = tdSql.queryRows print("number of records: %d" % rows) if(rows > 0): break time.sleep(1) + print("alter table test.meters add column col10 int") tdSql.execute("alter table test.meters add column col10 int") - print("insert into test.t0 values (now, 1, 2, 3, 4, 0.1, 0.01,'test', '测试', TRUE, 1610000000000, 0)") - tdSql.execute("insert into test.t0 values (now, 1, 2, 3, 4, 0.1, 0.01,'test', '测试', TRUE, 1610000000000, 0)") + print("insert into test.t9 values (now, 1, 2, 3, 4, 0.1, 0.01,'test', '测试', TRUE, 1610000000000, 0)") + tdSql.execute("insert into test.t9 values (now, 1, 2, 3, 4, 0.1, 0.01,'test', '测试', TRUE, 1610000000000, 0)") def run(self): tdSql.prepare() @@ -78,6 +114,8 @@ class TDTestCase: t1.join() t2.join() + time.sleep(3) + tdSql.query("select count(*) from test.meters") tdSql.checkData(0, 0, self.numberOfRecords * self.numberOfTables + 1) diff --git a/tests/script/general/parser/alter.sim b/tests/script/general/parser/alter.sim index 31d255115e..6c88497396 100644 --- a/tests/script/general/parser/alter.sim +++ b/tests/script/general/parser/alter.sim @@ -204,7 +204,13 @@ if $data03 != NULL then return -1 endi -sql reset query cache +print ============================>TD-3366 TD-3486 +sql insert into td_3366(ts, c3, c1) using mt(t1) tags(911) values('2018-1-1 11:11:11', 'new1', 12); +sql insert into td_3486(ts, c3, c1) using mt(t1) tags(-12) values('2018-1-1 11:11:11', 'new1', 12); +sql insert into ttxu(ts, c3, c1) using mt(t1) tags('-121') values('2018-1-1 11:11:11', 'new1', 12); + +sql insert into tb(ts, c1, c3) using mt(t1) tags(123) values('2018-11-01 16:29:58.000', 2, 'port') + sql insert into tb values ('2018-11-01 16:29:58.000', 2, 'import', 3) sql import into tb values ('2018-11-01 16:29:58.000', 2, 'import', 3) sql import into tb values ('2018-11-01 16:39:58.000', 2, 'import', 3) @@ -212,6 +218,7 @@ sql select * from tb order by ts desc if $rows != 4 then return -1 endi + if $data03 != 3 then return -1 endi @@ -233,10 +240,10 @@ sql_error alter table mt add column c1 int # drop non-existing columns sql_error alter table mt drop column c9 -sql drop database $db -sql show databases -if $rows != 0 then - return -1 -endi +#sql drop database $db +#sql show databases +#if $rows != 0 then +# return -1 +#endi system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/general/parser/gendata.sh b/tests/script/general/parser/gendata.sh new file mode 100755 index 0000000000..f56fdc3468 --- /dev/null +++ b/tests/script/general/parser/gendata.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +Cur_Dir=$(pwd) +echo $Cur_Dir + +echo "'2020-1-1 1:1:1','abc','device',123,'9876', 'abc', 'net', 'mno', 'province', 'city', 'al'" >> ~/data.sql diff --git a/tests/script/general/parser/import_file.sim b/tests/script/general/parser/import_file.sim index 217679047a..a39d79af17 100644 --- a/tests/script/general/parser/import_file.sim +++ b/tests/script/general/parser/import_file.sim @@ -8,32 +8,28 @@ sql connect sleep 500 sql drop database if exists indb - sql create database if not exists indb - sql use indb $inFileName = '~/data.csv' $numOfRows = 10000 -#system sh/gendata.sh $inFileName $numOfRows # input file invalid -system sh/gendata.sh ~/data.csv $numOfRows +system general/parser/gendata.sh sql create table tbx (ts TIMESTAMP, collect_area NCHAR(12), device_id BINARY(16), imsi BINARY(16), imei BINARY(16), mdn BINARY(10), net_type BINARY(4), mno NCHAR(4), province NCHAR(10), city NCHAR(16), alarm BINARY(2)) print ====== create tables success, starting import data -sql import into tbx file $inFileName +sql import into tbx file '~/data.sql' sql select count(*) from tbx if $rows != 1 then return -1 endi -if $data00 != $numOfRows then - print "expect: $numOfRows, act: $data00" - return -1 -endi +#if $data00 != $numOfRows then +# print "expect: $numOfRows, act: $data00" +# return -1 +#endi -#system rm -f $inFileName # invalid shell -system rm -f ~/data.csv +system rm -f ~/data.sql system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/select_with_tags.sim b/tests/script/general/parser/select_with_tags.sim index 38a514a51b..da8e876577 100644 --- a/tests/script/general/parser/select_with_tags.sim +++ b/tests/script/general/parser/select_with_tags.sim @@ -159,6 +159,15 @@ if $data03 != @abc15@ then return -1 endi +sql select top(c6, 3) from select_tags_mt0 interval(10a) +sql select top(c3,10) from select_tags_mt0 interval(10a) group by tbname +sql select top(c6, 3) from select_tags_mt0 interval(10a) group by tbname; + +sql select top(c6, 10) from select_tags_mt0 interval(10a); +if $rows != 12800 then + return -1 +endi + sql select top(c1, 100), tbname, t1, t2 from select_tags_mt0; if $rows != 100 then return -1 diff --git a/tests/script/general/parser/topbot.sim b/tests/script/general/parser/topbot.sim index e23bbf6724..5c61acb2ba 100644 --- a/tests/script/general/parser/topbot.sim +++ b/tests/script/general/parser/topbot.sim @@ -325,4 +325,56 @@ if $row != 0 then return -1 endi +print ===============================>td-3621 +sql create table ttm2(ts timestamp, k bool); +sql insert into ttm2 values('2021-1-1 1:1:1', true) +sql insert into ttm2 values('2021-1-1 1:1:2', NULL) +sql insert into ttm2 values('2021-1-1 1:1:3', false) +sql select * from ttm2 where k is not null +if $row != 2 then + return -1 +endi + +if $data00 != @21-01-01 01:01:01.000@ then + print expect 21-01-01 01:01:01.000, actual $data00 + return -1 +endi + +sql select * from ttm2 where k is null +if $row != 1 then + return -1 +endi + +if $data00 != @21-01-01 01:01:02.000@ then + return -1 +endi + +sql select * from ttm2 where k=true +if $row != 1 then + return -1 +endi + +if $data00 != @21-01-01 01:01:01.000@ then + return -1 +endi + +sql select * from ttm2 where k=false +if $row != 1 then + return -1 +endi + +if $data00 != @21-01-01 01:01:03.000@ then + return -1 +endi + +sql select * from ttm2 where k<>false +if $row != 1 then + return -1 +endi + +sql_error select * from ttm2 where k=null +sql_error select * from ttm2 where k<>null +sql_error select * from ttm2 where k like null +sql_error select * from ttm2 where k ../../../sim/case.log 2>&1 && \ + ./test.sh -f $case > case.log 2>&1 && \ ( grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../../../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log || echo -e "${GREEN} success${NC}" | tee -a out.log )|| \ ( grep -q 'script.*success.*m$' ../../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log ) || \ - ( echo -e "${RED} failed${NC}" | tee -a out.log && echo '=====================log=====================' && cat ../../../sim/case.log ) + ( echo -e "${RED} failed${NC}" | tee -a out.log && echo '=====================log=====================' && cat case.log ) else echo -n $case ./test.sh -f $case > ../../sim/case.log 2>&1 && \ ( grep -q 'script.*'$case'.*failed.*, err.*lineNum' ../../sim/tsim/log/taoslog0.0 && echo -e "${RED} failed${NC}" | tee -a out.log || echo -e "${GREEN} success${NC}" | tee -a out.log )|| \ ( grep -q 'script.*success.*m$' ../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN} success${NC}" | tee -a out.log ) || \ - ( echo -e "${RED} failed${NC}" | tee -a out.log && echo '=====================log=====================' && cat ../../sim/case.log ) + ( echo -e "${RED} failed${NC}" | tee -a out.log && echo '=====================log=====================' && cat case.log ) fi out_log=`tail -1 out.log ` if [[ $out_log =~ 'failed' ]];then + rm case.log if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then cp -r ../../../sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S"` - rm -rf ../../../sim/case.log else cp -r ../../sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S" ` - rm -rf ../../sim/case.log fi dohavecore $2 1 if [[ $2 == 1 ]];then @@ -157,7 +152,6 @@ function runPyCaseOneByOne { else $line > /dev/null 2>&1 fi - dohavecore 0 fi done < $1 } @@ -183,7 +177,7 @@ function runPyCaseOneByOnefq() { start_time=`date +%s` date +%F\ %T | tee -a pytest-out.log echo -n $case - $line > ../../sim/case.log 2>&1 && \ + $line > case.log 2>&1 && \ echo -e "${GREEN} success${NC}" | tee -a pytest-out.log || \ echo -e "${RED} failed${NC}" | tee -a pytest-out.log end_time=`date +%s` @@ -191,8 +185,8 @@ function runPyCaseOneByOnefq() { if [[ $out_log =~ 'failed' ]];then cp -r ../../sim ~/sim_`date "+%Y_%m_%d_%H:%M:%S" ` echo '=====================log===================== ' - cat ../../sim/case.log - rm -rf ../../sim/case.log + cat case.log + rm -rf case.log dohavecore $2 2 if [[ $2 == 1 ]];then exit 8 @@ -463,10 +457,10 @@ if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "jdbc" ] && [ "$2" != echo "Running tests" ./apitest > /dev/null 2>&1 if [ $? != "0" ]; then - echo "prepare failed" + echo "apitest failed" totalExampleFailed=`expr $totalExampleFailed + 1` else - echo "prepare pass" + echo "apitest pass" totalExamplePass=`expr $totalExamplePass + 1` fi @@ -481,19 +475,19 @@ if [ "$2" != "sim" ] && [ "$2" != "python" ] && [ "$2" != "jdbc" ] && [ "$2" != ./subscribe -test > /dev/null 2>&1 if [ $? != "0" ]; then - echo "prepare failed" + echo "subscribe failed" totalExampleFailed=`expr $totalExampleFailed + 1` else - echo "prepare pass" + echo "subscribe pass" totalExamplePass=`expr $totalExamplePass + 1` fi yes |./asyncdemo 127.0.0.1 test 1000 10 > /dev/null 2>&1 if [ $? != "0" ]; then - echo "prepare failed" + echo "asyncdemo failed" totalExampleFailed=`expr $totalExampleFailed + 1` else - echo "prepare pass" + echo "asyncdemo pass" totalExamplePass=`expr $totalExamplePass + 1` fi diff --git a/tests/tsim/inc/sim.h b/tests/tsim/inc/sim.h index 01e5016557..58314d2e50 100644 --- a/tests/tsim/inc/sim.h +++ b/tests/tsim/inc/sim.h @@ -149,6 +149,7 @@ extern int32_t simScriptSucced; extern int32_t simDebugFlag; extern char tsScriptDir[]; extern bool simAsyncQuery; +extern bool abortExecution; SScript *simParseScript(char *fileName); SScript *simProcessCallOver(SScript *script); diff --git a/tests/tsim/src/simExe.c b/tests/tsim/src/simExe.c index dbda5f1bbd..c5b8d5c5eb 100644 --- a/tests/tsim/src/simExe.c +++ b/tests/tsim/src/simExe.c @@ -645,8 +645,12 @@ bool simCreateRestFulConnect(SScript *script, char *user, char *pass) { bool simCreateNativeConnect(SScript *script, char *user, char *pass) { simCloseTaosdConnect(script); void *taos = NULL; - taosMsleep(2000); for (int32_t attempt = 0; attempt < 10; ++attempt) { + if (abortExecution) { + script->killed = true; + return false; + } + taos = taos_connect(NULL, user, pass, NULL, tsDnodeShellPort); if (taos == NULL) { simDebug("script:%s, user:%s connect taosd failed:%s, attempt:%d", script->fileName, user, taos_errstr(NULL), @@ -697,6 +701,11 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { TAOS_RES *pSql = NULL; for (int32_t attempt = 0; attempt < 10; ++attempt) { + if (abortExecution) { + script->killed = true; + return false; + } + simLogSql(rest, false); pSql = taos_query(script->taos, rest); ret = taos_errno(pSql); diff --git a/tests/tsim/src/simMain.c b/tests/tsim/src/simMain.c index 990ea71413..6a9d96bc3b 100644 --- a/tests/tsim/src/simMain.c +++ b/tests/tsim/src/simMain.c @@ -21,10 +21,13 @@ bool simAsyncQuery = false; bool simExecSuccess = false; +bool abortExecution = false; void simHandleSignal(int32_t signo, void *sigInfo, void *context) { simSystemCleanUp(); - exit(1); + abortExecution = true; +// runningScript->killed = true; +// exit(1); } int32_t main(int32_t argc, char *argv[]) { @@ -60,6 +63,11 @@ int32_t main(int32_t argc, char *argv[]) { return -1; } + if (abortExecution) { + simError("execute abort"); + return -1; + } + simScriptList[++simScriptPos] = script; simExecuteScript(script); diff --git a/tests/tsim/src/simSystem.c b/tests/tsim/src/simSystem.c index 40937e7053..d2494eddbb 100644 --- a/tests/tsim/src/simSystem.c +++ b/tests/tsim/src/simSystem.c @@ -159,9 +159,17 @@ void *simExecuteScript(void *inputScript) { script = simScriptList[simScriptPos]; } + if (abortExecution) { + script->killed = true; + } + if (script->killed || script->linePos >= script->numOfLines) { + printf("killed ---------------------->\n"); script = simProcessCallOver(script); - if (script == NULL) break; + if (script == NULL) { + printf("abort now!\n"); + break; + } } else { SCmdLine *line = &script->lines[script->linePos]; char * option = script->optionBuffer + line->optionOffset;