diff --git a/docs/en/07-develop/03-insert-data/02-influxdb-line.mdx b/docs/en/07-develop/03-insert-data/02-influxdb-line.mdx index 11db3daeb0..c91dbba5d2 100644 --- a/docs/en/07-develop/03-insert-data/02-influxdb-line.mdx +++ b/docs/en/07-develop/03-insert-data/02-influxdb-line.mdx @@ -37,7 +37,8 @@ meters,location=California.LosAngeles,groupid=2 current=13.4,voltage=223,phase=0 - All the data in `tag_set` will be converted to nchar type automatically . - Each data in `field_set` must be self-descriptive for its data type. For example 1.2f32 means a value 1.2 of float type. Without the "f" type suffix, it will be treated as type double. - Multiple kinds of precision can be used for the `timestamp` field. Time precision can be from nanosecond (ns) to hour (h). - +- You can configure smlChildTableName in taos.cfg to specify table names, for example, `smlChildTableName=tname`. You can insert `st,tname=cpul,t1=4 c1=3 1626006833639000000` and the cpu1 table will be automatically created. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored. +- It is assumed that the order of field_set in a supertable is consistent, meaning that the first record contains all fields and subsequent records store fields in the same order. If the order is not consistent, set smlDataFormat in taos.cfg to false. Otherwise, data will be written out of order and a database error will occur.(smlDataFormat in taos.cfg default to false after version of 3.0.1.3) ::: For more details please refer to [InfluxDB Line Protocol](https://docs.influxdata.com/influxdb/v2.0/reference/syntax/line-protocol/) and [TDengine Schemaless](/reference/schemaless/#Schemaless-Line-Protocol) @@ -64,3 +65,7 @@ For more details please refer to [InfluxDB Line Protocol](https://docs.influxdat + +## Query Examples +If you want query the data of `location=California.LosAngeles,groupid=2`,here is the query sql: +select * from `meters.voltage` where location="California.LosAngeles" and groupid=2 diff --git a/docs/en/07-develop/03-insert-data/03-opentsdb-telnet.mdx b/docs/en/07-develop/03-insert-data/03-opentsdb-telnet.mdx index db9bfd7364..d88a6335cb 100644 --- a/docs/en/07-develop/03-insert-data/03-opentsdb-telnet.mdx +++ b/docs/en/07-develop/03-insert-data/03-opentsdb-telnet.mdx @@ -31,7 +31,7 @@ For example: ```txt meters.current 1648432611250 11.3 location=California.LosAngeles groupid=3 ``` - +- The defult child table name is generated by rules.You can configure smlChildTableName in taos.cfg to specify chile table names, for example, `smlChildTableName=tname`. You can insert `meters.current 1648432611250 11.3 tname=cpu1 location=California.LosAngeles groupid=3` and the cpu1 table will be automatically created. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored. Please refer to [OpenTSDB Telnet API](http://opentsdb.net/docs/build/html/api_telnet/put.html) for more details. ## Examples @@ -79,3 +79,6 @@ taos> select tbname, * from `meters.current`; t_7e7b26dd860280242c6492a16... | 2022-03-28 09:56:51.250 | 12.600000000 | 2 | California.SanFrancisco | Query OK, 4 row(s) in set (0.005399s) ``` +## Query Examples +If you want query the data of `location=California.LosAngeles groupid=3`,here is the query sql: +select * from `meters.voltage` where location="California.LosAngeles" and groupid=3 diff --git a/docs/en/07-develop/03-insert-data/04-opentsdb-json.mdx b/docs/en/07-develop/03-insert-data/04-opentsdb-json.mdx index 23703f4087..e2e6d6fc9f 100644 --- a/docs/en/07-develop/03-insert-data/04-opentsdb-json.mdx +++ b/docs/en/07-develop/03-insert-data/04-opentsdb-json.mdx @@ -48,7 +48,7 @@ Please refer to [OpenTSDB HTTP API](http://opentsdb.net/docs/build/html/api_http - In JSON protocol, strings will be converted to nchar type and numeric values will be converted to double type. - Only data in array format is accepted and so an array must be used even if there is only one row. - +- The defult child table name is generated by rules.You can configure smlChildTableName in taos.cfg to specify chile table names, for example, `smlChildTableName=tname`. You can insert `"tags": { "host": "web02","dc": "lga","tname":"cpu1"}` and the cpu1 table will be automatically created. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored. ::: ## Examples @@ -94,3 +94,6 @@ taos> select * from `meters.current`; 2022-03-28 09:56:51.250 | 12.600000000 | 2.000000000 | California.SanFrancisco | Query OK, 2 row(s) in set (0.004076s) ``` +## Query Examples +If you want query the data of "tags": {"location": "California.LosAngeles", "groupid": 1},here is the query sql: +select * from `meters.voltage` where location="California.LosAngeles" and groupid=1 diff --git a/docs/en/12-taos-sql/10-function.md b/docs/en/12-taos-sql/10-function.md index 14dedfb216..243ede5fcb 100644 --- a/docs/en/12-taos-sql/10-function.md +++ b/docs/en/12-taos-sql/10-function.md @@ -867,6 +867,7 @@ INTERP(expr) - The number of rows in the result set of `INTERP` is determined by the parameter `EVERY`. Starting from timestamp1, one interpolation is performed for every time interval specified `EVERY` parameter. - Interpolation is performed based on `FILL` parameter. - `INTERP` can only be used to interpolate in single timeline. So it must be used with `partition by tbname` when it's used on a STable. +- Pseudo column `_irowts` can be used along with `INTERP` to return the timestamps associated with interpolation points(support after version 3.0.1.4). ### LAST diff --git a/docs/en/12-taos-sql/22-meta.md b/docs/en/12-taos-sql/22-meta.md index 9bda5a0a10..0a59777946 100644 --- a/docs/en/12-taos-sql/22-meta.md +++ b/docs/en/12-taos-sql/22-meta.md @@ -11,7 +11,15 @@ TDengine includes a built-in database named `INFORMATION_SCHEMA` to provide acce 4. Future versions of TDengine can add new columns to INFORMATION_SCHEMA tables without affecting existing business systems. 5. It is easier for users coming from other database management systems. For example, Oracle users can query data dictionary tables. -Note: SHOW statements are still supported for the convenience of existing users. +:::info + +- SHOW statements are still supported for the convenience of existing users. +- Some columns in the system table may be keywords, and you need to use the escape character '\`' when querying, for example, to query the VGROUPS in the database `test`: +```sql + select `vgroups` from ins_databases where name = 'test'; +``` + +::: This document introduces the tables of INFORMATION_SCHEMA and their structure. @@ -102,7 +110,11 @@ Provides information about user-created databases. Similar to SHOW DATABASES. | 24 | wal_retention_period | INT | WAL retention period | | 25 | wal_retention_size | INT | Maximum WAL size | | 26 | wal_roll_period | INT | WAL rotation period | -| 27 | wal_segment_size | WAL file size | +| 27 | wal_segment_size | BIGINT | WAL file size | +| 28 | stt_trigger | SMALLINT | The threshold for number of files to trigger file merging | +| 29 | table_prefix | SMALLINT | The prefix length in the table name that is ignored when distributing table to vnode based on table name | +| 30 | table_suffix | SMALLINT | The suffix length in the table name that is ignored when distributing table to vnode based on table name | +| 31 | tsdb_pagesize | INT | The page size for internal storage engine, its unit is KB | ## INS_FUNCTIONS diff --git a/docs/en/14-reference/13-schemaless/13-schemaless.md b/docs/en/14-reference/13-schemaless/13-schemaless.md index 4f50c38cbb..5b7924ce56 100644 --- a/docs/en/14-reference/13-schemaless/13-schemaless.md +++ b/docs/en/14-reference/13-schemaless/13-schemaless.md @@ -47,9 +47,8 @@ In the schemaless writing data line protocol, each data item in the field_set ne - `t`, `T`, `true`, `True`, `TRUE`, `f`, `F`, `false`, and `False` will be handled directly as BOOL types. -For example, the following data rows indicate that the t1 label is "3" (NCHAR), the t2 label is "4" (NCHAR), and the t3 label -is "t3" to the super table named `st` labeled "t3" (NCHAR), write c1 column as 3 (BIGINT), c2 column as false (BOOL), c3 column -is "passit" (BINARY), c4 column is 4 (DOUBLE), and the primary key timestamp is 1626006833639000000 in one row. +For example, the following data rows write c1 column as 3 (BIGINT), c2 column as false (BOOL), c3 column +as "passit" (BINARY), c4 column as 4 (DOUBLE), and the primary key timestamp as 1626006833639000000 to child table with the t1 label as "3" (NCHAR), the t2 label as "4" (NCHAR), and the t3 label as "t3" (NCHAR) and the super table named `st`. ```json st,t1=3,t2=4,t3=t3 c1=3i64,c3="passit",c2=false,c4=4f64 1626006833639000000 @@ -69,7 +68,7 @@ Schemaless writes process row data according to the following principles. Note that tag_key1, tag_key2 are not the original order of the tags entered by the user but the result of using the tag names in ascending order of the strings. Therefore, tag_key1 is not the first tag entered in the line protocol. The string's MD5 hash value "md5_val" is calculated after the ranking is completed. The calculation result is then combined with the string to generate the table name: "t_md5_val". "t_" is a fixed prefix that every table generated by this mapping relationship has. -You can configure smlChildTableName to specify table names, for example, `smlChildTableName=tname`. You can insert `st,tname=cpul,t1=4 c1=3 1626006833639000000` and the cpu1 table will be automatically created. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored. +You can configure smlChildTableName in taos.cfg to specify table names, for example, `smlChildTableName=tname`. You can insert `st,tname=cpul,t1=4 c1=3 1626006833639000000` and the cpu1 table will be automatically created. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored. 2. If the super table obtained by parsing the line protocol does not exist, this super table is created. 3. If the subtable obtained by the parse line protocol does not exist, Schemaless creates the sub-table according to the subtable name determined in steps 1 or 2. @@ -78,7 +77,7 @@ You can configure smlChildTableName to specify table names, for example, `smlChi NULL. 6. For BINARY or NCHAR columns, if the length of the value provided in a data row exceeds the column type limit, the maximum length of characters allowed to be stored in the column is automatically increased (only incremented and not decremented) to ensure complete preservation of the data. 7. Errors encountered throughout the processing will interrupt the writing process and return an error code. -8. It is assumed that the order of field_set in a supertable is consistent, meaning that the first record contains all fields and subsequent records store fields in the same order. If the order is not consistent, set smlDataFormat to false. Otherwise, data will be written out of order and a database error will occur. +8. It is assumed that the order of field_set in a supertable is consistent, meaning that the first record contains all fields and subsequent records store fields in the same order. If the order is not consistent, set smlDataFormat in taos.cfg to false. Otherwise, data will be written out of order and a database error will occur.(smlDataFormat in taos.cfg default to false after version of 3.0.1.3) :::tip All processing logic of schemaless will still follow TDengine's underlying restrictions on data structures, such as the total length of each row of data cannot exceed diff --git a/docs/zh/07-develop/03-insert-data/02-influxdb-line.mdx b/docs/zh/07-develop/03-insert-data/02-influxdb-line.mdx index f88284ad67..a107ffb1b6 100644 --- a/docs/zh/07-develop/03-insert-data/02-influxdb-line.mdx +++ b/docs/zh/07-develop/03-insert-data/02-influxdb-line.mdx @@ -37,7 +37,9 @@ meters,location=California.LosAngeles,groupid=2 current=13.4,voltage=223,phase=0 - tag_set 中的所有的数据自动转化为 nchar 数据类型; - field_set 中的每个数据项都需要对自身的数据类型进行描述, 比如 1.2f32 代表 float 类型的数值 1.2, 如果不带类型后缀会被当作 double 处理; - timestamp 支持多种时间精度。写入数据的时候需要用参数指定时间精度,支持从小时到纳秒的 6 种时间精度。 - +- 为了提高写入的效率,默认假设同一个超级表中 field_set 的顺序是一样的(第一条数据包含所有的 field,后面的数据按照这个顺序),如果顺序不一样,需要配置参数 smlDataFormat 为 false,否则,数据写入按照相同顺序写入,库中数据会异常。(3.0.1.3之后的版本 smlDataFormat 默认为 false) [TDengine 无模式写入参考指南](/reference/schemaless/#无模式写入行协议) +- 默认生产的子表名是根据规则生成的唯一ID值。为了让用户可以指定生成的表名,可以通过在taos.cfg里配置 smlChildTableName 参数来指定。 +举例如下:配置 smlChildTableName=tname 插入数据为 st,tname=cpu1,t1=4 c1=3 1626006833639000000 则创建的表名为 cpu1,注意如果多行数据 tname 相同,但是后面的 tag_set 不同,则使用第一行自动建表时指定的 tag_set,其他的行会忽略)。[TDengine 无模式写入参考指南](/reference/schemaless/#无模式写入行协议) ::: 要了解更多可参考:[InfluxDB Line 协议官方文档](https://docs.influxdata.com/influxdb/v2.0/reference/syntax/line-protocol/) 和 [TDengine 无模式写入参考指南](/reference/schemaless/#无模式写入行协议) @@ -64,3 +66,7 @@ meters,location=California.LosAngeles,groupid=2 current=13.4,voltage=223,phase=0 + +## 查询示例 +比如查询 location=California.LosAngeles,groupid=2 子表的数据可以通过如下sql: +select * from meters where location=California.LosAngeles and groupid=2 diff --git a/docs/zh/07-develop/03-insert-data/03-opentsdb-telnet.mdx b/docs/zh/07-develop/03-insert-data/03-opentsdb-telnet.mdx index 4f63e17635..58bca7f843 100644 --- a/docs/zh/07-develop/03-insert-data/03-opentsdb-telnet.mdx +++ b/docs/zh/07-develop/03-insert-data/03-opentsdb-telnet.mdx @@ -32,6 +32,8 @@ OpenTSDB 行协议同样采用一行字符串来表示一行数据。OpenTSDB meters.current 1648432611250 11.3 location=California.LosAngeles groupid=3 ``` +- 默认生产的子表名是根据规则生成的唯一ID值。为了让用户可以指定生成的表名,可以通过在taos.cfg里配置 smlChildTableName 参数来指定。 +举例如下:配置 smlChildTableName=tname 插入数据为 meters.current 1648432611250 11.3 tname=cpu1 location=California.LosAngeles groupid=3 则创建的表名为 cpu1,注意如果多行数据 tname 相同,但是后面的 tag_set 不同,则使用第一行自动建表时指定的 tag_set,其他的行会忽略)。 参考[OpenTSDB Telnet API 文档](http://opentsdb.net/docs/build/html/api_telnet/put.html)。 ## 示例代码 @@ -79,3 +81,6 @@ taos> select tbname, * from `meters.current`; t_7e7b26dd860280242c6492a16... | 2022-03-28 09:56:51.250 | 12.600000000 | 2 | California.SanFrancisco | Query OK, 4 row(s) in set (0.005399s) ``` +## 查询示例: +想要查询 location=California.LosAngeles groupid=3 的数据,可以通过如下sql: +select * from `meters.voltage` where location="California.LosAngeles" and groupid=3 diff --git a/docs/zh/07-develop/03-insert-data/04-opentsdb-json.mdx b/docs/zh/07-develop/03-insert-data/04-opentsdb-json.mdx index b0257b9cb7..aa3e5980cf 100644 --- a/docs/zh/07-develop/03-insert-data/04-opentsdb-json.mdx +++ b/docs/zh/07-develop/03-insert-data/04-opentsdb-json.mdx @@ -48,7 +48,8 @@ OpenTSDB JSON 格式协议采用一个 JSON 字符串表示一行或多行数据 - 对于 JSON 格式协议,TDengine 并不会自动把所有标签转成 nchar 类型, 字符串将将转为 nchar 类型, 数值将同样转换为 double 类型。 - TDengine 只接收 JSON **数组格式**的字符串,即使一行数据也需要转换成数组形式。 - +- 默认生产的子表名是根据规则生成的唯一ID值。为了让用户可以指定生成的表名,可以通过在taos.cfg里配置 smlChildTableName 参数来指定。 +举例如下:配置 smlChildTableName=tname 插入数据为 "tags": { "host": "web02","dc": "lga","tname":"cpu1"} 则创建的表名为 cpu1,注意如果多行数据 tname 相同,但是后面的 tag_set 不同,则使用第一行自动建表时指定的 tag_set,其他的行会忽略)。 ::: ## 示例代码 @@ -94,3 +95,7 @@ taos> select * from `meters.current`; 2022-03-28 09:56:51.250 | 12.600000000 | 2.000000000 | California.SanFrancisco | Query OK, 2 row(s) in set (0.004076s) ``` + +## 查询示例 +想要查询"tags": {"location": "California.LosAngeles", "groupid": 1} 的数据,可以通过如下sql: +select * from `meters.voltage` where location="California.LosAngeles" and groupid=1 diff --git a/docs/zh/12-taos-sql/10-function.md b/docs/zh/12-taos-sql/10-function.md index 2a4d82045f..4c33b5232a 100644 --- a/docs/zh/12-taos-sql/10-function.md +++ b/docs/zh/12-taos-sql/10-function.md @@ -868,6 +868,7 @@ INTERP(expr) - INTERP 根据 EVERY 字段来确定输出时间范围内的结果条数,即从 timestamp1 开始每隔固定长度的时间(EVERY 值)进行插值。如果没有指定 EVERY,则默认窗口大小为无穷大,即从 timestamp1 开始只有一个窗口。 - INTERP 根据 FILL 字段来决定在每个符合输出条件的时刻如何进行插值。 - INTERP 只能在一个时间序列内进行插值,因此当作用于超级表时必须跟 partition by tbname 一起使用。 +- INTERP 可以与伪列 _irowts 一起使用,返回插值点所对应的时间戳(3.0.1.4版本以后支持)。 ### LAST diff --git a/docs/zh/12-taos-sql/22-meta.md b/docs/zh/12-taos-sql/22-meta.md index c1ffc4a757..c192d0e5e8 100644 --- a/docs/zh/12-taos-sql/22-meta.md +++ b/docs/zh/12-taos-sql/22-meta.md @@ -12,7 +12,15 @@ TDengine 内置了一个名为 `INFORMATION_SCHEMA` 的数据库,提供对数 4. TDengine 在后续演进中可以灵活的添加已有 INFORMATION_SCHEMA 中表的列,而不用担心对既有业务系统造成影响 5. 与其他数据库系统更具互操作性。例如,Oracle 数据库用户熟悉查询 Oracle 数据字典中的表 -Note: 由于 SHOW 语句已经被开发者熟悉和广泛使用,所以它们仍然被保留。 +:::info + +- 由于 SHOW 语句已经被开发者熟悉和广泛使用,所以它们仍然被保留。 +- 系统表中的一些列可能是关键字,在查询时需要使用转义符'\`',例如查询数据库 test 有几个 VGROUP: +```sql + select `vgroups` from ins_databases where name = 'test'; +``` + +::: 本章将详细介绍 `INFORMATION_SCHEMA` 这个内置元数据库中的表和表结构。 @@ -103,7 +111,11 @@ Note: 由于 SHOW 语句已经被开发者熟悉和广泛使用,所以它们 | 24 | wal_retention_period | INT | WAL 的保存时长 | | 25 | wal_retention_size | INT | WAL 的保存上限 | | 26 | wal_roll_period | INT | wal 文件切换时长 | -| 27 | wal_segment_size | wal 单个文件大小 | +| 27 | wal_segment_size | BIGINT | wal 单个文件大小 | +| 28 | stt_trigger | SMALLINT | 触发文件合并的落盘文件的个数 | +| 29 | table_prefix | SMALLINT | 内部存储引擎根据表名分配存储该表数据的 VNODE 时要忽略的前缀的长度 | +| 30 | table_suffix | SMALLINT | 内部存储引擎根据表名分配存储该表数据的 VNODE 时要忽略的后缀的长度 | +| 31 | tsdb_pagesize | INT | 时序数据存储引擎中的页大小 | ## INS_FUNCTIONS diff --git a/docs/zh/14-reference/13-schemaless/13-schemaless.md b/docs/zh/14-reference/13-schemaless/13-schemaless.md index a33abafaf8..07d5b0692b 100644 --- a/docs/zh/14-reference/13-schemaless/13-schemaless.md +++ b/docs/zh/14-reference/13-schemaless/13-schemaless.md @@ -67,9 +67,12 @@ st,t1=3,t2=4,t3=t3 c1=3i64,c3="passit",c2=false,c4=4f64 1626006833639000000 "measurement,tag_key1=tag_value1,tag_key2=tag_value2" ``` +:::tip 需要注意的是,这里的 tag_key1, tag_key2 并不是用户输入的标签的原始顺序,而是使用了标签名称按照字符串升序排列后的结果。所以,tag_key1 并不是在行协议中输入的第一个标签。 排列完成以后计算该字符串的 MD5 散列值 "md5_val"。然后将计算的结果与字符串组合生成表名:“t_md5_val”。其中的 “t_” 是固定的前缀,每个通过该映射关系自动生成的表都具有该前缀。 -为了让用户可以指定生成的表名,可以通过配置 smlChildTableName 来指定(比如 配置 smlChildTableName=tname 插入数据为 st,tname=cpu1,t1=4 c1=3 1626006833639000000 则创建的表名为 cpu1,注意如果多行数据 tname 相同,但是后面的 tag_set 不同,则使用第一次自动建表时指定的 tag_set,其他的会忽略)。 +:::tip +为了让用户可以指定生成的表名,可以通过在taos.cfg里配置 smlChildTableName 参数来指定。 +举例如下:配置 smlChildTableName=tname 插入数据为 st,tname=cpu1,t1=4 c1=3 1626006833639000000 则创建的表名为 cpu1,注意如果多行数据 tname 相同,但是后面的 tag_set 不同,则使用第一行自动建表时指定的 tag_set,其他的行会忽略)。 2. 如果解析行协议获得的超级表不存在,则会创建这个超级表(不建议手动创建超级表,不然插入数据可能异常)。 3. 如果解析行协议获得子表不存在,则 Schemaless 会按照步骤 1 或 2 确定的子表名来创建子表。 diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 8c1d957381..81882ca1ac 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -136,7 +136,7 @@ int32_t qGetQueryTableSchemaVersion(qTaskInfo_t tinfo, char* dbName, char* table * @param handle * @return */ -int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, SLocalFetch *pLocal); +int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, bool* hasMore, SLocalFetch *pLocal); int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pBlock, uint64_t* useconds); /** diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 6a4ef3d821..5792f498ef 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -19,6 +19,7 @@ #include "functionMgt.h" #include "os.h" #include "query.h" +#include "qworker.h" #include "scheduler.h" #include "tcache.h" #include "tglobal.h" @@ -27,7 +28,6 @@ #include "trpc.h" #include "tsched.h" #include "ttime.h" -#include "qworker.h" #define TSC_VAR_NOT_RELEASE 1 #define TSC_VAR_RELEASED 0 @@ -70,11 +70,10 @@ static void deregisterRequest(SRequestObj *pRequest) { int32_t currentInst = atomic_sub_fetch_64((int64_t *)&pActivity->currentRequests, 1); int32_t num = atomic_sub_fetch_32(&pTscObj->numOfReqs, 1); - int64_t nowUs = taosGetTimestampUs(); - int64_t duration = nowUs - pRequest->metric.start; - tscDebug("0x%" PRIx64 " free Request from connObj: 0x%" PRIx64 ", reqId:0x%" PRIx64 " elapsed:%" PRIu64 - " ms, current:%d, app current:%d", - pRequest->self, pTscObj->id, pRequest->requestId, duration / 1000, num, currentInst); + int64_t duration = taosGetTimestampUs() - pRequest->metric.start; + tscDebug("0x%" PRIx64 " free Request from connObj: 0x%" PRIx64 ", reqId:0x%" PRIx64 " elapsed:%.2f ms, " + "current:%d, app current:%d", + pRequest->self, pTscObj->id, pRequest->requestId, duration / 1000.0, num, currentInst); if (QUERY_NODE_VNODE_MODIF_STMT == pRequest->stmtType) { tscPerf("insert duration %" PRId64 "us: syntax:%" PRId64 "us, ctg:%" PRId64 "us, semantic:%" PRId64 @@ -85,11 +84,12 @@ static void deregisterRequest(SRequestObj *pRequest) { atomic_add_fetch_64((int64_t *)&pActivity->insertElapsedTime, duration); } else if (QUERY_NODE_SELECT_STMT == pRequest->stmtType) { tscPerf("select duration %" PRId64 "us: syntax:%" PRId64 "us, ctg:%" PRId64 "us, semantic:%" PRId64 - "us, planner:%" PRId64 "us, exec:%" PRId64 "us", + "us, planner:%" PRId64 "us, exec:%" PRId64 "us, reqId:0x%"PRIx64, duration, pRequest->metric.syntaxEnd - pRequest->metric.syntaxStart, pRequest->metric.ctgEnd - pRequest->metric.ctgStart, pRequest->metric.semanticEnd - pRequest->metric.ctgEnd, pRequest->metric.planEnd - pRequest->metric.semanticEnd, - pRequest->metric.resultReady - pRequest->metric.planEnd); + pRequest->metric.resultReady - pRequest->metric.planEnd, pRequest->requestId); + atomic_add_fetch_64((int64_t *)&pActivity->queryElapsedTime, duration); } diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index be907ea1e2..594200b5fb 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -483,8 +483,8 @@ void setResPrecision(SReqResultInfo* pResInfo, int32_t precision) { int32_t buildVnodePolicyNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray* pMnodeList, SArray* pDbVgList) { SArray* nodeList = taosArrayInit(4, sizeof(SQueryNodeLoad)); - char *policy = (tsQueryPolicy == QUERY_POLICY_VNODE) ? "vnode" : "client"; - + char* policy = (tsQueryPolicy == QUERY_POLICY_VNODE) ? "vnode" : "client"; + int32_t dbNum = taosArrayGetSize(pDbVgList); for (int32_t i = 0; i < dbNum; ++i) { SArray* pVg = taosArrayGetP(pDbVgList, i); @@ -815,7 +815,7 @@ int32_t handleCreateTbExecRes(void* res, SCatalog* pCatalog) { int32_t handleQueryExecRsp(SRequestObj* pRequest) { if (NULL == pRequest->body.resInfo.execRes.res) { - return TSDB_CODE_SUCCESS; + return pRequest->code; } SCatalog* pCatalog = NULL; @@ -868,10 +868,12 @@ int32_t handleQueryExecRsp(SRequestObj* pRequest) { return code; } +//todo refacto the error code mgmt void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) { SRequestObj* pRequest = (SRequestObj*)param; - pRequest->code = code; + STscObj* pTscObj = pRequest->pTscObj; + pRequest->code = code; pRequest->metric.resultReady = taosGetTimestampUs(); if (pResult) { @@ -879,33 +881,32 @@ void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) { memcpy(&pRequest->body.resInfo.execRes, pResult, sizeof(*pResult)); } - if (TDMT_VND_SUBMIT == pRequest->type || TDMT_VND_DELETE == pRequest->type || - TDMT_VND_CREATE_TABLE == pRequest->type) { + int32_t type = pRequest->type; + if (TDMT_VND_SUBMIT == type || TDMT_VND_DELETE == type || TDMT_VND_CREATE_TABLE == type) { if (pResult) { pRequest->body.resInfo.numOfRows = pResult->numOfRows; - if (TDMT_VND_SUBMIT == pRequest->type) { - STscObj* pTscObj = pRequest->pTscObj; + + // record the insert rows + if (TDMT_VND_SUBMIT == type) { SAppClusterSummary* pActivity = &pTscObj->pAppInfo->summary; atomic_add_fetch_64((int64_t*)&pActivity->numOfInsertRows, pResult->numOfRows); } } schedulerFreeJob(&pRequest->body.queryJob, 0); - - pRequest->metric.execEnd = taosGetTimestampUs(); } taosMemoryFree(pResult); + tscDebug("0x%" PRIx64 " enter scheduler exec cb, code:%s, reqId:0x%" PRIx64, pRequest->self, tstrerror(code), + pRequest->requestId); - tscDebug("0x%" PRIx64 " enter scheduler exec cb, code:%d - %s, reqId:0x%" PRIx64, pRequest->self, code, - tstrerror(code), pRequest->requestId); - - STscObj* pTscObj = pRequest->pTscObj; if (code != TSDB_CODE_SUCCESS && NEED_CLIENT_HANDLE_ERROR(code) && pRequest->sqlstr != NULL) { - tscDebug("0x%" PRIx64 " client retry to handle the error, code:%d - %s, tryCount:%d, reqId:0x%" PRIx64, - pRequest->self, code, tstrerror(code), pRequest->retry, pRequest->requestId); + tscDebug("0x%" PRIx64 " client retry to handle the error, code:%s, tryCount:%d, reqId:0x%" PRIx64, + pRequest->self, tstrerror(code), pRequest->retry, pRequest->requestId); pRequest->prevCode = code; schedulerFreeJob(&pRequest->body.queryJob, 0); + qDestroyQuery(pRequest->pQuery); + pRequest->pQuery = NULL; doAsyncQuery(pRequest, true); return; } @@ -915,7 +916,11 @@ void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) { removeMeta(pTscObj, pRequest->targetTableList); } - handleQueryExecRsp(pRequest); + pRequest->metric.execEnd = taosGetTimestampUs(); + int32_t code1 = handleQueryExecRsp(pRequest); + if (pRequest->code == TSDB_CODE_SUCCESS && pRequest->code != code1) { + pRequest->code = code1; + } // return to client pRequest->body.queryFp(pRequest->body.param, pRequest, code); diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 650b16f855..5ec4ddf2e5 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -20,13 +20,13 @@ #include "functionMgt.h" #include "os.h" #include "query.h" +#include "qworker.h" #include "scheduler.h" #include "tglobal.h" #include "tmsg.h" #include "tref.h" #include "trpc.h" #include "version.h" -#include "qworker.h" #define TSC_VAR_NOT_RELEASE 1 #define TSC_VAR_RELEASED 0 @@ -700,6 +700,7 @@ void retrieveMetaCallback(SMetaData *pResultMeta, void *param, int32_t code) { SQuery *pQuery = pRequest->pQuery; pRequest->metric.ctgEnd = taosGetTimestampUs(); + qDebug("0x%" PRIx64 " start to semantic analysis, reqId:0x%" PRIx64, pRequest->self, pRequest->requestId); if (code == TSDB_CODE_SUCCESS) { code = qAnalyseSqlSemantic(pWrapper->pCtx, &pWrapper->catalogReq, pResultMeta, pQuery); @@ -723,13 +724,16 @@ void retrieveMetaCallback(SMetaData *pResultMeta, void *param, int32_t code) { destorySqlParseWrapper(pWrapper); - tscDebug("0x%" PRIx64 " analysis semantics completed, start async query, reqId:0x%" PRIx64, pRequest->self, - pRequest->requestId); + double el = (pRequest->metric.semanticEnd - pRequest->metric.ctgEnd)/1000.0; + tscDebug("0x%" PRIx64 " analysis semantics completed, start async query, elapsed time:%.2f ms, reqId:0x%" PRIx64, + pRequest->self, el, pRequest->requestId); + launchAsyncQuery(pRequest, pQuery, pResultMeta); } else { destorySqlParseWrapper(pWrapper); qDestroyQuery(pRequest->pQuery); pRequest->pQuery = NULL; + if (NEED_CLIENT_HANDLE_ERROR(code)) { tscDebug("0x%" PRIx64 " client retry to handle the error, code:%d - %s, tryCount:%d, reqId:0x%" PRIx64, pRequest->self, code, tstrerror(code), pRequest->retry, pRequest->requestId); diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 2a627af324..bf110f1ae3 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -719,8 +719,8 @@ struct SDiskDataBuilder { }; int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid, - STimeWindow *pTimeWindow, SVersionRange *pVerRange, void *pBlockLoadInfo, STSchema *pSchema, - int16_t *pCols, int32_t numOfCols, const char *idStr); + STimeWindow *pTimeWindow, SVersionRange *pVerRange, SSttBlockLoadInfo *pBlockLoadInfo, + bool destroyLoadInfo, const char *idStr); void tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter); bool tMergeTreeNext(SMergeTree *pMTree); TSDBROW tMergeTreeGetRow(SMergeTree *pMTree); diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 8d1525e081..aad141a404 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -690,7 +690,8 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma while (1) { uint64_t ts; - int32_t code = qExecTaskOpt(taskInfo, pResList, &ts, NULL); + bool hasMore = false; + int32_t code = qExecTaskOpt(taskInfo, pResList, &ts, &hasMore, NULL); if (code < 0) { if (code == TSDB_CODE_QRY_IN_EXEC) { break; diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 429f4bf414..2668f8d4f6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -420,6 +420,7 @@ typedef enum { typedef struct { SFSLASTNEXTROWSTATES state; // [input] STsdb *pTsdb; // [input] + STSchema *pTSchema;// [input] tb_uid_t suid; tb_uid_t uid; int32_t nFileSet; @@ -455,9 +456,10 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow) { code = tsdbDataFReaderOpen(&state->pDataFReader, state->pTsdb, pFileSet); if (code) goto _err; + SSttBlockLoadInfo* pLoadInfo = tCreateLastBlockLoadInfo(state->pTSchema, NULL, 0); tMergeTreeOpen(&state->mergeTree, 1, state->pDataFReader, state->suid, state->uid, &(STimeWindow){.skey = TSKEY_MIN, .ekey = TSKEY_MAX}, - &(SVersionRange){.minVer = 0, .maxVer = UINT64_MAX}, NULL, NULL, NULL, 0, NULL); + &(SVersionRange){.minVer = 0, .maxVer = UINT64_MAX}, pLoadInfo,true, NULL); bool hasVal = tMergeTreeNext(&state->mergeTree); if (!hasVal) { state->state = SFSLASTNEXTROW_FILESET; @@ -892,6 +894,7 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs pIter->fsLastState.state = (SFSLASTNEXTROWSTATES)SFSNEXTROW_FS; pIter->fsLastState.pTsdb = pTsdb; pIter->fsLastState.aDFileSet = pIter->pReadSnap->fs.aDFileSet; + pIter->fsLastState.pTSchema = pTSchema; pIter->fsLastState.suid = suid; pIter->fsLastState.uid = uid; diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index 126d2d729b..7e6a0d04ff 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -474,8 +474,8 @@ static FORCE_INLINE int32_t tLDataIterCmprFn(const void *p1, const void *p2) { } int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid, - STimeWindow *pTimeWindow, SVersionRange *pVerRange, void* pBlockLoadInfo, STSchema* pSchema, - int16_t* pCols, int32_t numOfCols, const char* idStr) { + STimeWindow *pTimeWindow, SVersionRange *pVerRange, SSttBlockLoadInfo *pBlockLoadInfo, + bool destroyLoadInfo, const char *idStr) { pMTree->backward = backward; pMTree->pIter = NULL; pMTree->pIterList = taosArrayInit(4, POINTER_BYTES); @@ -488,22 +488,12 @@ int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFRead tRBTreeCreate(&pMTree->rbt, tLDataIterCmprFn); int32_t code = TSDB_CODE_SUCCESS; - SSttBlockLoadInfo* pLoadInfo = NULL; - if (pBlockLoadInfo == NULL) { - ASSERT(0); - if (pMTree->pLoadInfo == NULL) { - pMTree->destroyLoadInfo = true; - pMTree->pLoadInfo = tCreateLastBlockLoadInfo(pSchema, pCols, numOfCols); - } - - pLoadInfo = pMTree->pLoadInfo; - } else { - pLoadInfo = pBlockLoadInfo; - } + pMTree->pLoadInfo = pBlockLoadInfo; + pMTree->destroyLoadInfo = destroyLoadInfo; for (int32_t i = 0; i < pFReader->pSet->nSttF; ++i) { // open all last file struct SLDataIter* pIter = NULL; - code = tLDataIterOpen(&pIter, pFReader, i, pMTree->backward, suid, uid, pTimeWindow, pVerRange, &pLoadInfo[i]); + code = tLDataIterOpen(&pIter, pFReader, i, pMTree->backward, suid, uid, pTimeWindow, pVerRange, &pMTree->pLoadInfo[i]); if (code != TSDB_CODE_SUCCESS) { goto _end; } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index b76b6b6280..2ea6a57b9e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -951,15 +951,22 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn return TSDB_CODE_SUCCESS; } -static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockIter, SBlockData* pBlockData) { +static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockIter, SBlockData* pBlockData, uint64_t uid) { int64_t st = taosGetTimestampUs(); + tBlockDataReset(pBlockData); + TABLEID tid = {.suid = pReader->suid, .uid = uid}; + int32_t code = tBlockDataInit(pBlockData, &tid, pReader->pSchema, &pReader->suppInfo.colIds[1], pReader->suppInfo.numOfCols-1); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; ASSERT(pBlockInfo != NULL); SDataBlk* pBlock = getCurrentBlock(pBlockIter); - int32_t code = tsdbReadDataBlock(pReader->pFileReader, pBlock, pBlockData); + code = tsdbReadDataBlock(pReader->pFileReader, pBlock, pBlockData); if (code != TSDB_CODE_SUCCESS) { tsdbError("%p error occurs in loading file block, global index:%d, table index:%d, brange:%" PRId64 "-%" PRId64 ", rows:%d, code:%s %s", @@ -1998,8 +2005,7 @@ static bool initLastBlockReader(SLastBlockReader* pLBlockReader, STableBlockScan int32_t code = tMergeTreeOpen(&pLBlockReader->mergeTree, (pLBlockReader->order == TSDB_ORDER_DESC), pReader->pFileReader, - pReader->suid, pScanInfo->uid, &w, &pLBlockReader->verRange, pLBlockReader->pInfo, - pReader->pSchema, pReader->suppInfo.colIds, pReader->suppInfo.numOfCols, pReader->idStr); + pReader->suid, pScanInfo->uid, &w, &pLBlockReader->verRange, pLBlockReader->pInfo, false, pReader->idStr); if (code != TSDB_CODE_SUCCESS) { return false; } @@ -2455,14 +2461,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { ASSERT(pBlockIter->numOfBlocks == 0); code = buildComposedDataBlock(pReader); } else if (fileBlockShouldLoad(pReader, pBlockInfo, pBlock, pScanInfo, keyInBuf, pLastBlockReader)) { - tBlockDataReset(&pStatus->fileBlockData); - TABLEID tid = {.suid = pReader->suid, .uid = pScanInfo->uid}; - code = tBlockDataInit(&pStatus->fileBlockData, &tid, pReader->pSchema, NULL, 0); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData); + code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData, pScanInfo->uid); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2936,14 +2935,7 @@ static int32_t checkForNeighborFileBlock(STsdbReader* pReader, STableBlockScanIn setFileBlockActiveInBlockIter(pBlockIter, neighborIndex, step); // 3. load the neighbor block, and set it to be the currently accessed file data block - tBlockDataReset(&pStatus->fileBlockData); - TABLEID tid = {.suid = pReader->suid, .uid = pFBlock->uid}; - int32_t code = tBlockDataInit(&pStatus->fileBlockData, &tid, pReader->pSchema, NULL, 0); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData); + int32_t code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData, pFBlock->uid); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -3701,15 +3693,7 @@ static SArray* doRetrieveDataBlock(STsdbReader* pReader) { SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(&pStatus->blockIter); STableBlockScanInfo* pBlockScanInfo = taosHashGet(pStatus->pTableMap, &pFBlock->uid, sizeof(pFBlock->uid)); - tBlockDataReset(&pStatus->fileBlockData); - TABLEID tid = {.suid = pReader->suid, .uid = pBlockScanInfo->uid}; - int32_t code = tBlockDataInit(&pStatus->fileBlockData, &tid, pReader->pSchema, NULL, 0); - if (code != TSDB_CODE_SUCCESS) { - terrno = code; - return NULL; - } - - code = doLoadFileBlockData(pReader, &pStatus->blockIter, &pStatus->fileBlockData); + int32_t code = doLoadFileBlockData(pReader, &pStatus->blockIter, &pStatus->fileBlockData, pBlockScanInfo->uid); if (code != TSDB_CODE_SUCCESS) { tBlockDataDestroy(&pStatus->fileBlockData, 1); terrno = code; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 373cb451f4..0fbbc25873 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -479,7 +479,7 @@ static void freeBlock(void* param) { blockDataDestroy(pBlock); } -int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, SLocalFetch* pLocal) { +int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, bool* hasMore, SLocalFetch* pLocal) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; int64_t threadId = taosGetSelfPthreadId(); @@ -536,6 +536,7 @@ int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, SL } } + *hasMore = (pRes != NULL); uint64_t el = (taosGetTimestampUs() - st); pTaskInfo->cost.elapsedTime += el; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 12d35be0b7..b4e2b73889 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2099,12 +2099,17 @@ static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp bool hasInterp = true; for (int32_t j = 0; j < pExprSup->numOfExprs; ++j) { SExprInfo* pExprInfo = &pExprSup->pExprInfo[j]; - int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId; - int32_t dstSlot = pExprInfo->base.resSchema.slotId; - // SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot); + int32_t dstSlot = pExprInfo->base.resSchema.slotId; SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot); + if (IS_TIMESTAMP_TYPE(pExprInfo->base.resSchema.type)) { + colDataAppend(pDst, rows, (char*)&pSliceInfo->current, false); + continue; + } + + int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId; + // SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot); switch (pSliceInfo->fillType) { case TSDB_FILL_NULL: { colDataAppendNULL(pDst, rows); @@ -2350,19 +2355,24 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { if (ts == pSliceInfo->current) { for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) { SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[j]; - int32_t dstSlot = pExprInfo->base.resSchema.slotId; - int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId; - SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot); + int32_t dstSlot = pExprInfo->base.resSchema.slotId; SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot); - if (colDataIsNull_s(pSrc, i)) { - colDataAppendNULL(pDst, pResBlock->info.rows); - continue; - } + if (IS_TIMESTAMP_TYPE(pExprInfo->base.resSchema.type)) { + colDataAppend(pDst, pResBlock->info.rows, (char *)&pSliceInfo->current, false); + } else { + int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId; + SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot); - char* v = colDataGetData(pSrc, i); - colDataAppend(pDst, pResBlock->info.rows, v, false); + if (colDataIsNull_s(pSrc, i)) { + colDataAppendNULL(pDst, pResBlock->info.rows); + continue; + } + + char* v = colDataGetData(pSrc, i); + colDataAppend(pDst, pResBlock->info.rows, v, false); + } } pResBlock->info.rows += 1; @@ -2482,14 +2492,24 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { if (ts == pSliceInfo->current && pSliceInfo->current <= pSliceInfo->win.ekey) { for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) { SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[j]; - int32_t dstSlot = pExprInfo->base.resSchema.slotId; - int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId; - SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot); + int32_t dstSlot = pExprInfo->base.resSchema.slotId; SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot); - char* v = colDataGetData(pSrc, i); - colDataAppend(pDst, pResBlock->info.rows, v, false); + if (IS_TIMESTAMP_TYPE(pExprInfo->base.resSchema.type)) { + colDataAppend(pDst, pResBlock->info.rows, (char *)&pSliceInfo->current, false); + } else { + int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId; + SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot); + + if (colDataIsNull_s(pSrc, i)) { + colDataAppendNULL(pDst, pResBlock->info.rows); + continue; + } + + char* v = colDataGetData(pSrc, i); + colDataAppend(pDst, pResBlock->info.rows, v, false); + } } pResBlock->info.rows += 1; diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 3c102938d0..ae9dd82a58 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -107,9 +107,12 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) { QW_TASK_DLOG("start to execTask, loopIdx:%d", i++); // if *taskHandle is NULL, it's killed right now + bool hasMore = false; + if (taskHandle) { qwDbgSimulateSleep(); - code = qExecTaskOpt(taskHandle, pResList, &useconds, &localFetch); + + code = qExecTaskOpt(taskHandle, pResList, &useconds, &hasMore, &localFetch); if (code) { if (code != TSDB_CODE_OPS_NOT_SUPPORT) { QW_TASK_ELOG("qExecTask failed, code:%x - %s", code, tstrerror(code)); @@ -122,20 +125,8 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) { ++execNum; - if (taosArrayGetSize(pResList) == 0) { - QW_TASK_DLOG("qExecTask end with empty res, useconds:%" PRIu64, useconds); - dsEndPut(sinkHandle, useconds); - - QW_ERR_JRET(qwHandleTaskComplete(QW_FPARAMS(), ctx)); - - if (queryStop) { - *queryStop = true; - } - - break; - } - - for (int32_t j = 0; j < taosArrayGetSize(pResList); ++j) { + size_t numOfResBlock = taosArrayGetSize(pResList); + for (int32_t j = 0; j < numOfResBlock; ++j) { SSDataBlock *pRes = taosArrayGetP(pResList, j); ASSERT(pRes->info.rows > 0); @@ -149,6 +140,23 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) { QW_TASK_DLOG("data put into sink, rows:%d, continueExecTask:%d", pRes->info.rows, qcontinue); } + if (numOfResBlock == 0 || (hasMore == false)) { + if (numOfResBlock == 0) { + QW_TASK_DLOG("qExecTask end with empty res, useconds:%" PRIu64, useconds); + } else { + QW_TASK_DLOG("qExecTask done", ""); + } + + dsEndPut(sinkHandle, useconds); + QW_ERR_JRET(qwHandleTaskComplete(QW_FPARAMS(), ctx)); + + if (queryStop) { + *queryStop = true; + } + + break; + } + if (!qcontinue) { if (queryStop) { *queryStop = true; diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index 5e47c0a0ed..69495c8b7a 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -47,8 +47,7 @@ void schUpdateJobErrCode(SSchJob *pJob, int32_t errCode) { return; _return: - - SCH_JOB_DLOG("job errCode updated to %x - %s", errCode, tstrerror(errCode)); + SCH_JOB_DLOG("job errCode updated to %s", tstrerror(errCode)); } bool schJobDone(SSchJob *pJob) { @@ -491,7 +490,7 @@ int32_t schProcessOnJobFailure(SSchJob *pJob, int32_t errCode) { int32_t code = atomic_load_32(&pJob->errCode); if (code) { - SCH_JOB_DLOG("job failed with error: %s", tstrerror(code)); + SCH_JOB_DLOG("job failed with error %s", tstrerror(code)); } schPostJobRes(pJob, 0); diff --git a/tests/system-test/2-query/interp.py b/tests/system-test/2-query/interp.py index 7bf0191ec1..bee20710b5 100644 --- a/tests/system-test/2-query/interp.py +++ b/tests/system-test/2-query/interp.py @@ -356,8 +356,261 @@ class TDTestCase: tdSql.query(f"select interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:16', '2020-02-01 00:00:19') every(1s) fill(linear)") tdSql.checkRows(0) + tdLog.printNoPrefix("==========step8:test _irowts with interp") - tdLog.printNoPrefix("==========step8:test intra block interpolation") + # fill null + tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:08', '2020-02-01 00:00:12') every(500a) fill(null)") + tdSql.checkRows(9) + tdSql.checkCols(2) + + tdSql.checkData(0, 0, '2020-02-01 00:00:08.000') + tdSql.checkData(1, 0, '2020-02-01 00:00:08.500') + tdSql.checkData(2, 0, '2020-02-01 00:00:09.000') + tdSql.checkData(3, 0, '2020-02-01 00:00:09.500') + tdSql.checkData(4, 0, '2020-02-01 00:00:10.000') + tdSql.checkData(5, 0, '2020-02-01 00:00:10.500') + tdSql.checkData(6, 0, '2020-02-01 00:00:11.000') + tdSql.checkData(7, 0, '2020-02-01 00:00:11.500') + tdSql.checkData(8, 0, '2020-02-01 00:00:12.000') + + tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(null)") + tdSql.checkRows(13) + tdSql.checkCols(2) + + tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') + tdSql.checkData(1, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(2, 0, '2020-02-01 00:00:06.000') + tdSql.checkData(3, 0, '2020-02-01 00:00:07.000') + tdSql.checkData(4, 0, '2020-02-01 00:00:08.000') + tdSql.checkData(5, 0, '2020-02-01 00:00:09.000') + tdSql.checkData(6, 0, '2020-02-01 00:00:10.000') + tdSql.checkData(7, 0, '2020-02-01 00:00:11.000') + tdSql.checkData(8, 0, '2020-02-01 00:00:12.000') + tdSql.checkData(9, 0, '2020-02-01 00:00:13.000') + tdSql.checkData(10, 0, '2020-02-01 00:00:14.000') + tdSql.checkData(11, 0, '2020-02-01 00:00:15.000') + tdSql.checkData(12, 0, '2020-02-01 00:00:16.000') + + tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:05', '2020-02-01 00:00:15') every(2s) fill(null)") + tdSql.checkRows(6) + tdSql.checkCols(2) + + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(1, 0, '2020-02-01 00:00:07.000') + tdSql.checkData(2, 0, '2020-02-01 00:00:09.000') + tdSql.checkData(3, 0, '2020-02-01 00:00:11.000') + tdSql.checkData(4, 0, '2020-02-01 00:00:13.000') + tdSql.checkData(5, 0, '2020-02-01 00:00:15.000') + + # fill value + tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:08', '2020-02-01 00:00:12') every(500a) fill(value, 1)") + tdSql.checkRows(9) + tdSql.checkCols(2) + + tdSql.checkData(0, 0, '2020-02-01 00:00:08.000') + tdSql.checkData(1, 0, '2020-02-01 00:00:08.500') + tdSql.checkData(2, 0, '2020-02-01 00:00:09.000') + tdSql.checkData(3, 0, '2020-02-01 00:00:09.500') + tdSql.checkData(4, 0, '2020-02-01 00:00:10.000') + tdSql.checkData(5, 0, '2020-02-01 00:00:10.500') + tdSql.checkData(6, 0, '2020-02-01 00:00:11.000') + tdSql.checkData(7, 0, '2020-02-01 00:00:11.500') + tdSql.checkData(8, 0, '2020-02-01 00:00:12.000') + + tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(value, 1)") + tdSql.checkRows(13) + tdSql.checkCols(2) + + tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') + tdSql.checkData(1, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(2, 0, '2020-02-01 00:00:06.000') + tdSql.checkData(3, 0, '2020-02-01 00:00:07.000') + tdSql.checkData(4, 0, '2020-02-01 00:00:08.000') + tdSql.checkData(5, 0, '2020-02-01 00:00:09.000') + tdSql.checkData(6, 0, '2020-02-01 00:00:10.000') + tdSql.checkData(7, 0, '2020-02-01 00:00:11.000') + tdSql.checkData(8, 0, '2020-02-01 00:00:12.000') + tdSql.checkData(9, 0, '2020-02-01 00:00:13.000') + tdSql.checkData(10, 0, '2020-02-01 00:00:14.000') + tdSql.checkData(11, 0, '2020-02-01 00:00:15.000') + tdSql.checkData(12, 0, '2020-02-01 00:00:16.000') + + tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:05', '2020-02-01 00:00:15') every(2s) fill(value, 1)") + tdSql.checkRows(6) + tdSql.checkCols(2) + + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(1, 0, '2020-02-01 00:00:07.000') + tdSql.checkData(2, 0, '2020-02-01 00:00:09.000') + tdSql.checkData(3, 0, '2020-02-01 00:00:11.000') + tdSql.checkData(4, 0, '2020-02-01 00:00:13.000') + tdSql.checkData(5, 0, '2020-02-01 00:00:15.000') + + # fill prev + tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:08', '2020-02-01 00:00:12') every(500a) fill(prev)") + tdSql.checkRows(9) + tdSql.checkCols(2) + + tdSql.checkData(0, 0, '2020-02-01 00:00:08.000') + tdSql.checkData(1, 0, '2020-02-01 00:00:08.500') + tdSql.checkData(2, 0, '2020-02-01 00:00:09.000') + tdSql.checkData(3, 0, '2020-02-01 00:00:09.500') + tdSql.checkData(4, 0, '2020-02-01 00:00:10.000') + tdSql.checkData(5, 0, '2020-02-01 00:00:10.500') + tdSql.checkData(6, 0, '2020-02-01 00:00:11.000') + tdSql.checkData(7, 0, '2020-02-01 00:00:11.500') + tdSql.checkData(8, 0, '2020-02-01 00:00:12.000') + + tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(prev)") + tdSql.checkRows(12) + tdSql.checkCols(2) + + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(1, 0, '2020-02-01 00:00:06.000') + tdSql.checkData(2, 0, '2020-02-01 00:00:07.000') + tdSql.checkData(3, 0, '2020-02-01 00:00:08.000') + tdSql.checkData(4, 0, '2020-02-01 00:00:09.000') + tdSql.checkData(5, 0, '2020-02-01 00:00:10.000') + tdSql.checkData(6, 0, '2020-02-01 00:00:11.000') + tdSql.checkData(7, 0, '2020-02-01 00:00:12.000') + tdSql.checkData(8, 0, '2020-02-01 00:00:13.000') + tdSql.checkData(9, 0, '2020-02-01 00:00:14.000') + tdSql.checkData(10, 0, '2020-02-01 00:00:15.000') + tdSql.checkData(11, 0, '2020-02-01 00:00:16.000') + + tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:05', '2020-02-01 00:00:15') every(2s) fill(prev)") + tdSql.checkRows(6) + tdSql.checkCols(2) + + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(1, 0, '2020-02-01 00:00:07.000') + tdSql.checkData(2, 0, '2020-02-01 00:00:09.000') + tdSql.checkData(3, 0, '2020-02-01 00:00:11.000') + tdSql.checkData(4, 0, '2020-02-01 00:00:13.000') + tdSql.checkData(5, 0, '2020-02-01 00:00:15.000') + + # fill next + tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:08', '2020-02-01 00:00:12') every(500a) fill(next)") + tdSql.checkRows(9) + tdSql.checkCols(2) + + tdSql.checkData(0, 0, '2020-02-01 00:00:08.000') + tdSql.checkData(1, 0, '2020-02-01 00:00:08.500') + tdSql.checkData(2, 0, '2020-02-01 00:00:09.000') + tdSql.checkData(3, 0, '2020-02-01 00:00:09.500') + tdSql.checkData(4, 0, '2020-02-01 00:00:10.000') + tdSql.checkData(5, 0, '2020-02-01 00:00:10.500') + tdSql.checkData(6, 0, '2020-02-01 00:00:11.000') + tdSql.checkData(7, 0, '2020-02-01 00:00:11.500') + tdSql.checkData(8, 0, '2020-02-01 00:00:12.000') + + tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(next)") + tdSql.checkRows(12) + tdSql.checkCols(2) + + tdSql.checkData(0, 0, '2020-02-01 00:00:04.000') + tdSql.checkData(1, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(2, 0, '2020-02-01 00:00:06.000') + tdSql.checkData(3, 0, '2020-02-01 00:00:07.000') + tdSql.checkData(4, 0, '2020-02-01 00:00:08.000') + tdSql.checkData(5, 0, '2020-02-01 00:00:09.000') + tdSql.checkData(6, 0, '2020-02-01 00:00:10.000') + tdSql.checkData(7, 0, '2020-02-01 00:00:11.000') + tdSql.checkData(8, 0, '2020-02-01 00:00:12.000') + tdSql.checkData(9, 0, '2020-02-01 00:00:13.000') + tdSql.checkData(10, 0, '2020-02-01 00:00:14.000') + tdSql.checkData(11, 0, '2020-02-01 00:00:15.000') + + tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:05', '2020-02-01 00:00:15') every(2s) fill(next)") + tdSql.checkRows(6) + tdSql.checkCols(2) + + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(1, 0, '2020-02-01 00:00:07.000') + tdSql.checkData(2, 0, '2020-02-01 00:00:09.000') + tdSql.checkData(3, 0, '2020-02-01 00:00:11.000') + tdSql.checkData(4, 0, '2020-02-01 00:00:13.000') + tdSql.checkData(5, 0, '2020-02-01 00:00:15.000') + + # fill linear + tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:08', '2020-02-01 00:00:12') every(500a) fill(linear)") + tdSql.checkRows(9) + tdSql.checkCols(2) + + tdSql.checkData(0, 0, '2020-02-01 00:00:08.000') + tdSql.checkData(1, 0, '2020-02-01 00:00:08.500') + tdSql.checkData(2, 0, '2020-02-01 00:00:09.000') + tdSql.checkData(3, 0, '2020-02-01 00:00:09.500') + tdSql.checkData(4, 0, '2020-02-01 00:00:10.000') + tdSql.checkData(5, 0, '2020-02-01 00:00:10.500') + tdSql.checkData(6, 0, '2020-02-01 00:00:11.000') + tdSql.checkData(7, 0, '2020-02-01 00:00:11.500') + tdSql.checkData(8, 0, '2020-02-01 00:00:12.000') + + tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(linear)") + tdSql.checkRows(11) + tdSql.checkCols(2) + + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(1, 0, '2020-02-01 00:00:06.000') + tdSql.checkData(2, 0, '2020-02-01 00:00:07.000') + tdSql.checkData(3, 0, '2020-02-01 00:00:08.000') + tdSql.checkData(4, 0, '2020-02-01 00:00:09.000') + tdSql.checkData(5, 0, '2020-02-01 00:00:10.000') + tdSql.checkData(6, 0, '2020-02-01 00:00:11.000') + tdSql.checkData(7, 0, '2020-02-01 00:00:12.000') + tdSql.checkData(8, 0, '2020-02-01 00:00:13.000') + tdSql.checkData(9, 0, '2020-02-01 00:00:14.000') + tdSql.checkData(10, 0, '2020-02-01 00:00:15.000') + + tdSql.query(f"select _irowts,interp(c0) from {dbname}.{tbname} range('2020-02-01 00:00:05', '2020-02-01 00:00:15') every(2s) fill(linear)") + tdSql.checkRows(6) + tdSql.checkCols(2) + + tdSql.checkData(0, 0, '2020-02-01 00:00:05.000') + tdSql.checkData(1, 0, '2020-02-01 00:00:07.000') + tdSql.checkData(2, 0, '2020-02-01 00:00:09.000') + tdSql.checkData(3, 0, '2020-02-01 00:00:11.000') + tdSql.checkData(4, 0, '2020-02-01 00:00:13.000') + tdSql.checkData(5, 0, '2020-02-01 00:00:15.000') + + # multiple _irowts + tdSql.query(f"select interp(c0),_irowts from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(linear)") + tdSql.checkRows(11) + tdSql.checkCols(2) + + tdSql.checkData(0, 1, '2020-02-01 00:00:05.000') + tdSql.checkData(1, 1, '2020-02-01 00:00:06.000') + tdSql.checkData(2, 1, '2020-02-01 00:00:07.000') + tdSql.checkData(3, 1, '2020-02-01 00:00:08.000') + tdSql.checkData(4, 1, '2020-02-01 00:00:09.000') + tdSql.checkData(5, 1, '2020-02-01 00:00:10.000') + tdSql.checkData(6, 1, '2020-02-01 00:00:11.000') + tdSql.checkData(7, 1, '2020-02-01 00:00:12.000') + tdSql.checkData(8, 1, '2020-02-01 00:00:13.000') + tdSql.checkData(9, 1, '2020-02-01 00:00:14.000') + tdSql.checkData(10, 1, '2020-02-01 00:00:15.000') + + tdSql.query(f"select _irowts, interp(c0), interp(c0), _irowts from {dbname}.{tbname} range('2020-02-01 00:00:04', '2020-02-01 00:00:16') every(1s) fill(linear)") + tdSql.checkRows(11) + tdSql.checkCols(4) + + cols = (0, 3) + for i in cols: + tdSql.checkData(0, i, '2020-02-01 00:00:05.000') + tdSql.checkData(1, i, '2020-02-01 00:00:06.000') + tdSql.checkData(2, i, '2020-02-01 00:00:07.000') + tdSql.checkData(3, i, '2020-02-01 00:00:08.000') + tdSql.checkData(4, i, '2020-02-01 00:00:09.000') + tdSql.checkData(5, i, '2020-02-01 00:00:10.000') + tdSql.checkData(6, i, '2020-02-01 00:00:11.000') + tdSql.checkData(7, i, '2020-02-01 00:00:12.000') + tdSql.checkData(8, i, '2020-02-01 00:00:13.000') + tdSql.checkData(9, i, '2020-02-01 00:00:14.000') + tdSql.checkData(10, i, '2020-02-01 00:00:15.000') + + + tdLog.printNoPrefix("==========step9:test intra block interpolation") tdSql.execute(f"drop database {dbname}"); tdSql.prepare() @@ -551,7 +804,7 @@ class TDTestCase: tdSql.checkData(0, 0, 15) tdSql.checkData(1, 0, 15) - tdLog.printNoPrefix("==========step9:test multi-interp cases") + tdLog.printNoPrefix("==========step10:test multi-interp cases") tdSql.query(f"select interp(c0),interp(c1),interp(c2),interp(c3) from {dbname}.{tbname} range('2020-02-09 00:00:05', '2020-02-13 00:00:05') every(1d) fill(null)") tdSql.checkRows(5) tdSql.checkCols(4) @@ -601,7 +854,7 @@ class TDTestCase: for i in range (tdSql.queryCols): tdSql.checkData(0, i, 13) - tdLog.printNoPrefix("==========step10:test error cases") + tdLog.printNoPrefix("==========step11:test error cases") tdSql.error(f"select interp(c0) from {dbname}.{tbname}") tdSql.error(f"select interp(c0) from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05')")