diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 32e9b11520..1b04e40f2a 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -333,6 +333,11 @@ pipeline { } stages { stage('run test') { + when { + allOf { + not { expression { env.CHANGE_BRANCH =~ /docs\// }} + } + } parallel { stage('windows test') { agent{label " windows10_01 || windows10_02 || windows10_03 || windows10_04 "} diff --git a/docs/en/12-taos-sql/14-limit.md b/docs/en/12-taos-sql/14-limit.md index db55cdd69e..e8bb77fc27 100644 --- a/docs/en/12-taos-sql/14-limit.md +++ b/docs/en/12-taos-sql/14-limit.md @@ -1,5 +1,5 @@ --- -title: Limits & Restrictions +title: Naming & Restrictions --- ## Naming Rules @@ -16,41 +16,21 @@ The legal character set is `[a-zA-Z0-9!?$%^&*()_–+={[}]:;@~#|<,>.?/]`. ## General Limits -- Maximum length of database name is 32 bytes. +- Maximum length of database name is 32 bytes, and it can't include "." or special characters. - Maximum length of table name is 192 bytes, excluding the database name prefix and the separator. -- Maximum length of each data row is 48K bytes since version 2.1.7.0 , before which the limit was 16K bytes. Please note that the upper limit includes the extra 2 bytes consumed by each column of BINARY/NCHAR type. +- Maximum length of each data row is 48K bytes. Please note that the upper limit includes the extra 2 bytes consumed by each column of BINARY/NCHAR type. - Maximum length of column name is 64. - Maximum number of columns is 4096. There must be at least 2 columns, and the first column must be timestamp. - Maximum length of tag name is 64. - Maximum number of tags is 128. There must be at least 1 tag. The total length of tag values should not exceed 16K bytes. - Maximum length of singe SQL statement is 1048576, i.e. 1 MB. It can be configured in the parameter `maxSQLLength` in the client side, the applicable range is [65480, 1048576]. -- At most 4096 columns (or 1024 prior to 2.1.7.0) can be returned by `SELECT`. Functions in the query statement constitute columns. An error is returned if the limit is exceeded. +- At most 4096 columns can be returned by `SELECT`. Functions in the query statement constitute columns. An error is returned if the limit is exceeded. - Maximum numbers of databases, STables, tables are dependent only on the system resources. -- Maximum of database name is 32 bytes, and it can't include "." or special characters. - Maximum number of replicas for a database is 3. - Maximum length of user name is 23 bytes. - Maximum length of password is 15 bytes. - Maximum number of rows depends only on the storage space. -- Maximum number of tables depends only on the number of nodes. -- Maximum number of databases depends only on the number of nodes. -- Maximum number of vnodes for a single database is 64. - -## Restrictions of `GROUP BY` - -`GROUP BY` can be performed on tags and `TBNAME`. It can be performed on data columns too, with the only restriction being it can only be performed on one data column and the number of unique values in that column is lower than 100,000. Please note that `GROUP BY` cannot be performed on float or double types. - -## Restrictions of `IS NOT NULL` - -`IS NOT NULL` can be used on any data type of columns. The non-empty string evaluation expression, i.e. `< > ""` can only be used on non-numeric data types. - -## Restrictions of `ORDER BY` - -- Only one `order by` is allowed for normal table and subtable. -- At most two `order by` are allowed for STable, and the second one must be `ts`. -- `order by tag` must be used with `group by tag` on same tag. This rule is also applicable to `tbname`. -- `order by column` must be used with `group by column` or `top/bottom` on same column. This rule is applicable to table and STable. -- `order by ts` is applicable to table and STable. -- If `order by ts` is used with `group by`, the result set is sorted using `ts` in each group. +- Maximum number of vnodes for a single database is 1024. ## Restrictions of Table/Column Names @@ -71,7 +51,3 @@ For example: The characters inside escape characters must be printable characters. ::: - -### Applicable Versions - -Escape character "\`" is available from version 2.3.0.1. diff --git a/docs/zh/12-taos-sql/12-interval.md b/docs/zh/12-taos-sql/12-interval.md index b0619ea5ce..0fece8a7d4 100644 --- a/docs/zh/12-taos-sql/12-interval.md +++ b/docs/zh/12-taos-sql/12-interval.md @@ -1,13 +1,80 @@ --- -sidebar_label: 按窗口切分聚合 -title: 按窗口切分聚合 +sidebar_label: 时序数据特色查询 +title: 时序数据特色查询 --- +TDengine是专为时序数据而研发的大数据平台，存储和计算都针对时序数据的特定进行了量身定制，在支持标准SQL的基础之上，还提供了一系列贴合时序业务场景的特色查询语法，极大的方便时序场景的应用开发。 -TDengine 支持按时间段窗口切分方式进行聚合结果查询，比如温度传感器每秒采集一次数据，但需查询每隔 10 分钟的温度平均值。这种场景下可以使用窗口子句来获得需要的查询结果。 -窗口子句用于针对查询的数据集合进行按照窗口切分成为查询子集并进行聚合，窗口包含时间窗口（time window）、状态窗口（status window）、会话窗口（session window）三种窗口。其中时间窗口又可划分为滑动时间窗口和翻转时间窗口。 +TDengine提供的特色查询包括标签切分查询和窗口切分查询。 -## 时间窗口 +## 标签切分查询 + +超级表查询中，当需要针对标签进行数据切分然后在切分出的数据空间内再进行一系列的计算时使用标签切分子句，标签切分的语句如下： + +```sql +PARTITION BY tag_list +``` +其中 `tag_list` 是标签列的列表，还可以包括tbname伪列。 + +TDengine按如下方式处理标签切分子句： + +标签切分子句位于 `WHERE` 子句之后，且不能和 `JOIN` 子句一起使用。 +标签切分子句将超级表数据按指定的标签组合进行切分，然后对每个切分的分片进行指定的计算。计算由之后的子句定义（窗口子句、`GROUP BY` 子句或`SELECT` 子句）。 +标签切分子句可以和窗口切分子句（或 `GROUP BY` 子句）一起使用，此时后面的子句作用在每个切分的分片上。例如，下面的示例将数据按标签 `location` 进行分组，并对每个组按10分钟进行降采样，取其最大值。 + +```sql +select max(current) from meters partition by location interval(10m) +``` + +## 窗口切分查询 + +TDengine 支持按时间段窗口切分方式进行聚合结果查询，比如温度传感器每秒采集一次数据，但需查询每隔 10 分钟的温度平均值。这种场景下可以使用窗口子句来获得需要的查询结果。窗口子句用于针对查询的数据集合按照窗口切分成为查询子集并进行聚合，窗口包含时间窗口（time window）、状态窗口（status window）、会话窗口（session window）三种窗口。其中时间窗口又可划分为滑动时间窗口和翻转时间窗口。窗口切分查询语法如下： + +```sql +SELECT function_list FROM tb_name + [WHERE where_condition] + [SESSION(ts_col, tol_val)] + [STATE_WINDOW(col)] + [INTERVAL(interval [, offset]) [SLIDING sliding]] + [FILL({NONE | VALUE | PREV | NULL | LINEAR | NEXT})] +``` + +在上述语法中的具体限制如下 + +### 窗口切分查询中使用函数的限制 +- 在聚合查询中，function_list 位置允许使用聚合和选择函数，并要求每个函数仅输出单个结果（例如：COUNT、AVG、SUM、STDDEV、LEASTSQUARES、PERCENTILE、MIN、MAX、FIRST、LAST），而不能使用具有多行输出结果的函数（例如：DIFF 以及四则运算）。 +- 此外 LAST_ROW 查询也不能与窗口聚合同时出现。 +- 标量函数（如：CEIL/FLOOR 等）也不能使用在窗口聚合查询中。 + +### 窗口子句的规则 +- 窗口子句位于标签切分子句之后，GROUP BY子句之前，且不可以和GROUP BY子句一起使用。 +- 窗口子句将数据按窗口进行切分，对每个窗口进行SELECT列表中的表达式的计算，SELECT列表中的表达式只能包含： + - 常量。 + - 聚集函数。 + - 包含上面表达式的表达式。 +- 窗口子句不可以和GROUP BY子句一起使用。 +- WHERE 语句可以指定查询的起止时间和其他过滤条件。 + +### FILL 子句 + FILL 语句指定某一窗口区间数据缺失的情况下的填充模式。填充模式包括以下几种： + 1. 不进行填充：NONE（默认填充模式）。 + 2. VALUE 填充：固定值填充，此时需要指定填充的数值。例如：FILL(VALUE, 1.23)。这里需要注意，最终填充的值受由相应列的类型决定，如FILL(VALUE, 1.23)，相应列为INT类型，则填充值为1。 + 3. PREV 填充：使用前一个非 NULL 值填充数据。例如：FILL(PREV)。 + 4. NULL 填充：使用 NULL 填充数据。例如：FILL(NULL)。 + 5. LINEAR 填充：根据前后距离最近的非 NULL 值做线性插值填充。例如：FILL(LINEAR)。 + 6. NEXT 填充：使用下一个非 NULL 值填充数据。例如：FILL(NEXT)。 + +:::info + +1. 使用 FILL 语句的时候可能生成大量的填充输出，务必指定查询的时间区间。针对每次查询，系统可返回不超过 1 千万条具有插值的结果。 +2. 在时间维度聚合中，返回的结果中时间序列严格单调递增。 +3. 如果查询对象是超级表，则聚合函数会作用于该超级表下满足值过滤条件的所有表的数据。如果查询中没有使用 GROUP BY 语句，则返回的结果按照时间序列严格单调递增；如果查询中使用了 GROUP BY 语句分组，则返回结果中每个 GROUP 内不按照时间序列严格单调递增。 + +::: + +### 时间窗口 + +时间窗口又可分为滑动时间窗口和翻转时间窗口。 INTERVAL 子句用于产生相等时间周期的窗口，SLIDING 用以指定窗口向前滑动的时间。每次执行的查询是一个时间窗口，时间窗口随着时间流动向前滑动。在定义连续查询的时候需要指定时间窗口（time window ）大小和每次前向增量时间（forward sliding times）。如图，[t0s, t0e] ，[t1s , t1e]， [t2s, t2e] 是分别是执行三次连续查询的时间窗口范围，窗口的前向滑动的时间范围 sliding time 标识 。查询过滤、聚合等操作按照每个时间窗口为独立的单位执行。当 SLIDING 与 INTERVAL 相等的时候，滑动窗口即为翻转窗口。 @@ -25,11 +92,12 @@ SLIDING 的向前滑动的时间不能超过一个窗口的时间范围。以下 SELECT COUNT(*) FROM temp_tb_1 INTERVAL(1m) SLIDING(2m); ``` -当 SLIDING 与 INTERVAL 取值相等的时候，滑动窗口即为翻转窗口。 -_ 聚合时间段的窗口宽度由关键词 INTERVAL 指定，最短时间间隔 10 毫秒（10a）；并且支持偏移 offset（偏移必须小于间隔），也即时间窗口划分与“UTC 时刻 0”相比的偏移量。SLIDING 语句用于指定聚合时间段的前向增量，也即每次窗口向前滑动的时长。 -_ 从 2.1.5.0 版本开始，INTERVAL 语句允许的最短时间间隔调整为 1 微秒（1u），当然如果所查询的 DATABASE 的时间精度设置为毫秒级，那么允许的最短时间间隔为 1 毫秒（1a）。 \* **注意**：用到 INTERVAL 语句时，除非极特殊的情况，都要求把客户端和服务端的 taos.cfg 配置文件中的 timezone 参数配置为相同的取值，以避免时间处理函数频繁进行跨时区转换而导致的严重性能影响。 +使用时间窗口需要注意： +- 聚合时间段的窗口宽度由关键词 INTERVAL 指定，最短时间间隔 10 毫秒（10a）；并且支持偏移 offset（偏移必须小于间隔），也即时间窗口划分与“UTC 时刻 0”相比的偏移量。SLIDING 语句用于指定聚合时间段的前向增量，也即每次窗口向前滑动的时长。 +- 使用 INTERVAL 语句时，除非极特殊的情况，都要求把客户端和服务端的 taos.cfg 配置文件中的 timezone 参数配置为相同的取值，以避免时间处理函数频繁进行跨时区转换而导致的严重性能影响。 +- 返回的结果中时间序列严格单调递增。 -## 状态窗口 +### 状态窗口 使用整数（布尔值）或字符串来标识产生记录时候设备的状态量。产生的记录如果具有相同的状态量数值则归属于同一个状态窗口，数值改变后该窗口关闭。如下图所示，根据状态量确定的状态窗口分别是[2019-04-28 14:22:07，2019-04-28 14:22:10]和[2019-04-28 14:22:11，2019-04-28 14:22:12]两个。（状态窗口暂不支持对超级表使用） @@ -41,7 +109,7 @@ _ 从 2.1.5.0 版本开始，INTERVAL 语句允许的最短时间间隔调整为 SELECT COUNT(*), FIRST(ts), status FROM temp_tb_1 STATE_WINDOW(status); ``` -## 会话窗口 +### 会话窗口 会话窗口根据记录的时间戳主键的值来确定是否属于同一个会话。如下图所示，如果设置时间戳的连续的间隔小于等于 12 秒，则以下 6 条记录构成 2 个会话窗口，分别是：[2019-04-28 14:22:10，2019-04-28 14:22:30]和[2019-04-28 14:23:10，2019-04-28 14:23:30]。因为 2019-04-28 14:22:30 与 2019-04-28 14:23:10 之间的时间间隔是 40 秒，超过了连续时间间隔（12 秒）。 @@ -54,48 +122,7 @@ SELECT COUNT(*), FIRST(ts), status FROM temp_tb_1 STATE_WINDOW(status); SELECT COUNT(*), FIRST(ts) FROM temp_tb_1 SESSION(ts, tol_val); ``` -这种类型的查询语法如下： - -``` -SELECT function_list FROM tb_name - [WHERE where_condition] - [SESSION(ts_col, tol_val)] - [STATE_WINDOW(col)] - [INTERVAL(interval [, offset]) [SLIDING sliding]] - [FILL({NONE | VALUE | PREV | NULL | LINEAR | NEXT})] - -SELECT function_list FROM stb_name - [WHERE where_condition] - [INTERVAL(interval [, offset]) [SLIDING sliding]] - [FILL({NONE | VALUE | PREV | NULL | LINEAR | NEXT})] - [GROUP BY tags] -``` - -- 在聚合查询中，function_list 位置允许使用聚合和选择函数，并要求每个函数仅输出单个结果（例如：COUNT、AVG、SUM、STDDEV、LEASTSQUARES、PERCENTILE、MIN、MAX、FIRST、LAST），而不能使用具有多行输出结果的函数（例如：DIFF 以及四则运算）。 -- 此外 LAST_ROW 查询也不能与窗口聚合同时出现。 -- 标量函数（如：CEIL/FLOOR 等）也不能使用在窗口聚合查询中。 -- - -- WHERE 语句可以指定查询的起止时间和其他过滤条件。 -- FILL 语句指定某一窗口区间数据缺失的情况下的填充模式。填充模式包括以下几种： - 1. 不进行填充：NONE（默认填充模式）。 - 2. VALUE 填充：固定值填充，此时需要指定填充的数值。例如：FILL(VALUE, 1.23)。 - 3. PREV 填充：使用前一个非 NULL 值填充数据。例如：FILL(PREV)。 - 4. NULL 填充：使用 NULL 填充数据。例如：FILL(NULL)。 - 5. LINEAR 填充：根据前后距离最近的非 NULL 值做线性插值填充。例如：FILL(LINEAR)。 - 6. NEXT 填充：使用下一个非 NULL 值填充数据。例如：FILL(NEXT)。 - -:::info - -1. 使用 FILL 语句的时候可能生成大量的填充输出，务必指定查询的时间区间。针对每次查询，系统可返回不超过 1 千万条具有插值的结果。 -2. 在时间维度聚合中，返回的结果中时间序列严格单调递增。 -3. 如果查询对象是超级表，则聚合函数会作用于该超级表下满足值过滤条件的所有表的数据。如果查询中没有使用 GROUP BY 语句，则返回的结果按照时间序列严格单调递增；如果查询中使用了 GROUP BY 语句分组，则返回结果中每个 GROUP 内不按照时间序列严格单调递增。 - -::: - -时间聚合也常被用于连续查询场景，可以参考文档 [连续查询(Continuous Query)](/develop/continuous-query)。 - -## 示例 +### 示例 智能电表的建表语句如下： diff --git a/docs/zh/12-taos-sql/14-limit.md b/docs/zh/12-taos-sql/14-limit.md index 7673e24a83..95d1837ac6 100644 --- a/docs/zh/12-taos-sql/14-limit.md +++ b/docs/zh/12-taos-sql/14-limit.md @@ -1,37 +1,43 @@ --- -sidebar_label: 边界限制 -title: 边界限制 +sidebar_label: 命名与边界限制 +title: 命名与边界限制 --- +## 名称命名规则 + +1. 合法字符：英文字符、数字和下划线 +2. 允许英文字符或下划线开头，不允许以数字开头 +3. 不区分大小写 +4. 转义后表（列）名规则： + 为了兼容支持更多形式的表（列）名，TDengine 引入新的转义符 "`"。可用让表名与关键词不冲突，同时不受限于上述表名称合法性约束检查。 + 转义后的表（列）名同样受到长度限制要求，且长度计算的时候不计算转义符。使用转义字符以后，不再对转义字符中的内容进行大小写统一。 + + 例如：\`aBc\` 和 \`abc\` 是不同的表（列）名，但是 abc 和 aBc 是相同的表（列）名。 + 需要注意的是转义字符中的内容必须是可打印字符。 + +## 密码合法字符集 + +`[a-zA-Z0-9!?$%^&*()_–+={[}]:;@~#|<,>.?/]` + +去掉了 `` ‘“`\ `` (单双引号、撇号、反斜杠、空格) + ## 一般限制 - 数据库名最大长度为 32。 - 表名最大长度为 192，不包括数据库名前缀和分隔符 - 每行数据最大长度 48KB （注意：数据行内每个 BINARY/NCHAR 类型的列还会额外占用 2 个字节的存储位置）。 -- 列名最大长度为 64，最多允许 4096 列，最少需要 2 列，第一列必须是时间戳。注：从 2.1.7.0 版本（不含)以前最多允许 4096 列 -- 标签名最大长度为 64，最多允许 128 个，至少要有 1 个标签，一个表中标签值的总长度不超过 16KB 。 +- 列名最大长度为 64 +- 最多允许 4096 列，最少需要 2 列，第一列必须是时间戳。 +- 标签名最大长度为 64 +- 最多允许 128 个，至少要有 1 个标签，一个表中标签值的总长度不超过 16KB 。 - SQL 语句最大长度 1048576 个字符，也可通过客户端配置参数 maxSQLLength 修改，取值范围 65480 ~ 1048576。 -- SELECT 语句的查询结果，最多允许返回 4096 列（语句中的函数调用可能也会占用一些列空间），超限时需要显式指定较少的返回数据列，以避免语句执行报错。注： 2.1.7.0 版本（不含）之前为最多允许 1024 列 +- SELECT 语句的查询结果，最多允许返回 4096 列（语句中的函数调用可能也会占用一些列空间），超限时需要显式指定较少的返回数据列，以避免语句执行报错。 - 库的数目，超级表的数目、表的数目，系统不做限制，仅受系统资源限制。 - -## GROUP BY 的限制 - -TAOS SQL 支持对标签、TBNAME 进行 GROUP BY 操作，也支持普通列进行 GROUP BY，前提是：仅限一列且该列的唯一值小于 10 万个。注意：group by 不支持 float,double 类型。 - -## IS NOT NULL 的限制 - -IS NOT NULL 与不为空的表达式适用范围。 - -IS NOT NULL 支持所有类型的列。不为空的表达式为 <\>""，仅对非数值类型的列适用。 - -## ORDER BY 的限制 - -- 非超级表只能有一个 order by. -- 超级表最多两个 order by， 并且第二个必须为 ts. -- order by tag，必须和 group by tag 一起，并且是同一个 tag。 tbname 和 tag 一样逻辑。 只适用于超级表 -- order by 普通列，必须和 group by 一起或者和 top/bottom 一起，并且是同一个普通列。 适用于超级表和普通表。如果同时存在 group by 和 top/bottom 一起，order by 优先必须和 group by 同一列。 -- order by ts. 适用于超级表和普通表。 -- order by ts 同时含有 group by 时 针对 group 内部用 ts 排序 +- 数据库的副本数只能设置为 1 或 3 +- 用户名的最大长度是 23 个字节 +- 用户密码的最大长度是 15 个字节 +- 总数据行数取决于可用资源 +- 单个数据库的虚拟结点数上限为 1024 ## 表(列)名合法性说明 @@ -49,6 +55,3 @@ IS NOT NULL 支持所有类型的列。不为空的表达式为 <\>""，仅对 转义字符中的内容必须是可打印字符。 ::: - -### 支持版本 -支持转义符的功能从 2.3.0.1 版本开始。 \ No newline at end of file diff --git a/docs/zh/12-taos-sql/20-keywords.md b/docs/zh/12-taos-sql/20-keywords.md index 202f223b45..d91c9be2ac 100644 --- a/docs/zh/12-taos-sql/20-keywords.md +++ b/docs/zh/12-taos-sql/20-keywords.md @@ -1,46 +1,8 @@ --- -sidebar_label: 参数限制与保留关键字 -title: TDengine 参数限制与保留关键字 +sidebar_label: 保留关键字 +title: TDengine 保留关键字 --- -## 名称命名规则 - -1. 合法字符：英文字符、数字和下划线 -2. 允许英文字符或下划线开头，不允许以数字开头 -3. 不区分大小写 -4. 转义后表（列）名规则： - 为了兼容支持更多形式的表（列）名，TDengine 引入新的转义符 "`"。可用让表名与关键词不冲突，同时不受限于上述表名称合法性约束检查。 - 转义后的表（列）名同样受到长度限制要求，且长度计算的时候不计算转义符。使用转义字符以后，不再对转义字符中的内容进行大小写统一。 - - 例如：\`aBc\` 和 \`abc\` 是不同的表（列）名，但是 abc 和 aBc 是相同的表（列）名。 - 需要注意的是转义字符中的内容必须是可打印字符。 - 支持转义符的功能从 2.3.0.1 版本开始。 - -## 密码合法字符集 - -`[a-zA-Z0-9!?$%^&*()_–+={[}]:;@~#|<,>.?/]` - -去掉了 `` ‘“`\ `` (单双引号、撇号、反斜杠、空格) - -- 数据库名：不能包含“.”以及特殊字符，不能超过 32 个字符 -- 表名：不能包含“.”以及特殊字符，与所属数据库名一起，不能超过 192 个字节 ，每行数据最大长度 48KB -- 表的列名：不能包含特殊字符，不能超过 64 个字节 -- 数据库名、表名、列名，都不能以数字开头，合法的可用字符集是“英文字符、数字和下划线” -- 表的列数：不能超过 1024 列，最少需要 2 列，第一列必须是时间戳（从 2.1.7.0 版本开始，改为最多支持 4096 列） -- 记录的最大长度：包括时间戳 8 字节，不能超过 48KB（每个 BINARY/NCHAR 类型的列还会额外占用 2 个 字节 的存储位置） -- 单条 SQL 语句默认最大字符串长度：1048576 字节，但可通过系统配置参数 maxSQLLength 修改，取值范围 65480 ~ 1048576 字节 -- 数据库副本数：不能超过 3 -- 用户名：不能超过 23 个 字节 -- 用户密码：不能超过 15 个 字节 -- 标签(Tags)数量：不能超过 128 个，可以 0 个 -- 标签的总长度：不能超过 16KB -- 记录条数：仅受存储空间限制 -- 表的个数：仅受节点个数限制 -- 库的个数：仅受节点个数限制 -- 单个库上虚拟节点个数：不能超过 64 个 -- 库的数目，超级表的数目、表的数目，系统不做限制，仅受系统资源限制 -- SELECT 语句的查询结果，最多允许返回 1024 列（语句中的函数调用可能也会占用一些列空间），超限时需要显式指定较少的返回数据列，以避免语句执行报错。（从 2.1.7.0 版本开始，改为最多允许 4096 列） - ## 保留关键字 目前 TDengine 有将近 200 个内部保留关键字，这些关键字无论大小写均不可以用作库名、表名、STable 名、数据列名及标签列名等。这些关键字列表如下： diff --git a/examples/c/tmq.c b/examples/c/tmq.c index 94e0b86821..38c3eba647 100644 --- a/examples/c/tmq.c +++ b/examples/c/tmq.c @@ -15,10 +15,10 @@ #include #include +#include #include #include #include "taos.h" -#include static int running = 1; static void msg_process(TAOS_RES* msg) { @@ -28,8 +28,8 @@ static void msg_process(TAOS_RES* msg) { printf("db: %s\n", tmq_get_db_name(msg)); printf("vg: %d\n", tmq_get_vgroup_id(msg)); if (tmq_get_res_type(msg) == TMQ_RES_TABLE_META) { - tmq_raw_data *raw = tmq_get_raw_meta(msg); - if(raw){ + tmq_raw_data* raw = tmq_get_raw_meta(msg); + if (raw) { TAOS* pConn = taos_connect("192.168.1.86", "root", "taosdata", NULL, 0); if (pConn == NULL) { return; @@ -55,7 +55,7 @@ static void msg_process(TAOS_RES* msg) { } tmq_free_raw_meta(raw); char* result = tmq_get_json_meta(msg); - if(result){ + if (result) { printf("meta result: %s\n", result); } tmq_free_json_meta(result); @@ -96,7 +96,9 @@ int32_t init_env() { } taos_free_result(pRes); - pRes = taos_query(pConn, "create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 nchar(8), t4 bool)"); + pRes = taos_query(pConn, + "create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 " + "nchar(8), t4 bool)"); if (taos_errno(pRes) != 0) { printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); return -1; @@ -152,6 +154,7 @@ int32_t init_env() { } taos_free_result(pRes); +#if 0 pRes = taos_query(pConn, "alter table st1 add column c4 bigint"); if (taos_errno(pRes) != 0) { printf("failed to alter super table st1, reason:%s\n", taos_errstr(pRes)); @@ -264,7 +267,9 @@ int32_t init_env() { } taos_free_result(pRes); - pRes = taos_query(pConn, "create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 nchar(8), t4 bool)"); + pRes = taos_query(pConn, + "create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 " + "nchar(8), t4 bool)"); if (taos_errno(pRes) != 0) { printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); return -1; @@ -277,6 +282,7 @@ int32_t init_env() { return -1; } taos_free_result(pRes); +#endif return 0; } @@ -296,8 +302,15 @@ int32_t create_topic() { } taos_free_result(pRes); - pRes = taos_query(pConn, "create topic topic_ctb_column with meta as database abc1"); - /*pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from st1");*/ + /*pRes = taos_query(pConn, "create topic topic_ctb_column with meta as database abc1");*/ + pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from st1"); + if (taos_errno(pRes) != 0) { + printf("failed to create topic topic_ctb_column, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create topic topic2 as select ts, c1, c2, c3 from st1"); if (taos_errno(pRes) != 0) { printf("failed to create topic topic_ctb_column, reason:%s\n", taos_errstr(pRes)); return -1; @@ -353,6 +366,7 @@ tmq_t* build_consumer() { tmq_conf_t* conf = tmq_conf_new(); tmq_conf_set(conf, "group.id", "tg2"); + tmq_conf_set(conf, "client.id", "my app 1"); tmq_conf_set(conf, "td.connect.user", "root"); tmq_conf_set(conf, "td.connect.pass", "taosdata"); tmq_conf_set(conf, "msg.with.table.name", "true"); diff --git a/include/common/systable.h b/include/common/systable.h index faea4dd3f8..ca097cd8d2 100644 --- a/include/common/systable.h +++ b/include/common/systable.h @@ -35,6 +35,7 @@ extern "C" { #define TSDB_INS_TABLE_USER_INDEXES "user_indexes" #define TSDB_INS_TABLE_USER_STABLES "user_stables" #define TSDB_INS_TABLE_USER_TABLES "user_tables" +#define TSDB_INS_TABLE_USER_TAGS "user_tags" #define TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED "user_table_distributed" #define TSDB_INS_TABLE_USER_USERS "user_users" #define TSDB_INS_TABLE_LICENCES "grants" diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 96014b1234..23d9c41a51 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -139,6 +139,7 @@ extern int32_t tsTransPullupInterval; extern int32_t tsMqRebalanceInterval; extern int32_t tsTtlUnit; extern int32_t tsTtlPushInterval; +extern int32_t tsGrantHBInterval; #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize) diff --git a/include/common/tgrant.h b/include/common/tgrant.h new file mode 100644 index 0000000000..ad23c661b1 --- /dev/null +++ b/include/common/tgrant.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_COMMON_GRANT_H_ +#define _TD_COMMON_GRANT_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "os.h" + +typedef enum { + TSDB_GRANT_ALL, + TSDB_GRANT_TIME, + TSDB_GRANT_USER, + TSDB_GRANT_DB, + TSDB_GRANT_TIMESERIES, + TSDB_GRANT_DNODE, + TSDB_GRANT_ACCT, + TSDB_GRANT_STORAGE, + TSDB_GRANT_SPEED, + TSDB_GRANT_QUERY_TIME, + TSDB_GRANT_CONNS, + TSDB_GRANT_STREAMS, + TSDB_GRANT_CPU_CORES, +} EGrantType; + +int32_t grantCheck(EGrantType grant); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_COMMON_GRANT_H_*/ \ No newline at end of file diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 5a0ef705fc..d195bf5407 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -75,13 +75,18 @@ typedef uint16_t tmsg_t; #define TSDB_IE_TYPE_DNODE_EXT 6 #define TSDB_IE_TYPE_DNODE_STATE 7 -enum { CONN_TYPE__QUERY = 1, CONN_TYPE__TMQ, CONN_TYPE__UDFD, CONN_TYPE__MAX }; +enum { + CONN_TYPE__QUERY = 1, + CONN_TYPE__TMQ, + CONN_TYPE__UDFD, + CONN_TYPE__MAX, +}; enum { HEARTBEAT_KEY_USER_AUTHINFO = 1, HEARTBEAT_KEY_DBINFO, HEARTBEAT_KEY_STBINFO, - HEARTBEAT_KEY_MQ_TMP, + HEARTBEAT_KEY_TMQ, }; typedef enum _mgmt_table { @@ -99,6 +104,7 @@ typedef enum _mgmt_table { TSDB_MGMT_TABLE_STB, TSDB_MGMT_TABLE_STREAMS, TSDB_MGMT_TABLE_TABLE, + TSDB_MGMT_TABLE_TAG, TSDB_MGMT_TABLE_USER, TSDB_MGMT_TABLE_GRANTS, TSDB_MGMT_TABLE_VGROUP, @@ -2146,6 +2152,15 @@ typedef struct { char cgroup[TSDB_CGROUP_LEN]; } SMqAskEpReq; +typedef struct { + int64_t consumerId; + int32_t epoch; +} SMqHbReq; + +typedef struct { + int8_t reserved; +} SMqHbRsp; + typedef struct { int32_t key; int32_t valueLen; @@ -2335,29 +2350,30 @@ static FORCE_INLINE int32_t tDecodeSClientHbKey(SDecoder* pDecoder, SClientHbKey return 0; } -typedef struct SMqHbVgInfo { +typedef struct { int32_t vgId; -} SMqHbVgInfo; + // TODO stas +} SMqReportVgInfo; -static FORCE_INLINE int32_t taosEncodeSMqVgInfo(void** buf, const SMqHbVgInfo* pVgInfo) { +static FORCE_INLINE int32_t taosEncodeSMqVgInfo(void** buf, const SMqReportVgInfo* pVgInfo) { int32_t tlen = 0; tlen += taosEncodeFixedI32(buf, pVgInfo->vgId); return tlen; } -static FORCE_INLINE void* taosDecodeSMqVgInfo(void* buf, SMqHbVgInfo* pVgInfo) { +static FORCE_INLINE void* taosDecodeSMqVgInfo(void* buf, SMqReportVgInfo* pVgInfo) { buf = taosDecodeFixedI32(buf, &pVgInfo->vgId); return buf; } -typedef struct SMqHbTopicInfo { +typedef struct { int32_t epoch; int64_t topicUid; char name[TSDB_TOPIC_FNAME_LEN]; SArray* pVgInfo; // SArray -} SMqHbTopicInfo; +} SMqTopicInfo; -static FORCE_INLINE int32_t taosEncodeSMqHbTopicInfoMsg(void** buf, const SMqHbTopicInfo* pTopicInfo) { +static FORCE_INLINE int32_t taosEncodeSMqTopicInfoMsg(void** buf, const SMqTopicInfo* pTopicInfo) { int32_t tlen = 0; tlen += taosEncodeFixedI32(buf, pTopicInfo->epoch); tlen += taosEncodeFixedI64(buf, pTopicInfo->topicUid); @@ -2365,35 +2381,35 @@ static FORCE_INLINE int32_t taosEncodeSMqHbTopicInfoMsg(void** buf, const SMqHbT int32_t sz = taosArrayGetSize(pTopicInfo->pVgInfo); tlen += taosEncodeFixedI32(buf, sz); for (int32_t i = 0; i < sz; i++) { - SMqHbVgInfo* pVgInfo = (SMqHbVgInfo*)taosArrayGet(pTopicInfo->pVgInfo, i); + SMqReportVgInfo* pVgInfo = (SMqReportVgInfo*)taosArrayGet(pTopicInfo->pVgInfo, i); tlen += taosEncodeSMqVgInfo(buf, pVgInfo); } return tlen; } -static FORCE_INLINE void* taosDecodeSMqHbTopicInfoMsg(void* buf, SMqHbTopicInfo* pTopicInfo) { +static FORCE_INLINE void* taosDecodeSMqTopicInfoMsg(void* buf, SMqTopicInfo* pTopicInfo) { buf = taosDecodeFixedI32(buf, &pTopicInfo->epoch); buf = taosDecodeFixedI64(buf, &pTopicInfo->topicUid); buf = taosDecodeStringTo(buf, pTopicInfo->name); int32_t sz; buf = taosDecodeFixedI32(buf, &sz); - pTopicInfo->pVgInfo = taosArrayInit(sz, sizeof(SMqHbVgInfo)); + pTopicInfo->pVgInfo = taosArrayInit(sz, sizeof(SMqReportVgInfo)); for (int32_t i = 0; i < sz; i++) { - SMqHbVgInfo vgInfo; + SMqReportVgInfo vgInfo; buf = taosDecodeSMqVgInfo(buf, &vgInfo); taosArrayPush(pTopicInfo->pVgInfo, &vgInfo); } return buf; } -typedef struct SMqHbMsg { +typedef struct { int32_t status; // ask hb endpoint int32_t epoch; int64_t consumerId; SArray* pTopics; // SArray -} SMqHbMsg; +} SMqReportReq; -static FORCE_INLINE int32_t taosEncodeSMqMsg(void** buf, const SMqHbMsg* pMsg) { +static FORCE_INLINE int32_t taosEncodeSMqReportMsg(void** buf, const SMqReportReq* pMsg) { int32_t tlen = 0; tlen += taosEncodeFixedI32(buf, pMsg->status); tlen += taosEncodeFixedI32(buf, pMsg->epoch); @@ -2401,22 +2417,22 @@ static FORCE_INLINE int32_t taosEncodeSMqMsg(void** buf, const SMqHbMsg* pMsg) { int32_t sz = taosArrayGetSize(pMsg->pTopics); tlen += taosEncodeFixedI32(buf, sz); for (int32_t i = 0; i < sz; i++) { - SMqHbTopicInfo* topicInfo = (SMqHbTopicInfo*)taosArrayGet(pMsg->pTopics, i); - tlen += taosEncodeSMqHbTopicInfoMsg(buf, topicInfo); + SMqTopicInfo* topicInfo = (SMqTopicInfo*)taosArrayGet(pMsg->pTopics, i); + tlen += taosEncodeSMqTopicInfoMsg(buf, topicInfo); } return tlen; } -static FORCE_INLINE void* taosDecodeSMqMsg(void* buf, SMqHbMsg* pMsg) { +static FORCE_INLINE void* taosDecodeSMqReportMsg(void* buf, SMqReportReq* pMsg) { buf = taosDecodeFixedI32(buf, &pMsg->status); buf = taosDecodeFixedI32(buf, &pMsg->epoch); buf = taosDecodeFixedI64(buf, &pMsg->consumerId); int32_t sz; buf = taosDecodeFixedI32(buf, &sz); - pMsg->pTopics = taosArrayInit(sz, sizeof(SMqHbTopicInfo)); + pMsg->pTopics = taosArrayInit(sz, sizeof(SMqTopicInfo)); for (int32_t i = 0; i < sz; i++) { - SMqHbTopicInfo topicInfo; - buf = taosDecodeSMqHbTopicInfoMsg(buf, &topicInfo); + SMqTopicInfo topicInfo; + buf = taosDecodeSMqTopicInfoMsg(buf, &topicInfo); taosArrayPush(pMsg->pTopics, &topicInfo); } return buf; @@ -2908,7 +2924,6 @@ typedef struct { SMqRspHead head; STqOffsetVal reqOffset; STqOffsetVal rspOffset; - int32_t skipLogNum; int32_t blockNum; int8_t withTbName; int8_t withSchema; @@ -2921,89 +2936,6 @@ typedef struct { int32_t tEncodeSMqDataRsp(SEncoder* pEncoder, const SMqDataRsp* pRsp); int32_t tDecodeSMqDataRsp(SDecoder* pDecoder, SMqDataRsp* pRsp); -#if 0 -typedef struct { - SMqRspHead head; - int64_t reqOffset; - int64_t rspOffset; - int32_t skipLogNum; - int32_t blockNum; - int8_t withTbName; - int8_t withSchema; - SArray* blockDataLen; // SArray - SArray* blockData; // SArray - SArray* blockTbName; // SArray - SArray* blockSchema; // SArray -} SMqDataBlkRsp; - -static FORCE_INLINE int32_t tEncodeSMqDataBlkRsp(void** buf, const SMqDataBlkRsp* pRsp) { - int32_t tlen = 0; - tlen += taosEncodeFixedI64(buf, pRsp->reqOffset); - tlen += taosEncodeFixedI64(buf, pRsp->rspOffset); - tlen += taosEncodeFixedI32(buf, pRsp->skipLogNum); - tlen += taosEncodeFixedI32(buf, pRsp->blockNum); - if (pRsp->blockNum != 0) { - tlen += taosEncodeFixedI8(buf, pRsp->withTbName); - tlen += taosEncodeFixedI8(buf, pRsp->withSchema); - - for (int32_t i = 0; i < pRsp->blockNum; i++) { - int32_t bLen = *(int32_t*)taosArrayGet(pRsp->blockDataLen, i); - void* data = taosArrayGetP(pRsp->blockData, i); - tlen += taosEncodeFixedI32(buf, bLen); - tlen += taosEncodeBinary(buf, data, bLen); - if (pRsp->withSchema) { - SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(pRsp->blockSchema, i); - tlen += taosEncodeSSchemaWrapper(buf, pSW); - } - if (pRsp->withTbName) { - char* tbName = (char*)taosArrayGetP(pRsp->blockTbName, i); - tlen += taosEncodeString(buf, tbName); - } - } - } - return tlen; -} - -static FORCE_INLINE void* tDecodeSMqDataBlkRsp(const void* buf, SMqDataBlkRsp* pRsp) { - buf = taosDecodeFixedI64(buf, &pRsp->reqOffset); - buf = taosDecodeFixedI64(buf, &pRsp->rspOffset); - buf = taosDecodeFixedI32(buf, &pRsp->skipLogNum); - buf = taosDecodeFixedI32(buf, &pRsp->blockNum); - if (pRsp->blockNum != 0) { - pRsp->blockData = taosArrayInit(pRsp->blockNum, sizeof(void*)); - pRsp->blockDataLen = taosArrayInit(pRsp->blockNum, sizeof(int32_t)); - buf = taosDecodeFixedI8(buf, &pRsp->withTbName); - buf = taosDecodeFixedI8(buf, &pRsp->withSchema); - if (pRsp->withTbName) { - pRsp->blockTbName = taosArrayInit(pRsp->blockNum, sizeof(void*)); - } - if (pRsp->withSchema) { - pRsp->blockSchema = taosArrayInit(pRsp->blockNum, sizeof(void*)); - } - - for (int32_t i = 0; i < pRsp->blockNum; i++) { - int32_t bLen = 0; - void* data = NULL; - buf = taosDecodeFixedI32(buf, &bLen); - buf = taosDecodeBinary(buf, &data, bLen); - taosArrayPush(pRsp->blockDataLen, &bLen); - taosArrayPush(pRsp->blockData, &data); - if (pRsp->withSchema) { - SSchemaWrapper* pSW = (SSchemaWrapper*)taosMemoryMalloc(sizeof(SSchemaWrapper)); - buf = taosDecodeSSchemaWrapper(buf, pSW); - taosArrayPush(pRsp->blockSchema, &pSW); - } - if (pRsp->withTbName) { - char* name = NULL; - buf = taosDecodeString(buf, &name); - taosArrayPush(pRsp->blockTbName, &name); - } - } - } - return (void*)buf; -} -#endif - typedef struct { SMqRspHead head; char cgroup[TSDB_CGROUP_LEN]; diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index a24eda33bb..a9d8871c59 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -144,6 +144,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_MQ_ASK_EP, "ask-ep", SMqAskEpReq, SMqAskEpRsp) TD_DEF_MSG_TYPE(TDMT_MND_MQ_CONSUMER_LOST, "consumer-lost", SMqConsumerLostMsg, NULL) TD_DEF_MSG_TYPE(TDMT_MND_MQ_CONSUMER_RECOVER, "consumer-recover", SMqConsumerRecoverMsg, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_MQ_HB, "consumer-hb", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_MQ_DO_REBALANCE, "do-rebalance", SMqDoRebalanceMsg, NULL) TD_DEF_MSG_TYPE(TDMT_MND_MQ_DROP_CGROUP, "drop-cgroup", SMqDropCGroupReq, SMqDropCGroupRsp) TD_DEF_MSG_TYPE(TDMT_MND_MQ_COMMIT_OFFSET, "mnode-commit-offset", SMqCMCommitOffsetReq, SMqCMCommitOffsetRsp) @@ -151,6 +152,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_TELEM_TIMER, "telem-tmr", SMTimerReq, SMTimerReq) TD_DEF_MSG_TYPE(TDMT_MND_TRANS_TIMER, "trans-tmr", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_TTL_TIMER, "ttl-tmr", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_GRANT_HB_TIMER, "grant-hb-tmr", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_KILL_TRANS, "kill-trans", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_KILL_QUERY, "kill-query", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_KILL_CONN, "kill-conn", NULL, NULL) diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 4d27325d75..8cb48cc9f0 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -172,13 +172,8 @@ typedef struct tExprNode { void tExprTreeDestroy(tExprNode *pNode, void (*fp)(void *)); -typedef enum { - SHOULD_FREE_COLDATA = 0x1, // the newly created column data needs to be destroyed. - DELEGATED_MGMT_COLDATA = 0x2, // input column data should not be released. -} ECOLDATA_MGMT_TYPE_E; - struct SScalarParam { - ECOLDATA_MGMT_TYPE_E type; + bool colAlloced; SColumnInfoData *columnData; SHashObj *pHashFilter; int32_t hashValueType; diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index c453ce98f8..1a1ae00ad1 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -170,6 +170,7 @@ typedef enum ENodeType { QUERY_NODE_SHOW_STABLES_STMT, QUERY_NODE_SHOW_STREAMS_STMT, QUERY_NODE_SHOW_TABLES_STMT, + QUERY_NODE_SHOW_TAGS_STMT, QUERY_NODE_SHOW_USERS_STMT, QUERY_NODE_SHOW_LICENCE_STMT, QUERY_NODE_SHOW_VGROUPS_STMT, diff --git a/include/libs/scalar/scalar.h b/include/libs/scalar/scalar.h index a6ed944632..1f20a64314 100644 --- a/include/libs/scalar/scalar.h +++ b/include/libs/scalar/scalar.h @@ -113,6 +113,10 @@ int32_t twaScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam * int32_t mavgScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t hllScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); int32_t csumScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t diffScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t stateCountScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t stateDurationScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); +int32_t histogramScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); #ifdef __cplusplus } diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 5c4d8ce250..8c69c0f2de 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -83,6 +83,7 @@ typedef struct { int32_t srcVgId; int32_t childId; int64_t sourceVer; + int64_t reqId; SArray* blocks; // SArray } SStreamDataBlock; @@ -324,6 +325,8 @@ static FORCE_INLINE int32_t streamTaskInput(SStreamTask* pTask, SStreamQueueItem if (pItem->type == STREAM_INPUT__DATA_SUBMIT) { SStreamDataSubmit* pSubmitClone = streamSubmitRefClone((SStreamDataSubmit*)pItem); if (pSubmitClone == NULL) { + qDebug("task %d %p submit enqueue failed since out of memory", pTask->taskId, pTask); + terrno = TSDB_CODE_OUT_OF_MEMORY; atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__FAILED); return -1; } @@ -412,6 +415,7 @@ typedef struct { typedef struct { int64_t streamId; + int64_t reqId; int32_t srcTaskId; int32_t srcNodeId; int32_t dstTaskId; diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index c226d7c8cc..06a58af0e3 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -134,7 +134,7 @@ typedef struct SSyncFSM { int32_t (*FpSnapshotDoRead)(struct SSyncFSM* pFsm, void* pReader, void** ppBuf, int32_t* len); int32_t (*FpSnapshotStartWrite)(struct SSyncFSM* pFsm, void* pWriterParam, void** ppWriter); - int32_t (*FpSnapshotStopWrite)(struct SSyncFSM* pFsm, void* pWriter, bool isApply); + int32_t (*FpSnapshotStopWrite)(struct SSyncFSM* pFsm, void* pWriter, bool isApply, SSnapshot* pSnapshot); int32_t (*FpSnapshotDoWrite)(struct SSyncFSM* pFsm, void* pWriter, void* pBuf, int32_t len); } SSyncFSM; diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 7441b38321..2ae1f7b854 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -135,7 +135,7 @@ void rpcSendRequestWithCtx(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg int32_t rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo); void rpcSendRecv(void *shandle, SEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp); void rpcSetDefaultAddr(void *thandle, const char *ip, const char *fqdn); -int64_t rpcAllocHandle(); +void* rpcAllocHandle(); #ifdef __cplusplus } diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 7e61456d45..d0321d3724 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -332,7 +332,6 @@ int32_t* taosGetErrno(); #define TSDB_CODE_VND_INVALID_TABLE_ACTION TAOS_DEF_ERROR_CODE(0, 0x0519) #define TSDB_CODE_VND_COL_ALREADY_EXISTS TAOS_DEF_ERROR_CODE(0, 0x051a) #define TSDB_CODE_VND_TABLE_COL_NOT_EXISTS TAOS_DEF_ERROR_CODE(0, 0x051b) -#define TSDB_CODE_VND_READ_END TAOS_DEF_ERROR_CODE(0, 0x051c) // tsdb #define TSDB_CODE_TDB_INVALID_TABLE_ID TAOS_DEF_ERROR_CODE(0, 0x0600) diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 6969e03e7c..b52df105b1 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -171,6 +171,7 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { pTscObj->pAppInfo->totalDnodes = pRsp->query->totalDnodes; pTscObj->pAppInfo->onlineDnodes = pRsp->query->onlineDnodes; pTscObj->connId = pRsp->query->connId; + tscTrace("conn %p hb rsp, dnodes %d/%d", pTscObj->connId, pTscObj->pAppInfo->onlineDnodes, pTscObj->pAppInfo->totalDnodes); if (pRsp->query->killRid) { tscDebug("request rid %" PRIx64 " need to be killed now", pRsp->query->killRid); @@ -294,6 +295,7 @@ static int32_t hbAsyncCallBack(void *param, SDataBuf *pMsg, int32_t code) { if (code != 0) { (*pInst)->onlineDnodes = ((*pInst)->totalDnodes ? 0 : -1); + tscDebug("hb rsp error %s, update server status %d/%d", tstrerror(code), (*pInst)->onlineDnodes, (*pInst)->totalDnodes); } if (rspNum) { @@ -571,7 +573,8 @@ int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req return TSDB_CODE_SUCCESS; } -void hbMgrInitMqHbHandle() { +static FORCE_INLINE void hbMgrInitHandle() { + // init all handle clientHbMgr.reqHandle[CONN_TYPE__QUERY] = hbQueryHbReqHandle; clientHbMgr.reqHandle[CONN_TYPE__TMQ] = hbMqHbReqHandle; @@ -579,11 +582,6 @@ void hbMgrInitMqHbHandle() { clientHbMgr.rspHandle[CONN_TYPE__TMQ] = hbMqHbRspHandle; } -static FORCE_INLINE void hbMgrInitHandle() { - // init all handle - hbMgrInitMqHbHandle(); -} - SClientHbBatchReq *hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { SClientHbBatchReq *pBatchReq = taosMemoryCalloc(1, sizeof(SClientHbBatchReq)); if (pBatchReq == NULL) { diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 3506b5cfdf..73374c7f77 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1276,7 +1276,12 @@ int32_t doProcessMsgFromServer(void* param) { assert(pMsg->info.ahandle != NULL); STscObj* pTscObj = NULL; - tscDebug("processMsgFromServer message: %s, code: %s", TMSG_INFO(pMsg->msgType), tstrerror(pMsg->code)); + STraceId* trace = &pMsg->info.traceId; + char tbuf[40] = {0}; + TRACE_TO_STR(trace, tbuf); + + tscDebug("processMsgFromServer handle %p, message: %s, code: %s, gtid: %s", pMsg->info.handle, TMSG_INFO(pMsg->msgType), tstrerror(pMsg->code), + tbuf); if (pSendInfo->requestObjRefId != 0) { SRequestObj* pRequest = (SRequestObj*)taosAcquireRef(clientReqRefPool, pSendInfo->requestObjRefId); diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 62052457fd..77b31011a3 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -142,6 +142,7 @@ void taos_close(TAOS *taos) { int taos_errno(TAOS_RES *res) { if (res == NULL || TD_RES_TMQ_META(res)) { + if (terrno == TSDB_CODE_RPC_REDIRECT) terrno = TSDB_CODE_RPC_NETWORK_UNAVAIL; return terrno; } @@ -149,11 +150,13 @@ int taos_errno(TAOS_RES *res) { return 0; } - return ((SRequestObj *)res)->code; + return ((SRequestObj *)res)->code == TSDB_CODE_RPC_REDIRECT ? TSDB_CODE_RPC_NETWORK_UNAVAIL + : ((SRequestObj *)res)->code; } const char *taos_errstr(TAOS_RES *res) { if (res == NULL || TD_RES_TMQ_META(res)) { + if (terrno == TSDB_CODE_RPC_REDIRECT) terrno = TSDB_CODE_RPC_NETWORK_UNAVAIL; return (const char *)tstrerror(terrno); } @@ -165,7 +168,8 @@ const char *taos_errstr(TAOS_RES *res) { if (NULL != pRequest->msgBuf && (strlen(pRequest->msgBuf) > 0 || pRequest->code == TSDB_CODE_RPC_FQDN_ERROR)) { return pRequest->msgBuf; } else { - return (const char *)tstrerror(pRequest->code); + return pRequest->code == TSDB_CODE_RPC_REDIRECT ? (const char *)tstrerror(TSDB_CODE_RPC_NETWORK_UNAVAIL) + : (const char *)tstrerror(pRequest->code); } } diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 46b21086ce..d9e8c32825 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -50,13 +50,16 @@ struct tmq_list_t { }; struct tmq_conf_t { - char clientId[256]; - char groupId[TSDB_CGROUP_LEN]; - int8_t autoCommit; - int8_t resetOffset; - int8_t withTbName; - int8_t spEnable; - int32_t spBatchSize; + char clientId[256]; + char groupId[TSDB_CGROUP_LEN]; + int8_t autoCommit; + int8_t resetOffset; + int8_t withTbName; + int8_t ssEnable; + int32_t ssBatchSize; + + bool hbBgEnable; + uint16_t port; int32_t autoCommitInterval; char* ip; @@ -68,14 +71,17 @@ struct tmq_conf_t { struct tmq_t { // conf - char groupId[TSDB_CGROUP_LEN]; - char clientId[256]; - int8_t withTbName; - int8_t useSnapshot; - int8_t autoCommit; - int32_t autoCommitInterval; - int32_t resetOffsetCfg; - int64_t consumerId; + char groupId[TSDB_CGROUP_LEN]; + char clientId[256]; + int8_t withTbName; + int8_t useSnapshot; + int8_t autoCommit; + int32_t autoCommitInterval; + int32_t resetOffsetCfg; + int64_t consumerId; + + bool hbBgEnable; + tmq_commit_cb* commitCb; void* commitCbUserParam; @@ -89,7 +95,8 @@ struct tmq_t { int64_t pollCnt; // timer - tmr_h hbTimer; + tmr_h hbLiveTimer; + tmr_h epTimer; tmr_h reportTimer; tmr_h commitTimer; @@ -118,7 +125,7 @@ enum { }; enum { - TMQ_DELAYED_TASK__HB = 1, + TMQ_DELAYED_TASK__ASK_EP = 1, TMQ_DELAYED_TASK__REPORT, TMQ_DELAYED_TASK__COMMIT, }; @@ -291,18 +298,31 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value if (strcmp(key, "experimental.snapshot.enable") == 0) { if (strcmp(value, "true") == 0) { - conf->spEnable = true; + conf->ssEnable = true; return TMQ_CONF_OK; } else if (strcmp(value, "false") == 0) { - conf->spEnable = false; + conf->ssEnable = false; return TMQ_CONF_OK; } else { return TMQ_CONF_INVALID; } } + if (strcmp(key, "enable.heartbeat.background") == 0) { + if (strcmp(value, "true") == 0) { + conf->hbBgEnable = true; + return TMQ_CONF_OK; + } else if (strcmp(value, "false") == 0) { + conf->hbBgEnable = false; + return TMQ_CONF_OK; + } else { + return TMQ_CONF_INVALID; + } + return TMQ_CONF_OK; + } + if (strcmp(key, "experimental.snapshot.batch.size") == 0) { - conf->spBatchSize = atoi(value); + conf->ssBatchSize = atoi(value); return TMQ_CONF_OK; } @@ -751,10 +771,10 @@ END: } #endif -void tmqAssignDelayedHbTask(void* param, void* tmrId) { +void tmqAssignAskEpTask(void* param, void* tmrId) { tmq_t* tmq = (tmq_t*)param; int8_t* pTaskType = taosAllocateQitem(sizeof(int8_t), DEF_QITEM); - *pTaskType = TMQ_DELAYED_TASK__HB; + *pTaskType = TMQ_DELAYED_TASK__ASK_EP; taosWriteQitem(tmq->delayedTask, pTaskType); tsem_post(&tmq->rspSem); } @@ -775,6 +795,46 @@ void tmqAssignDelayedReportTask(void* param, void* tmrId) { tsem_post(&tmq->rspSem); } +int32_t tmqHbCb(void* param, SDataBuf* pMsg, int32_t code) { + if (pMsg && pMsg->pData) taosMemoryFree(pMsg->pData); + return 0; +} + +void tmqSendHbReq(void* param, void* tmrId) { + // TODO replace with ref + tmq_t* tmq = (tmq_t*)param; + int64_t consumerId = tmq->consumerId; + int32_t epoch = tmq->epoch; + SMqHbReq* pReq = taosMemoryMalloc(sizeof(SMqHbReq)); + if (pReq == NULL) goto OVER; + pReq->consumerId = consumerId; + pReq->epoch = epoch; + + SMsgSendInfo* sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); + if (sendInfo == NULL) { + taosMemoryFree(pReq); + } + sendInfo->msgInfo = (SDataBuf){ + .pData = pReq, + .len = sizeof(SMqHbReq), + .handle = NULL, + }; + + sendInfo->requestId = generateRequestId(); + sendInfo->requestObjRefId = 0; + sendInfo->param = NULL; + sendInfo->fp = tmqHbCb; + sendInfo->msgType = TDMT_MND_MQ_HB; + + SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp); + + int64_t transporterId = 0; + asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, &transporterId, sendInfo); + +OVER: + taosTmrReset(tmqSendHbReq, 1000, tmq, tmqMgmt.timer, &tmq->hbLiveTimer); +} + int32_t tmqHandleAllDelayedTask(tmq_t* tmq) { STaosQall* qall = taosAllocateQall(); taosReadAllQitems(tmq->delayedTask, qall); @@ -783,9 +843,9 @@ int32_t tmqHandleAllDelayedTask(tmq_t* tmq) { taosGetQitem(qall, (void**)&pTaskType); if (pTaskType == NULL) break; - if (*pTaskType == TMQ_DELAYED_TASK__HB) { + if (*pTaskType == TMQ_DELAYED_TASK__ASK_EP) { tmqAskEp(tmq, true); - taosTmrReset(tmqAssignDelayedHbTask, 1000, tmq, tmqMgmt.timer, &tmq->hbTimer); + taosTmrReset(tmqAssignAskEpTask, 1000, tmq, tmqMgmt.timer, &tmq->epTimer); } else if (*pTaskType == TMQ_DELAYED_TASK__COMMIT) { tmqCommitInner2(tmq, NULL, 1, 1, tmq->commitCb, tmq->commitCbUserParam); taosTmrReset(tmqAssignDelayedCommitTask, tmq->autoCommitInterval, tmq, tmqMgmt.timer, &tmq->commitTimer); @@ -929,13 +989,15 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { strcpy(pTmq->clientId, conf->clientId); strcpy(pTmq->groupId, conf->groupId); pTmq->withTbName = conf->withTbName; - pTmq->useSnapshot = conf->spEnable; + pTmq->useSnapshot = conf->ssEnable; pTmq->autoCommit = conf->autoCommit; pTmq->autoCommitInterval = conf->autoCommitInterval; pTmq->commitCb = conf->commitCb; pTmq->commitCbUserParam = conf->commitCbUserParam; pTmq->resetOffsetCfg = conf->resetOffset; + pTmq->hbBgEnable = conf->hbBgEnable; + // assign consumerId pTmq->consumerId = tGenIdPI64(); @@ -953,6 +1015,10 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { goto FAIL; } + if (pTmq->hbBgEnable) { + pTmq->hbLiveTimer = taosTmrStart(tmqSendHbReq, 1000, pTmq, tmqMgmt.timer); + } + tscInfo("consumer %ld is setup, consumer group %s", pTmq->consumerId, pTmq->groupId); return pTmq; @@ -980,6 +1046,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { int32_t code = -1; req.consumerId = tmq->consumerId; + tstrncpy(req.clientId, tmq->clientId, 256); tstrncpy(req.cgroup, tmq->groupId, TSDB_CGROUP_LEN); req.topicNames = taosArrayInit(sz, sizeof(void*)); if (req.topicNames == NULL) goto FAIL; @@ -1049,9 +1116,9 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { taosMsleep(500); } - // init hb timer - if (tmq->hbTimer == NULL) { - tmq->hbTimer = taosTmrStart(tmqAssignDelayedHbTask, 1000, tmq, tmqMgmt.timer); + // init ep timer + if (tmq->epTimer == NULL) { + tmq->epTimer = taosTmrStart(tmqAssignAskEpTask, 1000, tmq, tmqMgmt.timer); } // init auto commit timer @@ -1074,14 +1141,6 @@ void tmq_conf_set_auto_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb, void* para conf->commitCbUserParam = param; } -#if 0 -int32_t tmqGetSkipLogNum(tmq_message_t* tmq_message) { - if (tmq_message == NULL) return 0; - SMqPollRsp* pRsp = &tmq_message->msg; - return pRsp->skipLogNum; -} -#endif - int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { SMqPollCbParam* pParam = (SMqPollCbParam*)param; SMqClientVg* pVg = pParam->pVg; @@ -1224,9 +1283,6 @@ bool tmqUpdateEp2(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { offsetNew = *pOffset; } - /*tscDebug("consumer:%" PRId64 ", (epoch %d) offset of vgId:%d updated to %" PRId64 ", vgKey is %s", - * tmq->consumerId, epoch,*/ - /*pVgEp->vgId, offset, vgKey);*/ SMqClientVg clientVg = { .pollCnt = 0, .currentOffsetNew = offsetNew, @@ -1863,7 +1919,8 @@ int32_t tmq_get_raw_meta(TAOS_RES* res, tmq_raw_data *raw) { return TSDB_CODE_INVALID_PARA; } -static char *buildCreateTableJson(SSchemaWrapper *schemaRow, SSchemaWrapper* schemaTag, char* name, int64_t id, int8_t t){ +static char* buildCreateTableJson(SSchemaWrapper* schemaRow, SSchemaWrapper* schemaTag, char* name, int64_t id, + int8_t t) { char* string = NULL; cJSON* json = cJSON_CreateObject(); if (json == NULL) { @@ -1872,32 +1929,32 @@ static char *buildCreateTableJson(SSchemaWrapper *schemaRow, SSchemaWrapper* sch cJSON* type = cJSON_CreateString("create"); cJSON_AddItemToObject(json, "type", type); -// char uid[32] = {0}; -// sprintf(uid, "%"PRIi64, id); -// cJSON* id_ = cJSON_CreateString(uid); -// cJSON_AddItemToObject(json, "id", id_); + // char uid[32] = {0}; + // sprintf(uid, "%"PRIi64, id); + // cJSON* id_ = cJSON_CreateString(uid); + // cJSON_AddItemToObject(json, "id", id_); cJSON* tableName = cJSON_CreateString(name); cJSON_AddItemToObject(json, "tableName", tableName); cJSON* tableType = cJSON_CreateString(t == TSDB_NORMAL_TABLE ? "normal" : "super"); cJSON_AddItemToObject(json, "tableType", tableType); -// cJSON* version = cJSON_CreateNumber(1); -// cJSON_AddItemToObject(json, "version", version); + // cJSON* version = cJSON_CreateNumber(1); + // cJSON_AddItemToObject(json, "version", version); cJSON* columns = cJSON_CreateArray(); - for(int i = 0; i < schemaRow->nCols; i++){ - cJSON* column = cJSON_CreateObject(); - SSchema *s = schemaRow->pSchema + i; - cJSON* cname = cJSON_CreateString(s->name); + for (int i = 0; i < schemaRow->nCols; i++) { + cJSON* column = cJSON_CreateObject(); + SSchema* s = schemaRow->pSchema + i; + cJSON* cname = cJSON_CreateString(s->name); cJSON_AddItemToObject(column, "name", cname); cJSON* ctype = cJSON_CreateNumber(s->type); cJSON_AddItemToObject(column, "type", ctype); - if(s->type == TSDB_DATA_TYPE_BINARY){ + if (s->type == TSDB_DATA_TYPE_BINARY) { int32_t length = s->bytes - VARSTR_HEADER_SIZE; - cJSON* cbytes = cJSON_CreateNumber(length); + cJSON* cbytes = cJSON_CreateNumber(length); cJSON_AddItemToObject(column, "length", cbytes); - }else if (s->type == TSDB_DATA_TYPE_NCHAR){ - int32_t length = (s->bytes - VARSTR_HEADER_SIZE)/TSDB_NCHAR_SIZE; - cJSON* cbytes = cJSON_CreateNumber(length); + } else if (s->type == TSDB_DATA_TYPE_NCHAR) { + int32_t length = (s->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; + cJSON* cbytes = cJSON_CreateNumber(length); cJSON_AddItemToObject(column, "length", cbytes); } cJSON_AddItemToArray(columns, column); @@ -1905,20 +1962,20 @@ static char *buildCreateTableJson(SSchemaWrapper *schemaRow, SSchemaWrapper* sch cJSON_AddItemToObject(json, "columns", columns); cJSON* tags = cJSON_CreateArray(); - for(int i = 0; schemaTag && i < schemaTag->nCols; i++){ - cJSON* tag = cJSON_CreateObject(); - SSchema *s = schemaTag->pSchema + i; - cJSON* tname = cJSON_CreateString(s->name); + for (int i = 0; schemaTag && i < schemaTag->nCols; i++) { + cJSON* tag = cJSON_CreateObject(); + SSchema* s = schemaTag->pSchema + i; + cJSON* tname = cJSON_CreateString(s->name); cJSON_AddItemToObject(tag, "name", tname); cJSON* ttype = cJSON_CreateNumber(s->type); cJSON_AddItemToObject(tag, "type", ttype); - if(s->type == TSDB_DATA_TYPE_BINARY){ + if (s->type == TSDB_DATA_TYPE_BINARY) { int32_t length = s->bytes - VARSTR_HEADER_SIZE; - cJSON* cbytes = cJSON_CreateNumber(length); + cJSON* cbytes = cJSON_CreateNumber(length); cJSON_AddItemToObject(tag, "length", cbytes); - }else if (s->type == TSDB_DATA_TYPE_NCHAR){ - int32_t length = (s->bytes - VARSTR_HEADER_SIZE)/TSDB_NCHAR_SIZE; - cJSON* cbytes = cJSON_CreateNumber(length); + } else if (s->type == TSDB_DATA_TYPE_NCHAR) { + int32_t length = (s->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; + cJSON* cbytes = cJSON_CreateNumber(length); cJSON_AddItemToObject(tag, "length", cbytes); } cJSON_AddItemToArray(tags, tag); @@ -1930,10 +1987,10 @@ static char *buildCreateTableJson(SSchemaWrapper *schemaRow, SSchemaWrapper* sch return string; } -static char *buildAlterSTableJson(void* alterData, int32_t alterDataLen){ - SMAlterStbReq req = {0}; - cJSON* json = NULL; - char* string = NULL; +static char* buildAlterSTableJson(void* alterData, int32_t alterDataLen) { + SMAlterStbReq req = {0}; + cJSON* json = NULL; + char* string = NULL; if (tDeserializeSMAlterStbReq(alterData, alterDataLen, &req) != 0) { goto end; @@ -1945,8 +2002,8 @@ static char *buildAlterSTableJson(void* alterData, int32_t alterDataLen){ } cJSON* type = cJSON_CreateString("alter"); cJSON_AddItemToObject(json, "type", type); -// cJSON* uid = cJSON_CreateNumber(id); -// cJSON_AddItemToObject(json, "uid", uid); + // cJSON* uid = cJSON_CreateNumber(id); + // cJSON_AddItemToObject(json, "uid", uid); SName name = {0}; tNameFromString(&name, req.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); cJSON* tableName = cJSON_CreateString(name.tname); @@ -1959,53 +2016,53 @@ static char *buildAlterSTableJson(void* alterData, int32_t alterDataLen){ switch (req.alterType) { case TSDB_ALTER_TABLE_ADD_TAG: case TSDB_ALTER_TABLE_ADD_COLUMN: { - TAOS_FIELD *field = taosArrayGet(req.pFields, 0); - cJSON* colName = cJSON_CreateString(field->name); + TAOS_FIELD* field = taosArrayGet(req.pFields, 0); + cJSON* colName = cJSON_CreateString(field->name); cJSON_AddItemToObject(json, "colName", colName); cJSON* colType = cJSON_CreateNumber(field->type); cJSON_AddItemToObject(json, "colType", colType); - if(field->type == TSDB_DATA_TYPE_BINARY){ + if (field->type == TSDB_DATA_TYPE_BINARY) { int32_t length = field->bytes - VARSTR_HEADER_SIZE; - cJSON* cbytes = cJSON_CreateNumber(length); + cJSON* cbytes = cJSON_CreateNumber(length); cJSON_AddItemToObject(json, "colLength", cbytes); - }else if (field->type == TSDB_DATA_TYPE_NCHAR){ - int32_t length = (field->bytes - VARSTR_HEADER_SIZE)/TSDB_NCHAR_SIZE; - cJSON* cbytes = cJSON_CreateNumber(length); + } else if (field->type == TSDB_DATA_TYPE_NCHAR) { + int32_t length = (field->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; + cJSON* cbytes = cJSON_CreateNumber(length); cJSON_AddItemToObject(json, "colLength", cbytes); } break; } case TSDB_ALTER_TABLE_DROP_TAG: - case TSDB_ALTER_TABLE_DROP_COLUMN:{ - TAOS_FIELD *field = taosArrayGet(req.pFields, 0); - cJSON* colName = cJSON_CreateString(field->name); + case TSDB_ALTER_TABLE_DROP_COLUMN: { + TAOS_FIELD* field = taosArrayGet(req.pFields, 0); + cJSON* colName = cJSON_CreateString(field->name); cJSON_AddItemToObject(json, "colName", colName); break; } case TSDB_ALTER_TABLE_UPDATE_TAG_BYTES: - case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES:{ - TAOS_FIELD *field = taosArrayGet(req.pFields, 0); - cJSON* colName = cJSON_CreateString(field->name); + case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: { + TAOS_FIELD* field = taosArrayGet(req.pFields, 0); + cJSON* colName = cJSON_CreateString(field->name); cJSON_AddItemToObject(json, "colName", colName); cJSON* colType = cJSON_CreateNumber(field->type); cJSON_AddItemToObject(json, "colType", colType); - if(field->type == TSDB_DATA_TYPE_BINARY){ + if (field->type == TSDB_DATA_TYPE_BINARY) { int32_t length = field->bytes - VARSTR_HEADER_SIZE; - cJSON* cbytes = cJSON_CreateNumber(length); + cJSON* cbytes = cJSON_CreateNumber(length); cJSON_AddItemToObject(json, "colLength", cbytes); - }else if (field->type == TSDB_DATA_TYPE_NCHAR){ - int32_t length = (field->bytes - VARSTR_HEADER_SIZE)/TSDB_NCHAR_SIZE; - cJSON* cbytes = cJSON_CreateNumber(length); + } else if (field->type == TSDB_DATA_TYPE_NCHAR) { + int32_t length = (field->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; + cJSON* cbytes = cJSON_CreateNumber(length); cJSON_AddItemToObject(json, "colLength", cbytes); } break; } case TSDB_ALTER_TABLE_UPDATE_TAG_NAME: - case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME:{ - TAOS_FIELD *oldField = taosArrayGet(req.pFields, 0); - TAOS_FIELD *newField = taosArrayGet(req.pFields, 1); - cJSON* colName = cJSON_CreateString(oldField->name); + case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME: { + TAOS_FIELD* oldField = taosArrayGet(req.pFields, 0); + TAOS_FIELD* newField = taosArrayGet(req.pFields, 1); + cJSON* colName = cJSON_CreateString(oldField->name); cJSON_AddItemToObject(json, "colName", colName); cJSON* colNewName = cJSON_CreateString(newField->name); cJSON_AddItemToObject(json, "colNewName", colNewName); @@ -2016,19 +2073,19 @@ static char *buildAlterSTableJson(void* alterData, int32_t alterDataLen){ } string = cJSON_PrintUnformatted(json); - end: +end: cJSON_Delete(json); tFreeSMAltertbReq(&req); return string; } -static char *processCreateStb(SMqMetaRsp *metaRsp){ +static char* processCreateStb(SMqMetaRsp* metaRsp) { SVCreateStbReq req = {0}; SDecoder coder; - char* string = NULL; + char* string = NULL; // decode and process req - void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); + void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); int32_t len = metaRsp->metaRspLen - sizeof(SMsgHead); tDecoderInit(&coder, data, len); @@ -2039,18 +2096,18 @@ static char *processCreateStb(SMqMetaRsp *metaRsp){ tDecoderClear(&coder); return string; - _err: +_err: tDecoderClear(&coder); return string; } -static char *processAlterStb(SMqMetaRsp *metaRsp){ +static char* processAlterStb(SMqMetaRsp* metaRsp) { SVCreateStbReq req = {0}; SDecoder coder; - char* string = NULL; + char* string = NULL; // decode and process req - void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); + void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); int32_t len = metaRsp->metaRspLen - sizeof(SMsgHead); tDecoderInit(&coder, data, len); @@ -2061,24 +2118,24 @@ static char *processAlterStb(SMqMetaRsp *metaRsp){ tDecoderClear(&coder); return string; - _err: +_err: tDecoderClear(&coder); return string; } -static char *buildCreateCTableJson(STag* pTag, char* sname, char* name, SArray* tagName, int64_t id){ - char* string = NULL; +static char* buildCreateCTableJson(STag* pTag, char* sname, char* name, SArray* tagName, int64_t id) { + char* string = NULL; SArray* pTagVals = NULL; - cJSON* json = cJSON_CreateObject(); + cJSON* json = cJSON_CreateObject(); if (json == NULL) { return string; } cJSON* type = cJSON_CreateString("create"); cJSON_AddItemToObject(json, "type", type); -// char cid[32] = {0}; -// sprintf(cid, "%"PRIi64, id); -// cJSON* cid_ = cJSON_CreateString(cid); -// cJSON_AddItemToObject(json, "id", cid_); + // char cid[32] = {0}; + // sprintf(cid, "%"PRIi64, id); + // cJSON* cid_ = cJSON_CreateString(cid); + // cJSON_AddItemToObject(json, "id", cid_); cJSON* tableName = cJSON_CreateString(name); cJSON_AddItemToObject(json, "tableName", tableName); @@ -2086,10 +2143,10 @@ static char *buildCreateCTableJson(STag* pTag, char* sname, char* name, SArray* cJSON_AddItemToObject(json, "tableType", tableType); cJSON* using = cJSON_CreateString(sname); cJSON_AddItemToObject(json, "using", using); -// cJSON* version = cJSON_CreateNumber(1); -// cJSON_AddItemToObject(json, "version", version); + // cJSON* version = cJSON_CreateNumber(1); + // cJSON_AddItemToObject(json, "version", version); - cJSON* tags = cJSON_CreateArray(); + cJSON* tags = cJSON_CreateArray(); int32_t code = tTagToValArray(pTag, &pTagVals); if (code) { goto end; @@ -2097,18 +2154,18 @@ static char *buildCreateCTableJson(STag* pTag, char* sname, char* name, SArray* if (tTagIsJson(pTag)) { STag* p = (STag*)pTag; - if(p->nTag == 0){ + if (p->nTag == 0) { goto end; } - char* pJson = parseTagDatatoJson(pTag); - cJSON* tag = cJSON_CreateObject(); + char* pJson = parseTagDatatoJson(pTag); + cJSON* tag = cJSON_CreateObject(); STagVal* pTagVal = taosArrayGet(pTagVals, 0); char* ptname = taosArrayGet(tagName, 0); cJSON* tname = cJSON_CreateString(ptname); cJSON_AddItemToObject(tag, "name", tname); -// cJSON* cid_ = cJSON_CreateString(""); -// cJSON_AddItemToObject(tag, "cid", cid_); + // cJSON* cid_ = cJSON_CreateString(""); + // cJSON_AddItemToObject(tag, "cid", cid_); cJSON* ttype = cJSON_CreateNumber(TSDB_DATA_TYPE_JSON); cJSON_AddItemToObject(tag, "type", ttype); cJSON* tvalue = cJSON_CreateString(pJson); @@ -2118,7 +2175,7 @@ static char *buildCreateCTableJson(STag* pTag, char* sname, char* name, SArray* goto end; } - for(int i = 0; i < taosArrayGetSize(pTagVals); i++){ + for (int i = 0; i < taosArrayGetSize(pTagVals); i++) { STagVal* pTagVal = (STagVal*)taosArrayGet(pTagVals, i); cJSON* tag = cJSON_CreateObject(); @@ -2126,8 +2183,8 @@ static char *buildCreateCTableJson(STag* pTag, char* sname, char* name, SArray* char* ptname = taosArrayGet(tagName, i); cJSON* tname = cJSON_CreateString(ptname); cJSON_AddItemToObject(tag, "name", tname); -// cJSON* cid = cJSON_CreateNumber(pTagVal->cid); -// cJSON_AddItemToObject(tag, "cid", cid); + // cJSON* cid = cJSON_CreateNumber(pTagVal->cid); + // cJSON_AddItemToObject(tag, "cid", cid); cJSON* ttype = cJSON_CreateNumber(pTagVal->type); cJSON_AddItemToObject(tag, "type", ttype); @@ -2146,7 +2203,7 @@ static char *buildCreateCTableJson(STag* pTag, char* sname, char* name, SArray* cJSON_AddItemToArray(tags, tag); } - end: +end: cJSON_AddItemToObject(json, "tags", tags); string = cJSON_PrintUnformatted(json); cJSON_Delete(json); @@ -2154,13 +2211,13 @@ static char *buildCreateCTableJson(STag* pTag, char* sname, char* name, SArray* return string; } -static char *processCreateTable(SMqMetaRsp *metaRsp){ +static char* processCreateTable(SMqMetaRsp* metaRsp) { SDecoder decoder = {0}; SVCreateTbBatchReq req = {0}; - SVCreateTbReq *pCreateReq; - char *string = NULL; + SVCreateTbReq* pCreateReq; + char* string = NULL; // decode - void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); + void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); int32_t len = metaRsp->metaRspLen - sizeof(SMsgHead); tDecoderInit(&decoder, data, len); if (tDecodeSVCreateTbBatchReq(&decoder, &req) < 0) { @@ -2170,27 +2227,29 @@ static char *processCreateTable(SMqMetaRsp *metaRsp){ // loop to create table for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { pCreateReq = req.pReqs + iReq; - if(pCreateReq->type == TSDB_CHILD_TABLE){ - string = buildCreateCTableJson((STag*)pCreateReq->ctb.pTag, pCreateReq->ctb.name, pCreateReq->name, pCreateReq->ctb.tagName, pCreateReq->uid); - }else if(pCreateReq->type == TSDB_NORMAL_TABLE){ - string = buildCreateTableJson(&pCreateReq->ntb.schemaRow, NULL, pCreateReq->name, pCreateReq->uid, TSDB_NORMAL_TABLE); + if (pCreateReq->type == TSDB_CHILD_TABLE) { + string = buildCreateCTableJson((STag*)pCreateReq->ctb.pTag, pCreateReq->ctb.name, pCreateReq->name, + pCreateReq->ctb.tagName, pCreateReq->uid); + } else if (pCreateReq->type == TSDB_NORMAL_TABLE) { + string = + buildCreateTableJson(&pCreateReq->ntb.schemaRow, NULL, pCreateReq->name, pCreateReq->uid, TSDB_NORMAL_TABLE); } } tDecoderClear(&decoder); - _exit: +_exit: tDecoderClear(&decoder); return string; } -static char *processAlterTable(SMqMetaRsp *metaRsp){ - SDecoder decoder = {0}; - SVAlterTbReq vAlterTbReq = {0}; - char *string = NULL; +static char* processAlterTable(SMqMetaRsp* metaRsp) { + SDecoder decoder = {0}; + SVAlterTbReq vAlterTbReq = {0}; + char* string = NULL; // decode - void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); + void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); int32_t len = metaRsp->metaRspLen - sizeof(SMsgHead); tDecoderInit(&decoder, data, len); if (tDecodeSVAlterTbReq(&decoder, &vAlterTbReq) < 0) { @@ -2203,8 +2262,8 @@ static char *processAlterTable(SMqMetaRsp *metaRsp){ } cJSON* type = cJSON_CreateString("alter"); cJSON_AddItemToObject(json, "type", type); -// cJSON* uid = cJSON_CreateNumber(id); -// cJSON_AddItemToObject(json, "uid", uid); + // cJSON* uid = cJSON_CreateNumber(id); + // cJSON_AddItemToObject(json, "uid", uid); cJSON* tableName = cJSON_CreateString(vAlterTbReq.tbName); cJSON_AddItemToObject(json, "tableName", tableName); cJSON* tableType = cJSON_CreateString(vAlterTbReq.action == TSDB_ALTER_TABLE_UPDATE_TAG_VAL ? "child" : "normal"); @@ -2219,55 +2278,55 @@ static char *processAlterTable(SMqMetaRsp *metaRsp){ cJSON* colType = cJSON_CreateNumber(vAlterTbReq.type); cJSON_AddItemToObject(json, "colType", colType); - if(vAlterTbReq.type == TSDB_DATA_TYPE_BINARY){ + if (vAlterTbReq.type == TSDB_DATA_TYPE_BINARY) { int32_t length = vAlterTbReq.bytes - VARSTR_HEADER_SIZE; - cJSON* cbytes = cJSON_CreateNumber(length); + cJSON* cbytes = cJSON_CreateNumber(length); cJSON_AddItemToObject(json, "colLength", cbytes); - }else if (vAlterTbReq.type == TSDB_DATA_TYPE_NCHAR){ - int32_t length = (vAlterTbReq.bytes - VARSTR_HEADER_SIZE)/TSDB_NCHAR_SIZE; - cJSON* cbytes = cJSON_CreateNumber(length); + } else if (vAlterTbReq.type == TSDB_DATA_TYPE_NCHAR) { + int32_t length = (vAlterTbReq.bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; + cJSON* cbytes = cJSON_CreateNumber(length); cJSON_AddItemToObject(json, "colLength", cbytes); } break; } - case TSDB_ALTER_TABLE_DROP_COLUMN:{ + case TSDB_ALTER_TABLE_DROP_COLUMN: { cJSON* colName = cJSON_CreateString(vAlterTbReq.colName); cJSON_AddItemToObject(json, "colName", colName); break; } - case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES:{ + case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: { cJSON* colName = cJSON_CreateString(vAlterTbReq.colName); cJSON_AddItemToObject(json, "colName", colName); cJSON* colType = cJSON_CreateNumber(vAlterTbReq.colModType); cJSON_AddItemToObject(json, "colType", colType); - if(vAlterTbReq.colModType == TSDB_DATA_TYPE_BINARY){ + if (vAlterTbReq.colModType == TSDB_DATA_TYPE_BINARY) { int32_t length = vAlterTbReq.colModBytes - VARSTR_HEADER_SIZE; - cJSON* cbytes = cJSON_CreateNumber(length); + cJSON* cbytes = cJSON_CreateNumber(length); cJSON_AddItemToObject(json, "colLength", cbytes); - }else if (vAlterTbReq.colModType == TSDB_DATA_TYPE_NCHAR){ - int32_t length = (vAlterTbReq.colModBytes - VARSTR_HEADER_SIZE)/TSDB_NCHAR_SIZE; - cJSON* cbytes = cJSON_CreateNumber(length); + } else if (vAlterTbReq.colModType == TSDB_DATA_TYPE_NCHAR) { + int32_t length = (vAlterTbReq.colModBytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; + cJSON* cbytes = cJSON_CreateNumber(length); cJSON_AddItemToObject(json, "colLength", cbytes); } break; } - case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME:{ + case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME: { cJSON* colName = cJSON_CreateString(vAlterTbReq.colName); cJSON_AddItemToObject(json, "colName", colName); cJSON* colNewName = cJSON_CreateString(vAlterTbReq.colNewName); cJSON_AddItemToObject(json, "colNewName", colNewName); break; } - case TSDB_ALTER_TABLE_UPDATE_TAG_VAL:{ + case TSDB_ALTER_TABLE_UPDATE_TAG_VAL: { cJSON* tagName = cJSON_CreateString(vAlterTbReq.tagName); cJSON_AddItemToObject(json, "colName", tagName); bool isNull = vAlterTbReq.isNull; - if(vAlterTbReq.tagType == TSDB_DATA_TYPE_JSON){ - STag *jsonTag = (STag *)vAlterTbReq.pTagVal; - if(jsonTag->nTag == 0) isNull = true; + if (vAlterTbReq.tagType == TSDB_DATA_TYPE_JSON) { + STag* jsonTag = (STag*)vAlterTbReq.pTagVal; + if (jsonTag->nTag == 0) isNull = true; } - if (!isNull){ + if (!isNull) { char* buf = NULL; if (vAlterTbReq.tagType == TSDB_DATA_TYPE_JSON) { @@ -2292,18 +2351,18 @@ static char *processAlterTable(SMqMetaRsp *metaRsp){ } string = cJSON_PrintUnformatted(json); - _exit: +_exit: tDecoderClear(&decoder); return string; } -static char *processDropSTable(SMqMetaRsp *metaRsp){ - SDecoder decoder = {0}; - SVDropStbReq req = {0}; - char *string = NULL; +static char* processDropSTable(SMqMetaRsp* metaRsp) { + SDecoder decoder = {0}; + SVDropStbReq req = {0}; + char* string = NULL; // decode - void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); + void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); int32_t len = metaRsp->metaRspLen - sizeof(SMsgHead); tDecoderInit(&decoder, data, len); if (tDecodeSVDropStbReq(&decoder, &req) < 0) { @@ -2323,18 +2382,18 @@ static char *processDropSTable(SMqMetaRsp *metaRsp){ string = cJSON_PrintUnformatted(json); - _exit: +_exit: tDecoderClear(&decoder); return string; } -static char *processDropTable(SMqMetaRsp *metaRsp){ - SDecoder decoder = {0}; - SVDropTbBatchReq req = {0}; - char *string = NULL; +static char* processDropTable(SMqMetaRsp* metaRsp) { + SDecoder decoder = {0}; + SVDropTbBatchReq req = {0}; + char* string = NULL; // decode - void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); + void* data = POINTER_SHIFT(metaRsp->metaRsp, sizeof(SMsgHead)); int32_t len = metaRsp->metaRspLen - sizeof(SMsgHead); tDecoderInit(&decoder, data, len); if (tDecodeSVDropTbBatchReq(&decoder, &req) < 0) { @@ -2347,10 +2406,10 @@ static char *processDropTable(SMqMetaRsp *metaRsp){ } cJSON* type = cJSON_CreateString("drop"); cJSON_AddItemToObject(json, "type", type); -// cJSON* uid = cJSON_CreateNumber(id); -// cJSON_AddItemToObject(json, "uid", uid); -// cJSON* tableType = cJSON_CreateString("normal"); -// cJSON_AddItemToObject(json, "tableType", tableType); + // cJSON* uid = cJSON_CreateNumber(id); + // cJSON_AddItemToObject(json, "uid", uid); + // cJSON* tableType = cJSON_CreateString("normal"); + // cJSON_AddItemToObject(json, "tableType", tableType); cJSON* tableNameList = cJSON_CreateArray(); for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { @@ -2363,55 +2422,53 @@ static char *processDropTable(SMqMetaRsp *metaRsp){ string = cJSON_PrintUnformatted(json); - _exit: +_exit: tDecoderClear(&decoder); return string; } -char *tmq_get_json_meta(TAOS_RES *res){ +char* tmq_get_json_meta(TAOS_RES* res) { if (!TD_RES_TMQ_META(res)) { return NULL; } SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res; - if(pMetaRspObj->metaRsp.resMsgType == TDMT_VND_CREATE_STB){ + if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_CREATE_STB) { return processCreateStb(&pMetaRspObj->metaRsp); - }else if(pMetaRspObj->metaRsp.resMsgType == TDMT_VND_ALTER_STB){ + } else if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_ALTER_STB) { return processAlterStb(&pMetaRspObj->metaRsp); - }else if(pMetaRspObj->metaRsp.resMsgType == TDMT_VND_DROP_STB){ + } else if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_DROP_STB) { return processDropSTable(&pMetaRspObj->metaRsp); - }else if(pMetaRspObj->metaRsp.resMsgType == TDMT_VND_CREATE_TABLE){ + } else if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_CREATE_TABLE) { return processCreateTable(&pMetaRspObj->metaRsp); - }else if(pMetaRspObj->metaRsp.resMsgType == TDMT_VND_ALTER_TABLE){ + } else if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_ALTER_TABLE) { return processAlterTable(&pMetaRspObj->metaRsp); - }else if(pMetaRspObj->metaRsp.resMsgType == TDMT_VND_DROP_TABLE){ + } else if (pMetaRspObj->metaRsp.resMsgType == TDMT_VND_DROP_TABLE) { return processDropTable(&pMetaRspObj->metaRsp); } return NULL; } -void tmq_free_json_meta(char* jsonMeta){ - taosMemoryFreeClear(jsonMeta); -} +void tmq_free_json_meta(char* jsonMeta) { taosMemoryFreeClear(jsonMeta); } -static int32_t taosCreateStb(TAOS *taos, void *meta, int32_t metaLen){ +static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) { SVCreateStbReq req = {0}; SDecoder coder; SMCreateStbReq pReq = {0}; - int32_t code = TSDB_CODE_SUCCESS; - SRequestObj* pRequest = NULL; + int32_t code = TSDB_CODE_SUCCESS; + SRequestObj* pRequest = NULL; code = buildRequest(*(int64_t*)taos, "", 0, NULL, false, &pRequest); if (code != TSDB_CODE_SUCCESS) { goto end; } - if(!pRequest->pDb){ + if (!pRequest->pDb) { code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; goto end; } // decode and process req - void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); + void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); int32_t len = metaLen - sizeof(SMsgHead); tDecoderInit(&coder, data, len); if (tDecodeSVCreateStbReq(&coder, &req) < 0) { @@ -2420,16 +2477,16 @@ static int32_t taosCreateStb(TAOS *taos, void *meta, int32_t metaLen){ } // build create stable pReq.pColumns = taosArrayInit(req.schemaRow.nCols, sizeof(SField)); - for(int32_t i = 0; i < req.schemaRow.nCols; i++){ + for (int32_t i = 0; i < req.schemaRow.nCols; i++) { SSchema* pSchema = req.schemaRow.pSchema + i; - SField field = {.type = pSchema->type, .bytes = pSchema->bytes}; + SField field = {.type = pSchema->type, .bytes = pSchema->bytes}; strcpy(field.name, pSchema->name); taosArrayPush(pReq.pColumns, &field); } pReq.pTags = taosArrayInit(req.schemaTag.nCols, sizeof(SField)); - for(int32_t i = 0; i < req.schemaTag.nCols; i++){ + for (int32_t i = 0; i < req.schemaTag.nCols; i++) { SSchema* pSchema = req.schemaTag.pSchema + i; - SField field = {.type = pSchema->type, .bytes = pSchema->bytes}; + SField field = {.type = pSchema->type, .bytes = pSchema->bytes}; strcpy(field.name, pSchema->name); taosArrayPush(pReq.pTags, &field); } @@ -2444,7 +2501,7 @@ static int32_t taosCreateStb(TAOS *taos, void *meta, int32_t metaLen){ pReq.igExists = true; STscObj* pTscObj = pRequest->pTscObj; - SName tableName; + SName tableName; tNameExtractFullName(toName(pTscObj->acctId, pRequest->pDb, req.name, &tableName), pReq.name); SCmdMsgInfo pCmdMsg = {0}; @@ -2458,7 +2515,7 @@ static int32_t taosCreateStb(TAOS *taos, void *meta, int32_t metaLen){ } tSerializeSMCreateStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq); - SQuery pQuery = {0}; + SQuery pQuery = {0}; pQuery.execMode = QUERY_EXEC_MODE_RPC; pQuery.pCmdMsg = &pCmdMsg; pQuery.msgType = pQuery.pCmdMsg->msgType; @@ -2468,18 +2525,18 @@ static int32_t taosCreateStb(TAOS *taos, void *meta, int32_t metaLen){ code = pRequest->code; taosMemoryFree(pCmdMsg.pMsg); - end: +end: destroyRequest(pRequest); tFreeSMCreateStbReq(&pReq); tDecoderClear(&coder); return code; } -static int32_t taosDropStb(TAOS *taos, void *meta, int32_t metaLen){ +static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) { SVDropStbReq req = {0}; SDecoder coder; SMDropStbReq pReq = {0}; - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; SRequestObj* pRequest = NULL; code = buildRequest(*(int64_t*)taos, "", 0, NULL, false, &pRequest); @@ -2487,12 +2544,12 @@ static int32_t taosDropStb(TAOS *taos, void *meta, int32_t metaLen){ goto end; } - if(!pRequest->pDb){ + if (!pRequest->pDb) { code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; goto end; } // decode and process req - void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); + void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); int32_t len = metaLen - sizeof(SMsgHead); tDecoderInit(&coder, data, len); if (tDecodeSVDropStbReq(&coder, &req) < 0) { @@ -2506,7 +2563,7 @@ static int32_t taosDropStb(TAOS *taos, void *meta, int32_t metaLen){ pReq.suid = req.suid; STscObj* pTscObj = pRequest->pTscObj; - SName tableName; + SName tableName; tNameExtractFullName(toName(pTscObj->acctId, pRequest->pDb, req.name, &tableName), pReq.name); SCmdMsgInfo pCmdMsg = {0}; @@ -2520,7 +2577,7 @@ static int32_t taosDropStb(TAOS *taos, void *meta, int32_t metaLen){ } tSerializeSMDropStbReq(pCmdMsg.pMsg, pCmdMsg.msgLen, &pReq); - SQuery pQuery = {0}; + SQuery pQuery = {0}; pQuery.execMode = QUERY_EXEC_MODE_RPC; pQuery.pCmdMsg = &pCmdMsg; pQuery.msgType = pQuery.pCmdMsg->msgType; @@ -2530,7 +2587,7 @@ static int32_t taosDropStb(TAOS *taos, void *meta, int32_t metaLen){ code = pRequest->code; taosMemoryFree(pCmdMsg.pMsg); - end: +end: destroyRequest(pRequest); tDecoderClear(&coder); return code; @@ -2543,7 +2600,7 @@ typedef struct SVgroupCreateTableBatch { } SVgroupCreateTableBatch; static void destroyCreateTbReqBatch(void* data) { - SVgroupCreateTableBatch* pTbBatch = (SVgroupCreateTableBatch*) data; + SVgroupCreateTableBatch* pTbBatch = (SVgroupCreateTableBatch*)data; taosArrayDestroy(pTbBatch->req.pArray); } @@ -2560,13 +2617,12 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) { goto end; } - - if(!pRequest->pDb){ + if (!pRequest->pDb) { code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; goto end; } // decode and process req - void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); + void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); int32_t len = metaLen - sizeof(SMsgHead); tDecoderInit(&coder, data, len); if (tDecodeSVCreateTbBatchReq(&coder, &req) < 0) { @@ -2591,15 +2647,15 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) { taosHashSetFreeFp(pVgroupHashmap, destroyCreateTbReqBatch); SRequestConnInfo conn = {.pTrans = pTscObj->pAppInfo->pTransporter, - .requestId = pRequest->requestId, - .requestObjRefId = pRequest->self, - .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)}; + .requestId = pRequest->requestId, + .requestObjRefId = pRequest->self, + .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)}; // loop to create table for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { pCreateReq = req.pReqs + iReq; SVgroupInfo pInfo = {0}; - SName pName; + SName pName; toName(pTscObj->acctId, pRequest->pDb, pCreateReq->name, &pName); code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &pInfo); if (code != TSDB_CODE_SUCCESS) { @@ -2631,7 +2687,7 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) { pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; pQuery->msgType = TDMT_VND_CREATE_TABLE; pQuery->stableQuery = false; - pQuery->pRoot = nodesMakeNode(QUERY_NODE_CREATE_TABLE_STMT); + pQuery->pRoot = nodesMakeNode(QUERY_NODE_CREATE_TABLE_STMT); code = rewriteToVnodeModifyOpStmt(pQuery, pBufArray); if (code != TSDB_CODE_SUCCESS) { @@ -2639,10 +2695,10 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) { } launchQueryImpl(pRequest, pQuery, false, NULL); - pQuery = NULL; // no need to free in the end - code = pRequest->code; + pQuery = NULL; // no need to free in the end + code = pRequest->code; - end: +end: taosHashCleanup(pVgroupHashmap); destroyRequest(pRequest); tDecoderClear(&coder); @@ -2674,12 +2730,12 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) { goto end; } - if(!pRequest->pDb){ + if (!pRequest->pDb) { code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; goto end; } // decode and process req - void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); + void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); int32_t len = metaLen - sizeof(SMsgHead); tDecoderInit(&coder, data, len); if (tDecodeSVDropTbBatchReq(&coder, &req) < 0) { @@ -2704,16 +2760,16 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) { taosHashSetFreeFp(pVgroupHashmap, destroyDropTbReqBatch); SRequestConnInfo conn = {.pTrans = pTscObj->pAppInfo->pTransporter, - .requestId = pRequest->requestId, - .requestObjRefId = pRequest->self, - .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)}; + .requestId = pRequest->requestId, + .requestObjRefId = pRequest->self, + .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)}; // loop to create table for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { pDropReq = req.pReqs + iReq; pDropReq->igNotExists = true; SVgroupInfo pInfo = {0}; - SName pName; + SName pName; toName(pTscObj->acctId, pRequest->pDb, pDropReq->name, &pName); code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &pInfo); if (code != TSDB_CODE_SUCCESS) { @@ -2743,7 +2799,7 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) { pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; pQuery->msgType = TDMT_VND_DROP_TABLE; pQuery->stableQuery = false; - pQuery->pRoot = nodesMakeNode(QUERY_NODE_DROP_TABLE_STMT); + pQuery->pRoot = nodesMakeNode(QUERY_NODE_DROP_TABLE_STMT); code = rewriteToVnodeModifyOpStmt(pQuery, pBufArray); if (code != TSDB_CODE_SUCCESS) { @@ -2751,10 +2807,10 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) { } launchQueryImpl(pRequest, pQuery, false, NULL); - pQuery = NULL; // no need to free in the end - code = pRequest->code; + pQuery = NULL; // no need to free in the end + code = pRequest->code; - end: +end: taosHashCleanup(pVgroupHashmap); destroyRequest(pRequest); tDecoderClear(&coder); @@ -2762,27 +2818,27 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) { return code; } -static int32_t taosAlterTable(TAOS *taos, void *meta, int32_t metaLen){ - SVAlterTbReq req = {0}; - SDecoder coder = {0}; - int32_t code = TSDB_CODE_SUCCESS; - SRequestObj *pRequest = NULL; - SQuery *pQuery = NULL; - SArray *pArray = NULL; - SVgDataBlocks *pVgData = NULL; +static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) { + SVAlterTbReq req = {0}; + SDecoder coder = {0}; + int32_t code = TSDB_CODE_SUCCESS; + SRequestObj* pRequest = NULL; + SQuery* pQuery = NULL; + SArray* pArray = NULL; + SVgDataBlocks* pVgData = NULL; - code = buildRequest(*(int64_t*) taos, "", 0, NULL, false, &pRequest); + code = buildRequest(*(int64_t*)taos, "", 0, NULL, false, &pRequest); if (code != TSDB_CODE_SUCCESS) { goto end; } - if(!pRequest->pDb){ + if (!pRequest->pDb) { code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; goto end; } // decode and process req - void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); + void* data = POINTER_SHIFT(meta, sizeof(SMsgHead)); int32_t len = metaLen - sizeof(SMsgHead); tDecoderInit(&coder, data, len); if (tDecodeSVAlterTbReq(&coder, &req) < 0) { @@ -2791,24 +2847,24 @@ static int32_t taosAlterTable(TAOS *taos, void *meta, int32_t metaLen){ } // do not deal TSDB_ALTER_TABLE_UPDATE_OPTIONS - if(req.action == TSDB_ALTER_TABLE_UPDATE_OPTIONS){ + if (req.action == TSDB_ALTER_TABLE_UPDATE_OPTIONS) { goto end; } STscObj* pTscObj = pRequest->pTscObj; - SCatalog *pCatalog = NULL; + SCatalog* pCatalog = NULL; code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCatalog); if (code != TSDB_CODE_SUCCESS) { goto end; } SRequestConnInfo conn = {.pTrans = pTscObj->pAppInfo->pTransporter, - .requestId = pRequest->requestId, - .requestObjRefId = pRequest->self, - .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)}; + .requestId = pRequest->requestId, + .requestObjRefId = pRequest->self, + .mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp)}; SVgroupInfo pInfo = {0}; - SName pName = {0}; + SName pName = {0}; toName(pTscObj->acctId, pRequest->pDb, req.tbName, &pName); code = catalogGetTableHashVgroup(pCatalog, &conn, &pName, &pInfo); if (code != TSDB_CODE_SUCCESS) { @@ -2842,7 +2898,7 @@ static int32_t taosAlterTable(TAOS *taos, void *meta, int32_t metaLen){ pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; pQuery->msgType = TDMT_VND_ALTER_TABLE; pQuery->stableQuery = false; - pQuery->pRoot = nodesMakeNode(QUERY_NODE_ALTER_TABLE_STMT); + pQuery->pRoot = nodesMakeNode(QUERY_NODE_ALTER_TABLE_STMT); code = rewriteToVnodeModifyOpStmt(pQuery, pArray); if (code != TSDB_CODE_SUCCESS) { @@ -2850,17 +2906,17 @@ static int32_t taosAlterTable(TAOS *taos, void *meta, int32_t metaLen){ } launchQueryImpl(pRequest, pQuery, false, NULL); - pQuery = NULL; // no need to free in the end + pQuery = NULL; // no need to free in the end pVgData = NULL; - pArray = NULL; - code = pRequest->code; - if (code == TSDB_CODE_VND_TABLE_NOT_EXIST){ + pArray = NULL; + code = pRequest->code; + if (code == TSDB_CODE_VND_TABLE_NOT_EXIST) { code = 0; } - end: +end: taosArrayDestroy(pArray); - if(pVgData) taosMemoryFreeClear(pVgData->pData); + if (pVgData) taosMemoryFreeClear(pVgData->pData); taosMemoryFreeClear(pVgData); destroyRequest(pRequest); tDecoderClear(&coder); @@ -2889,12 +2945,17 @@ int32_t taos_write_raw_meta(TAOS *taos, tmq_raw_data raw_meta){ return TSDB_CODE_INVALID_PARA; } -void tmq_free_raw_meta(tmq_raw_data *rawMeta){ +void tmq_free_raw_meta(tmq_raw_data* rawMeta) { + // taosMemoryFreeClear(rawMeta); } void tmq_commit_async(tmq_t* tmq, const TAOS_RES* msg, tmq_commit_cb* cb, void* param) { + // tmqCommitInner2(tmq, msg, 0, 1, cb, param); } -int32_t tmq_commit_sync(tmq_t* tmq, const TAOS_RES* msg) { return tmqCommitInner2(tmq, msg, 0, 0, NULL, NULL); } \ No newline at end of file +int32_t tmq_commit_sync(tmq_t* tmq, const TAOS_RES* msg) { + // + return tmqCommitInner2(tmq, msg, 0, 0, NULL, NULL); +} diff --git a/source/common/src/systable.c b/source/common/src/systable.c index c9c1c1b395..1f9966fd54 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -153,6 +153,15 @@ static const SSysDbTableSchema userTblsSchema[] = { {.name = "type", .bytes = 21 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, }; +static const SSysDbTableSchema userTagsSchema[] = { + {.name = "table_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "stable_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "tag_name", .bytes = TSDB_COL_NAME_LEN - 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "tag_type", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, + {.name = "tag_value", .bytes = TSDB_MAX_TAGS_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, +}; + static const SSysDbTableSchema userTblDistSchema[] = { {.name = "db_name", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "table_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR}, @@ -255,6 +264,7 @@ static const SSysTableMeta infosMeta[] = { {TSDB_INS_TABLE_USER_STABLES, userStbsSchema, tListLen(userStbsSchema)}, {TSDB_PERFS_TABLE_STREAMS, streamSchema, tListLen(streamSchema)}, {TSDB_INS_TABLE_USER_TABLES, userTblsSchema, tListLen(userTblsSchema)}, + {TSDB_INS_TABLE_USER_TAGS, userTagsSchema, tListLen(userTagsSchema)}, // {TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED, userTblDistSchema, tListLen(userTblDistSchema)}, {TSDB_INS_TABLE_USER_USERS, userUsersSchema, tListLen(userUsersSchema)}, {TSDB_INS_TABLE_LICENCES, grantsSchema, tListLen(grantsSchema)}, diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index cd21de52ed..d9ac84ec06 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1479,10 +1479,12 @@ static int32_t colDataMoveVarData(SColumnInfoData* pColInfoData, size_t start, s } beigin++; } + if (dataOffset > 0) { memmove(pColInfoData->pData, pColInfoData->pData + dataOffset, dataLen); - memmove(pColInfoData->varmeta.offset, &pColInfoData->varmeta.offset[start], (end - start) * sizeof(int32_t)); } + + memmove(pColInfoData->varmeta.offset, &pColInfoData->varmeta.offset[start], (end - start) * sizeof(int32_t)); return dataLen; } @@ -1752,7 +1754,7 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf) int32_t colNum = taosArrayGetSize(pDataBlock->pDataBlock); int32_t rows = pDataBlock->info.rows; int32_t len = 0; - len += snprintf(dumpBuf + len, size - len, "%s |block type %d |child id %d|group id:%" PRIu64 "| uid:%ld|======\n", "dumpBlockData", + len += snprintf(dumpBuf + len, size - len, "===stream===%s |block type %d |child id %d|group id:%" PRIu64 "| uid:%ld|\n", flag, (int32_t)pDataBlock->info.type, pDataBlock->info.childId, pDataBlock->info.groupId, pDataBlock->info.uid); if (len >= size - 1) return dumpBuf; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index e4032118c6..c4f3ca0bdf 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -184,6 +184,7 @@ int32_t tsTransPullupInterval = 2; int32_t tsMqRebalanceInterval = 2; int32_t tsTtlUnit = 86400; int32_t tsTtlPushInterval = 60; +int32_t tsGrantHBInterval = 60; void taosAddDataDir(int32_t index, char *v1, int32_t level, int32_t primary) { tstrncpy(tsDiskCfg[index].dir, v1, TSDB_FILENAME_LEN); diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index aeb83d3425..b79c412914 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -4919,8 +4919,8 @@ int tDecodeSVCreateStbReq(SDecoder *pCoder, SVCreateStbReq *pReq) { if (tDecodeCStr(pCoder, &pReq->name) < 0) return -1; if (tDecodeI64(pCoder, &pReq->suid) < 0) return -1; if (tDecodeI8(pCoder, &pReq->rollup) < 0) return -1; - if (tDecodeSSchemaWrapper(pCoder, &pReq->schemaRow) < 0) return -1; - if (tDecodeSSchemaWrapper(pCoder, &pReq->schemaTag) < 0) return -1; + if (tDecodeSSchemaWrapperEx(pCoder, &pReq->schemaRow) < 0) return -1; + if (tDecodeSSchemaWrapperEx(pCoder, &pReq->schemaTag) < 0) return -1; if (pReq->rollup) { if (tDecodeSRSmaParam(pCoder, &pReq->rsmaParam) < 0) return -1; } @@ -5648,7 +5648,6 @@ int32_t tDecodeDeleteRes(SDecoder *pCoder, SDeleteRes *pRes) { int32_t tEncodeSMqDataRsp(SEncoder *pEncoder, const SMqDataRsp *pRsp) { if (tEncodeSTqOffsetVal(pEncoder, &pRsp->reqOffset) < 0) return -1; if (tEncodeSTqOffsetVal(pEncoder, &pRsp->rspOffset) < 0) return -1; - if (tEncodeI32(pEncoder, pRsp->skipLogNum) < 0) return -1; if (tEncodeI32(pEncoder, pRsp->blockNum) < 0) return -1; if (pRsp->blockNum != 0) { if (tEncodeI8(pEncoder, pRsp->withTbName) < 0) return -1; @@ -5674,7 +5673,6 @@ int32_t tEncodeSMqDataRsp(SEncoder *pEncoder, const SMqDataRsp *pRsp) { int32_t tDecodeSMqDataRsp(SDecoder *pDecoder, SMqDataRsp *pRsp) { if (tDecodeSTqOffsetVal(pDecoder, &pRsp->reqOffset) < 0) return -1; if (tDecodeSTqOffsetVal(pDecoder, &pRsp->rspOffset) < 0) return -1; - if (tDecodeI32(pDecoder, &pRsp->skipLogNum) < 0) return -1; if (tDecodeI32(pDecoder, &pRsp->blockNum) < 0) return -1; if (pRsp->blockNum != 0) { pRsp->blockData = taosArrayInit(pRsp->blockNum, sizeof(void *)); diff --git a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h index 4479c06bea..dc4412b77b 100644 --- a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h +++ b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h @@ -46,6 +46,7 @@ int32_t dmProcessAuthRsp(SDnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t dmProcessGrantRsp(SDnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t dmProcessServerRunStatus(SDnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg); +int32_t dmProcessGrantReq(SRpcMsg *pMsg); // dmWorker.c int32_t dmPutNodeMsgToMgmtQueue(SDnodeMgmt *pMgmt, SRpcMsg *pMsg); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 5f982ad3a4..62f1f9e315 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -331,7 +331,8 @@ SArray *dmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_DND_SYSTABLE_RETRIEVE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; // Requests handled by MNODE - if (dmSetMgmtHandle(pArray, TDMT_MND_GRANT_RSP, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_GRANT, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; + // if (dmSetMgmtHandle(pArray, TDMT_MND_GRANT_RSP, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_AUTH_RSP, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; code = 0; diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index 89e8aa976e..9d9217267a 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -144,6 +144,9 @@ static void dmProcessMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { case TDMT_DND_SYSTABLE_RETRIEVE: code = dmProcessRetrieve(pMgmt, pMsg); break; + case TDMT_MND_GRANT: + code = dmProcessGrantReq(pMsg); + break; default: terrno = TSDB_CODE_MSG_NOT_PROCESSED; break; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index b9a02728fc..8db43a460e 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -196,6 +196,7 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_TOPIC, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_SUBSCRIBE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_MQ_ASK_EP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_MQ_HB, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_MQ_DROP_CGROUP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_MQ_DROP_CGROUP_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_MQ_COMMIT_OFFSET, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; @@ -205,7 +206,7 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_MND_HEARTBEAT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_STATUS, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_SYSTABLE_RETRIEVE, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_GRANT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + // if (dmSetMgmtHandle(pArray, TDMT_MND_GRANT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_AUTH, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_SHOW_VARIABLES, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_SERVER_VERSION, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c b/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c index 6814643b59..1c7edbe6be 100644 --- a/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c +++ b/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c @@ -81,6 +81,7 @@ int32_t qmPutRpcMsgToQueue(SQnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) { taosWriteQitem(pMgmt->queryWorker.queue, pMsg); return 0; case READ_QUEUE: + case FETCH_QUEUE: dTrace("msg:%p, is created and will put into qnode-fetch queue", pMsg); taosWriteQitem(pMgmt->fetchWorker.queue, pMsg); return 0; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 4a60dbfe0f..93df7f8ab2 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -153,9 +153,15 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp switch (qtype) { case QUERY_QUEUE: - vnodePreprocessQueryMsg(pVnode->pImpl, pMsg); - dGTrace("vgId:%d, msg:%p put into vnode-query queue", pVnode->vgId, pMsg); - taosWriteQitem(pVnode->pQueryQ, pMsg); + if ((pMsg->msgType == TDMT_SCH_QUERY) && (grantCheck(TSDB_GRANT_TIME) != TSDB_CODE_SUCCESS)) { + terrno = TSDB_CODE_GRANT_EXPIRED; + code = terrno; + dDebug("vgId:%d, msg:%p put into vnode-query queue failed since %s", pVnode->vgId, pMsg, terrstr()); + } else { + vnodePreprocessQueryMsg(pVnode->pImpl, pMsg); + dGTrace("vgId:%d, msg:%p put into vnode-query queue", pVnode->vgId, pMsg); + taosWriteQitem(pVnode->pQueryQ, pMsg); + } break; case STREAM_QUEUE: dGTrace("vgId:%d, msg:%p put into vnode-stream queue", pVnode->vgId, pMsg); @@ -166,8 +172,14 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp taosWriteQitem(pVnode->pFetchQ, pMsg); break; case WRITE_QUEUE: - dGTrace("vgId:%d, msg:%p put into vnode-write queue", pVnode->vgId, pMsg); - taosWriteQitem(pVnode->pWriteQ, pMsg); + if ((pMsg->msgType == TDMT_VND_SUBMIT) && (grantCheck(TSDB_GRANT_STORAGE) != TSDB_CODE_SUCCESS)) { + terrno = TSDB_CODE_VND_NO_WRITE_AUTH; + code = terrno; + dDebug("vgId:%d, msg:%p put into vnode-write queue failed since %s", pVnode->vgId, pMsg, terrstr()); + } else { + dGTrace("vgId:%d, msg:%p put into vnode-write queue", pVnode->vgId, pMsg); + taosWriteQitem(pVnode->pWriteQ, pMsg); + } break; case SYNC_QUEUE: dGTrace("vgId:%d, msg:%p put into vnode-sync queue", pVnode->vgId, pMsg); diff --git a/source/dnode/mnode/impl/inc/mndCluster.h b/source/dnode/mnode/impl/inc/mndCluster.h index 5b7bac4486..0de253fb6a 100644 --- a/source/dnode/mnode/impl/inc/mndCluster.h +++ b/source/dnode/mnode/impl/inc/mndCluster.h @@ -26,6 +26,7 @@ int32_t mndInitCluster(SMnode *pMnode); void mndCleanupCluster(SMnode *pMnode); int32_t mndGetClusterName(SMnode *pMnode, char *clusterName, int32_t len); int64_t mndGetClusterId(SMnode *pMnode); +int64_t mndGetClusterCreateTime(SMnode *pMnode); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndDnode.h b/source/dnode/mnode/impl/inc/mndDnode.h index cf1e7422be..ebbabdfa33 100644 --- a/source/dnode/mnode/impl/inc/mndDnode.h +++ b/source/dnode/mnode/impl/inc/mndDnode.h @@ -29,6 +29,7 @@ void mndReleaseDnode(SMnode *pMnode, SDnodeObj *pDnode); SEpSet mndGetDnodeEpset(SDnodeObj *pDnode); int32_t mndGetDnodeSize(SMnode *pMnode); bool mndIsDnodeOnline(SDnodeObj *pDnode, int64_t curMs); +void mndGetDnodeData(SMnode *pMnode, SArray *pDnodeEps); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndGrant.h b/source/dnode/mnode/impl/inc/mndGrant.h index 525dd2c2e5..9c8e853a21 100644 --- a/source/dnode/mnode/impl/inc/mndGrant.h +++ b/source/dnode/mnode/impl/inc/mndGrant.h @@ -22,27 +22,10 @@ #include "mndInt.h" -typedef enum { - TSDB_GRANT_ALL, - TSDB_GRANT_TIME, - TSDB_GRANT_USER, - TSDB_GRANT_DB, - TSDB_GRANT_TIMESERIES, - TSDB_GRANT_DNODE, - TSDB_GRANT_ACCT, - TSDB_GRANT_STORAGE, - TSDB_GRANT_SPEED, - TSDB_GRANT_QUERY_TIME, - TSDB_GRANT_CONNS, - TSDB_GRANT_STREAMS, - TSDB_GRANT_CPU_CORES, -} EGrantType; - -int32_t mndInitGrant(); +int32_t mndInitGrant(SMnode *pMnode); void mndCleanupGrant(); void grantParseParameter(); -int32_t grantCheck(EGrantType grant); -void grantReset(EGrantType grant, uint64_t value); +void grantReset(SMnode *pMnode, EGrantType grant, uint64_t value); void grantAdd(EGrantType grant, uint64_t value); void grantRestore(EGrantType grant, uint64_t value); diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index 8ad1ac56e4..f72b69a7de 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -24,6 +24,7 @@ #include "tcache.h" #include "tdatablock.h" #include "tglobal.h" +#include "tgrant.h" #include "tqueue.h" #include "ttime.h" #include "version.h" diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index bb3377d16a..a82bf739f5 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -79,6 +79,23 @@ int64_t mndGetClusterId(SMnode *pMnode) { return clusterId; } +int64_t mndGetClusterCreateTime(SMnode *pMnode) { + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; + int64_t createTime = INT64_MAX; + + while (1) { + SClusterObj *pCluster = NULL; + pIter = sdbFetch(pSdb, SDB_CLUSTER, pIter, (void **)&pCluster); + if (pIter == NULL) break; + + createTime = pCluster->createdTime; + sdbRelease(pSdb, pCluster); + } + + return createTime; +} + static SSdbRaw *mndClusterActionEncode(SClusterObj *pCluster) { terrno = TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 27b785f4d2..a60db8a8c2 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -48,6 +48,7 @@ static void mndCancelGetNextConsumer(SMnode *pMnode, void *pIter); static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg); static int32_t mndProcessAskEpReq(SRpcMsg *pMsg); +static int32_t mndProcessMqHbReq(SRpcMsg *pMsg); static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg); static int32_t mndProcessConsumerLostMsg(SRpcMsg *pMsg); static int32_t mndProcessConsumerRecoverMsg(SRpcMsg *pMsg); @@ -62,6 +63,7 @@ int32_t mndInitConsumer(SMnode *pMnode) { .deleteFp = (SdbDeleteFp)mndConsumerActionDelete}; mndSetMsgHandle(pMnode, TDMT_MND_SUBSCRIBE, mndProcessSubscribeReq); + mndSetMsgHandle(pMnode, TDMT_MND_MQ_HB, mndProcessMqHbReq); mndSetMsgHandle(pMnode, TDMT_MND_MQ_ASK_EP, mndProcessAskEpReq); mndSetMsgHandle(pMnode, TDMT_MND_MQ_TIMER, mndProcessMqTimerMsg); mndSetMsgHandle(pMnode, TDMT_MND_MQ_CONSUMER_LOST, mndProcessConsumerLostMsg); @@ -90,7 +92,9 @@ static int32_t mndProcessConsumerLostMsg(SRpcMsg *pMsg) { SMnode *pMnode = pMsg->info.node; SMqConsumerLostMsg *pLostMsg = pMsg->pCont; SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, pLostMsg->consumerId); - ASSERT(pConsumer); + if (pConsumer == NULL) { + return 0; + } mInfo("receive consumer lost msg, consumer id %" PRId64 ", status %s", pLostMsg->consumerId, mndConsumerStatusName(pConsumer->status)); @@ -255,24 +259,15 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) { return 0; } -static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { - SMnode *pMnode = pMsg->info.node; - SMqAskEpReq *pReq = (SMqAskEpReq *)pMsg->pCont; - SMqAskEpRsp rsp = {0}; - int64_t consumerId = be64toh(pReq->consumerId); - int32_t epoch = ntohl(pReq->epoch); +static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) { + SMnode *pMnode = pMsg->info.node; + SMqHbReq *pReq = (SMqHbReq *)pMsg->pCont; + int64_t consumerId = be64toh(pReq->consumerId); - SMqConsumerObj *pConsumer = mndAcquireConsumer(pMsg->info.node, consumerId); - if (pConsumer == NULL) { - terrno = TSDB_CODE_MND_CONSUMER_NOT_EXIST; - return -1; - } + SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, consumerId); - ASSERT(strcmp(pReq->cgroup, pConsumer->cgroup) == 0); - /*int32_t hbStatus = atomic_load_32(&pConsumer->hbStatus);*/ atomic_store_32(&pConsumer->hbStatus, 0); - // 1. check consumer status int32_t status = atomic_load_32(&pConsumer->status); if (status == MQ_CONSUMER_STATUS__LOST_REBD) { @@ -286,6 +281,46 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, pRpcMsg); } + mndReleaseConsumer(pMnode, pConsumer); + + return 0; +} + +static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { + SMnode *pMnode = pMsg->info.node; + SMqAskEpReq *pReq = (SMqAskEpReq *)pMsg->pCont; + SMqAskEpRsp rsp = {0}; + int64_t consumerId = be64toh(pReq->consumerId); + int32_t epoch = ntohl(pReq->epoch); + + SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, consumerId); + if (pConsumer == NULL) { + terrno = TSDB_CODE_MND_CONSUMER_NOT_EXIST; + return -1; + } + + ASSERT(strcmp(pReq->cgroup, pConsumer->cgroup) == 0); + +#if 1 + atomic_store_32(&pConsumer->hbStatus, 0); +#endif + + // 1. check consumer status + int32_t status = atomic_load_32(&pConsumer->status); + +#if 1 + if (status == MQ_CONSUMER_STATUS__LOST_REBD) { + SMqConsumerRecoverMsg *pRecoverMsg = rpcMallocCont(sizeof(SMqConsumerRecoverMsg)); + + pRecoverMsg->consumerId = consumerId; + SRpcMsg *pRpcMsg = taosMemoryCalloc(1, sizeof(SRpcMsg)); + pRpcMsg->msgType = TDMT_MND_MQ_CONSUMER_RECOVER; + pRpcMsg->pCont = pRecoverMsg; + pRpcMsg->contLen = sizeof(SMqConsumerRecoverMsg); + tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, pRpcMsg); + } +#endif + if (status != MQ_CONSUMER_STATUS__READY) { terrno = TSDB_CODE_MND_CONSUMER_NOT_READY; return -1; @@ -417,6 +452,7 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { int32_t code = -1; SArray *newSub = subscribe.topicNames; taosArraySortString(newSub, taosArrayCompareString); + taosArrayRemoveDuplicate(newSub, taosArrayCompareString, taosMemoryFree); int32_t newTopicNum = taosArrayGetSize(newSub); // check topic existance @@ -874,8 +910,8 @@ static int32_t mndRetrieveConsumer(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock * colDataAppend(pColInfo, numOfRows, (const char *)cgroup, false); // client id - char clientId[TSDB_CGROUP_LEN + VARSTR_HEADER_SIZE] = {0}; - tstrncpy(varDataVal(clientId), pConsumer->clientId, TSDB_CGROUP_LEN); + char clientId[256 + VARSTR_HEADER_SIZE] = {0}; + tstrncpy(varDataVal(clientId), pConsumer->clientId, 256); varDataSetLen(clientId, strlen(varDataVal(clientId))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)clientId, false); diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 715340f688..b4b1b383e7 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -509,6 +509,11 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) { SUserObj *pUser = NULL; SCreateDbReq createReq = {0}; + if ((terrno = grantCheck(TSDB_GRANT_DB)) != 0) { + code = terrno; + goto _OVER; + } + if (tDeserializeSCreateDbReq(pReq->pCont, pReq->contLen, &createReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index c26424e049..abac0573da 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -199,6 +199,7 @@ int32_t tEncodeSMqConsumerObj(void **buf, const SMqConsumerObj *pConsumer) { int32_t tlen = 0; int32_t sz; tlen += taosEncodeFixedI64(buf, pConsumer->consumerId); + tlen += taosEncodeString(buf, pConsumer->clientId); tlen += taosEncodeString(buf, pConsumer->cgroup); tlen += taosEncodeFixedI8(buf, pConsumer->updateType); tlen += taosEncodeFixedI32(buf, pConsumer->epoch); @@ -264,6 +265,7 @@ int32_t tEncodeSMqConsumerObj(void **buf, const SMqConsumerObj *pConsumer) { void *tDecodeSMqConsumerObj(const void *buf, SMqConsumerObj *pConsumer) { int32_t sz; buf = taosDecodeFixedI64(buf, &pConsumer->consumerId); + buf = taosDecodeStringTo(buf, pConsumer->clientId); buf = taosDecodeStringTo(buf, pConsumer->cgroup); buf = taosDecodeFixedI8(buf, &pConsumer->updateType); buf = taosDecodeFixedI32(buf, &pConsumer->epoch); diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 3e5d378bb1..f26c1a7d32 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -262,7 +262,7 @@ bool mndIsDnodeOnline(SDnodeObj *pDnode, int64_t curMs) { return true; } -static void mndGetDnodeData(SMnode *pMnode, SArray *pDnodeEps) { +void mndGetDnodeData(SMnode *pMnode, SArray *pDnodeEps) { SSdb *pSdb = pMnode->pSdb; int32_t numOfEps = 0; @@ -621,6 +621,11 @@ static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) { SDnodeObj *pDnode = NULL; SCreateDnodeReq createReq = {0}; + if ((terrno = grantCheck(TSDB_GRANT_DNODE)) != 0) { + code = terrno; + goto _OVER; + } + if (tDeserializeSCreateDnodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndGrant.c b/source/dnode/mnode/impl/src/mndGrant.c index d1d43c841c..1148fd740b 100644 --- a/source/dnode/mnode/impl/src/mndGrant.c +++ b/source/dnode/mnode/impl/src/mndGrant.c @@ -118,17 +118,21 @@ static int32_t mndRetrieveGrant(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl return numOfRows; } +static int32_t mndProcessGrantHB(SRpcMsg *pReq) { return TSDB_CODE_SUCCESS; } + int32_t mndInitGrant(SMnode *pMnode) { mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_GRANTS, mndRetrieveGrant); + mndSetMsgHandle(pMnode, TDMT_MND_GRANT_HB_TIMER, mndProcessGrantHB); return 0; } void mndCleanupGrant() {} void grantParseParameter() { mError("can't parsed parameter k"); } int32_t grantCheck(EGrantType grant) { return TSDB_CODE_SUCCESS; } -void grantReset(EGrantType grant, uint64_t value) {} +void grantReset(SMnode *pMnode, EGrantType grant, uint64_t value) {} void grantAdd(EGrantType grant, uint64_t value) {} void grantRestore(EGrantType grant, uint64_t value) {} +int32_t dmProcessGrantReq(SRpcMsg *pMsg) { return TSDB_CODE_SUCCESS; } #endif diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 041fc2a2d1..443d70aef9 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -90,6 +90,16 @@ static void mndPullupTelem(SMnode *pMnode) { } } +static void mndGrantHeartBeat(SMnode *pMnode) { + int32_t contLen = 0; + void *pReq = mndBuildTimerMsg(&contLen); + if (pReq != NULL) { + SRpcMsg rpcMsg = { + .msgType = TDMT_MND_GRANT_HB_TIMER, .pCont = pReq, .contLen = contLen, .info.ahandle = (void *)0x9527}; + tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg); + } +} + static void *mndThreadFp(void *param) { SMnode *pMnode = param; int64_t lastTime = 0; @@ -115,6 +125,10 @@ static void *mndThreadFp(void *param) { if (lastTime % (tsTelemInterval * 10) == 0) { mndPullupTelem(pMnode); } + + if (lastTime % (tsGrantHBInterval * 10) == 0) { + mndGrantHeartBeat(pMnode); + } } return NULL; @@ -402,6 +416,9 @@ int32_t mndStart(SMnode *pMnode) { } mndSetRestore(pMnode, true); } + + grantReset(pMnode, TSDB_GRANT_ALL, 0); + return mndInitTimer(pMnode); } diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index a1da68606b..5871d56a8f 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -388,67 +388,7 @@ static void mndCancelGetNextApp(SMnode *pMnode, void *pIter) { } static SClientHbRsp *mndMqHbBuildRsp(SMnode *pMnode, SClientHbReq *pReq) { -#if 0 - SClientHbRsp* pRsp = taosMemoryMalloc(sizeof(SClientHbRsp)); - if (pRsp == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - pRsp->connKey = pReq->connKey; - SMqHbBatchRsp batchRsp; - batchRsp.batchRsps = taosArrayInit(0, sizeof(SMqHbRsp)); - if (batchRsp.batchRsps == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - SClientHbKey connKey = pReq->connKey; - SHashObj* pObj = pReq->info; - SKv* pKv = taosHashGet(pObj, "mq-tmp", strlen("mq-tmp") + 1); - if (pKv == NULL) { - taosMemoryFree(pRsp); - return NULL; - } - SMqHbMsg mqHb; - taosDecodeSMqMsg(pKv->value, &mqHb); - /*int64_t clientUid = htonl(pKv->value);*/ - /*if (mqHb.epoch )*/ - int sz = taosArrayGetSize(mqHb.pTopics); - SMqConsumerObj* pConsumer = mndAcquireConsumer(pMnode, mqHb.consumerId); - for (int i = 0; i < sz; i++) { - SMqHbOneTopicBatchRsp innerBatchRsp; - innerBatchRsp.rsps = taosArrayInit(sz, sizeof(SMqHbRsp)); - if (innerBatchRsp.rsps == NULL) { - //TODO - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - SMqHbTopicInfo* topicInfo = taosArrayGet(mqHb.pTopics, i); - SMqConsumerTopic* pConsumerTopic = taosHashGet(pConsumer->topicHash, topicInfo->name, strlen(topicInfo->name)+1); - if (pConsumerTopic->epoch != topicInfo->epoch) { - //add new vgids into rsp - int vgSz = taosArrayGetSize(topicInfo->pVgInfo); - for (int j = 0; j < vgSz; j++) { - SMqHbRsp innerRsp; - SMqHbVgInfo* pVgInfo = taosArrayGet(topicInfo->pVgInfo, i); - SVgObj* pVgObj = mndAcquireVgroup(pMnode, pVgInfo->vgId); - innerRsp.epSet = mndGetVgroupEpset(pMnode, pVgObj); - taosArrayPush(innerBatchRsp.rsps, &innerRsp); - } - } - taosArrayPush(batchRsp.batchRsps, &innerBatchRsp); - } - int32_t tlen = taosEncodeSMqHbBatchRsp(NULL, &batchRsp); - void* buf = taosMemoryMalloc(tlen); - if (buf == NULL) { - //TODO - return NULL; - } - void* abuf = buf; - taosEncodeSMqHbBatchRsp(&abuf, &batchRsp); - pRsp->body = buf; - pRsp->bodyLen = tlen; - return pRsp; -#endif + // return NULL; } diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 2a8cbc4425..0711f7e06d 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -76,6 +76,8 @@ static int32_t convertToRetrieveType(char *name, int32_t len) { type = TSDB_MGMT_TABLE_STB; } else if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, len) == 0) { type = TSDB_MGMT_TABLE_TABLE; + } else if (strncasecmp(name, TSDB_INS_TABLE_USER_TAGS, len) == 0) { + type = TSDB_MGMT_TABLE_TAG; } else if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED, len) == 0) { // type = TSDB_MGMT_TABLE_DIST; } else if (strncasecmp(name, TSDB_INS_TABLE_USER_USERS, len) == 0) { diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 88629ebc69..bff3f19e99 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -709,7 +709,7 @@ static int32_t mndProcessCreateSmaReq(SRpcMsg *pReq) { _OVER: if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { - mError("sma:%s, failed to create since %s", createReq.name, terrstr(terrno)); + mError("sma:%s, failed to create since %s", createReq.name, terrstr()); } mndReleaseStb(pMnode, pStb); diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index bff33af5af..1306a0252f 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -1689,6 +1689,9 @@ static int32_t mndAlterStb(SMnode *pMnode, SRpcMsg *pReq, const SMAlterStbReq *p _OVER: taosMemoryFreeClear(stbObj.pTags); taosMemoryFreeClear(stbObj.pColumns); + if (pAlter->commentLen > 0) { + taosMemoryFreeClear(stbObj.comment); + } return code; } @@ -1733,7 +1736,7 @@ _OVER: mndReleaseStb(pMnode, pStb); mndReleaseDb(pMnode, pDb); - taosArrayDestroy(alterReq.pFields); + tFreeSMAltertbReq(&alterReq); return code; } diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 2811aeb43a..36362bed87 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -143,7 +143,7 @@ int32_t mndSnapshotStartWrite(struct SSyncFSM *pFsm, void *pParam, void **ppWrit return sdbStartWrite(pMnode->pSdb, (SSdbIter **)ppWriter); } -int32_t mndSnapshotStopWrite(struct SSyncFSM *pFsm, void *pWriter, bool isApply) { +int32_t mndSnapshotStopWrite(struct SSyncFSM *pFsm, void *pWriter, bool isApply, SSnapshot *pSnapshot) { mInfo("stop to apply snapshot to sdb, apply:%d", isApply); SMnode *pMnode = pFsm->data; return sdbStopWrite(pMnode->pSdb, pWriter, isApply); diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 921dba422d..0452659d47 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -363,6 +363,11 @@ static int32_t mndProcessCreateUserReq(SRpcMsg *pReq) { goto _OVER; } + if ((terrno = grantCheck(TSDB_GRANT_USER)) != 0) { + code = terrno; + goto _OVER; + } + code = mndCreateUser(pMnode, pOperUser->acct, &createReq, pReq); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 0b24b570a5..3eb3a6cd1f 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -639,6 +639,7 @@ static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p if (pShow->pIter == NULL) break; if (pDb != NULL && pVgroup->dbUid != pDb->uid) { + sdbRelease(pSdb, pVgroup); continue; } diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index f0a6b6505d..97eff4804d 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -27,6 +27,7 @@ #include "wal.h" #include "tcommon.h" +#include "tgrant.h" #include "tfs.h" #include "tmsg.h" #include "trow.h" @@ -116,11 +117,11 @@ int32_t metaTbCursorNext(SMTbCursor *pTbCur); // typedef struct STsdb STsdb; typedef struct STsdbReader STsdbReader; -#define BLOCK_LOAD_OFFSET_ORDER 1 +#define BLOCK_LOAD_OFFSET_ORDER 1 #define BLOCK_LOAD_TABLESEQ_ORDER 2 -#define BLOCK_LOAD_EXTERN_ORDER 3 +#define BLOCK_LOAD_EXTERN_ORDER 3 -#define LASTROW_RETRIEVE_TYPE_ALL 0x1 +#define LASTROW_RETRIEVE_TYPE_ALL 0x1 #define LASTROW_RETRIEVE_TYPE_SINGLE 0x2 int32_t tsdbSetTableId(STsdbReader *pReader, int64_t uid); @@ -138,7 +139,7 @@ void *tsdbGetIdx(SMeta *pMeta); void *tsdbGetIvtIdx(SMeta *pMeta); int32_t tsdbLastRowReaderOpen(void *pVnode, int32_t type, SArray *pTableIdList, int32_t numOfCols, void **pReader); -int32_t tsdbRetrieveLastRow(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, SArray* pTableUids); +int32_t tsdbRetrieveLastRow(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, SArray *pTableUids); int32_t tsdbLastrowReaderClose(void *pReader); int32_t tsdbGetTableSchema(SVnode *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid); @@ -191,7 +192,7 @@ int32_t vnodeSnapReaderClose(SVSnapReader *pReader); int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData); // SVSnapWriter int32_t vnodeSnapWriterOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapWriter **ppWriter); -int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback); +int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback, SSnapshot *pSnapshot); int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData); // structs @@ -237,8 +238,8 @@ typedef struct { uint64_t groupId; } STableKeyInfo; -#define TABLE_ROLLUP_ON ((int8_t)0x1) -#define TABLE_IS_ROLLUP(FLG) (((FLG) & (TABLE_ROLLUP_ON)) != 0) +#define TABLE_ROLLUP_ON ((int8_t)0x1) +#define TABLE_IS_ROLLUP(FLG) (((FLG) & (TABLE_ROLLUP_ON)) != 0) #define TABLE_SET_ROLLUP(FLG) ((FLG) |= TABLE_ROLLUP_ON) struct SMetaEntry { int64_t version; diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h index e767d94ebd..1c8434594d 100644 --- a/source/dnode/vnode/src/inc/sma.h +++ b/source/dnode/vnode/src/inc/sma.h @@ -32,6 +32,8 @@ extern "C" { #define smaTrace(...) do { if (smaDebugFlag & DEBUG_TRACE) { taosPrintLog("SMA ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); }} while(0) // clang-format on +#define RSMA_TASK_INFO_HASH_SLOT 8 + typedef struct SSmaEnv SSmaEnv; typedef struct SSmaStat SSmaStat; typedef struct STSmaStat STSmaStat; @@ -41,7 +43,7 @@ typedef struct SRSmaInfo SRSmaInfo; typedef struct SRSmaInfoItem SRSmaInfoItem; struct SSmaEnv { - TdThreadRwlock lock; + SRWLatch lock; int8_t type; SSmaStat *pStat; }; @@ -52,7 +54,7 @@ typedef struct { void *tmrHandle; // shared by all fetch tasks } SSmaMgmt; -#define SMA_ENV_LOCK(env) ((env)->lock) +#define SMA_ENV_LOCK(env) (&(env)->lock) #define SMA_ENV_TYPE(env) ((env)->type) #define SMA_ENV_STAT(env) ((env)->pStat) @@ -64,10 +66,14 @@ struct STSmaStat { struct SRSmaStat { SSma *pSma; - int64_t submitVer; - int64_t refId; // shared by fetch tasks - int8_t triggerStat; // shared by fetch tasks - SHashObj *rsmaInfoHash; // key: stbUid, value: SRSmaInfo; + int64_t commitAppliedVer; // vnode applied version for async commit + int64_t commitSubmitVer; // rsma submit version for async commit + int64_t submitVer; // latest submit version + int64_t refId; // shared by fetch tasks + int8_t triggerStat; // shared by fetch tasks + int8_t commitStat; // 0 not in committing, 1 in committing + SHashObj *rsmaInfoHash; // key: stbUid, value: SRSmaInfo; + SHashObj *iRsmaInfoHash; // key: stbUid, value: SRSmaInfo; immutable rsmaInfoHash }; struct SSmaStat { @@ -78,12 +84,29 @@ struct SSmaStat { T_REF_DECLARE() }; -#define SMA_TSMA_STAT(s) (&(s)->tsmaStat) -#define SMA_RSMA_STAT(s) (&(s)->rsmaStat) -#define RSMA_INFO_HASH(r) ((r)->rsmaInfoHash) -#define RSMA_TRIGGER_STAT(r) (&(r)->triggerStat) -#define RSMA_REF_ID(r) ((r)->refId) -#define RSMA_SUBMIT_VER(r) ((r)->submitVer) +#define SMA_TSMA_STAT(s) (&(s)->tsmaStat) +#define SMA_RSMA_STAT(s) (&(s)->rsmaStat) +#define RSMA_INFO_HASH(r) ((r)->rsmaInfoHash) +#define RSMA_IMU_INFO_HASH(r) ((r)->iRsmaInfoHash) +#define RSMA_TRIGGER_STAT(r) (&(r)->triggerStat) +#define RSMA_COMMIT_STAT(r) (&(r)->commitStat) +#define RSMA_REF_ID(r) ((r)->refId) +#define RSMA_SUBMIT_VER(r) ((r)->submitVer) + +struct SRSmaInfoItem { + void *taskInfo; // qTaskInfo_t + int64_t refId; + tmr_h tmrId; + int32_t maxDelay; + int8_t level; + int8_t triggerStat; +}; + +struct SRSmaInfo { + STSchema *pTSchema; + int64_t suid; + SRSmaInfoItem items[TSDB_RETENTION_L2]; +}; enum { TASK_TRIGGER_STAT_INIT = 0, @@ -94,6 +117,14 @@ enum { TASK_TRIGGER_STAT_DROPPED = 5, }; +enum { + RSMA_ROLE_CREATE = 0, + RSMA_ROLE_DROP = 1, + RSMA_ROLE_FETCH = 2, + RSMA_ROLE_SUBMIT = 3, + RSMA_ROLE_ITERATE = 4, +}; + void tdDestroySmaEnv(SSmaEnv *pSmaEnv); void *tdFreeSmaEnv(SSmaEnv *pSmaEnv); @@ -112,33 +143,6 @@ int32_t tdCheckAndInitSmaEnv(SSma *pSma, int8_t smaType); int32_t tdLockSma(SSma *pSma); int32_t tdUnLockSma(SSma *pSma); -static FORCE_INLINE int32_t tdRLockSmaEnv(SSmaEnv *pEnv) { - int code = taosThreadRwlockRdlock(&(pEnv->lock)); - if (code != 0) { - terrno = TAOS_SYSTEM_ERROR(code); - return -1; - } - return 0; -} - -static FORCE_INLINE int32_t tdWLockSmaEnv(SSmaEnv *pEnv) { - int code = taosThreadRwlockWrlock(&(pEnv->lock)); - if (code != 0) { - terrno = TAOS_SYSTEM_ERROR(code); - return -1; - } - return 0; -} - -static FORCE_INLINE int32_t tdUnLockSmaEnv(SSmaEnv *pEnv) { - int code = taosThreadRwlockUnlock(&(pEnv->lock)); - if (code != 0) { - terrno = TAOS_SYSTEM_ERROR(code); - return -1; - } - return 0; -} - static FORCE_INLINE int8_t tdSmaStat(STSmaStat *pTStat) { if (pTStat) { return atomic_load_8(&pTStat->state); @@ -184,10 +188,12 @@ static FORCE_INLINE void tdSmaStatSetDropped(STSmaStat *pTStat) { } } +int32_t tdCloneRSmaInfo(SSma *pSma, SRSmaInfo *pDest, SRSmaInfo *pSrc); +void tdFreeQTaskInfo(qTaskInfo_t *taskHandle, int32_t vgId, int32_t level); static int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType); void *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType); -void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo); -int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat); +void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree); +int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash); int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, const char *tbName); int32_t tdProcessRSmaRestoreImpl(SSma *pSma); diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index c62b7e95bf..757749a9b6 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -52,7 +52,6 @@ typedef struct { int64_t reqOffset; int64_t processedVer; int32_t epoch; - int32_t skipLogNum; // rpc info int64_t reqId; SRpcHandleInfo rpcInfo; diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 4df9f96514..3f667a7465 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -24,12 +24,12 @@ extern "C" { // tsdbDebug ================ // clang-format off -#define tsdbFatal(...) do { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TSDB FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} while(0) -#define tsdbError(...) do { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TSDB ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} while(0) -#define tsdbWarn(...) do { if (tsdbDebugFlag & DEBUG_WARN) { taosPrintLog("TSDB WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} while(0) -#define tsdbInfo(...) do { if (tsdbDebugFlag & DEBUG_INFO) { taosPrintLog("TSDB ", DEBUG_INFO, 255, __VA_ARGS__); }} while(0) -#define tsdbDebug(...) do { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSDB ", DEBUG_DEBUG, tsdbDebugFlag, __VA_ARGS__); }} while(0) -#define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TSDB ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); }} while(0) +#define tsdbFatal(...) do { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TSD FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} while(0) +#define tsdbError(...) do { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TSD ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} while(0) +#define tsdbWarn(...) do { if (tsdbDebugFlag & DEBUG_WARN) { taosPrintLog("TSD WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} while(0) +#define tsdbInfo(...) do { if (tsdbDebugFlag & DEBUG_INFO) { taosPrintLog("TSD ", DEBUG_INFO, 255, __VA_ARGS__); }} while(0) +#define tsdbDebug(...) do { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSD ", DEBUG_DEBUG, tsdbDebugFlag, __VA_ARGS__); }} while(0) +#define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TSD ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); }} while(0) // clang-format on typedef struct TSDBROW TSDBROW; @@ -115,7 +115,6 @@ int32_t tGetBlock(uint8_t *p, void *ph); int32_t tBlockCmprFn(const void *p1, const void *p2); bool tBlockHasSma(SBlock *pBlock); // SBlockIdx -void tBlockIdxReset(SBlockIdx *pBlockIdx); int32_t tPutBlockIdx(uint8_t *p, void *ph); int32_t tGetBlockIdx(uint8_t *p, void *ph); int32_t tCmprBlockIdx(void const *lhs, void const *rhs); @@ -126,6 +125,8 @@ void tColDataClear(void *ph); int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal); int32_t tColDataGetValue(SColData *pColData, int32_t iRow, SColVal *pColVal); int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest); +int32_t tPutColData(uint8_t *p, SColData *pColData); +int32_t tGetColData(uint8_t *p, SColData *pColData); // SBlockData #define tBlockDataFirstRow(PBLOCKDATA) tsdbRowFromBlockData(PBLOCKDATA, 0) #define tBlockDataLastRow(PBLOCKDATA) tsdbRowFromBlockData(PBLOCKDATA, (PBLOCKDATA)->nRow - 1) @@ -134,14 +135,17 @@ int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest); int32_t tBlockDataInit(SBlockData *pBlockData); void tBlockDataReset(SBlockData *pBlockData); int32_t tBlockDataSetSchema(SBlockData *pBlockData, STSchema *pTSchema); +int32_t tBlockDataCorrectSchema(SBlockData *pBlockData, SBlockData *pBlockDataFrom); void tBlockDataClearData(SBlockData *pBlockData); -void tBlockDataClear(SBlockData *pBlockData); +void tBlockDataClear(SBlockData *pBlockData, int8_t deepClear); int32_t tBlockDataAddColData(SBlockData *pBlockData, int32_t iColData, SColData **ppColData); int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema); int32_t tBlockDataMerge(SBlockData *pBlockData1, SBlockData *pBlockData2, SBlockData *pBlockData); int32_t tBlockDataCopy(SBlockData *pBlockDataSrc, SBlockData *pBlockDataDest); SColData *tBlockDataGetColDataByIdx(SBlockData *pBlockData, int32_t idx); void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColData); +int32_t tPutBlockData(uint8_t *p, SBlockData *pBlockData); +int32_t tGetBlockData(uint8_t *p, SBlockData *pBlockData); // SDelIdx int32_t tPutDelIdx(uint8_t *p, void *ph); int32_t tGetDelIdx(uint8_t *p, void *ph); @@ -202,7 +206,7 @@ int32_t tsdbFSStateUpsertDelFile(STsdbFSState *pState, SDelFile *pDelFile); int32_t tsdbFSStateUpsertDFileSet(STsdbFSState *pState, SDFileSet *pSet); void tsdbFSStateDeleteDFileSet(STsdbFSState *pState, int32_t fid); SDelFile *tsdbFSStateGetDelFile(STsdbFSState *pState); -SDFileSet *tsdbFSStateGetDFileSet(STsdbFSState *pState, int32_t fid); +SDFileSet *tsdbFSStateGetDFileSet(STsdbFSState *pState, int32_t fid, int32_t flag); // tsdbReaderWriter.c ============================================================================================== // SDataFWriter int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pSet); @@ -357,10 +361,6 @@ struct TSDBROW { struct SBlockIdx { int64_t suid; int64_t uid; - TSKEY minKey; - TSKEY maxKey; - int64_t minVersion; - int64_t maxVersion; int64_t offset; int64_t size; }; diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 7b298ba830..fb403f79a7 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -99,7 +99,8 @@ STSchema* metaGetTbTSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver); int32_t metaGetTbTSchemaEx(SMeta* pMeta, tb_uid_t suid, tb_uid_t uid, int32_t sver, STSchema** ppTSchema); int metaGetTableEntryByName(SMetaReader* pReader, const char* name); tb_uid_t metaGetTableEntryUidByName(SMeta* pMeta, const char* name); -int metaGetTbNum(SMeta* pMeta); +int64_t metaGetTbNum(SMeta* pMeta); +int64_t metaGetTimeSeriesNum(SMeta* pMeta); SMCtbCursor* metaOpenCtbCursor(SMeta* pMeta, tb_uid_t uid); void metaCloseCtbCursor(SMCtbCursor* pCtbCur); tb_uid_t metaCtbCursorNext(SMCtbCursor* pCtbCur); @@ -164,9 +165,12 @@ void smaCleanUp(); int32_t smaOpen(SVnode* pVnode); int32_t smaClose(SSma* pSma); int32_t smaBegin(SSma* pSma); -int32_t smaPreCommit(SSma* pSma); -int32_t smaCommit(SSma* pSma); -int32_t smaPostCommit(SSma* pSma); +int32_t smaSyncPreCommit(SSma* pSma); +int32_t smaSyncCommit(SSma* pSma); +int32_t smaSyncPostCommit(SSma* pSma); +int32_t smaAsyncPreCommit(SSma* pSma); +int32_t smaAsyncCommit(SSma* pSma); +int32_t smaAsyncPostCommit(SSma* pSma); int32_t tdProcessTSmaCreate(SSma* pSma, int64_t version, const char* msg); int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg); @@ -309,6 +313,7 @@ void smaHandleRes(void* pVnode, int64_t smaId, const SArray* data); struct SSnapDataHdr { int8_t type; + int64_t index; int64_t size; uint8_t data[]; }; diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index e1236c2853..776a3c5b7f 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -205,7 +205,7 @@ _query: } tDecoderInit(&dc, pData, nData); - tDecodeSSchemaWrapper(&dc, &schema); + tDecodeSSchemaWrapperEx(&dc, &schema); pSchema = tCloneSSchemaWrapper(&schema); tDecoderClear(&dc); @@ -463,12 +463,18 @@ _err: return code; } -int metaGetTbNum(SMeta *pMeta) { +// N.B. Called by statusReq per second +int64_t metaGetTbNum(SMeta *pMeta) { // TODO - // ASSERT(0); return 0; } +// N.B. Called by statusReq per second +int64_t metaGetTimeSeriesNum(SMeta *pMeta) { + // TODO + return 400; +} + typedef struct { SMeta *pMeta; TBC *pCur; diff --git a/source/dnode/vnode/src/meta/metaSnapshot.c b/source/dnode/vnode/src/meta/metaSnapshot.c index ac84842e85..85261d302e 100644 --- a/source/dnode/vnode/src/meta/metaSnapshot.c +++ b/source/dnode/vnode/src/meta/metaSnapshot.c @@ -26,60 +26,72 @@ struct SMetaSnapReader { int32_t metaSnapReaderOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapReader** ppReader) { int32_t code = 0; int32_t c = 0; - SMetaSnapReader* pMetaSnapReader = NULL; + SMetaSnapReader* pReader = NULL; // alloc - pMetaSnapReader = (SMetaSnapReader*)taosMemoryCalloc(1, sizeof(*pMetaSnapReader)); - if (pMetaSnapReader == NULL) { + pReader = (SMetaSnapReader*)taosMemoryCalloc(1, sizeof(*pReader)); + if (pReader == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } - pMetaSnapReader->pMeta = pMeta; - pMetaSnapReader->sver = sver; - pMetaSnapReader->ever = ever; + pReader->pMeta = pMeta; + pReader->sver = sver; + pReader->ever = ever; // impl - code = tdbTbcOpen(pMeta->pTbDb, &pMetaSnapReader->pTbc, NULL); + code = tdbTbcOpen(pMeta->pTbDb, &pReader->pTbc, NULL); if (code) { + taosMemoryFree(pReader); goto _err; } - code = tdbTbcMoveTo(pMetaSnapReader->pTbc, &(STbDbKey){.version = sver, .uid = INT64_MIN}, sizeof(STbDbKey), &c); + code = tdbTbcMoveTo(pReader->pTbc, &(STbDbKey){.version = sver, .uid = INT64_MIN}, sizeof(STbDbKey), &c); if (code) { + taosMemoryFree(pReader); goto _err; } - *ppReader = pMetaSnapReader; + metaInfo("vgId:%d vnode snapshot meta reader opened", TD_VID(pMeta->pVnode)); + + *ppReader = pReader; return code; _err: - metaError("vgId:%d meta snap reader open failed since %s", TD_VID(pMeta->pVnode), tstrerror(code)); + metaError("vgId:%d vnode snapshot meta reader open failed since %s", TD_VID(pMeta->pVnode), tstrerror(code)); *ppReader = NULL; return code; } int32_t metaSnapReaderClose(SMetaSnapReader** ppReader) { + int32_t code = 0; + tdbTbcClose((*ppReader)->pTbc); taosMemoryFree(*ppReader); *ppReader = NULL; - return 0; + + return code; } int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData) { + int32_t code = 0; const void* pKey = NULL; const void* pData = NULL; int32_t nKey = 0; int32_t nData = 0; - int32_t code = 0; + STbDbKey key; + *ppData = NULL; for (;;) { - code = tdbTbcGet(pReader->pTbc, &pKey, &nKey, &pData, &nData); - if (code || ((STbDbKey*)pData)->version > pReader->ever) { - code = TSDB_CODE_VND_READ_END; + if (tdbTbcGet(pReader->pTbc, &pKey, &nKey, &pData, &nData)) { goto _exit; } - if (((STbDbKey*)pData)->version < pReader->sver) { + key = ((STbDbKey*)pKey)[0]; + if (key.version > pReader->ever) { + goto _exit; + } + + if (key.version < pReader->sver) { tdbTbcMoveToNext(pReader->pTbc); continue; } @@ -88,17 +100,28 @@ int32_t metaSnapRead(SMetaSnapReader* pReader, uint8_t** ppData) { break; } - // copy the data - if (tRealloc(ppData, sizeof(SSnapDataHdr) + nData) < 0) { + ASSERT(pData && nData); + + *ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + nData); + if (*ppData == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - return code; + goto _err; } - ((SSnapDataHdr*)(*ppData))->type = 0; // TODO: use macro - ((SSnapDataHdr*)(*ppData))->size = nData; - memcpy(((SSnapDataHdr*)(*ppData))->data, pData, nData); + + SSnapDataHdr* pHdr = (SSnapDataHdr*)(*ppData); + pHdr->type = 0; // TODO: use macro + pHdr->size = nData; + memcpy(pHdr->data, pData, nData); + + metaInfo("vgId:%d vnode snapshot meta read data, version:%" PRId64 " uid:%" PRId64 " nData:%d", + TD_VID(pReader->pMeta->pVnode), key.version, key.uid, nData); _exit: return code; + +_err: + metaError("vgId:%d vnode snapshot meta read data failed since %s", TD_VID(pReader->pMeta->pVnode), tstrerror(code)); + return code; } // SMetaSnapWriter ======================================== @@ -108,18 +131,6 @@ struct SMetaSnapWriter { int64_t ever; }; -static int32_t metaSnapRollback(SMetaSnapWriter* pWriter) { - int32_t code = 0; - // TODO - return code; -} - -static int32_t metaSnapCommit(SMetaSnapWriter* pWriter) { - int32_t code = 0; - // TODO - return code; -} - int32_t metaSnapWriterOpen(SMeta* pMeta, int64_t sver, int64_t ever, SMetaSnapWriter** ppWriter) { int32_t code = 0; SMetaSnapWriter* pWriter; @@ -148,10 +159,9 @@ int32_t metaSnapWriterClose(SMetaSnapWriter** ppWriter, int8_t rollback) { SMetaSnapWriter* pWriter = *ppWriter; if (rollback) { - code = metaSnapRollback(pWriter); - if (code) goto _err; + ASSERT(0); } else { - code = metaSnapCommit(pWriter); + code = metaCommit(pWriter->pMeta); if (code) goto _err; } taosMemoryFree(pWriter); @@ -170,15 +180,16 @@ int32_t metaSnapWrite(SMetaSnapWriter* pWriter, uint8_t* pData, uint32_t nData) SMetaEntry metaEntry = {0}; SDecoder* pDecoder = &(SDecoder){0}; - tDecoderInit(pDecoder, pData, nData); + tDecoderInit(pDecoder, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr)); metaDecodeEntry(pDecoder, &metaEntry); code = metaHandleEntry(pMeta, &metaEntry); if (code) goto _err; + tDecoderClear(pDecoder); return code; _err: - metaError("vgId:%d meta snapshot write failed since %s", TD_VID(pMeta->pVnode), tstrerror(code)); + metaError("vgId:%d vnode snapshot meta write failed since %s", TD_VID(pMeta->pVnode), tstrerror(code)); return code; } \ No newline at end of file diff --git a/source/dnode/vnode/src/sma/smaCommit.c b/source/dnode/vnode/src/sma/smaCommit.c index e241c14fc7..cdaaf2bbdb 100644 --- a/source/dnode/vnode/src/sma/smaCommit.c +++ b/source/dnode/vnode/src/sma/smaCommit.c @@ -15,9 +15,13 @@ #include "sma.h" -static int32_t tdProcessRSmaPreCommitImpl(SSma *pSma); -static int32_t tdProcessRSmaCommitImpl(SSma *pSma); -static int32_t tdProcessRSmaPostCommitImpl(SSma *pSma); +static int32_t tdProcessRSmaSyncPreCommitImpl(SSma *pSma); +static int32_t tdProcessRSmaSyncCommitImpl(SSma *pSma); +static int32_t tdProcessRSmaSyncPostCommitImpl(SSma *pSma); +static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma); +static int32_t tdProcessRSmaAsyncCommitImpl(SSma *pSma); +static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma); +static int32_t tdCleanupQTaskInfoFiles(SSma *pSma, SRSmaStat *pRSmaStat); /** * @brief Only applicable to Rollup SMA @@ -25,7 +29,7 @@ static int32_t tdProcessRSmaPostCommitImpl(SSma *pSma); * @param pSma * @return int32_t */ -int32_t smaPreCommit(SSma *pSma) { return tdProcessRSmaPreCommitImpl(pSma); } +int32_t smaSyncPreCommit(SSma *pSma) { return tdProcessRSmaSyncPreCommitImpl(pSma); } /** * @brief Only applicable to Rollup SMA @@ -33,7 +37,7 @@ int32_t smaPreCommit(SSma *pSma) { return tdProcessRSmaPreCommitImpl(pSma); } * @param pSma * @return int32_t */ -int32_t smaCommit(SSma *pSma) { return tdProcessRSmaCommitImpl(pSma); } +int32_t smaSyncCommit(SSma *pSma) { return tdProcessRSmaSyncCommitImpl(pSma); } /** * @brief Only applicable to Rollup SMA @@ -41,7 +45,31 @@ int32_t smaCommit(SSma *pSma) { return tdProcessRSmaCommitImpl(pSma); } * @param pSma * @return int32_t */ -int32_t smaPostCommit(SSma *pSma) { return tdProcessRSmaPostCommitImpl(pSma); } +int32_t smaSyncPostCommit(SSma *pSma) { return tdProcessRSmaSyncPostCommitImpl(pSma); } + +/** + * @brief Only applicable to Rollup SMA + * + * @param pSma + * @return int32_t + */ +int32_t smaAsyncPreCommit(SSma *pSma) { return tdProcessRSmaAsyncPreCommitImpl(pSma); } + +/** + * @brief Only applicable to Rollup SMA + * + * @param pSma + * @return int32_t + */ +int32_t smaAsyncCommit(SSma *pSma) { return tdProcessRSmaAsyncCommitImpl(pSma); } + +/** + * @brief Only applicable to Rollup SMA + * + * @param pSma + * @return int32_t + */ +int32_t smaAsyncPostCommit(SSma *pSma) { return tdProcessRSmaAsyncPostCommitImpl(pSma); } /** * @brief set rsma trigger stat active @@ -62,18 +90,17 @@ int32_t smaBegin(SSma *pSma) { atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_PAUSED, TASK_TRIGGER_STAT_ACTIVE); switch (rsmaTriggerStat) { case TASK_TRIGGER_STAT_PAUSED: { - smaDebug("vgId:%d rsma trigger stat from paused to active", SMA_VID(pSma)); + smaDebug("vgId:%d, rsma trigger stat from paused to active", SMA_VID(pSma)); break; } case TASK_TRIGGER_STAT_INIT: { atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_ACTIVE); - smaDebug("vgId:%d rsma trigger stat from init to active", SMA_VID(pSma)); + smaDebug("vgId:%d, rsma trigger stat from init to active", SMA_VID(pSma)); break; } default: { atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_ACTIVE); - smaWarn("vgId:%d rsma trigger stat %" PRIi8 " is unexpected", SMA_VID(pSma), rsmaTriggerStat); - ASSERT(0); + smaError("vgId:%d, rsma trigger stat %" PRIi8 " is unexpected", SMA_VID(pSma), rsmaTriggerStat); break; } } @@ -81,7 +108,7 @@ int32_t smaBegin(SSma *pSma) { } /** - * @brief pre-commit for rollup sma. + * @brief pre-commit for rollup sma(sync commit). * 1) set trigger stat of rsma timer TASK_TRIGGER_STAT_PAUSED. * 2) wait all triggered fetch tasks finished * 3) perform persist task for qTaskInfo @@ -89,7 +116,7 @@ int32_t smaBegin(SSma *pSma) { * @param pSma * @return int32_t */ -static int32_t tdProcessRSmaPreCommitImpl(SSma *pSma) { +static int32_t tdProcessRSmaSyncPreCommitImpl(SSma *pSma) { SSmaEnv *pSmaEnv = SMA_RSMA_ENV(pSma); if (!pSmaEnv) { return TSDB_CODE_SUCCESS; @@ -98,8 +125,7 @@ static int32_t tdProcessRSmaPreCommitImpl(SSma *pSma) { SSmaStat *pStat = SMA_ENV_STAT(pSmaEnv); SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pStat); - - // step 1: set persistence task paused + // step 1: set rsma stat paused atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_PAUSED); // step 2: wait all triggered fetch tasks finished @@ -119,7 +145,9 @@ static int32_t tdProcessRSmaPreCommitImpl(SSma *pSma) { } // step 3: perform persist task for qTaskInfo - tdRSmaPersistExecImpl(pRSmaStat); + pRSmaStat->commitAppliedVer = pSma->pVnode->state.applied; + pRSmaStat->commitSubmitVer = pRSmaStat->submitVer; + tdRSmaPersistExecImpl(pRSmaStat, RSMA_INFO_HASH(pRSmaStat)); smaDebug("vgId:%d, rsma pre commit success", SMA_VID(pSma)); @@ -132,7 +160,7 @@ static int32_t tdProcessRSmaPreCommitImpl(SSma *pSma) { * @param pSma * @return int32_t */ -static int32_t tdProcessRSmaCommitImpl(SSma *pSma) { +static int32_t tdProcessRSmaSyncCommitImpl(SSma *pSma) { SSmaEnv *pSmaEnv = SMA_RSMA_ENV(pSma); if (!pSmaEnv) { return TSDB_CODE_SUCCESS; @@ -140,21 +168,9 @@ static int32_t tdProcessRSmaCommitImpl(SSma *pSma) { return TSDB_CODE_SUCCESS; } -/** - * @brief post-commit for rollup sma - * 1) clean up the outdated qtaskinfo files - * - * @param pSma - * @return int32_t - */ -static int32_t tdProcessRSmaPostCommitImpl(SSma *pSma) { - SVnode *pVnode = pSma->pVnode; - - if (!VND_IS_RSMA(pVnode)) { - return TSDB_CODE_SUCCESS; - } - - int64_t committed = pVnode->state.committed; +static int32_t tdCleanupQTaskInfoFiles(SSma *pSma, SRSmaStat *pRSmaStat) { + SVnode *pVnode = pSma->pVnode; + int64_t committed = pRSmaStat->commitAppliedVer; TdDirPtr pDir = NULL; TdDirEntryPtr pDirEntry = NULL; char dir[TSDB_FILENAME_LEN]; @@ -222,5 +238,159 @@ static int32_t tdProcessRSmaPostCommitImpl(SSma *pSma) { taosCloseDir(&pDir); regfree(®ex); + + return TSDB_CODE_SUCCESS; +} + +/** + * @brief post-commit for rollup sma + * 1) clean up the outdated qtaskinfo files + * + * @param pSma + * @return int32_t + */ +static int32_t tdProcessRSmaSyncPostCommitImpl(SSma *pSma) { + SVnode *pVnode = pSma->pVnode; + if (!VND_IS_RSMA(pVnode)) { + return TSDB_CODE_SUCCESS; + } + + SSmaEnv *pSmaEnv = SMA_RSMA_ENV(pSma); + SRSmaStat *pRSmaStat = SMA_RSMA_STAT(SMA_ENV_STAT(pSmaEnv)); + + // cleanup outdated qtaskinfo files + tdCleanupQTaskInfoFiles(pSma, pRSmaStat); + + return TSDB_CODE_SUCCESS; +} + +/** + * @brief Rsma async commit implementation + * 1) set rsma stat TASK_TRIGGER_STAT_PAUSED + * 2) Wait all running fetch task finish to fetch and put submitMsg into level 2/3 wQueue(blocking level 1 write) + * 3) + * + * @param pSma + * @return int32_t + */ +static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) { + SSmaEnv *pSmaEnv = SMA_RSMA_ENV(pSma); + if (!pSmaEnv) { + return TSDB_CODE_SUCCESS; + } + + SSmaStat *pStat = SMA_ENV_STAT(pSmaEnv); + SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pStat); + + // step 1: set rsma stat + atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_PAUSED); + atomic_store_8(RSMA_COMMIT_STAT(pRSmaStat), 1); + + // step 2: wait all triggered fetch tasks finished + int32_t nLoops = 0; + while (1) { + if (T_REF_VAL_GET(pStat) == 0) { + smaDebug("vgId:%d, rsma fetch tasks all finished", SMA_VID(pSma)); + break; + } else { + smaDebug("vgId:%d, rsma fetch tasks not all finished yet", SMA_VID(pSma)); + } + ++nLoops; + if (nLoops > 1000) { + sched_yield(); + nLoops = 0; + } + } + + // step 3: swap rsmaInfoHash and iRsmaInfoHash + ASSERT(!RSMA_IMU_INFO_HASH(pRSmaStat)); + ASSERT(RSMA_INFO_HASH(pRSmaStat)); + + RSMA_IMU_INFO_HASH(pRSmaStat) = RSMA_INFO_HASH(pRSmaStat); + RSMA_INFO_HASH(pRSmaStat) = + taosHashInit(RSMA_TASK_INFO_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK); + + if (!RSMA_INFO_HASH(pRSmaStat)) { + smaError("vgId:%d, rsma async commit failed since %s", SMA_VID(pSma), terrstr()); + return TSDB_CODE_FAILED; + } + + // step 4: others + pRSmaStat->commitAppliedVer = pSma->pVnode->state.applied; + pRSmaStat->commitSubmitVer = pRSmaStat->submitVer; + + return TSDB_CODE_SUCCESS; +} + +/** + * @brief commit for rollup sma + * + * @param pSma + * @return int32_t + */ +static int32_t tdProcessRSmaAsyncCommitImpl(SSma *pSma) { + SSmaEnv *pSmaEnv = SMA_RSMA_ENV(pSma); + if (!pSmaEnv) { + return TSDB_CODE_SUCCESS; + } + + SSmaStat *pStat = SMA_ENV_STAT(pSmaEnv); + SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pStat); + + // perform persist task for qTaskInfo + tdRSmaPersistExecImpl(pRSmaStat, RSMA_IMU_INFO_HASH(pRSmaStat)); + + return TSDB_CODE_SUCCESS; +} + +/** + * @brief Migrate rsmaInfo from iRsmaInfo to rsmaInfo if rsmaInfoHash not empty. + * + * @param pSma + * @return int32_t + */ +static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma) { + SSmaEnv *pSmaEnv = SMA_RSMA_ENV(pSma); + if (!pSmaEnv) { + return TSDB_CODE_SUCCESS; + } + + SSmaStat *pStat = SMA_ENV_STAT(pSmaEnv); + SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pStat); + + // step 1: merge rsmaInfoHash and iRsmaInfoHash + taosWLockLatch(SMA_ENV_LOCK(pSmaEnv)); + + if (taosHashGetSize(RSMA_INFO_HASH(pRSmaStat)) <= 0) { + // TODO: optimization - just switch the hash pointer if rsmaInfoHash is empty + } + + void *pIter = taosHashIterate(RSMA_IMU_INFO_HASH(pRSmaStat), NULL); + while (pIter) { + tb_uid_t *pSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL); + + if (!taosHashGet(RSMA_INFO_HASH(pRSmaStat), pSuid, sizeof(tb_uid_t))) { + taosHashPut(RSMA_INFO_HASH(pRSmaStat), pSuid, sizeof(tb_uid_t), pIter, sizeof(pIter)); + smaDebug("vgId:%d, rsma async post commit, migrated from iRsmaInfoHash for table:%" PRIi64, SMA_VID(pSma), + *pSuid); + } else { + // free the resources + SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)pIter; + tdFreeRSmaInfo(pSma, pRSmaInfo, false); + smaDebug("vgId:%d, rsma async post commit, free rsma info since already COW for table:%" PRIi64, SMA_VID(pSma), + *pSuid); + } + + pIter = taosHashIterate(RSMA_IMU_INFO_HASH(pRSmaStat), pIter); + } + + taosHashCleanup(RSMA_IMU_INFO_HASH(pRSmaStat)); + RSMA_IMU_INFO_HASH(pRSmaStat) = NULL; + + taosWUnLockLatch(SMA_ENV_LOCK(pSmaEnv)); + + // step 2: cleanup outdated qtaskinfo files + tdCleanupQTaskInfoFiles(pSma, pRSmaStat); + return TSDB_CODE_SUCCESS; } diff --git a/source/dnode/vnode/src/sma/smaEnv.c b/source/dnode/vnode/src/sma/smaEnv.c index 4a7d4db874..577d5fd4fa 100644 --- a/source/dnode/vnode/src/sma/smaEnv.c +++ b/source/dnode/vnode/src/sma/smaEnv.c @@ -17,7 +17,6 @@ typedef struct SSmaStat SSmaStat; -#define RSMA_TASK_INFO_HASH_SLOT 8 #define SMA_MGMT_REF_NUM 10240 extern SSmaMgmt smaMgmt; @@ -109,12 +108,7 @@ static SSmaEnv *tdNewSmaEnv(const SSma *pSma, int8_t smaType, const char *path) SMA_ENV_TYPE(pEnv) = smaType; - int code = taosThreadRwlockInit(&(pEnv->lock), NULL); - if (code) { - terrno = TAOS_SYSTEM_ERROR(code); - taosMemoryFree(pEnv); - return NULL; - } + taosInitRWLatch(&(pEnv->lock)); if (tdInitSmaStat(&SMA_ENV_STAT(pEnv), smaType, pSma) != TSDB_CODE_SUCCESS) { tdFreeSmaEnv(pEnv); @@ -148,7 +142,6 @@ static int32_t tdInitSmaEnv(SSma *pSma, int8_t smaType, const char *path, SSmaEn void tdDestroySmaEnv(SSmaEnv *pSmaEnv) { if (pSmaEnv) { pSmaEnv->pStat = tdFreeSmaState(pSmaEnv->pStat, SMA_ENV_TYPE(pSmaEnv)); - taosThreadRwlockDestroy(&(pSmaEnv->lock)); } } @@ -260,7 +253,7 @@ static void tdDestroyRSmaStat(void *pRSmaStat) { void *infoHash = taosHashIterate(RSMA_INFO_HASH(pStat), NULL); while (infoHash) { SRSmaInfo *pSmaInfo = *(SRSmaInfo **)infoHash; - tdFreeRSmaInfo(pSma, pSmaInfo); + tdFreeRSmaInfo(pSma, pSmaInfo, true); infoHash = taosHashIterate(RSMA_INFO_HASH(pStat), infoHash); } } @@ -311,7 +304,6 @@ int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType) { if (taosRemoveRef(smaMgmt.rsetId, RSMA_REF_ID(pRSmaStat)) < 0) { smaError("vgId:%d, remove refId:%" PRIi64 " from rsmaRef:%" PRIi32 " failed since %s", SMA_VID(pRSmaStat->pSma), RSMA_REF_ID(pRSmaStat), smaMgmt.rsetId, terrstr()); - ASSERT(0); } else { smaDebug("vgId:%d, remove refId:%" PRIi64 " from rsmaRef:%" PRIi32 " succeed", SMA_VID(pRSmaStat->pSma), RSMA_REF_ID(pRSmaStat), smaMgmt.rsetId); diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index a6d56acbfb..32d6dee57f 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -48,20 +48,7 @@ static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables); static int32_t tdRSmaRestoreQTaskInfoReload(SSma *pSma, int64_t *committed); static int32_t tdRSmaRestoreTSDataReload(SSma *pSma, int64_t committed); -struct SRSmaInfoItem { - void *taskInfo; // qTaskInfo_t - int64_t refId; - tmr_h tmrId; - int32_t maxDelay; - int8_t level; - int8_t triggerStat; -}; -struct SRSmaInfo { - STSchema *pTSchema; - int64_t suid; - SRSmaInfoItem items[TSDB_RETENTION_L2]; -}; static SRSmaInfo *tdGetRSmaInfoByItem(SRSmaInfoItem *pItem) { // adapt accordingly if definition of SRSmaInfo update @@ -102,8 +89,9 @@ static FORCE_INLINE int32_t tdRSmaQTaskInfoContLen(int32_t lenWithHead) { static FORCE_INLINE void tdRSmaQTaskInfoIterDestroy(SRSmaQTaskInfoIter *pIter) { taosMemoryFreeClear(pIter->pBuf); } -static FORCE_INLINE void tdFreeTaskHandle(qTaskInfo_t *taskHandle, int32_t vgId, int32_t level) { +void tdFreeQTaskInfo(qTaskInfo_t *taskHandle, int32_t vgId, int32_t level) { // Note: free/kill may in RC + if (!taskHandle) return; qTaskInfo_t otaskHandle = atomic_load_ptr(taskHandle); if (otaskHandle && atomic_val_compare_exchange_ptr(taskHandle, otaskHandle, NULL)) { smaDebug("vgId:%d, free qTaskInfo_t %p of level %d", vgId, otaskHandle, level); @@ -111,25 +99,36 @@ static FORCE_INLINE void tdFreeTaskHandle(qTaskInfo_t *taskHandle, int32_t vgId, } else { smaDebug("vgId:%d, not free qTaskInfo_t %p of level %d", vgId, otaskHandle, level); } + // TODO: clear files related to qTaskInfo? } -void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo) { +/** + * @brief general function to free rsmaInfo + * + * @param pSma + * @param pInfo + * @param isDeepFree Only stop tmrId and free pTSchema for deep free + * @return void* + */ +void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree) { if (pInfo) { for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { SRSmaInfoItem *pItem = &pInfo->items[i]; if (pItem->taskInfo) { - if (pItem->tmrId) { + if (isDeepFree && pItem->tmrId) { smaDebug("vgId:%d, table %" PRIi64 " stop fetch timer %p level %d", SMA_VID(pSma), pInfo->suid, pItem->tmrId, i + 1); taosTmrStopA(&pItem->tmrId); } - tdFreeTaskHandle(&pItem->taskInfo, SMA_VID(pSma), i + 1); + tdFreeQTaskInfo(&pItem->taskInfo, SMA_VID(pSma), i + 1); } else { smaDebug("vgId:%d, table %" PRIi64 " no need to destroy rsma info level %d since empty taskInfo", SMA_VID(pSma), pInfo->suid, i + 1); } } - taosMemoryFree(pInfo->pTSchema); + if (isDeepFree) { + taosMemoryFree(pInfo->pTSchema); + } taosMemoryFree(pInfo); } @@ -151,7 +150,7 @@ static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids) if (!suid || !tbUids) { terrno = TSDB_CODE_INVALID_PTR; - smaError("vgId:%d, failed to get rsma info for uid:%" PRIi64 " since %s", SMA_VID(pSma), *suid, terrstr(terrno)); + smaError("vgId:%d, failed to get rsma info for uid:%" PRIi64 " since %s", SMA_VID(pSma), *suid, terrstr()); return TSDB_CODE_FAILED; } @@ -165,7 +164,7 @@ static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids) if (pRSmaInfo->items[0].taskInfo) { if ((qUpdateQualifiedTableId(pRSmaInfo->items[0].taskInfo, tbUids, true) < 0)) { - smaError("vgId:%d, update tbUidList failed for uid:%" PRIi64 " since %s", SMA_VID(pSma), *suid, terrstr(terrno)); + smaError("vgId:%d, update tbUidList failed for uid:%" PRIi64 " since %s", SMA_VID(pSma), *suid, terrstr()); return TSDB_CODE_FAILED; } else { smaDebug("vgId:%d, update tbUidList succeed for qTaskInfo:%p with suid:%" PRIi64 ", uid:%" PRIi64, SMA_VID(pSma), @@ -175,7 +174,7 @@ static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids) if (pRSmaInfo->items[1].taskInfo) { if ((qUpdateQualifiedTableId(pRSmaInfo->items[1].taskInfo, tbUids, true) < 0)) { - smaError("vgId:%d, update tbUidList failed for uid:%" PRIi64 " since %s", SMA_VID(pSma), *suid, terrstr(terrno)); + smaError("vgId:%d, update tbUidList failed for uid:%" PRIi64 " since %s", SMA_VID(pSma), *suid, terrstr()); return TSDB_CODE_FAILED; } else { smaDebug("vgId:%d, update tbUidList succeed for qTaskInfo:%p with suid:%" PRIi64 ", uid:%" PRIi64, SMA_VID(pSma), @@ -257,22 +256,22 @@ int32_t tdFetchTbUidList(SSma *pSma, STbUidStore **ppStore, tb_uid_t suid, tb_ui static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat *pStat, SRSmaInfo *pRSmaInfo, int8_t idx) { - SRetention *pRetention = SMA_RETENTION(pSma); - STsdbCfg *pTsdbCfg = SMA_TSDB_CFG(pSma); + if ((param->qmsgLen > 0) && param->qmsg[idx]) { + SRetention *pRetention = SMA_RETENTION(pSma); + STsdbCfg *pTsdbCfg = SMA_TSDB_CFG(pSma); + SVnode *pVnode = pSma->pVnode; + SReadHandle handle = { + .meta = pVnode->pMeta, + .vnode = pVnode, + .initTqReader = 1, + }; - SReadHandle handle = { - .meta = pSma->pVnode->pMeta, - .vnode = pSma->pVnode, - .initTqReader = 1, - }; - - if (param->qmsg[idx]) { SRSmaInfoItem *pItem = &(pRSmaInfo->items[idx]); pItem->refId = RSMA_REF_ID(pStat); pItem->taskInfo = qCreateStreamExecTaskInfo(param->qmsg[idx], &handle); if (!pItem->taskInfo) { terrno = TSDB_CODE_RSMA_QTASKINFO_CREATE; - goto _err; + return TSDB_CODE_FAILED; } pItem->triggerStat = TASK_TRIGGER_STAT_INACTIVE; if (param->maxdelay[idx] < TSDB_MIN_ROLLUP_MAX_DELAY) { @@ -286,13 +285,11 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat pItem->maxDelay = TSDB_MAX_ROLLUP_MAX_DELAY; } pItem->level = idx == 0 ? TSDB_RETENTION_L1 : TSDB_RETENTION_L2; - smaInfo("vgId:%d table:%" PRIi64 " level:%" PRIi8 " maxdelay:%" PRIi64 " watermark:%" PRIi64 + smaInfo("vgId:%d, table:%" PRIi64 " level:%" PRIi8 " maxdelay:%" PRIi64 " watermark:%" PRIi64 ", finally maxdelay:%" PRIi32, - SMA_VID(pSma), pRSmaInfo->suid, idx + 1, param->maxdelay[idx], param->watermark[idx], pItem->maxDelay); + TD_VID(pVnode), pRSmaInfo->suid, idx + 1, param->maxdelay[idx], param->watermark[idx], pItem->maxDelay); } return TSDB_CODE_SUCCESS; -_err: - return TSDB_CODE_FAILED; } /** @@ -357,7 +354,7 @@ int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con return TSDB_CODE_SUCCESS; _err: - tdFreeRSmaInfo(pSma, pRSmaInfo); + tdFreeRSmaInfo(pSma, pRSmaInfo, true); return TSDB_CODE_FAILED; } @@ -562,7 +559,9 @@ static int32_t tdRSmaFetchAndSubmitResult(SRSmaInfoItem *pItem, STSchema *pTSche SSDataBlock *output = NULL; uint64_t ts; if (qExecTask(pItem->taskInfo, &output, &ts) < 0) { - ASSERT(false); + smaError("vgId:%d, qExecTask for rsma table %" PRIi64 "l evel %" PRIi8 " failed since %s", SMA_VID(pSma), suid, + pItem->level, terrstr()); + goto _err; } if (!output) { break; @@ -572,7 +571,7 @@ static int32_t tdRSmaFetchAndSubmitResult(SRSmaInfoItem *pItem, STSchema *pTSche pResult = taosArrayInit(1, sizeof(SSDataBlock)); if (!pResult) { terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; + goto _err; } } @@ -649,9 +648,18 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType return TSDB_CODE_SUCCESS; } +/** + * @brief During async commit, the SRSmaInfo object would be COW from iRSmaInfoHash and write lock should be applied. + * + * @param pSma + * @param suid + * @return SRSmaInfo* + */ static SRSmaInfo *tdGetRSmaInfoBySuid(SSma *pSma, int64_t suid) { SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); SRSmaStat *pStat = NULL; + SRSmaInfo *pRSmaInfo = NULL; + if (!pEnv) { // only applicable when rsma env exists return NULL; @@ -662,11 +670,37 @@ static SRSmaInfo *tdGetRSmaInfoBySuid(SSma *pSma, int64_t suid) { return NULL; } - SRSmaInfo *pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)); - if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) { + pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)); + if (pRSmaInfo && (pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) { + return pRSmaInfo; + } + + if (RSMA_COMMIT_STAT(pStat) == 0) { return NULL; } - return pRSmaInfo; + + // clone the SRSmaInfo from iRsmaInfoHash to rsmaInfoHash if in committing stat + SRSmaInfo *pCowRSmaInfo = NULL; + // lock + taosWLockLatch(SMA_ENV_LOCK(pEnv)); + void *iRSmaInfo = taosHashGet(RSMA_IMU_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)); + if (iRSmaInfo) { + SRSmaInfo *pIRSmaInfo = *(SRSmaInfo **)iRSmaInfo; + if (pIRSmaInfo) { + if (tdCloneRSmaInfo(pSma, pCowRSmaInfo, pIRSmaInfo) < 0) { + taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); + smaError("vgId:%d, clone rsma info failed for suid:%" PRIu64 " since %s", SMA_VID(pSma), suid, terrstr()); + return NULL; + } + if (taosHashPut(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t), &pCowRSmaInfo, sizeof(pCowRSmaInfo)) < 0) { + taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); + return NULL; + } + } + } + // unlock + taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); + return pCowRSmaInfo; } static int32_t tdExecuteRSma(SSma *pSma, const void *pMsg, int32_t inputType, tb_uid_t suid) { @@ -891,10 +925,17 @@ int32_t tdProcessRSmaRestoreImpl(SSma *pSma) { return TSDB_CODE_SUCCESS; _err: - smaError("vgId:%d failed to restore rsma task since %s", SMA_VID(pSma), terrstr()); + smaError("vgId:%d, failed to restore rsma task since %s", SMA_VID(pSma), terrstr()); return TSDB_CODE_FAILED; } +/** + * @brief Restore from SRSmaQTaskInfoItem + * + * @param pSma + * @param pItem + * @return int32_t + */ static int32_t tdRSmaQTaskInfoItemRestore(SSma *pSma, const SRSmaQTaskInfoItem *pItem) { SRSmaInfo *pRSmaInfo = NULL; void *qTaskInfo = NULL; @@ -920,7 +961,7 @@ static int32_t tdRSmaQTaskInfoItemRestore(SSma *pSma, const SRSmaQTaskInfoItem * if (qDeserializeTaskStatus(qTaskInfo, pItem->qTaskInfo, pItem->len) < 0) { smaError("vgId:%d, restore rsma task failed for table:%" PRIi64 " level %d since %s", SMA_VID(pSma), pItem->suid, - pItem->type, terrstr(terrno)); + pItem->type, terrstr()); return TSDB_CODE_FAILED; } smaDebug("vgId:%d, restore rsma task success for table:%" PRIi64 " level %d", SMA_VID(pSma), pItem->suid, @@ -1067,26 +1108,27 @@ static int32_t tdRSmaQTaskInfoRestore(SSma *pSma, SRSmaQTaskInfoIter *pIter) { return TSDB_CODE_SUCCESS; } -int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat) { +int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash) { SSma *pSma = pRSmaStat->pSma; SVnode *pVnode = pSma->pVnode; int32_t vid = SMA_VID(pSma); int64_t toffset = 0; bool isFileCreated = false; - if (taosHashGetSize(RSMA_INFO_HASH(pRSmaStat)) <= 0) { + if (taosHashGetSize(pInfoHash) <= 0) { return TSDB_CODE_SUCCESS; } - void *infoHash = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), NULL); + void *infoHash = taosHashIterate(pInfoHash, NULL); if (!infoHash) { return TSDB_CODE_SUCCESS; } STFile tFile = {0}; - if (RSMA_SUBMIT_VER(pRSmaStat) > 0) { +#if 0 + if (pRSmaStat->commitAppliedVer > 0) { char qTaskInfoFName[TSDB_FILENAME_LEN]; - tdRSmaQTaskInfoGetFName(vid, pSma->pVnode->state.applied, qTaskInfoFName); + tdRSmaQTaskInfoGetFName(vid, pRSmaStat->commitAppliedVer, qTaskInfoFName); if (tdInitTFile(&tFile, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFName) < 0) { smaError("vgId:%d, rsma persit, init %s failed since %s", vid, qTaskInfoFName, terrstr()); goto _err; @@ -1099,6 +1141,7 @@ int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat) { isFileCreated = true; } +#endif while (infoHash) { SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)infoHash; @@ -1114,7 +1157,7 @@ int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat) { int8_t type = (int8_t)(i + 1); if (qSerializeTaskStatus(taskInfo, &pOutput, &len) < 0) { smaError("vgId:%d, rsma, table %" PRIi64 " level %d serialize qTaskInfo failed since %s", vid, pRSmaInfo->suid, - i + 1, terrstr(terrno)); + i + 1, terrstr()); goto _err; } if (!pOutput || len <= 0) { @@ -1130,7 +1173,7 @@ int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat) { if (!isFileCreated) { char qTaskInfoFName[TSDB_FILENAME_LEN]; - tdRSmaQTaskInfoGetFName(vid, pSma->pVnode->state.applied, qTaskInfoFName); + tdRSmaQTaskInfoGetFName(vid, pRSmaStat->commitAppliedVer, qTaskInfoFName); if (tdInitTFile(&tFile, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFName) < 0) { smaError("vgId:%d, rsma persit, init %s failed since %s", vid, qTaskInfoFName, terrstr()); goto _err; @@ -1163,11 +1206,11 @@ int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat) { taosMemoryFree(pOutput); } - infoHash = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), infoHash); + infoHash = taosHashIterate(pInfoHash, infoHash); } if (isFileCreated) { - tFile.info.qTaskInfo.submitVer = atomic_load_64(&pRSmaStat->submitVer); + tFile.info.qTaskInfo.submitVer = atomic_load_64(&pRSmaStat->commitSubmitVer); if (tdUpdateTFileHeader(&tFile) < 0) { smaError("vgId:%d, rsma, failed to update tfile %s header since %s", vid, TD_TFILE_FULL_NAME(&tFile), tstrerror(terrno)); @@ -1217,6 +1260,10 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data since stat is %" PRIi8 ", rsetId rsetId:%" PRIi64 " refId:%d", SMA_VID(pSma), pItem->level, rsmaTriggerStat, smaMgmt.rsetId, pItem->refId); + if (rsmaTriggerStat == TASK_TRIGGER_STAT_PAUSED) { + taosTmrReset(tdRSmaFetchTrigger, pItem->maxDelay > 5000 ? 5000 : pItem->maxDelay, pItem, smaMgmt.tmrHandle, + &pItem->tmrId); + } return; } default: diff --git a/source/dnode/vnode/src/sma/smaUtil.c b/source/dnode/vnode/src/sma/smaUtil.c index 22d82e8df0..13befabed5 100644 --- a/source/dnode/vnode/src/sma/smaUtil.c +++ b/source/dnode/vnode/src/sma/smaUtil.c @@ -313,4 +313,99 @@ int32_t tdReleaseSmaRef(int32_t rsetId, int64_t refId, const char *tags, int32_t return TSDB_CODE_SUCCESS; } + +static int32_t tdCloneQTaskInfo(SSma *pSma, qTaskInfo_t dstTaskInfo, qTaskInfo_t srcTaskInfo, SRSmaParam *param, + tb_uid_t suid, int8_t idx) { + SVnode *pVnode = pSma->pVnode; + char *pOutput = NULL; + int32_t len = 0; + + if (qSerializeTaskStatus(srcTaskInfo, &pOutput, &len) < 0) { + smaError("vgId:%d, rsma clone, table %" PRIi64 " serialize qTaskInfo failed since %s", TD_VID(pVnode), suid, + terrstr()); + goto _err; + } + + SReadHandle handle = { + .meta = pVnode->pMeta, + .vnode = pVnode, + .initTqReader = 1, + }; + ASSERT(!dstTaskInfo); + dstTaskInfo = qCreateStreamExecTaskInfo(param->qmsg[idx], &handle); + if (!dstTaskInfo) { + terrno = TSDB_CODE_RSMA_QTASKINFO_CREATE; + goto _err; + } + + if (qDeserializeTaskStatus(dstTaskInfo, pOutput, len) < 0) { + smaError("vgId:%d, rsma clone, restore rsma task for table:%" PRIi64 " failed since %s", TD_VID(pVnode), suid, + terrstr()); + goto _err; + } + + smaError("vgId:%d, rsma clone, restore rsma task for table:%" PRIi64 " succeed", TD_VID(pVnode), suid); + + taosMemoryFreeClear(pOutput); + return TSDB_CODE_SUCCESS; +_err: + taosMemoryFreeClear(pOutput); + tdFreeQTaskInfo(dstTaskInfo, TD_VID(pVnode), idx + 1); + return TSDB_CODE_FAILED; +} + +/** + * @brief pTSchema is shared + * + * @param pSma + * @param pDest + * @param pSrc + * @return int32_t + */ +int32_t tdCloneRSmaInfo(SSma *pSma, SRSmaInfo *pDest, SRSmaInfo *pSrc) { + SVnode *pVnode = pSma->pVnode; + SRSmaParam *param = NULL; + if (!pSrc) { + return TSDB_CODE_SUCCESS; + } + + if (!pDest) { + pDest = taosMemoryCalloc(1, sizeof(SRSmaInfo)); + if (!pDest) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } + } + + memcpy(pDest, pSrc, sizeof(SRSmaInfo)); + + SMetaReader mr = {0}; + metaReaderInit(&mr, SMA_META(pSma), 0); + smaDebug("vgId:%d, rsma clone, suid is %" PRIi64, TD_VID(pVnode), pSrc->suid); + if (metaGetTableEntryByUid(&mr, pSrc->suid) < 0) { + smaError("vgId:%d, rsma clone, failed to get table meta for %" PRIi64 " since %s", TD_VID(pVnode), pSrc->suid, + terrstr()); + goto _err; + } + ASSERT(mr.me.type == TSDB_SUPER_TABLE); + ASSERT(mr.me.uid == pSrc->suid); + if (TABLE_IS_ROLLUP(mr.me.flags)) { + param = &mr.me.stbEntry.rsmaParam; + for (int i = 0; i < TSDB_RETENTION_L2; ++i) { + SRSmaInfoItem *pItem = &pSrc->items[i]; + if (pItem->taskInfo) { + tdCloneQTaskInfo(pSma, pDest->items[i].taskInfo, pItem->taskInfo, param, pSrc->suid, i); + } + } + smaDebug("vgId:%d, rsma clone env success for %" PRIi64, TD_VID(pVnode), pSrc->suid); + } + + metaReaderClear(&mr); + return TSDB_CODE_SUCCESS; +_err: + metaReaderClear(&mr); + tdFreeRSmaInfo(pSma, pDest, false); + return TSDB_CODE_FAILED; +} + // ... \ No newline at end of file diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index f36e6f874f..fb05aeecd9 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -635,6 +635,8 @@ int32_t tqProcessStreamTrigger(STQ* pTq, SSubmitReq* pReq) { pSubmit = streamDataSubmitNew(pReq); if (pSubmit == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + qError("failed to create data submit for stream since out of memory"); failed = true; } @@ -644,12 +646,16 @@ int32_t tqProcessStreamTrigger(STQ* pTq, SSubmitReq* pReq) { SStreamTask* pTask = *(SStreamTask**)pIter; if (!pTask->isDataScan) continue; + qDebug("data submit enqueue stream task: %d", pTask->taskId); + if (!failed) { if (streamTaskInput(pTask, (SStreamQueueItem*)pSubmit) < 0) { + qError("stream task input failed, task id %d", pTask->taskId); continue; } if (streamLaunchByWrite(pTask, TD_VID(pTq->pVnode)) < 0) { + qError("stream task launch failed, task id %d", pTask->taskId); continue; } } else { diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index 0bdbe82b77..618b32239a 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -64,9 +64,16 @@ int64_t tqScan(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVa qTaskInfo_t task = pExec->execCol.task[0]; if (qStreamPrepareScan(task, pOffset) < 0) { - ASSERT(pOffset->type == TMQ_OFFSET__LOG); - pRsp->rspOffset = *pOffset; - return 0; + if (pOffset->type == TMQ_OFFSET__LOG) { + pRsp->rspOffset = *pOffset; + return 0; + } else { + tqOffsetResetToLog(pOffset, pHandle->snapshotVer + 1); + if (qStreamPrepareScan(task, pOffset) < 0) { + pRsp->rspOffset = *pOffset; + return 0; + } + } } int32_t rowCnt = 0; @@ -217,7 +224,6 @@ int32_t tqLogScanExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataR } if (pRsp->blockNum == 0) { - pRsp->skipLogNum++; return -1; } diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index e9e5f6cd8b..4b2160434f 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -242,6 +242,8 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) void* data = taosMemoryMalloc(msgLen); if (data == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + qError("failed to copy data for stream since out of memory"); return -1; } memcpy(data, msg, msgLen); diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 605f4bf3d5..723caca5d7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -476,9 +476,9 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) { if (--state->iFileSet >= 0) { pFileSet = (SDFileSet *)taosArrayGet(state->aDFileSet, state->iFileSet); } else { - // tBlockDataClear(&state->blockData); + // tBlockDataClear(&state->blockData, 1); if (state->pBlockData) { - tBlockDataClear(state->pBlockData); + tBlockDataClear(state->pBlockData, 1); state->pBlockData = NULL; } @@ -500,9 +500,7 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) { if (code) goto _err; /* if (state->pBlockIdx) { */ - /* tBlockIdxReset(state->blockIdx); */ /* } */ - /* tBlockIdxReset(state->blockIdx); */ /* code = tMapDataSearch(&state->blockIdxMap, state->pBlockIdxExp, tGetBlockIdx, tCmprBlockIdx, * &state->blockIdx); */ @@ -582,8 +580,8 @@ _err: state->aBlockIdx = NULL; } if (state->pBlockData) { - // tBlockDataClear(&state->blockData); - tBlockDataClear(state->pBlockData); + // tBlockDataClear(&state->blockData, 1); + tBlockDataClear(state->pBlockData, 1); state->pBlockData = NULL; } @@ -609,8 +607,8 @@ int32_t clearNextRowFromFS(void *iter) { state->aBlockIdx = NULL; } if (state->pBlockData) { - // tBlockDataClear(&state->blockData); - tBlockDataClear(state->pBlockData); + // tBlockDataClear(&state->blockData, 1); + tBlockDataClear(state->pBlockData, 1); state->pBlockData = NULL; } diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 8a189d98b3..470bff1eae 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -263,7 +263,7 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { taosArrayClear(pCommitter->aBlockIdx); tMapDataReset(&pCommitter->oBlockMap); tBlockDataReset(&pCommitter->oBlockData); - pRSet = tsdbFSStateGetDFileSet(pTsdb->fs->nState, pCommitter->commitFid); + pRSet = tsdbFSStateGetDFileSet(pTsdb->fs->nState, pCommitter->commitFid, TD_EQ); if (pRSet) { code = tsdbDataFReaderOpen(&pCommitter->pReader, pTsdb, pRSet); if (code) goto _err; @@ -284,16 +284,7 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { .fLast = {.commitID = pCommitter->commitID, .size = 0}, .fSma = pRSet->fSma}; } else { - STfs *pTfs = pTsdb->pVnode->pTfs; - SDiskID did = {.level = 0, .id = 0}; - - // TODO: alloc a new disk - // tfsAllocDisk(pTfs, 0, &did); - - // create the directory - tfsMkdirRecurAt(pTfs, pTsdb->path, did); - - wSet = (SDFileSet){.diskId = did, + wSet = (SDFileSet){.diskId = (SDiskID){.level = 0, .id = 0}, .fid = pCommitter->commitFid, .fHead = {.commitID = pCommitter->commitID, .offset = 0, .size = 0}, .fData = {.commitID = pCommitter->commitID, .size = 0}, @@ -1001,10 +992,10 @@ _exit: static void tsdbCommitDataEnd(SCommitter *pCommitter) { taosArrayDestroy(pCommitter->aBlockIdx); tMapDataClear(&pCommitter->oBlockMap); - tBlockDataClear(&pCommitter->oBlockData); + tBlockDataClear(&pCommitter->oBlockData, 1); taosArrayDestroy(pCommitter->aBlockIdxN); tMapDataClear(&pCommitter->nBlockMap); - tBlockDataClear(&pCommitter->nBlockData); + tBlockDataClear(&pCommitter->nBlockData, 1); tTSchemaDestroy(pCommitter->skmTable.pTSchema); tTSchemaDestroy(pCommitter->skmRow.pTSchema); } diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 53b6735c30..70fbd90646 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -698,6 +698,6 @@ void tsdbFSStateDeleteDFileSet(STsdbFSState *pState, int32_t fid) { SDelFile *tsdbFSStateGetDelFile(STsdbFSState *pState) { return pState->pDelFile; } -SDFileSet *tsdbFSStateGetDFileSet(STsdbFSState *pState, int32_t fid) { - return (SDFileSet *)taosArraySearch(pState->aDFileSet, &(SDFileSet){.fid = fid}, tDFileSetCmprFn, TD_EQ); +SDFileSet *tsdbFSStateGetDFileSet(STsdbFSState *pState, int32_t fid, int32_t flag) { + return (SDFileSet *)taosArraySearch(pState->aDFileSet, &(SDFileSet){.fid = fid}, tDFileSetCmprFn, flag); } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index f0aea0cefb..0b05bb7224 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -63,10 +63,10 @@ typedef struct SBlockLoadSuppInfo { } SBlockLoadSuppInfo; typedef struct SFilesetIter { - int32_t numOfFiles; // number of total files - int32_t index; // current accessed index in the list - SArray* pFileList; // data file list - int32_t order; + int32_t numOfFiles; // number of total files + int32_t index; // current accessed index in the list + SArray* pFileList; // data file list + int32_t order; } SFilesetIter; typedef struct SFileDataBlockInfo { @@ -830,8 +830,8 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; - int32_t code = tsdbReadColData(pReader->pFileReader, &pBlockScanInfo->blockIdx, pBlock, pSupInfo->colIds, numOfCols, - pBlockData, NULL, NULL); + int32_t code = tsdbReadColData(pReader->pFileReader, &pBlockScanInfo->blockIdx, pBlock, pSupInfo->colIds, numOfCols, + pBlockData, NULL, NULL); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -1991,7 +1991,7 @@ static TSDBKEY getCurrentKeyInBuf(SDataBlockIter* pBlockIter, STsdbReader* pRead static int32_t moveToNextFile(STsdbReader* pReader, int32_t* numOfBlocks) { SReaderStatus* pStatus = &pReader->status; - SArray* pIndexList = taosArrayInit(4, sizeof(SBlockIdx)); + SArray* pIndexList = taosArrayInit(4, sizeof(SBlockIdx)); while (1) { bool hasNext = filesetIteratorNext(&pStatus->fileIter, pReader); @@ -3006,14 +3006,14 @@ SArray* tsdbRetrieveDataBlock(STsdbReader* pReader, SArray* pIdList) { code = doLoadFileBlockData(pReader, &pStatus->blockIter, pBlockScanInfo, &pStatus->fileBlockData); if (code != TSDB_CODE_SUCCESS) { - tBlockDataClear(&pStatus->fileBlockData); + tBlockDataClear(&pStatus->fileBlockData, 1); terrno = code; return NULL; } copyBlockDataToSDataBlock(pReader, pBlockScanInfo); - tBlockDataClear(&pStatus->fileBlockData); + tBlockDataClear(&pStatus->fileBlockData, 1); return pReader->pResBlock->pDataBlock; } @@ -3132,8 +3132,8 @@ int32_t tsdbGetFileBlocksDistInfo(STsdbReader* pReader, STableBlockDistInfo* pTa hasNext = (pBlockIter->numOfBlocks > 0); } -// tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, %s", pReader, numOfBlocks, numOfTables, -// pReader->pFileGroup->fid, pReader->idStr); + // tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, %s", pReader, numOfBlocks, numOfTables, + // pReader->pFileGroup->fid, pReader->idStr); } return code; @@ -3186,6 +3186,7 @@ int32_t tsdbGetTableSchema(SVnode* pVnode, int64_t uid, STSchema** pSchema, int6 *suid = 0; if (mr.me.type == TSDB_CHILD_TABLE) { + tDecoderClear(&mr.coder); *suid = mr.me.ctbEntry.suid; code = metaGetTableEntryByUid(&mr, *suid); if (code != TSDB_CODE_SUCCESS) { @@ -3204,4 +3205,3 @@ int32_t tsdbGetTableSchema(SVnode* pVnode, int64_t uid, STSchema** pSchema, int6 return TSDB_CODE_SUCCESS; } - diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 4e7a9d3b04..0ed2e12542 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -979,21 +979,21 @@ int32_t tsdbReadColData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *pBl code = tBlockDataCopy(pBlockData, pBlockData2); if (code) { - tBlockDataClear(pBlockData1); - tBlockDataClear(pBlockData2); + tBlockDataClear(pBlockData1, 1); + tBlockDataClear(pBlockData2, 1); goto _err; } code = tBlockDataMerge(pBlockData1, pBlockData2, pBlockData); if (code) { - tBlockDataClear(pBlockData1); - tBlockDataClear(pBlockData2); + tBlockDataClear(pBlockData1, 1); + tBlockDataClear(pBlockData2, 1); goto _err; } } - tBlockDataClear(pBlockData1); - tBlockDataClear(pBlockData2); + tBlockDataClear(pBlockData1, 1); + tBlockDataClear(pBlockData2, 1); } tFree(pBuf1); @@ -1115,29 +1115,29 @@ int32_t tsdbReadBlockData(SDataFReader *pReader, SBlockIdx *pBlockIdx, SBlock *p for (iSubBlock = 1; iSubBlock < pBlock->nSubBlock; iSubBlock++) { code = tsdbReadSubBlockData(pReader, pBlockIdx, pBlock, iSubBlock, pBlockData1, ppBuf1, ppBuf2); if (code) { - tBlockDataClear(pBlockData1); - tBlockDataClear(pBlockData2); + tBlockDataClear(pBlockData1, 1); + tBlockDataClear(pBlockData2, 1); goto _err; } code = tBlockDataCopy(pBlockData, pBlockData2); if (code) { - tBlockDataClear(pBlockData1); - tBlockDataClear(pBlockData2); + tBlockDataClear(pBlockData1, 1); + tBlockDataClear(pBlockData2, 1); goto _err; } // merge two block data code = tBlockDataMerge(pBlockData1, pBlockData2, pBlockData); if (code) { - tBlockDataClear(pBlockData1); - tBlockDataClear(pBlockData2); + tBlockDataClear(pBlockData1, 1); + tBlockDataClear(pBlockData2, 1); goto _err; } } - tBlockDataClear(pBlockData1); - tBlockDataClear(pBlockData2); + tBlockDataClear(pBlockData1, 1); + tBlockDataClear(pBlockData2, 1); } ASSERT(pBlock->nRow == pBlockData->nRow); diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index 06d4a86116..4886d874a9 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -29,29 +29,27 @@ struct STsdbSnapReader { SBlockIdx* pBlockIdx; SMapData mBlock; // SMapData int32_t iBlock; - SBlockData blkData; + SBlockData oBlockData; + SBlockData nBlockData; // for del file int8_t delDone; SDelFReader* pDelFReader; + SArray* aDelIdx; // SArray int32_t iDelIdx; - SArray* aDelIdx; // SArray SArray* aDelData; // SArray }; static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { int32_t code = 0; + STsdb* pTsdb = pReader->pTsdb; while (true) { if (pReader->pDataFReader == NULL) { - SDFileSet* pSet = NULL; + SDFileSet* pSet = tsdbFSStateGetDFileSet(pTsdb->fs->cState, pReader->fid, TD_GT); - // search the next data file set to read (todo) - if (0 /* TODO */) { - code = TSDB_CODE_VND_READ_END; - goto _exit; - } + if (pSet == NULL) goto _exit; - // open + pReader->fid = pSet->fid; code = tsdbDataFReaderOpen(&pReader->pDataFReader, pReader->pTsdb, pSet); if (code) goto _err; @@ -61,6 +59,8 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { pReader->iBlockIdx = 0; pReader->pBlockIdx = NULL; + + tsdbInfo("vgId:%d vnode snapshot tsdb open data file to read, fid:%d", TD_VID(pTsdb->pVnode), pReader->fid); } while (true) { @@ -73,17 +73,15 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { pReader->pBlockIdx = (SBlockIdx*)taosArrayGet(pReader->aBlockIdx, pReader->iBlockIdx); pReader->iBlockIdx++; - // SBlock code = tsdbReadBlock(pReader->pDataFReader, pReader->pBlockIdx, &pReader->mBlock, NULL); if (code) goto _err; pReader->iBlock = 0; } + SBlock block; + SBlock* pBlock = █ while (true) { - SBlock block; - SBlock* pBlock = █ - if (pReader->iBlock >= pReader->mBlock.nItem) { pReader->pBlockIdx = NULL; break; @@ -92,15 +90,60 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { tMapDataGetItemByIdx(&pReader->mBlock, pReader->iBlock, pBlock, tGetBlock); pReader->iBlock++; - if ((pBlock->minVersion >= pReader->sver && pBlock->minVersion <= pReader->ever) || - (pBlock->maxVersion >= pReader->sver && pBlock->maxVersion <= pReader->ever)) { - // overlap (todo) + if (pBlock->minVersion > pReader->ever || pBlock->maxVersion < pReader->sver) continue; - code = tsdbReadBlockData(pReader->pDataFReader, pReader->pBlockIdx, pBlock, &pReader->blkData, NULL, NULL); + code = tsdbReadBlockData(pReader->pDataFReader, pReader->pBlockIdx, pBlock, &pReader->oBlockData, NULL, NULL); + if (code) goto _err; + + // filter + tBlockDataReset(&pReader->nBlockData); + for (int32_t iColData = 0; iColData < taosArrayGetSize(pReader->oBlockData.aIdx); iColData++) { + SColData* pColDataO = tBlockDataGetColDataByIdx(&pReader->oBlockData, iColData); + SColData* pColDataN = NULL; + + code = tBlockDataAddColData(&pReader->nBlockData, taosArrayGetSize(pReader->nBlockData.aIdx), &pColDataN); if (code) goto _err; - goto _exit; + tColDataInit(pColDataN, pColDataO->cid, pColDataO->type, pColDataO->smaOn); } + + for (int32_t iRow = 0; iRow < pReader->oBlockData.nRow; iRow++) { + TSDBROW row = tsdbRowFromBlockData(&pReader->oBlockData, iRow); + int64_t version = TSDBROW_VERSION(&row); + + if (version < pReader->sver || version > pReader->ever) continue; + + code = tBlockDataAppendRow(&pReader->nBlockData, &row, NULL); + if (code) goto _err; + } + + // org data + // compress data (todo) + int32_t size = sizeof(TABLEID) + tPutBlockData(NULL, &pReader->nBlockData); + + *ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + size); + if (*ppData == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + + SSnapDataHdr* pHdr = (SSnapDataHdr*)(*ppData); + pHdr->type = 1; + pHdr->size = size; + + TABLEID* pId = (TABLEID*)(&pHdr[1]); + pId->suid = pReader->pBlockIdx->suid; + pId->uid = pReader->pBlockIdx->uid; + + tPutBlockData((uint8_t*)(&pId[1]), &pReader->nBlockData); + + tsdbInfo("vgId:%d vnode snapshot read data, fid:%d suid:%" PRId64 " uid:%" PRId64 + " iBlock:%d minVersion:%d maxVersion:%d nRow:%d out of %d size:%d", + TD_VID(pTsdb->pVnode), pReader->fid, pReader->pBlockIdx->suid, pReader->pBlockIdx->uid, + pReader->iBlock - 1, pBlock->minVersion, pBlock->maxVersion, pReader->nBlockData.nRow, pBlock->nRow, + size); + + goto _exit; } } } @@ -109,7 +152,7 @@ _exit: return code; _err: - tsdbError("vgId:%d snap read data failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); + tsdbError("vgId:%d vnode snapshot tsdb read data failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); return code; } @@ -120,7 +163,6 @@ static int32_t tsdbSnapReadDel(STsdbSnapReader* pReader, uint8_t** ppData) { if (pReader->pDelFReader == NULL) { if (pDelFile == NULL) { - code = TSDB_CODE_VND_READ_END; goto _exit; } @@ -135,15 +177,20 @@ static int32_t tsdbSnapReadDel(STsdbSnapReader* pReader, uint8_t** ppData) { pReader->iDelIdx = 0; } - while (pReader->iDelIdx < taosArrayGetSize(pReader->aDelIdx)) { + while (true) { + if (pReader->iDelIdx >= taosArrayGetSize(pReader->aDelIdx)) { + tsdbDelFReaderClose(&pReader->pDelFReader); + break; + } + SDelIdx* pDelIdx = (SDelIdx*)taosArrayGet(pReader->aDelIdx, pReader->iDelIdx); - int32_t size = 0; pReader->iDelIdx++; code = tsdbReadDelData(pReader->pDelFReader, pDelIdx, pReader->aDelData, NULL); if (code) goto _err; + int32_t size = 0; for (int32_t iDelData = 0; iDelData < taosArrayGetSize(pReader->aDelData); iDelData++) { SDelData* pDelData = (SDelData*)taosArrayGet(pReader->aDelData, iDelData); @@ -152,46 +199,44 @@ static int32_t tsdbSnapReadDel(STsdbSnapReader* pReader, uint8_t** ppData) { } } - if (size > 0) { - int64_t n = 0; + if (size == 0) continue; - size = size + sizeof(SSnapDataHdr) + sizeof(TABLEID); - code = tRealloc(ppData, size); - if (code) goto _err; - - // SSnapDataHdr - SSnapDataHdr* pSnapDataHdr = (SSnapDataHdr*)(*ppData + n); - pSnapDataHdr->type = 1; - pSnapDataHdr->size = size; // TODO: size here may incorrect - n += sizeof(SSnapDataHdr); - - // TABLEID - TABLEID* pId = (TABLEID*)(*ppData + n); - pId->suid = pDelIdx->suid; - pId->uid = pDelIdx->uid; - n += sizeof(*pId); - - // DATA - for (int32_t iDelData = 0; iDelData < taosArrayGetSize(pReader->aDelData); iDelData++) { - SDelData* pDelData = (SDelData*)taosArrayGet(pReader->aDelData, iDelData); - - if (pDelData->version >= pReader->sver && pDelData->version <= pReader->ever) { - n += tPutDelData(*ppData + n, pDelData); - } - } - - goto _exit; + // org data + size = sizeof(TABLEID) + size; + *ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + size); + if (*ppData == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; } - } - code = TSDB_CODE_VND_READ_END; - tsdbDelFReaderClose(&pReader->pDelFReader); + SSnapDataHdr* pHdr = (SSnapDataHdr*)(*ppData); + pHdr->type = 2; + pHdr->size = size; + + TABLEID* pId = (TABLEID*)(&pHdr[1]); + pId->suid = pDelIdx->suid; + pId->uid = pDelIdx->uid; + int32_t n = sizeof(SSnapDataHdr) + sizeof(TABLEID); + for (int32_t iDelData = 0; iDelData < taosArrayGetSize(pReader->aDelData); iDelData++) { + SDelData* pDelData = (SDelData*)taosArrayGet(pReader->aDelData, iDelData); + + if (pDelData->version < pReader->sver) continue; + if (pDelData->version > pReader->ever) continue; + + n += tPutDelData((*ppData) + n, pDelData); + } + + tsdbInfo("vgId:%d vnode snapshot tsdb read del data, suid:%" PRId64 " uid:%d" PRId64 " size:%d", + TD_VID(pTsdb->pVnode), pDelIdx->suid, pDelIdx->uid, size); + + break; + } _exit: return code; _err: - tsdbError("vgId:%d snap read del failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + tsdbError("vgId:%d vnode snapshot tsdb read del failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); return code; } @@ -209,15 +254,16 @@ int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapRe pReader->sver = sver; pReader->ever = ever; + pReader->fid = INT32_MIN; pReader->aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); if (pReader->aBlockIdx == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } - pReader->mBlock = tMapDataInit(); - - code = tBlockDataInit(&pReader->blkData); + code = tBlockDataInit(&pReader->oBlockData); + if (code) goto _err; + code = tBlockDataInit(&pReader->nBlockData); if (code) goto _err; pReader->aDelIdx = taosArrayInit(0, sizeof(SDelIdx)); @@ -225,18 +271,18 @@ int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapRe code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } - pReader->aDelData = taosArrayInit(0, sizeof(SDelData)); if (pReader->aDelData == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } + tsdbInfo("vgId:%d vnode snapshot tsdb reader opened", TD_VID(pTsdb->pVnode)); *ppReader = pReader; return code; _err: - tsdbError("vgId:%d snapshot reader open failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + tsdbError("vgId:%d vnode snapshot tsdb reader open failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); *ppReader = NULL; return code; } @@ -245,37 +291,43 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) { int32_t code = 0; STsdbSnapReader* pReader = *ppReader; - taosArrayDestroy(pReader->aDelData); - taosArrayDestroy(pReader->aDelIdx); - if (pReader->pDelFReader) { - tsdbDelFReaderClose(&pReader->pDelFReader); - } - tBlockDataClear(&pReader->blkData); - tMapDataClear(&pReader->mBlock); - taosArrayDestroy(pReader->aBlockIdx); if (pReader->pDataFReader) { tsdbDataFReaderClose(&pReader->pDataFReader); } + taosArrayDestroy(pReader->aBlockIdx); + tMapDataClear(&pReader->mBlock); + tBlockDataClear(&pReader->oBlockData, 1); + tBlockDataClear(&pReader->nBlockData, 1); + + if (pReader->pDelFReader) { + tsdbDelFReaderClose(&pReader->pDelFReader); + } + taosArrayDestroy(pReader->aDelIdx); + taosArrayDestroy(pReader->aDelData); + + tsdbInfo("vgId:%d vnode snapshot tsdb reader closed", TD_VID(pReader->pTsdb->pVnode)); + taosMemoryFree(pReader); *ppReader = NULL; - return code; } int32_t tsdbSnapRead(STsdbSnapReader* pReader, uint8_t** ppData) { int32_t code = 0; + *ppData = NULL; + // read data file if (!pReader->dataDone) { code = tsdbSnapReadData(pReader, ppData); if (code) { - if (code == TSDB_CODE_VND_READ_END) { - pReader->dataDone = 1; - } else { - goto _err; - } + goto _err; } else { - goto _exit; + if (*ppData) { + goto _exit; + } else { + pReader->dataDone = 1; + } } } @@ -283,23 +335,21 @@ int32_t tsdbSnapRead(STsdbSnapReader* pReader, uint8_t** ppData) { if (!pReader->delDone) { code = tsdbSnapReadDel(pReader, ppData); if (code) { - if (code == TSDB_CODE_VND_READ_END) { - pReader->delDone = 1; - } else { - goto _err; - } + goto _err; } else { - goto _exit; + if (*ppData) { + goto _exit; + } else { + pReader->delDone = 1; + } } } - code = TSDB_CODE_VND_READ_END; - _exit: return code; _err: - tsdbError("vgId:%d snapshot read failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); + tsdbError("vgId:%d vnode snapshot tsdb read failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); return code; } @@ -315,74 +365,40 @@ struct STsdbSnapWriter { int32_t minRow; int32_t maxRow; int8_t cmprAlg; + int64_t commitID; // for data file + SBlockData bData; + int32_t fid; SDataFReader* pDataFReader; - SArray* aBlockIdx; + SArray* aBlockIdx; // SArray int32_t iBlockIdx; SBlockIdx* pBlockIdx; - SMapData mBlock; + SMapData mBlock; // SMapData int32_t iBlock; - SBlock* pBlock; - SBlock block; - SBlockData blockData; + SBlockData* pBlockData; int32_t iRow; + SBlockData bDataR; SDataFWriter* pDataFWriter; - SArray* aBlockIdxN; - SBlockIdx* pBlockIdxN; - SBlockIdx blockIdx; - SMapData mBlockN; - SBlock* pBlockN; - SBlock blockN; - SBlockData nBlockData; + SBlockIdx* pBlockIdxW; // NULL when no committing table + SBlock blockW; + SBlockData bDataW; + SBlockIdx blockIdxW; + + SMapData mBlockW; // SMapData + SArray* aBlockIdxW; // SArray // for del file SDelFReader* pDelFReader; SDelFWriter* pDelFWriter; int32_t iDelIdx; - SArray* aDelIdx; + SArray* aDelIdxR; SArray* aDelData; - SArray* aDelIdxN; + SArray* aDelIdxW; }; -static int32_t tsdbSnapRollback(STsdbSnapWriter* pWriter) { - int32_t code = 0; - // TODO - return code; -} - -static int32_t tsdbSnapCommit(STsdbSnapWriter* pWriter) { - int32_t code = 0; - // TODO - return code; -} - -static int32_t tsdbSnapWriteDataEnd(STsdbSnapWriter* pWriter) { - int32_t code = 0; - STsdb* pTsdb = pWriter->pTsdb; - - if (pWriter->pDataFWriter == NULL) goto _exit; - - // TODO - - code = tsdbDataFWriterClose(&pWriter->pDataFWriter, 0); - if (code) goto _err; - - if (pWriter->pDataFReader) { - code = tsdbDataFReaderClose(&pWriter->pDataFReader); - if (code) goto _err; - } - -_exit: - return code; - -_err: - tsdbError("vgId:%d tsdb snapshot writer data end failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - return code; -} - static int32_t tsdbSnapWriteAppendData(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { int32_t code = 0; int32_t iRow = 0; // todo @@ -390,7 +406,7 @@ static int32_t tsdbSnapWriteAppendData(STsdbSnapWriter* pWriter, uint8_t* pData, SBlockData* pBlockData = NULL; // todo while (iRow < nRow) { - code = tBlockDataAppendRow(&pWriter->nBlockData, &tsdbRowFromBlockData(pBlockData, iRow), NULL); + code = tBlockDataAppendRow(&pWriter->bDataW, &tsdbRowFromBlockData(pBlockData, iRow), NULL); if (code) goto _err; } @@ -401,160 +417,488 @@ _err: return code; } -static int32_t tsdbSnapWriteTableDataEnd(STsdbSnapWriter* pWrite) { +static int32_t tsdbSnapWriteTableDataEnd(STsdbSnapWriter* pWriter) { int32_t code = 0; - // TODO - return code; -} -static int32_t tsdbSnapWriteTableData(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { - int32_t code = 0; - TABLEID id = {0}; // TODO + ASSERT(pWriter->pDataFWriter); - // skip - while (pWriter->pBlockIdx && tTABLEIDCmprFn(&id, pWriter->pBlockIdx) < 0) { - code = tsdbSnapWriteTableDataEnd(pWriter); - if (code) goto _err; + if (pWriter->pBlockIdxW == NULL) goto _exit; - pWriter->iBlockIdx++; - if (pWriter->iBlockIdx < taosArrayGetSize(pWriter->aBlockIdx)) { - pWriter->pBlockIdx = (SBlockIdx*)taosArrayGet(pWriter->aBlockIdx, pWriter->iBlockIdx); - } else { - pWriter->pBlockIdx = NULL; + // consume remain rows + if (pWriter->pBlockData) { + ASSERT(pWriter->iRow < pWriter->pBlockData->nRow); + while (pWriter->iRow < pWriter->pBlockData->nRow) { + code = tBlockDataAppendRow(&pWriter->bDataW, &tsdbRowFromBlockData(pWriter->pBlockData, pWriter->iRow), NULL); + if (code) goto _err; + + if (pWriter->bDataW.nRow >= pWriter->maxRow * 4 / 5) { + pWriter->blockW.last = 0; + code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW, + &pWriter->blockW, pWriter->cmprAlg); + if (code) goto _err; + + code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutBlock); + if (code) goto _err; + + tBlockReset(&pWriter->blockW); + tBlockDataClearData(&pWriter->bDataW); + } + + pWriter->iRow++; } } - // new or merge - if (pWriter->pBlockIdx == NULL || tTABLEIDCmprFn(&id, pWriter->pBlockIdx) < 0) { - int32_t c; - - if (pWriter->pBlockIdxN && ((c = tTABLEIDCmprFn(&id, pWriter->pBlockIdxN)) != 0)) { - ASSERT(c > 0); - - code = tsdbSnapWriteTableDataEnd(pWriter); - if (code) goto _err; - } - - if (pWriter->pBlockIdxN == NULL) { - pWriter->pBlockIdx = &pWriter->blockIdx; - pWriter->pBlockIdx->suid = id.suid; - pWriter->pBlockIdx->uid = id.uid; - } - - // loop to write the data - TSDBROW* pRow = NULL; // todo - int32_t nRow = 0; // todo - SBlockData* pBlockData = NULL; // todo - for (int32_t iRow = 0; iRow < nRow; iRow++) { - code = tBlockDataAppendRow(&pWriter->nBlockData, &tsdbRowFromBlockData(pBlockData, iRow), NULL); - if (code) goto _err; - - if (pWriter->nBlockData.nRow > pWriter->maxRow * 4 / 5) { - code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->nBlockData, NULL, NULL, pWriter->pBlockIdxN, - pWriter->pBlockN, pWriter->cmprAlg); - if (code) goto _err; + // write remain data if has + if (pWriter->bDataW.nRow > 0) { + pWriter->blockW.last = 0; + if (pWriter->bDataW.nRow < pWriter->minRow) { + if (pWriter->iBlock > pWriter->mBlock.nItem) { + pWriter->blockW.last = 1; } } - } else { - // skip - while (true) { - if (pWriter->pBlock == NULL) break; - if (pWriter->pBlock->last) break; - if (tBlockCmprFn(&(SBlock){.minKey = {0}, .maxKey = {0}}, pWriter->pBlock) >= 0) break; - code = tMapDataPutItem(&pWriter->mBlockN, pWriter->pBlock, tPutBlock); + code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW, + &pWriter->blockW, pWriter->cmprAlg); + if (code) goto _err; + + code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutBlock); + if (code) goto _err; + } + + while (true) { + if (pWriter->iBlock >= pWriter->mBlock.nItem) break; + + SBlock block; + tMapDataGetItemByIdx(&pWriter->mBlock, pWriter->iBlock, &block, tGetBlock); + + if (block.last) { + code = tsdbReadBlockData(pWriter->pDataFReader, pWriter->pBlockIdx, &block, &pWriter->bDataR, NULL, NULL); + if (code) goto _err; + + tBlockReset(&block); + block.last = 1; + code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataR, NULL, NULL, pWriter->pBlockIdxW, &block, + pWriter->cmprAlg); if (code) goto _err; } - if (pWriter->pBlock) { - if (pWriter->pBlock->last) { - // load the last block and merge with the data (todo) - } else { - int32_t c = tBlockCmprFn(&(SBlock){0 /*TODO*/}, pWriter->pBlock); + code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutBlock); + if (code) goto _err; - if (c > 0) { - // commit until pWriter->pBlock (todo) + pWriter->iBlock++; + } + + // SBlock + code = tsdbWriteBlock(pWriter->pDataFWriter, &pWriter->mBlockW, NULL, pWriter->pBlockIdxW); + if (code) goto _err; + + // SBlockIdx + if (taosArrayPush(pWriter->aBlockIdxW, pWriter->pBlockIdxW) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + +_exit: + return code; + +_err: + return code; +} + +static int32_t tsdbSnapMoveWriteTableData(STsdbSnapWriter* pWriter, SBlockIdx* pBlockIdx) { + int32_t code = 0; + + code = tsdbReadBlock(pWriter->pDataFReader, pBlockIdx, &pWriter->mBlock, NULL); + if (code) goto _err; + + // SBlockData + SBlock block; + tMapDataReset(&pWriter->mBlockW); + for (int32_t iBlock = 0; iBlock < pWriter->mBlock.nItem; iBlock++) { + tMapDataGetItemByIdx(&pWriter->mBlock, iBlock, &block, tGetBlock); + + if (block.last) { + code = tsdbReadBlockData(pWriter->pDataFReader, pBlockIdx, &block, &pWriter->bDataR, NULL, NULL); + if (code) goto _err; + + tBlockReset(&block); + block.last = 1; + code = + tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataR, NULL, NULL, pBlockIdx, &block, pWriter->cmprAlg); + if (code) goto _err; + } + + code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutBlock); + if (code) goto _err; + } + + // SBlock + SBlockIdx blockIdx = {.suid = pBlockIdx->suid, .uid = pBlockIdx->uid}; + code = tsdbWriteBlock(pWriter->pDataFWriter, &pWriter->mBlockW, NULL, &blockIdx); + if (code) goto _err; + + // SBlockIdx + if (taosArrayPush(pWriter->aBlockIdxW, &blockIdx) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + +_exit: + return code; + +_err: + return code; +} + +static int32_t tsdbSnapWriteTableDataImpl(STsdbSnapWriter* pWriter) { + int32_t code = 0; + SBlockData* pBlockData = &pWriter->bData; + int32_t iRow = 0; + TSDBROW row; + TSDBROW* pRow = &row; + + // correct schema + code = tBlockDataCorrectSchema(&pWriter->bDataW, pBlockData); + if (code) goto _err; + + // loop to merge + *pRow = tsdbRowFromBlockData(pBlockData, iRow); + while (true) { + if (pRow == NULL) break; + + if (pWriter->pBlockData) { + ASSERT(pWriter->iRow < pWriter->pBlockData->nRow); + + int32_t c = tsdbRowCmprFn(pRow, &tsdbRowFromBlockData(pWriter->pBlockData, pWriter->iRow)); + + ASSERT(c); + + if (c < 0) { + code = tBlockDataAppendRow(&pWriter->bDataW, pRow, NULL); + if (code) goto _err; + + iRow++; + if (iRow < pWriter->pBlockData->nRow) { + *pRow = tsdbRowFromBlockData(pBlockData, iRow); } else { - // load the block and merge with the data (todo) + pRow = NULL; + } + } else if (c > 0) { + code = tBlockDataAppendRow(&pWriter->bDataW, &tsdbRowFromBlockData(pWriter->pBlockData, pWriter->iRow), NULL); + if (code) goto _err; + + pWriter->iRow++; + if (pWriter->iRow >= pWriter->pBlockData->nRow) { + pWriter->pBlockData = NULL; } } } else { - int32_t nRow = 0; - SBlockData* pBlockData = NULL; + TSDBKEY key = TSDBROW_KEY(pRow); - for (int32_t iRow = 0; iRow < nRow; iRow++) { - code = tBlockDataAppendRow(&pWriter->nBlockData, &tsdbRowFromBlockData(pBlockData, iRow), NULL); - if (code) goto _err; + while (true) { + if (pWriter->iBlock >= pWriter->mBlock.nItem) break; - if (pWriter->nBlockData.nRow >= pWriter->maxRow * 4 / 5) { - code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->nBlockData, NULL, NULL, pWriter->pBlockIdxN, - pWriter->pBlockN, pWriter->cmprAlg); + SBlock block; + int32_t c; + + tMapDataGetItemByIdx(&pWriter->mBlock, pWriter->iBlock, &block, tGetBlock); + + if (block.last) { + pWriter->pBlockData = &pWriter->bDataR; + + code = tsdbReadBlockData(pWriter->pDataFReader, pWriter->pBlockIdx, &block, pWriter->pBlockData, NULL, NULL); + if (code) goto _err; + pWriter->iRow = 0; + + pWriter->iBlock++; + break; + } + + c = tsdbKeyCmprFn(&block.maxKey, &key); + + ASSERT(c); + + if (c < 0) { + if (pWriter->bDataW.nRow) { + pWriter->blockW.last = 0; + code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW, + &pWriter->blockW, pWriter->cmprAlg); + if (code) goto _err; + + code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutBlock); + if (code) goto _err; + + tBlockReset(&pWriter->blockW); + tBlockDataClearData(&pWriter->bDataW); + } + + code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutBlock); if (code) goto _err; - tBlockDataClearData(&pWriter->nBlockData); + pWriter->iBlock++; + } else { + c = tsdbKeyCmprFn(&tBlockDataLastKey(pBlockData), &block.minKey); + + ASSERT(c); + + if (c > 0) { + pWriter->pBlockData = &pWriter->bDataR; + code = + tsdbReadBlockData(pWriter->pDataFReader, pWriter->pBlockIdx, &block, pWriter->pBlockData, NULL, NULL); + if (code) goto _err; + pWriter->iRow = 0; + + pWriter->iBlock++; + } + break; } } + + if (pWriter->pBlockData) continue; + + code = tBlockDataAppendRow(&pWriter->bDataW, pRow, NULL); + if (code) goto _err; + + iRow++; + if (iRow < pBlockData->nRow) { + *pRow = tsdbRowFromBlockData(pBlockData, iRow); + } else { + pRow = NULL; + } } + + _check_write: + if (pWriter->bDataW.nRow < pWriter->maxRow * 4 / 5) continue; + + _write_block: + code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdx, &pWriter->blockW, + pWriter->cmprAlg); + if (code) goto _err; + + code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutBlock); + if (code) goto _err; + + tBlockReset(&pWriter->blockW); + tBlockDataClearData(&pWriter->bDataW); } return code; _err: - tsdbError("vgId:%d tsdb snapshot write table data failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); + tsdbError("vgId:%d vnode snapshot tsdb write table data impl failed since %s", TD_VID(pWriter->pTsdb->pVnode), + tstrerror(code)); + return code; +} + +static int32_t tsdbSnapWriteTableData(STsdbSnapWriter* pWriter, TABLEID id) { + int32_t code = 0; + SBlockData* pBlockData = &pWriter->bData; + TSDBKEY keyFirst = tBlockDataFirstKey(pBlockData); + TSDBKEY keyLast = tBlockDataLastKey(pBlockData); + + // end last table write if should + if (pWriter->pBlockIdxW) { + int32_t c = tTABLEIDCmprFn(pWriter->pBlockIdxW, &id); + if (c < 0) { + // end + code = tsdbSnapWriteTableDataEnd(pWriter); + if (code) goto _err; + + // reset + pWriter->pBlockIdxW = NULL; + } else if (c > 0) { + ASSERT(0); + } + } + + // start new table data write if need + if (pWriter->pBlockIdxW == NULL) { + // write table data ahead + while (true) { + if (pWriter->iBlockIdx >= taosArrayGetSize(pWriter->aBlockIdx)) break; + + SBlockIdx* pBlockIdx = (SBlockIdx*)taosArrayGet(pWriter->aBlockIdx, pWriter->iBlockIdx); + int32_t c = tTABLEIDCmprFn(pBlockIdx, &id); + + if (c >= 0) break; + + code = tsdbSnapMoveWriteTableData(pWriter, pBlockIdx); + if (code) goto _err; + + pWriter->iBlockIdx++; + } + + // reader + pWriter->pBlockIdx = NULL; + if (pWriter->iBlockIdx < taosArrayGetSize(pWriter->aBlockIdx)) { + ASSERT(pWriter->pDataFReader); + + SBlockIdx* pBlockIdx = (SBlockIdx*)taosArrayGet(pWriter->aBlockIdx, pWriter->iBlock); + int32_t c = tTABLEIDCmprFn(pBlockIdx, &id); + + ASSERT(c >= 0); + + if (c == 0) { + pWriter->pBlockIdx = pBlockIdx; + pWriter->iBlockIdx++; + } + } + + if (pWriter->pBlockIdx) { + code = tsdbReadBlock(pWriter->pDataFReader, pWriter->pBlockIdx, &pWriter->mBlock, NULL); + if (code) goto _err; + } else { + tMapDataReset(&pWriter->mBlock); + } + pWriter->iBlock = 0; + pWriter->pBlockData = NULL; + pWriter->iRow = 0; + + // writer + pWriter->pBlockIdxW = &pWriter->blockIdxW; + pWriter->pBlockIdxW->suid = id.suid; + pWriter->pBlockIdxW->uid = id.uid; + + tBlockReset(&pWriter->blockW); + tBlockDataReset(&pWriter->bDataW); + tMapDataReset(&pWriter->mBlockW); + } + + ASSERT(pWriter->pBlockIdxW && pWriter->pBlockIdxW->suid == id.suid && pWriter->pBlockIdxW->uid == id.uid); + ASSERT(pWriter->pBlockIdx == NULL || (pWriter->pBlockIdx->suid == id.suid && pWriter->pBlockIdx->uid == id.uid)); + + code = tsdbSnapWriteTableDataImpl(pWriter); + if (code) goto _err; + +_exit: + return code; + +_err: + tsdbError("vgId:%d vnode snapshot tsdb write data impl failed since %s", TD_VID(pWriter->pTsdb->pVnode), + tstrerror(code)); + return code; +} + +static int32_t tsdbSnapWriteDataEnd(STsdbSnapWriter* pWriter) { + int32_t code = 0; + STsdb* pTsdb = pWriter->pTsdb; + + if (pWriter->pDataFWriter == NULL) goto _exit; + + code = tsdbSnapWriteTableDataEnd(pWriter); + if (code) goto _err; + + while (pWriter->iBlockIdx < taosArrayGetSize(pWriter->aBlockIdx)) { + code = tsdbSnapMoveWriteTableData(pWriter, (SBlockIdx*)taosArrayGet(pWriter->aBlockIdx, pWriter->iBlockIdx)); + if (code) goto _err; + + pWriter->iBlockIdx++; + } + + code = tsdbWriteBlockIdx(pWriter->pDataFWriter, pWriter->aBlockIdxW, NULL); + if (code) goto _err; + + code = tsdbFSStateUpsertDFileSet(pTsdb->fs->nState, tsdbDataFWriterGetWSet(pWriter->pDataFWriter)); + if (code) goto _err; + + code = tsdbDataFWriterClose(&pWriter->pDataFWriter, 1); + if (code) goto _err; + + if (pWriter->pDataFReader) { + code = tsdbDataFReaderClose(&pWriter->pDataFReader); + if (code) goto _err; + } + +_exit: + tsdbError("vgId:%d vnode snapshot tsdb writer data end", TD_VID(pTsdb->pVnode)); + return code; + +_err: + tsdbError("vgId:%d vnode snapshot tsdb writer data end failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); return code; } static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { int32_t code = 0; STsdb* pTsdb = pWriter->pTsdb; - int64_t skey; // todo - int64_t ekey; // todo + TABLEID id = *(TABLEID*)(pData + sizeof(SSnapDataHdr)); + int64_t n; - int32_t fid = tsdbKeyFid(skey, pWriter->minutes, pWriter->precision); - ASSERT(fid == tsdbKeyFid(ekey, pWriter->minutes, pWriter->precision)); + // decode + SBlockData* pBlockData = &pWriter->bData; + n = tGetBlockData(pData + sizeof(SSnapDataHdr) + sizeof(TABLEID), pBlockData); + ASSERT(n + sizeof(SSnapDataHdr) + sizeof(TABLEID) == nData); - // begin + // open file + TSDBKEY keyFirst = tBlockDataFirstKey(pBlockData); + TSDBKEY keyLast = tBlockDataLastKey(pBlockData); + + int32_t fid = tsdbKeyFid(keyFirst.ts, pWriter->minutes, pWriter->precision); + ASSERT(fid == tsdbKeyFid(keyLast.ts, pWriter->minutes, pWriter->precision)); if (pWriter->pDataFWriter == NULL || pWriter->fid != fid) { + // end last file data write if need code = tsdbSnapWriteDataEnd(pWriter); if (code) goto _err; pWriter->fid = fid; - SDFileSet* pSet = tsdbFSStateGetDFileSet(pTsdb->fs->nState, fid); - // reader + + // read + SDFileSet* pSet = tsdbFSStateGetDFileSet(pTsdb->fs->nState, fid, TD_EQ); if (pSet) { - // open code = tsdbDataFReaderOpen(&pWriter->pDataFReader, pTsdb, pSet); if (code) goto _err; - // SBlockIdx code = tsdbReadBlockIdx(pWriter->pDataFReader, pWriter->aBlockIdx, NULL); if (code) goto _err; } else { + ASSERT(pWriter->pDataFReader == NULL); taosArrayClear(pWriter->aBlockIdx); } pWriter->iBlockIdx = 0; + pWriter->pBlockIdx = NULL; + tMapDataReset(&pWriter->mBlock); + pWriter->iBlock = 0; + pWriter->pBlockData = NULL; + pWriter->iRow = 0; + tBlockDataReset(&pWriter->bDataR); - // writer - SDFileSet wSet = {0}; - if (pSet == NULL) { - wSet = (SDFileSet){0}; // todo + // write + SDFileSet wSet; + + if (pSet) { + wSet = (SDFileSet){.diskId = pSet->diskId, + .fid = fid, + .fHead = {.commitID = pWriter->commitID, .offset = 0, .size = 0}, + .fData = pSet->fData, + .fLast = {.commitID = pWriter->commitID, .size = 0}, + .fSma = pSet->fSma}; } else { - wSet = (SDFileSet){0}; // todo + wSet = (SDFileSet){.diskId = (SDiskID){.level = 0, .id = 0}, + .fid = fid, + .fHead = {.commitID = pWriter->commitID, .offset = 0, .size = 0}, + .fData = {.commitID = pWriter->commitID, .size = 0}, + .fLast = {.commitID = pWriter->commitID, .size = 0}, + .fSma = {.commitID = pWriter->commitID, .size = 0}}; } code = tsdbDataFWriterOpen(&pWriter->pDataFWriter, pTsdb, &wSet); if (code) goto _err; - taosArrayClear(pWriter->aBlockIdxN); + taosArrayClear(pWriter->aBlockIdxW); + tMapDataReset(&pWriter->mBlockW); + pWriter->pBlockIdxW = NULL; + tBlockDataReset(&pWriter->bDataW); } - code = tsdbSnapWriteTableData(pWriter, pData, nData); + code = tsdbSnapWriteTableData(pWriter, id); if (code) goto _err; + tsdbInfo("vgId:%d vnode snapshot tsdb write data, fid:%d suid:%" PRId64 " uid:%" PRId64 " nRow:%d", + TD_VID(pTsdb->pVnode), fid, id.suid, id.suid, pBlockData->nRow); return code; _err: - tsdbError("vgId:%d tsdb snapshot write data failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + tsdbError("vgId:%d vnode snapshot tsdb write data failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); return code; } @@ -570,28 +914,28 @@ static int32_t tsdbSnapWriteDel(STsdbSnapWriter* pWriter, uint8_t* pData, uint32 code = tsdbDelFReaderOpen(&pWriter->pDelFReader, pDelFile, pTsdb, NULL); if (code) goto _err; - code = tsdbReadDelIdx(pWriter->pDelFReader, pWriter->aDelIdx, NULL); + code = tsdbReadDelIdx(pWriter->pDelFReader, pWriter->aDelIdxR, NULL); if (code) goto _err; } // writer - SDelFile delFile = {.commitID = pTsdb->pVnode->state.commitID, .offset = 0, .size = 0}; + SDelFile delFile = {.commitID = pWriter->commitID, .offset = 0, .size = 0}; code = tsdbDelFWriterOpen(&pWriter->pDelFWriter, &delFile, pTsdb); if (code) goto _err; } // process the del data - TABLEID id = {0}; // todo + TABLEID id = *(TABLEID*)(pData + sizeof(SSnapDataHdr)); while (true) { SDelIdx* pDelIdx = NULL; - int64_t n = 0; + int64_t n = sizeof(SSnapDataHdr) + sizeof(TABLEID); SDelData delData; SDelIdx delIdx; int8_t toBreak = 0; - if (pWriter->iDelIdx < taosArrayGetSize(pWriter->aDelIdx)) { - pDelIdx = taosArrayGet(pWriter->aDelIdx, pWriter->iDelIdx); + if (pWriter->iDelIdx < taosArrayGetSize(pWriter->aDelIdxR)) { + pDelIdx = (SDelIdx*)taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx); } if (pDelIdx) { @@ -632,7 +976,7 @@ static int32_t tsdbSnapWriteDel(STsdbSnapWriter* pWriter, uint8_t* pData, uint32 code = tsdbWriteDelData(pWriter->pDelFWriter, pWriter->aDelData, NULL, &delIdx); if (code) goto _err; - if (taosArrayPush(pWriter->aDelIdxN, &delIdx) == NULL) { + if (taosArrayPush(pWriter->aDelIdxW, &delIdx) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } @@ -644,7 +988,7 @@ _exit: return code; _err: - tsdbError("vgId:%d tsdb snapshot write del failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + tsdbError("vgId:%d vnode snapshot tsdb write del failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); return code; } @@ -653,8 +997,9 @@ static int32_t tsdbSnapWriteDelEnd(STsdbSnapWriter* pWriter) { STsdb* pTsdb = pWriter->pTsdb; if (pWriter->pDelFWriter == NULL) goto _exit; - for (; pWriter->iDelIdx < taosArrayGetSize(pWriter->aDelIdx); pWriter->iDelIdx++) { - SDelIdx* pDelIdx = (SDelIdx*)taosArrayGet(pWriter->aDelIdx, pWriter->iDelIdx); + + for (; pWriter->iDelIdx < taosArrayGetSize(pWriter->aDelIdxR); pWriter->iDelIdx++) { + SDelIdx* pDelIdx = (SDelIdx*)taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx); code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->aDelData, NULL); if (code) goto _err; @@ -663,7 +1008,7 @@ static int32_t tsdbSnapWriteDelEnd(STsdbSnapWriter* pWriter) { code = tsdbWriteDelData(pWriter->pDelFWriter, pWriter->aDelData, NULL, &delIdx); if (code) goto _err; - if (taosArrayPush(pWriter->aDelIdx, &delIdx) == NULL) { + if (taosArrayPush(pWriter->aDelIdxR, &delIdx) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } @@ -684,10 +1029,11 @@ static int32_t tsdbSnapWriteDelEnd(STsdbSnapWriter* pWriter) { } _exit: + tsdbInfo("vgId:%d vnode snapshot tsdb write del end", TD_VID(pTsdb->pVnode)); return code; _err: - tsdbError("vgId:%d tsdb snapshow write del end failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + tsdbError("vgId:%d vnode snapshot tsdb write del end failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); return code; } @@ -705,6 +1051,54 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr pWriter->sver = sver; pWriter->ever = ever; + // config + pWriter->minutes = pTsdb->keepCfg.days; + pWriter->precision = pTsdb->keepCfg.precision; + pWriter->minRow = pTsdb->pVnode->config.tsdbCfg.minRows; + pWriter->maxRow = pTsdb->pVnode->config.tsdbCfg.maxRows; + pWriter->cmprAlg = pTsdb->pVnode->config.tsdbCfg.compression; + pWriter->commitID = pTsdb->pVnode->state.commitID; + + // for data file + code = tBlockDataInit(&pWriter->bData); + + if (code) goto _err; + pWriter->aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); + if (pWriter->aBlockIdx == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + code = tBlockDataInit(&pWriter->bDataR); + if (code) goto _err; + + pWriter->aBlockIdxW = taosArrayInit(0, sizeof(SBlockIdx)); + if (pWriter->aBlockIdxW == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + code = tBlockDataInit(&pWriter->bDataW); + if (code) goto _err; + + // for del file + pWriter->aDelIdxR = taosArrayInit(0, sizeof(SDelIdx)); + if (pWriter->aDelIdxR == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + pWriter->aDelData = taosArrayInit(0, sizeof(SDelData)); + if (pWriter->aDelData == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + pWriter->aDelIdxW = taosArrayInit(0, sizeof(SDelIdx)); + if (pWriter->aDelIdxW == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + + code = tsdbFSBegin(pTsdb->fs); + if (code) goto _err; + *ppWriter = pWriter; return code; @@ -719,7 +1113,7 @@ int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback) { STsdbSnapWriter* pWriter = *ppWriter; if (rollback) { - code = tsdbSnapRollback(pWriter); + code = tsdbFSRollback(pWriter->pTsdb->fs); if (code) goto _err; } else { code = tsdbSnapWriteDataEnd(pWriter); @@ -728,7 +1122,7 @@ int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback) { code = tsdbSnapWriteDelEnd(pWriter); if (code) goto _err; - code = tsdbSnapCommit(pWriter); + code = tsdbFSCommit(pWriter->pTsdb->fs); if (code) goto _err; } @@ -738,29 +1132,35 @@ int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback) { return code; _err: - tsdbError("vgId:%d tsdb snapshot writer close failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); + tsdbError("vgId:%d vnode snapshot tsdb writer close failed since %s", TD_VID(pWriter->pTsdb->pVnode), + tstrerror(code)); return code; } int32_t tsdbSnapWrite(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { - int32_t code = 0; - int8_t type = pData[0]; + int32_t code = 0; + SSnapDataHdr* pHdr = (SSnapDataHdr*)pData; // ts data - if (type == 0) { - code = tsdbSnapWriteData(pWriter, pData + 1, nData - 1); + if (pHdr->type == 1) { + code = tsdbSnapWriteData(pWriter, pData, nData); if (code) goto _err; + + goto _exit; } else { - code = tsdbSnapWriteDataEnd(pWriter); - if (code) goto _err; + if (pWriter->pDataFWriter) { + code = tsdbSnapWriteDataEnd(pWriter); + if (code) goto _err; + } } // del data - if (type == 1) { - code = tsdbSnapWriteDel(pWriter, pData + 1, nData - 1); + if (pHdr->type == 2) { + code = tsdbSnapWriteDel(pWriter, pData, nData); if (code) goto _err; } +_exit: return code; _err: diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index dc6c823368..f318c69c6f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -36,15 +36,15 @@ int32_t tMapDataPutItem(SMapData *pMapData, void *pItem, int32_t (*tPutItemFn)(u // alloc code = tRealloc((uint8_t **)&pMapData->aOffset, sizeof(int32_t) * pMapData->nItem); - if (code) goto _err; + if (code) goto _exit; code = tRealloc(&pMapData->pData, pMapData->nData); - if (code) goto _err; + if (code) goto _exit; // put pMapData->aOffset[nItem] = offset; tPutItemFn(pMapData->pData + offset, pItem); -_err: +_exit: return code; } @@ -189,25 +189,12 @@ static FORCE_INLINE int32_t tGetTSDBKEY(uint8_t *p, TSDBKEY *pKey) { } // SBlockIdx ====================================================== -void tBlockIdxReset(SBlockIdx *pBlockIdx) { - pBlockIdx->minKey = TSKEY_MAX; - pBlockIdx->maxKey = TSKEY_MIN; - pBlockIdx->minVersion = VERSION_MAX; - pBlockIdx->maxVersion = VERSION_MIN; - pBlockIdx->offset = -1; - pBlockIdx->size = -1; -} - int32_t tPutBlockIdx(uint8_t *p, void *ph) { int32_t n = 0; SBlockIdx *pBlockIdx = (SBlockIdx *)ph; n += tPutI64(p ? p + n : p, pBlockIdx->suid); n += tPutI64(p ? p + n : p, pBlockIdx->uid); - n += tPutI64(p ? p + n : p, pBlockIdx->minKey); - n += tPutI64(p ? p + n : p, pBlockIdx->maxKey); - n += tPutI64v(p ? p + n : p, pBlockIdx->minVersion); - n += tPutI64v(p ? p + n : p, pBlockIdx->maxVersion); n += tPutI64v(p ? p + n : p, pBlockIdx->offset); n += tPutI64v(p ? p + n : p, pBlockIdx->size); @@ -220,10 +207,6 @@ int32_t tGetBlockIdx(uint8_t *p, void *ph) { n += tGetI64(p + n, &pBlockIdx->suid); n += tGetI64(p + n, &pBlockIdx->uid); - n += tGetI64(p + n, &pBlockIdx->minKey); - n += tGetI64(p + n, &pBlockIdx->maxKey); - n += tGetI64v(p + n, &pBlockIdx->minVersion); - n += tGetI64v(p + n, &pBlockIdx->maxVersion); n += tGetI64v(p + n, &pBlockIdx->offset); n += tGetI64v(p + n, &pBlockIdx->size); @@ -921,6 +904,76 @@ _exit: return code; } +int32_t tPutColData(uint8_t *p, SColData *pColData) { + int32_t n = 0; + + n += tPutI16v(p ? p + n : p, pColData->cid); + n += tPutI8(p ? p + n : p, pColData->type); + n += tPutI8(p ? p + n : p, pColData->smaOn); + n += tPutI32v(p ? p + n : p, pColData->nVal); + n += tPutU8(p ? p + n : p, pColData->flag); + + if (pColData->flag == HAS_NONE || pColData->flag == HAS_NULL) goto _exit; + if (pColData->flag != HAS_VALUE) { + // bitmap + + int32_t size = BIT2_SIZE(pColData->nVal); + if (p) { + memcpy(p + n, pColData->pBitMap, size); + } + n += size; + } + if (IS_VAR_DATA_TYPE(pColData->type)) { + // offset + + int32_t size = sizeof(int32_t) * pColData->nVal; + if (p) { + memcpy(p + n, pColData->aOffset, size); + } + n += size; + } + n += tPutI32v(p ? p + n : p, pColData->nData); + if (p) { + memcpy(p + n, pColData->pData, pColData->nData); + } + n += pColData->nData; + +_exit: + return n; +} + +int32_t tGetColData(uint8_t *p, SColData *pColData) { + int32_t n = 0; + + n += tGetI16v(p + n, &pColData->cid); + n += tGetI8(p + n, &pColData->type); + n += tGetI8(p + n, &pColData->smaOn); + n += tGetI32v(p + n, &pColData->nVal); + n += tGetU8(p + n, &pColData->flag); + + if (pColData->flag == HAS_NONE || pColData->flag == HAS_NULL) goto _exit; + if (pColData->flag != HAS_VALUE) { + // bitmap + + int32_t size = BIT2_SIZE(pColData->nVal); + pColData->pBitMap = p + n; + n += size; + } + if (IS_VAR_DATA_TYPE(pColData->type)) { + // offset + + int32_t size = sizeof(int32_t) * pColData->nVal; + pColData->aOffset = (int32_t *)(p + n); + n += size; + } + n += tGetI32v(p + n, &pColData->nData); + pColData->pData = p + n; + n += pColData->nData; + +_exit: + return n; +} + static FORCE_INLINE int32_t tColDataCmprFn(const void *p1, const void *p2) { SColData *pColData1 = (SColData *)p1; SColData *pColData2 = (SColData *)p2; @@ -962,11 +1015,11 @@ void tBlockDataReset(SBlockData *pBlockData) { taosArrayClear(pBlockData->aIdx); } -void tBlockDataClear(SBlockData *pBlockData) { +void tBlockDataClear(SBlockData *pBlockData, int8_t deepClear) { tFree((uint8_t *)pBlockData->aVersion); tFree((uint8_t *)pBlockData->aTSKEY); taosArrayDestroy(pBlockData->aIdx); - taosArrayDestroyEx(pBlockData->aColData, tColDataClear); + taosArrayDestroyEx(pBlockData->aColData, deepClear ? tColDataClear : NULL); } int32_t tBlockDataSetSchema(SBlockData *pBlockData, STSchema *pTSchema) { @@ -1079,6 +1132,46 @@ _err: return code; } +int32_t tBlockDataCorrectSchema(SBlockData *pBlockData, SBlockData *pBlockDataFrom) { + int32_t code = 0; + + int32_t iColData = 0; + for (int32_t iColDataFrom = 0; iColDataFrom < taosArrayGetSize(pBlockDataFrom->aIdx); iColDataFrom++) { + SColData *pColDataFrom = tBlockDataGetColDataByIdx(pBlockDataFrom, iColDataFrom); + + while (true) { + SColData *pColData; + if (iColData < taosArrayGetSize(pBlockData->aIdx)) { + pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); + } else { + pColData = NULL; + } + + if (pColData == NULL || pColData->cid > pColDataFrom->cid) { + code = tBlockDataAddColData(pBlockData, iColData, &pColData); + if (code) goto _exit; + + tColDataInit(pColData, pColDataFrom->cid, pColDataFrom->type, pColDataFrom->smaOn); + for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) { + code = tColDataAppendValue(pColData, &COL_VAL_NONE(pColData->cid, pColData->type)); + if (code) goto _exit; + } + + iColData++; + break; + } else if (pColData->cid == pColDataFrom->cid) { + iColData++; + break; + } else { + iColData++; + } + } + } + +_exit: + return code; +} + int32_t tBlockDataMerge(SBlockData *pBlockData1, SBlockData *pBlockData2, SBlockData *pBlockData) { int32_t code = 0; @@ -1239,6 +1332,52 @@ void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColD *ppColData = NULL; } +int32_t tPutBlockData(uint8_t *p, SBlockData *pBlockData) { + int32_t n = 0; + + n += tPutI32v(p ? p + n : p, pBlockData->nRow); + if (p) { + memcpy(p + n, pBlockData->aVersion, sizeof(int64_t) * pBlockData->nRow); + } + n = n + sizeof(int64_t) * pBlockData->nRow; + if (p) { + memcpy(p + n, pBlockData->aTSKEY, sizeof(TSKEY) * pBlockData->nRow); + } + n = n + sizeof(TSKEY) * pBlockData->nRow; + + int32_t nCol = taosArrayGetSize(pBlockData->aIdx); + n += tPutI32v(p ? p + n : p, nCol); + for (int32_t iCol = 0; iCol < nCol; iCol++) { + SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iCol); + n += tPutColData(p ? p + n : p, pColData); + } + + return n; +} + +int32_t tGetBlockData(uint8_t *p, SBlockData *pBlockData) { + int32_t n = 0; + + tBlockDataReset(pBlockData); + + n += tGetI32v(p + n, &pBlockData->nRow); + pBlockData->aVersion = (int64_t *)(p + n); + n = n + sizeof(int64_t) * pBlockData->nRow; + pBlockData->aTSKEY = (TSKEY *)(p + n); + n = n + sizeof(TSKEY) * pBlockData->nRow; + + int32_t nCol; + n += tGetI32v(p + n, &nCol); + for (int32_t iCol = 0; iCol < nCol; iCol++) { + SColData *pColData; + + if (tBlockDataAddColData(pBlockData, iCol, &pColData)) return -1; + n += tGetColData(p + n, pColData); + } + + return n; +} + // ALGORITHM ============================== void tsdbCalcColDataSMA(SColData *pColData, SColumnDataAgg *pColAgg) { SColVal colVal; diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index ebbb691e28..c969861241 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -233,7 +233,8 @@ int vnodeCommit(SVnode *pVnode) { walBeginSnapshot(pVnode->pWal, pVnode->state.applied); // preCommit - smaPreCommit(pVnode->pSma); + // smaSyncPreCommit(pVnode->pSma); + smaAsyncPreCommit(pVnode->pSma); // commit each sub-system if (metaCommit(pVnode->pMeta) < 0) { @@ -242,6 +243,8 @@ int vnodeCommit(SVnode *pVnode) { } if (VND_IS_RSMA(pVnode)) { + smaAsyncCommit(pVnode->pSma); + if (tsdbCommit(VND_RSMA0(pVnode)) < 0) { ASSERT(0); return -1; @@ -276,7 +279,8 @@ int vnodeCommit(SVnode *pVnode) { pVnode->state.committed = info.state.committed; // postCommit - smaPostCommit(pVnode->pSma); + // smaSyncPostCommit(pVnode->pSma); + smaAsyncPostCommit(pVnode->pSma); // apply the commit (TODO) walEndSnapshot(pVnode->pWal); diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 5b807d60e3..0e443ee3b8 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -239,9 +239,9 @@ int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad) { pLoad->vgId = TD_VID(pVnode); pLoad->syncState = syncGetMyRole(pVnode->sync); pLoad->numOfTables = metaGetTbNum(pVnode->pMeta); - pLoad->numOfTimeSeries = 400; - pLoad->totalStorage = 300; - pLoad->compStorage = 200; + pLoad->numOfTimeSeries = metaGetTimeSeriesNum(pVnode->pMeta); + pLoad->totalStorage = (int64_t)3 * 1073741824; + pLoad->compStorage = (int64_t)2 * 1073741824; pLoad->pointsWritten = 100; pLoad->numOfSelectReqs = 1; pLoad->numOfInsertReqs = 3; diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index 27f30ec787..59242cbd8a 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -20,13 +20,13 @@ struct SVSnapReader { SVnode *pVnode; int64_t sver; int64_t ever; + int64_t index; // meta int8_t metaDone; SMetaSnapReader *pMetaReader; // tsdb int8_t tsdbDone; STsdbSnapReader *pTsdbReader; - uint8_t *pData; }; int32_t vnodeSnapReaderOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapReader **ppReader) { @@ -42,12 +42,7 @@ int32_t vnodeSnapReaderOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapRe pReader->sver = sver; pReader->ever = ever; - code = metaSnapReaderOpen(pVnode->pMeta, sver, ever, &pReader->pMetaReader); - if (code) goto _err; - - code = tsdbSnapReaderOpen(pVnode->pTsdb, sver, ever, &pReader->pTsdbReader); - if (code) goto _err; - + vInfo("vgId:%d vnode snapshot reader opened, sver:%" PRId64 " ever:%" PRId64, TD_VID(pVnode), sver, ever); *ppReader = pReader; return code; @@ -60,54 +55,85 @@ _err: int32_t vnodeSnapReaderClose(SVSnapReader *pReader) { int32_t code = 0; - tFree(pReader->pData); - if (pReader->pTsdbReader) tsdbSnapReaderClose(&pReader->pTsdbReader); - if (pReader->pMetaReader) metaSnapReaderClose(&pReader->pMetaReader); - taosMemoryFree(pReader); + if (pReader->pTsdbReader) { + tsdbSnapReaderClose(&pReader->pTsdbReader); + } + if (pReader->pMetaReader) { + metaSnapReaderClose(&pReader->pMetaReader); + } + + vInfo("vgId:%d vnode snapshot reader closed", TD_VID(pReader->pVnode)); + taosMemoryFree(pReader); return code; } int32_t vnodeSnapRead(SVSnapReader *pReader, uint8_t **ppData, uint32_t *nData) { int32_t code = 0; + // META ============== if (!pReader->metaDone) { - code = metaSnapRead(pReader->pMetaReader, &pReader->pData); + // open reader if not + if (pReader->pMetaReader == NULL) { + code = metaSnapReaderOpen(pReader->pVnode->pMeta, pReader->sver, pReader->ever, &pReader->pMetaReader); + if (code) goto _err; + } + + code = metaSnapRead(pReader->pMetaReader, ppData); if (code) { - if (code == TSDB_CODE_VND_READ_END) { + goto _err; + } else { + if (*ppData) { + goto _exit; + } else { pReader->metaDone = 1; - } else { - goto _err; + code = metaSnapReaderClose(&pReader->pMetaReader); + if (code) goto _err; } - } else { - *ppData = pReader->pData; - *nData = sizeof(SSnapDataHdr) + ((SSnapDataHdr *)pReader->pData)->size; - goto _exit; } } + // TSDB ============== if (!pReader->tsdbDone) { - code = tsdbSnapRead(pReader->pTsdbReader, &pReader->pData); + // open if not + if (pReader->pTsdbReader == NULL) { + code = tsdbSnapReaderOpen(pReader->pVnode->pTsdb, pReader->sver, pReader->ever, &pReader->pTsdbReader); + if (code) goto _err; + } + + code = tsdbSnapRead(pReader->pTsdbReader, ppData); if (code) { - if (code == TSDB_CODE_VND_READ_END) { - pReader->tsdbDone = 1; - } else { - goto _err; - } + goto _err; } else { - *ppData = pReader->pData; - *nData = sizeof(SSnapDataHdr) + ((SSnapDataHdr *)pReader->pData)->size; - goto _exit; + if (*ppData) { + goto _exit; + } else { + pReader->tsdbDone = 1; + code = tsdbSnapReaderClose(&pReader->pTsdbReader); + if (code) goto _err; + } } } - code = TSDB_CODE_VND_READ_END; + *ppData = NULL; + *nData = 0; _exit: + if (*ppData) { + SSnapDataHdr *pHdr = (SSnapDataHdr *)(*ppData); + + pReader->index++; + *nData = sizeof(SSnapDataHdr) + pHdr->size; + pHdr->index = pReader->index; + vInfo("vgId:%d vnode snapshot read data,index:%" PRId64 " type:%d nData:%d ", TD_VID(pReader->pVnode), + pReader->index, pHdr->type, *nData); + } else { + vInfo("vgId:%d vnode snapshot read data end, index:%" PRId64, TD_VID(pReader->pVnode), pReader->index); + } return code; _err: - vError("vgId:% snapshot read failed since %s", TD_VID(pReader->pVnode), tstrerror(code)); + vError("vgId:% vnode snapshot read failed since %s", TD_VID(pReader->pVnode), tstrerror(code)); return code; } @@ -116,24 +142,13 @@ struct SVSnapWriter { SVnode *pVnode; int64_t sver; int64_t ever; + int64_t index; // meta SMetaSnapWriter *pMetaSnapWriter; // tsdb STsdbSnapWriter *pTsdbSnapWriter; }; -static int32_t vnodeSnapRollback(SVSnapWriter *pWriter) { - int32_t code = 0; - // TODO - return code; -} - -static int32_t vnodeSnapCommit(SVSnapWriter *pWriter) { - int32_t code = 0; - // TODO - return code; -} - int32_t vnodeSnapWriterOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapWriter **ppWriter) { int32_t code = 0; SVSnapWriter *pWriter = NULL; @@ -148,62 +163,102 @@ int32_t vnodeSnapWriterOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapWr pWriter->sver = sver; pWriter->ever = ever; + vInfo("vgId:%d vnode snapshot writer opened", TD_VID(pVnode)); + *ppWriter = pWriter; return code; _err: vError("vgId:%d vnode snapshot writer open failed since %s", TD_VID(pVnode), tstrerror(code)); + *ppWriter = NULL; return code; } -int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback) { +int32_t vnodeSnapWriterClose(SVSnapWriter *pWriter, int8_t rollback, SSnapshot *pSnapshot) { int32_t code = 0; + SVnode *pVnode = pWriter->pVnode; - if (rollback) { - code = vnodeSnapRollback(pWriter); - if (code) goto _err; - } else { - code = vnodeSnapCommit(pWriter); + if (pWriter->pMetaSnapWriter) { + code = metaSnapWriterClose(&pWriter->pMetaSnapWriter, rollback); if (code) goto _err; } + if (pWriter->pTsdbSnapWriter) { + code = tsdbSnapWriterClose(&pWriter->pTsdbSnapWriter, rollback); + if (code) goto _err; + } + + if (!rollback) { + SVnodeInfo info = {0}; + char dir[TSDB_FILENAME_LEN]; + + pVnode->state.committed = pWriter->ever; + pVnode->state.applied = pWriter->ever; + pVnode->state.applyTerm = pSnapshot->lastApplyTerm; + pVnode->state.commitTerm = pSnapshot->lastApplyTerm; + + info.config = pVnode->config; + info.state.committed = pVnode->state.applied; + info.state.commitTerm = pVnode->state.applyTerm; + info.state.commitID = pVnode->state.commitID; + snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pVnode->pTfs), TD_DIRSEP, pVnode->path); + code = vnodeSaveInfo(dir, &info); + if (code) goto _err; + + code = vnodeCommitInfo(dir, &info); + if (code) goto _err; + } else { + ASSERT(0); + } + +_exit: + vInfo("vgId:%d vnode snapshot writer closed, rollback:%d", TD_VID(pVnode), rollback); taosMemoryFree(pWriter); return code; _err: - vError("vgId:%d vnode snapshow writer close failed since %s", TD_VID(pWriter->pVnode), tstrerror(code)); + vError("vgId:%d vnode snapshot writer close failed since %s", TD_VID(pWriter->pVnode), tstrerror(code)); return code; } int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) { int32_t code = 0; - SSnapDataHdr *pSnapDataHdr = (SSnapDataHdr *)pData; + SSnapDataHdr *pHdr = (SSnapDataHdr *)pData; SVnode *pVnode = pWriter->pVnode; - ASSERT(pSnapDataHdr->size + sizeof(SSnapDataHdr) == nData); + ASSERT(pHdr->size + sizeof(SSnapDataHdr) == nData); + ASSERT(pHdr->index == pWriter->index + 1); + pWriter->index = pHdr->index; - if (pSnapDataHdr->type == 0) { + vInfo("vgId:%d vnode snapshot write data, index:%" PRId64 " type:%d nData:%d", TD_VID(pVnode), pHdr->index, + pHdr->type, nData); + + if (pHdr->type == 0) { // meta + if (pWriter->pMetaSnapWriter == NULL) { code = metaSnapWriterOpen(pVnode->pMeta, pWriter->sver, pWriter->ever, &pWriter->pMetaSnapWriter); if (code) goto _err; } - code = metaSnapWrite(pWriter->pMetaSnapWriter, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr)); + code = metaSnapWrite(pWriter->pMetaSnapWriter, pData, nData); if (code) goto _err; } else { // tsdb + if (pWriter->pTsdbSnapWriter == NULL) { code = tsdbSnapWriterOpen(pVnode->pTsdb, pWriter->sver, pWriter->ever, &pWriter->pTsdbSnapWriter); if (code) goto _err; } - code = tsdbSnapWrite(pWriter->pTsdbSnapWriter, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr)); + code = tsdbSnapWrite(pWriter->pTsdbSnapWriter, pData, nData); if (code) goto _err; } +_exit: return code; _err: - vError("vgId:%d vnode snapshot write failed since %s", TD_VID(pVnode), tstrerror(code)); + vError("vgId:%d vnode snapshot write failed since %s, index:%" PRId64 " type:%d nData:%d", TD_VID(pVnode), + tstrerror(code), pHdr->index, pHdr->type, nData); return code; } \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index cca212a4e4..e5322f1bd3 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -422,8 +422,8 @@ static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t version, void *p goto _err; } - taosMemoryFree(req.schemaRow.pSchema); - taosMemoryFree(req.schemaTag.pSchema); + // taosMemoryFree(req.schemaRow.pSchema); + // taosMemoryFree(req.schemaTag.pSchema); tDecoderClear(&coder); return 0; @@ -630,6 +630,9 @@ _exit: tEncoderInit(&ec, pRsp->pCont, pRsp->contLen); tEncodeSVAlterTbRsp(&ec, &vAlterTbRsp); tEncoderClear(&ec); + if (vMetaRsp.pSchemas) { + taosMemoryFree(vMetaRsp.pSchemas); + } return 0; } @@ -689,6 +692,7 @@ _exit: tEncoderInit(&encoder, pRsp->pCont, pRsp->contLen); tEncodeSVDropTbBatchRsp(&encoder, &rsp); tEncoderClear(&encoder); + taosArrayDestroy(rsp.pArray); return 0; } @@ -900,7 +904,7 @@ static int32_t vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t version, void * _err: tDecoderClear(&coder); vError("vgId:%d, failed to create tsma %s:%" PRIi64 " version %" PRIi64 "for table %" PRIi64 " since %s", - TD_VID(pVnode), req.indexName, req.indexUid, version, req.tableUid, terrstr(terrno)); + TD_VID(pVnode), req.indexName, req.indexUid, version, req.tableUid, terrstr()); return -1; } diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 81177fd5d8..712cee9fd0 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -515,10 +515,10 @@ static int32_t vnodeSnapshotStartWrite(struct SSyncFSM *pFsm, void *pParam, void #endif } -static int32_t vnodeSnapshotStopWrite(struct SSyncFSM *pFsm, void *pWriter, bool isApply) { +static int32_t vnodeSnapshotStopWrite(struct SSyncFSM *pFsm, void *pWriter, bool isApply, SSnapshot *pSnapshot) { #ifdef USE_TSDB_SNAPSHOT SVnode *pVnode = pFsm->data; - int32_t code = vnodeSnapWriterClose(pWriter, !isApply); + int32_t code = vnodeSnapWriterClose(pWriter, !isApply, pSnapshot); return code; #else taosMemoryFree(pWriter); diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index e77df8f7f2..920acbac2e 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -163,7 +163,7 @@ int32_t ctgInitGetQnodeTask(SCtgJob *pJob, int32_t taskIdx, void* param) { taosArrayPush(pJob->pTasks, &task); - qDebug("QID:0x%" PRIx64 " the %d task type %s initialized", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type)); + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type)); return TSDB_CODE_SUCCESS; } @@ -178,7 +178,7 @@ int32_t ctgInitGetDnodeTask(SCtgJob *pJob, int32_t taskIdx, void* param) { taosArrayPush(pJob->pTasks, &task); - qDebug("QID:0x%" PRIx64 " the %d task type %s initialized", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type)); + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type)); return TSDB_CODE_SUCCESS; } @@ -264,7 +264,7 @@ int32_t ctgInitGetSvrVerTask(SCtgJob *pJob, int32_t taskIdx, void* param) { taosArrayPush(pJob->pTasks, &task); - qDebug("QID:0x%" PRIx64 " [%dth] task type %s initialized", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type)); + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type)); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index a996a70973..1f0f074a0f 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -272,7 +272,7 @@ void ctgFreeHandle(SCatalog* pCtg) { taosMemoryFree(pCtg); - ctgInfo("handle freed, culsterId:0x%" PRIx64, clusterId); + ctgInfo("handle freed, clusterId:0x%" PRIx64, clusterId); } void ctgClearHandle(SCatalog* pCtg) { @@ -303,7 +303,7 @@ void ctgClearHandle(SCatalog* pCtg) { CTG_CACHE_STAT_INC(numOfClear, 1); - ctgInfo("handle cleared, culsterId:0x%" PRIx64, clusterId); + ctgInfo("handle cleared, clusterId:0x%" PRIx64, clusterId); } void ctgFreeSUseDbOutput(SUseDbOutput* pOutput) { diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index f9aba30a46..330ccd68a9 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -96,9 +96,9 @@ static FORCE_INLINE SResultRow* getResultRowByPos(SDiskbasedBuf* pBuf, SResultRo } void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SHashObj* pHashmap, int32_t order); -void initMultiResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList); - void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo); + +void initMultiResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList); bool hasDataInGroupInfo(SGroupResInfo* pGroupResInfo); int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index cd50a13edd..2ffb1be260 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -247,6 +247,16 @@ typedef struct SLoadRemoteDataInfo { uint64_t totalElapsed; // total elapsed time } SLoadRemoteDataInfo; +typedef struct SLimitInfo { + SLimit limit; + SLimit slimit; + uint64_t currentGroupId; + int64_t remainGroupOffset; + int64_t numOfOutputGroups; + int64_t remainOffset; + int64_t numOfOutputRows; +} SLimitInfo; + typedef struct SExchangeInfo { SArray* pSources; SArray* pSourceDataInfo; @@ -257,6 +267,7 @@ typedef struct SExchangeInfo { int32_t current; SLoadRemoteDataInfo loadInfo; uint64_t self; + SLimitInfo limitInfo; } SExchangeInfo; typedef struct SColMatchInfo { @@ -542,15 +553,7 @@ typedef struct SProjectOperatorInfo { SNode* pFilterNode; // filter info, which is push down by optimizer SSDataBlock* existDataBlock; SArray* pPseudoColInfo; - SLimit limit; - SLimit slimit; - - uint64_t groupId; - int64_t curSOffset; - int64_t curGroupOutput; - - int64_t curOffset; - int64_t curOutput; + SLimitInfo limitInfo; } SProjectOperatorInfo; typedef struct SIndefOperatorInfo { @@ -791,6 +794,9 @@ int32_t initAggInfo(SExprSupp *pSup, SAggSupporter* pAggSup, SExprInfo* pExprInf const char* pkey); void initResultSizeInfo(SOperatorInfo* pOperator, int32_t numOfRows); void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, SDiskbasedBuf* pBuf); +int32_t handleLimitOffset(SOperatorInfo *pOperator, SLimitInfo* pLimitInfo, SSDataBlock* pBlock, bool holdDataInBuf); +bool hasLimitOffsetInfo(SLimitInfo* pLimitInfo); +void initLimitInfo(const SNode* pLimit, const SNode* pSLimit, SLimitInfo* pLimitInfo); void doApplyFunctions(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, STimeWindow* pWin, SColumnInfoData* pTimeWindowData, int32_t offset, int32_t forwardStep, TSKEY* tsCol, int32_t numOfTotal, int32_t numOfOutput, int32_t order); @@ -837,7 +843,7 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhysiNode *pNode, SExecTaskInfo* pTaskInfo); SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhysiNode* pProjPhyNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* pSortPhyNode, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* pSortNode, SExecTaskInfo* pTaskInfo); SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** dowStreams, size_t numStreams, SMergePhysiNode* pMergePhysiNode, SExecTaskInfo* pTaskInfo); SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t numOfDownstream, SExprInfo* pExprInfo, int32_t num, SArray* pSortInfo, SArray* pGroupInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createLastrowScanOperator(SLastRowScanPhysiNode* pTableScanNode, SReadHandle* readHandle, SExecTaskInfo* pTaskInfo); diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c index e53c9fae6f..a575e355f1 100644 --- a/source/libs/executor/src/dataInserter.c +++ b/source/libs/executor/src/dataInserter.c @@ -90,6 +90,7 @@ _return: tsem_post(&pInserter->ready); + taosMemoryFree(pMsg->pData); taosMemoryFree(param); return TSDB_CODE_SUCCESS; @@ -283,6 +284,8 @@ static int32_t destroyDataSinker(SDataSinkHandle* pHandle) { atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pInserter->cachedSize); taosArrayDestroy(pInserter->pDataBlocks); taosMemoryFree(pInserter->pSchema); + taosMemoryFree(pInserter->pParam); + taosHashCleanup(pInserter->pCols); taosThreadMutexDestroy(&pInserter->mutex); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index a8ff3188c8..f10dbf5349 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -72,8 +72,12 @@ size_t getResultRowSize(SqlFunctionCtx* pCtx, int32_t numOfOutput) { void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo) { assert(pGroupResInfo != NULL); - taosArrayDestroy(pGroupResInfo->pRows); - pGroupResInfo->pRows = NULL; + for(int32_t i = 0; i < taosArrayGetSize(pGroupResInfo->pRows); ++i) { + SResKeyPos* pRes = taosArrayGetP(pGroupResInfo->pRows, i); + taosMemoryFree(pRes); + } + + pGroupResInfo->pRows = taosArrayDestroy(pGroupResInfo->pRows); pGroupResInfo->index = 0; } @@ -115,9 +119,6 @@ void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SHashObj* pHashmap, int p->groupId = *(uint64_t*)key; p->pos = *(SResultRowPosition*)pData; memcpy(p->key, (char*)key + sizeof(uint64_t), keyLen - sizeof(uint64_t)); -#ifdef BUF_PAGE_DEBUG - qDebug("page_groupRes, groupId:%" PRIu64 ",pageId:%d,offset:%d\n", p->groupId, p->pos.pageId, p->pos.offset); -#endif taosArrayPush(pGroupResInfo->pRows, &p); } @@ -901,4 +902,23 @@ STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowI } return w; +} + +bool hasLimitOffsetInfo(SLimitInfo* pLimitInfo) { + return (pLimitInfo->limit.limit != -1 || pLimitInfo->limit.offset != -1 || pLimitInfo->slimit.limit != -1 || + pLimitInfo->slimit.offset != -1); +} + + +static int64_t getLimit(const SNode* pLimit) { return NULL == pLimit ? -1 : ((SLimitNode*)pLimit)->limit; } +static int64_t getOffset(const SNode* pLimit) { return NULL == pLimit ? -1 : ((SLimitNode*)pLimit)->offset; } + +void initLimitInfo(const SNode* pLimit, const SNode* pSLimit, SLimitInfo* pLimitInfo) { + SLimit limit = {.limit = getLimit(pLimit), .offset = getOffset(pLimit)}; + SLimit slimit = {.limit = getLimit(pSLimit), .offset = getOffset(pSLimit)}; + + pLimitInfo->limit = limit; + pLimitInfo->slimit= slimit; + pLimitInfo->remainOffset = limit.offset; + pLimitInfo->remainGroupOffset = slimit.offset; } \ No newline at end of file diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index 37d2521e5d..9d3d62cabf 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -332,6 +332,8 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, const STqOffsetVal* pOffset) { STableKeyInfo* pTableInfo = taosArrayGet(pTaskInfo->tableqinfoList.pTableList, 0); uid = pTableInfo->uid; ts = INT64_MIN; + } else { + return -1; } } /*if (pTaskInfo->streamInfo.lastStatus.type != TMQ_OFFSET__SNAPSHOT_DATA ||*/ diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index c8f2083456..8381fae019 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -43,6 +43,11 @@ #define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP) +enum { + PROJECT_RETRIEVE_CONTINUE = 0x1, + PROJECT_RETRIEVE_DONE = 0x2, +}; + #if 0 static UNUSED_FUNC void *u_malloc (size_t __size) { uint32_t v = taosRand(); @@ -372,6 +377,10 @@ void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow colDataAppendInt64(pColData, 4, &pQueryWindow->ekey); } +void cleanupExecTimeWindowInfo(SColumnInfoData* pColData) { + colDataDestroy(pColData); +} + void doApplyFunctions(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, STimeWindow* pWin, SColumnInfoData* pTimeWindowData, int32_t offset, int32_t forwardStep, TSKEY* tsCol, int32_t numOfTotal, int32_t numOfOutput, int32_t order) { @@ -624,7 +633,8 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc int32_t startOffset = createNewColModel ? 0 : pResult->info.rows; ASSERT(pResult->info.capacity > 0); colDataMergeCol(pResColData, startOffset, &pResult->info.capacity, &idata, dest.numOfRows); - + colDataDestroy(&idata); + numOfRows = dest.numOfRows; taosArrayDestroy(pBlockList); } else if (pExpr[k].pExpr->nodeType == QUERY_NODE_FUNCTION) { @@ -679,6 +689,7 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc int32_t startOffset = createNewColModel ? 0 : pResult->info.rows; ASSERT(pResult->info.capacity > 0); colDataMergeCol(pResColData, startOffset, &pResult->info.capacity, &idata, dest.numOfRows); + colDataDestroy(&idata); numOfRows = dest.numOfRows; taosArrayDestroy(pBlockList); @@ -1506,15 +1517,11 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprI int32_t numOfExprs) { int32_t numOfRows = getNumOfTotalRes(pGroupResInfo); int32_t start = pGroupResInfo->index; -#ifdef BUF_PAGE_DEBUG - qDebug("\npage_copytoblock rows:%d", numOfRows); -#endif + for (int32_t i = start; i < numOfRows; i += 1) { SResKeyPos* pPos = taosArrayGetP(pGroupResInfo->pRows, i); SFilePage* page = getBufPage(pBuf, pPos->pos.pageId); -#ifdef BUF_PAGE_DEBUG - qDebug("page_copytoblock pos pageId:%d, offset:%d", pPos->pos.pageId, pPos->pos.offset); -#endif + SResultRow* pRow = (SResultRow*)((char*)page + pPos->pos.offset); doUpdateNumOfRows(pRow, numOfExprs, rowCellOffset); @@ -1980,6 +1987,7 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { qDebug("%s fetch rsp received, index:%d, rows:%d", pSourceDataInfo->taskId, index, pRsp->numOfRows); } else { pSourceDataInfo->code = code; + qDebug("%s fetch rsp received, index:%d, error:%d", pSourceDataInfo->taskId, index, tstrerror(code)); } pSourceDataInfo->status = EX_SOURCE_DATA_READY; @@ -2221,6 +2229,8 @@ static SSDataBlock* concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SEx if (completed == totalSources) { return setAllSourcesCompleted(pOperator, startTs); } + + sched_yield(); } _error: @@ -2342,7 +2352,7 @@ static int32_t prepareLoadRemoteData(SOperatorInfo* pOperator) { return TSDB_CODE_SUCCESS; } -static SSDataBlock* doLoadRemoteData(SOperatorInfo* pOperator) { +static SSDataBlock* doLoadRemoteDataImpl(SOperatorInfo* pOperator) { SExchangeInfo* pExchangeInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -2368,6 +2378,44 @@ static SSDataBlock* doLoadRemoteData(SOperatorInfo* pOperator) { } } +static SSDataBlock* doLoadRemoteData(SOperatorInfo* pOperator) { + SExchangeInfo* pExchangeInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + while(1) { + SSDataBlock* pBlock = doLoadRemoteDataImpl(pOperator); + if (pBlock == NULL) { + return NULL; + } + + ASSERT(pBlock == pExchangeInfo->pResult); + + SLimitInfo* pLimitInfo = &pExchangeInfo->limitInfo; + if (hasLimitOffsetInfo(pLimitInfo)) { + int32_t status = handleLimitOffset(pOperator, pLimitInfo, pExchangeInfo->pResult, false); + if (status == PROJECT_RETRIEVE_CONTINUE) { + continue; + } else if (status == PROJECT_RETRIEVE_DONE) { + size_t rows = pExchangeInfo->pResult->info.rows; + pExchangeInfo->limitInfo.numOfOutputRows += rows; + + if (rows == 0) { + doSetOperatorCompleted(pOperator); + return NULL; + } else { + return pExchangeInfo->pResult; + } + } + } else { + return pExchangeInfo->pResult; + } + } +} + static int32_t initDataSource(int32_t numOfSources, SExchangeInfo* pInfo, const char* id) { pInfo->pSourceDataInfo = taosArrayInit(numOfSources, sizeof(SSourceDataInfo)); if (pInfo->pSourceDataInfo == NULL) { @@ -2407,6 +2455,7 @@ static int32_t initExchangeOperator(SExchangePhysiNode* pExNode, SExchangeInfo* taosArrayPush(pInfo->pSources, pNode); } + initLimitInfo(pExNode->node.pLimit, pExNode->node.pSlimit, &pInfo->limitInfo); pInfo->self = taosAddRef(exchangeObjRefPool, pInfo); return initDataSource(numOfSources, pInfo, id); @@ -3150,68 +3199,60 @@ int32_t aggDecodeResultRow(SOperatorInfo* pOperator, char* result) { return TDB_CODE_SUCCESS; } -enum { - PROJECT_RETRIEVE_CONTINUE = 0x1, - PROJECT_RETRIEVE_DONE = 0x2, -}; - -static int32_t handleLimitOffset(SOperatorInfo* pOperator, SSDataBlock* pBlock) { - SProjectOperatorInfo* pProjectInfo = pOperator->info; - SOptrBasicInfo* pInfo = &pProjectInfo->binfo; - SSDataBlock* pRes = pInfo->pRes; - - if (pProjectInfo->curSOffset > 0) { - if (pProjectInfo->groupId == 0) { // it is the first group - pProjectInfo->groupId = pBlock->info.groupId; - blockDataCleanup(pInfo->pRes); +int32_t handleLimitOffset(SOperatorInfo* pOperator, SLimitInfo* pLimitInfo, SSDataBlock* pBlock, bool holdDataInBuf) { + if (pLimitInfo->remainGroupOffset > 0) { + if (pLimitInfo->currentGroupId == 0) { // it is the first group + pLimitInfo->currentGroupId = pBlock->info.groupId; + blockDataCleanup(pBlock); return PROJECT_RETRIEVE_CONTINUE; - } else if (pProjectInfo->groupId != pBlock->info.groupId) { - pProjectInfo->curSOffset -= 1; + } else if (pLimitInfo->currentGroupId != pBlock->info.groupId) { + // now it is the data from a new group + pLimitInfo->remainGroupOffset -= 1; // ignore data block in current group - if (pProjectInfo->curSOffset > 0) { - blockDataCleanup(pInfo->pRes); + if (pLimitInfo->remainGroupOffset > 0) { + blockDataCleanup(pBlock); return PROJECT_RETRIEVE_CONTINUE; } } // set current group id of the project operator - pProjectInfo->groupId = pBlock->info.groupId; + pLimitInfo->currentGroupId = pBlock->info.groupId; } - if (pProjectInfo->groupId != 0 && pProjectInfo->groupId != pBlock->info.groupId) { - pProjectInfo->curGroupOutput += 1; - if ((pProjectInfo->slimit.limit > 0) && (pProjectInfo->slimit.limit <= pProjectInfo->curGroupOutput)) { + if (pLimitInfo->currentGroupId != 0 && pLimitInfo->currentGroupId != pBlock->info.groupId) { + pLimitInfo->numOfOutputGroups += 1; + if ((pLimitInfo->slimit.limit > 0) && (pLimitInfo->slimit.limit <= pLimitInfo->numOfOutputGroups)) { pOperator->status = OP_EXEC_DONE; - blockDataCleanup(pRes); + blockDataCleanup(pBlock); return PROJECT_RETRIEVE_DONE; } // reset the value for a new group data - pProjectInfo->curOffset = 0; - pProjectInfo->curOutput = 0; + pLimitInfo->numOfOutputRows = 0; + pLimitInfo->remainOffset = pLimitInfo->limit.offset; } // here we reach the start position, according to the limit/offset requirements. // set current group id - pProjectInfo->groupId = pBlock->info.groupId; + pLimitInfo->currentGroupId = pBlock->info.groupId; - if (pProjectInfo->curOffset >= pRes->info.rows) { - pProjectInfo->curOffset -= pRes->info.rows; - blockDataCleanup(pRes); + if (pLimitInfo->remainOffset >= pBlock->info.rows) { + pLimitInfo->remainOffset -= pBlock->info.rows; + blockDataCleanup(pBlock); return PROJECT_RETRIEVE_CONTINUE; - } else if (pProjectInfo->curOffset < pRes->info.rows && pProjectInfo->curOffset > 0) { - blockDataTrimFirstNRows(pRes, pProjectInfo->curOffset); - pProjectInfo->curOffset = 0; + } else if (pLimitInfo->remainOffset < pBlock->info.rows && pLimitInfo->remainOffset > 0) { + blockDataTrimFirstNRows(pBlock, pLimitInfo->remainOffset); + pLimitInfo->remainOffset = 0; } // check for the limitation in each group - if (pProjectInfo->limit.limit >= 0 && pProjectInfo->curOutput + pRes->info.rows >= pProjectInfo->limit.limit) { - int32_t keepRows = (int32_t)(pProjectInfo->limit.limit - pProjectInfo->curOutput); - blockDataKeepFirstNRows(pRes, keepRows); - if (pProjectInfo->slimit.limit > 0 && pProjectInfo->slimit.limit <= pProjectInfo->curGroupOutput) { + if (pLimitInfo->limit.limit >= 0 && pLimitInfo->numOfOutputRows + pBlock->info.rows >= pLimitInfo->limit.limit) { + int32_t keepRows = (int32_t)(pLimitInfo->limit.limit - pLimitInfo->numOfOutputRows); + blockDataKeepFirstNRows(pBlock, keepRows); + if (pLimitInfo->slimit.limit > 0 && pLimitInfo->slimit.limit <= pLimitInfo->numOfOutputGroups) { pOperator->status = OP_EXEC_DONE; } @@ -3221,8 +3262,8 @@ static int32_t handleLimitOffset(SOperatorInfo* pOperator, SSDataBlock* pBlock) // todo optimize performance // If there are slimit/soffset value exists, multi-round result can not be packed into one group, since the // they may not belong to the same group the limit/offset value is not valid in this case. - if (pRes->info.rows >= pOperator->resultInfo.threshold || pProjectInfo->slimit.offset != -1 || - pProjectInfo->slimit.limit != -1) { + if ((!holdDataInBuf) || (pBlock->info.rows >= pOperator->resultInfo.threshold) || pLimitInfo->slimit.offset != -1 || + pLimitInfo->slimit.limit != -1) { return PROJECT_RETRIEVE_DONE; } else { // not full enough, continue to accumulate the output data in the buffer. return PROJECT_RETRIEVE_CONTINUE; @@ -3308,7 +3349,7 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { longjmp(pTaskInfo->env, code); } - int32_t status = handleLimitOffset(pOperator, pBlock); + int32_t status = handleLimitOffset(pOperator, &pProjectInfo->limitInfo, pInfo->pRes, true); // filter shall be applied after apply functions and limit/offset on the result doFilter(pProjectInfo->pFilterNode, pInfo->pRes); @@ -3320,9 +3361,9 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { } } - pProjectInfo->curOutput += pInfo->pRes->info.rows; - size_t rows = pInfo->pRes->info.rows; + pProjectInfo->limitInfo.numOfOutputRows += rows; + pOperator->resultInfo.totalRows += rows; if (pOperator->cost.openCost == 0) { @@ -3700,7 +3741,7 @@ void destroyAggOperatorInfo(void* param, int32_t numOfOutput) { cleanupBasicInfo(&pInfo->binfo); cleanupAggSup(&pInfo->aggSup); - taosArrayDestroyEx(pInfo->groupResInfo.pRows, freeItem); + cleanupGroupResInfo(&pInfo->groupResInfo); taosMemoryFreeClear(param); } @@ -3747,7 +3788,7 @@ void doDestroyExchangeOperatorInfo(void* param) { taosArrayDestroy(pExInfo->pSources); taosArrayDestroy(pExInfo->pSourceDataInfo); if (pExInfo->pResult != NULL) { - blockDataDestroy(pExInfo->pResult); + pExInfo->pResult = blockDataDestroy(pExInfo->pResult); } tsem_destroy(&pExInfo->ready); @@ -3766,10 +3807,6 @@ static SArray* setRowTsColumnOutputInfo(SqlFunctionCtx* pCtx, int32_t numOfCols) return pList; } -static int64_t getLimit(SNode* pLimit) { return NULL == pLimit ? -1 : ((SLimitNode*)pLimit)->limit; } - -static int64_t getOffset(SNode* pLimit) { return NULL == pLimit ? -1 : ((SLimitNode*)pLimit)->offset; } - SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhysiNode* pProjPhyNode, SExecTaskInfo* pTaskInfo) { SProjectOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SProjectOperatorInfo)); @@ -3782,13 +3819,8 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhys SExprInfo* pExprInfo = createExprInfo(pProjPhyNode->pProjections, NULL, &numOfCols); SSDataBlock* pResBlock = createResDataBlock(pProjPhyNode->node.pOutputDataBlockDesc); - SLimit limit = {.limit = getLimit(pProjPhyNode->node.pLimit), .offset = getOffset(pProjPhyNode->node.pLimit)}; - SLimit slimit = {.limit = getLimit(pProjPhyNode->node.pSlimit), .offset = getOffset(pProjPhyNode->node.pSlimit)}; + initLimitInfo(pProjPhyNode->node.pLimit, pProjPhyNode->node.pSlimit, &pInfo->limitInfo); - pInfo->limit = limit; - pInfo->slimit = slimit; - pInfo->curOffset = limit.offset; - pInfo->curSOffset = slimit.offset; pInfo->binfo.pRes = pResBlock; pInfo->pFilterNode = pProjPhyNode->node.pConditions; diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index ee20bc7ba6..2964948e70 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -29,7 +29,7 @@ static void* getCurrentDataGroupInfo(const SPartitionOperatorInfo* pInfo, SDataGroupInfo** pGroupInfo, int32_t len); static int32_t* setupColumnOffset(const SSDataBlock* pBlock, int32_t rowCapacity); static int32_t setGroupResultOutputBuf(SOperatorInfo* pOperator, SOptrBasicInfo* binfo, int32_t numOfCols, char* pData, int16_t bytes, - int32_t groupId, SDiskbasedBuf* pBuf, SAggSupporter* pAggSup); + uint64_t groupId, SDiskbasedBuf* pBuf, SAggSupporter* pAggSup); static void destroyGroupOperatorInfo(void* param, int32_t numOfOutput) { SGroupbyOperatorInfo* pInfo = (SGroupbyOperatorInfo*)param; @@ -264,7 +264,7 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) { } len = buildGroupKeys(pInfo->keyBuf, pInfo->pGroupColVals); - int32_t ret = setGroupResultOutputBuf(pOperator, &(pInfo->binfo), pOperator->exprSupp.numOfExprs, pInfo->keyBuf, len, 0, pInfo->aggSup.pResultBuf, &pInfo->aggSup); + int32_t ret = setGroupResultOutputBuf(pOperator, &(pInfo->binfo), pOperator->exprSupp.numOfExprs, pInfo->keyBuf, len, pBlock->info.groupId, pInfo->aggSup.pResultBuf, &pInfo->aggSup); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); } @@ -282,7 +282,7 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) { len = buildGroupKeys(pInfo->keyBuf, pInfo->pGroupColVals); int32_t ret = setGroupResultOutputBuf(pOperator, &(pInfo->binfo), pOperator->exprSupp.numOfExprs, pInfo->keyBuf, len, - 0, pInfo->aggSup.pResultBuf, &pInfo->aggSup); + pBlock->info.groupId, pInfo->aggSup.pResultBuf, &pInfo->aggSup); if (ret != TSDB_CODE_SUCCESS) { longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); } @@ -293,6 +293,29 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) { } } +static SSDataBlock* buildGroupResultDataBlock(SOperatorInfo* pOperator) { + SGroupbyOperatorInfo* pInfo = pOperator->info; + + SSDataBlock* pRes = pInfo->binfo.pRes; + while(1) { + doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); + doFilter(pInfo->pCondition, pRes); + + bool hasRemain = hasDataInGroupInfo(&pInfo->groupResInfo); + if (!hasRemain) { + doSetOperatorCompleted(pOperator); + break; + } + + if (pRes->info.rows > 0) { + break; + } + } + + pOperator->resultInfo.totalRows += pRes->info.rows; + return (pRes->info.rows == 0)? NULL:pRes; +} + static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) { if (pOperator->status == OP_EXEC_DONE) { return NULL; @@ -304,22 +327,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) { SSDataBlock* pRes = pInfo->binfo.pRes; if (pOperator->status == OP_RES_TO_RETURN) { - while(1) { - doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - doFilter(pInfo->pCondition, pRes); - - bool hasRemain = hasDataInGroupInfo(&pInfo->groupResInfo); - if (!hasRemain) { - doSetOperatorCompleted(pOperator); - break; - } - - if (pRes->info.rows > 0) { - break; - } - } - pOperator->resultInfo.totalRows += pRes->info.rows; - return (pRes->info.rows == 0)? NULL:pRes; + return buildGroupResultDataBlock(pOperator); } int32_t order = TSDB_ORDER_ASC; @@ -373,26 +381,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) { initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, 0); pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0; - - while(1) { - doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - doFilter(pInfo->pCondition, pRes); - - bool hasRemain = hasDataInGroupInfo(&pInfo->groupResInfo); - if (!hasRemain) { - doSetOperatorCompleted(pOperator); - break; - } - - if (pRes->info.rows > 0) { - break; - } - } - - size_t rows = pRes->info.rows; - pOperator->resultInfo.totalRows += rows; - - return (rows == 0)? NULL:pRes; + return buildGroupResultDataBlock(pOperator); } SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, @@ -451,19 +440,19 @@ static void doHashPartition(SOperatorInfo* pOperator, SSDataBlock* pBlock) { recordNewGroupKeys(pInfo->pGroupCols, pInfo->pGroupColVals, pBlock, j); int32_t len = buildGroupKeys(pInfo->keyBuf, pInfo->pGroupColVals); - SDataGroupInfo* pGInfo = NULL; - void *pPage = getCurrentDataGroupInfo(pInfo, &pGInfo, len); + SDataGroupInfo* pGroupInfo = NULL; + void *pPage = getCurrentDataGroupInfo(pInfo, &pGroupInfo, len); - pGInfo->numOfRows += 1; - if (pGInfo->groupId == 0) { - pGInfo->groupId = calcGroupId(pInfo->keyBuf, len); + pGroupInfo->numOfRows += 1; + + // group id + if (pGroupInfo->groupId == 0) { + pGroupInfo->groupId = calcGroupId(pInfo->keyBuf, len); } // number of rows int32_t* rows = (int32_t*) pPage; - // group id - size_t numOfCols = pOperator->exprSupp.numOfExprs; for(int32_t i = 0; i < numOfCols; ++i) { SExprInfo* pExpr = &pOperator->exprSupp.pExprInfo[i]; @@ -614,7 +603,13 @@ static void clearPartitionOperator(SPartitionOperatorInfo* pInfo) { static int compareDataGroupInfo(const void* group1, const void* group2) { const SDataGroupInfo* pGroupInfo1 = group1; const SDataGroupInfo* pGroupInfo2 = group2; - return pGroupInfo1->groupId - pGroupInfo2->groupId; + + if (pGroupInfo1->groupId == pGroupInfo2->groupId) { + ASSERT(0); + return 0; + } + + return (pGroupInfo1->groupId < pGroupInfo2->groupId)? -1:1; } static SSDataBlock* buildPartitionResult(SOperatorInfo* pOperator) { @@ -800,7 +795,7 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartition } int32_t setGroupResultOutputBuf(SOperatorInfo* pOperator, SOptrBasicInfo* binfo, int32_t numOfCols, char* pData, int16_t bytes, - int32_t groupId, SDiskbasedBuf* pBuf, SAggSupporter* pAggSup) { + uint64_t groupId, SDiskbasedBuf* pBuf, SAggSupporter* pAggSup) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SResultRowInfo* pResultRowInfo = &binfo->resultRowInfo; SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index a81a796fa7..b80b9af237 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -521,8 +521,9 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { int32_t code = tsdbReaderOpen(pInfo->readHandle.vnode, &pInfo->cond, tableList, (STsdbReader**)&pInfo->dataReader, GET_TASKID(pTaskInfo)); - if (code != 0) { - // TODO + if (code != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, code); + return NULL; } } @@ -1142,7 +1143,7 @@ static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsCol[rowId], &pInfo->interval, TSDB_ORDER_ASC); // must check update info first. bool update = updateInfoIsUpdated(pInfo->pUpdateInfo, pBlock->info.uid, tsCol[rowId]); - if ( (update || (isSignleIntervalWindow(pInfo) && isCloseWindow(&win, &pInfo->twAggSup)) ) && out) { + if ((update || (isSignleIntervalWindow(pInfo) && isCloseWindow(&win, &pInfo->twAggSup))) && out) { taosArrayPush(pInfo->tsArray, &rowId); } } @@ -1595,7 +1596,8 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys pInfo->pUpdateRes = createResDataBlock(pDescNode); pInfo->pCondition = pScanPhyNode->node.pConditions; pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; - pInfo->sessionSup = (SessionWindowSupporter){.pStreamAggSup = NULL, .gap = -1, .parentType = QUERY_NODE_PHYSICAL_PLAN}; + pInfo->sessionSup = + (SessionWindowSupporter){.pStreamAggSup = NULL, .gap = -1, .parentType = QUERY_NODE_PHYSICAL_PLAN}; pInfo->groupId = 0; pInfo->pPullDataRes = createPullDataBlock(); pInfo->pStreamScanOp = pOperator; @@ -1629,7 +1631,8 @@ static void destroySysScanOperator(void* param, int32_t numOfOutput) { blockDataDestroy(pInfo->pRes); const char* name = tNameGetTableName(&pInfo->name); - if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, TSDB_TABLE_FNAME_LEN) == 0 || pInfo->pCur != NULL) { + if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, TSDB_TABLE_FNAME_LEN) == 0 || + strncasecmp(name, TSDB_INS_TABLE_USER_TAGS, TSDB_TABLE_FNAME_LEN) == 0 || pInfo->pCur != NULL) { metaCloseTbCursor(pInfo->pCur); pInfo->pCur = NULL; } @@ -1776,14 +1779,14 @@ static SSDataBlock* doFilterResult(SSysTableScanInfo* pInfo) { return pInfo->pRes->info.rows == 0 ? NULL : pInfo->pRes; } -static SSDataBlock* buildSysTableMetaBlock() { +static SSDataBlock* buildInfoSchemaTableMetaBlock(char* tableName) { size_t size = 0; const SSysTableMeta* pMeta = NULL; getInfosDbMeta(&pMeta, &size); int32_t index = 0; for (int32_t i = 0; i < size; ++i) { - if (strcmp(pMeta[i].name, TSDB_INS_TABLE_USER_TABLES) == 0) { + if (strcmp(pMeta[i].name, tableName) == 0) { index = i; break; } @@ -1799,185 +1802,340 @@ static SSDataBlock* buildSysTableMetaBlock() { return pBlock; } +static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SSysTableScanInfo* pInfo = pOperator->info; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + if (pInfo->pCur == NULL) { + pInfo->pCur = metaOpenTbCursor(pInfo->readHandle.meta); + } + + blockDataCleanup(pInfo->pRes); + int32_t numOfRows = 0; + + const char* db = NULL; + int32_t vgId = 0; + vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId); + + SName sn = {0}; + char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB); + + tNameGetDbName(&sn, varDataVal(dbname)); + varDataSetLen(dbname, strlen(varDataVal(dbname))); + + SSDataBlock* p = buildInfoSchemaTableMetaBlock(TSDB_INS_TABLE_USER_TAGS); + blockDataEnsureCapacity(p, pOperator->resultInfo.capacity); + + int32_t ret = 0; + while ((ret = metaTbCursorNext(pInfo->pCur)) == 0) { + if (pInfo->pCur->mr.me.type != TSDB_CHILD_TABLE) { + continue; + } + + char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(tableName, pInfo->pCur->mr.me.name); + + SMetaReader smr = {0}; + metaReaderInit(&smr, pInfo->readHandle.meta, 0); + + uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid; + int32_t code = metaGetTableEntryByUid(&smr, suid); + if (code != TSDB_CODE_SUCCESS) { + qError("failed to get super table meta, uid:0x%" PRIx64 ", code:%s, %s", suid, tstrerror(terrno), + GET_TASKID(pTaskInfo)); + metaReaderClear(&smr); + metaCloseTbCursor(pInfo->pCur); + pInfo->pCur = NULL; + longjmp(pTaskInfo->env, terrno); + } + + char stableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(stableName, smr.me.name); + + int32_t numOfTags = smr.me.stbEntry.schemaTag.nCols; + for (int32_t i = 0; i < numOfTags; ++i) { + SColumnInfoData* pColInfoData = NULL; + + // table name + pColInfoData = taosArrayGet(p->pDataBlock, 0); + colDataAppend(pColInfoData, numOfRows, tableName, false); + + // database name + pColInfoData = taosArrayGet(p->pDataBlock, 1); + colDataAppend(pColInfoData, numOfRows, dbname, false); + + // super table name + pColInfoData = taosArrayGet(p->pDataBlock, 2); + colDataAppend(pColInfoData, numOfRows, stableName, false); + + char tagName[TSDB_COL_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(tagName, smr.me.stbEntry.schemaTag.pSchema[i].name); + pColInfoData = taosArrayGet(p->pDataBlock, 3); + colDataAppend(pColInfoData, numOfRows, tagName, false); + + int8_t tagType = smr.me.stbEntry.schemaTag.pSchema[i].type; + pColInfoData = taosArrayGet(p->pDataBlock, 4); + colDataAppend(pColInfoData, numOfRows, (char*)&tagType, false); + + STagVal tagVal = {0}; + tagVal.cid = smr.me.stbEntry.schemaTag.pSchema[i].colId; + char* tagData = NULL; + uint32_t tagLen = 0; + + if (tagType == TSDB_DATA_TYPE_JSON) { + tagData = (char*)pInfo->pCur->mr.me.ctbEntry.pTags; + } else { + bool exist = tTagGet((STag*)pInfo->pCur->mr.me.ctbEntry.pTags, &tagVal); + if (exist) { + if (IS_VAR_DATA_TYPE(tagType)) { + tagData = (char*)tagVal.pData; + tagLen = tagVal.nData; + } else { + tagData = (char*)&tagVal.i64; + tagLen = tDataTypes[tagType].bytes; + } + } + } + + char* tagVarChar = NULL; + if (tagData != NULL) { + if (tagType == TSDB_DATA_TYPE_JSON) { + char* tagJson = parseTagDatatoJson(tagData); + tagVarChar = taosMemoryMalloc(strlen(tagJson) + VARSTR_HEADER_SIZE); + memcpy(varDataVal(tagVarChar), tagJson, strlen(tagJson)); + varDataSetLen(tagVarChar, strlen(tagJson)); + taosMemoryFree(tagJson); + } else { + int32_t bufSize = IS_VAR_DATA_TYPE(tagType) ? (tagLen + VARSTR_HEADER_SIZE) + : (3 + DBL_MANT_DIG - DBL_MIN_EXP + VARSTR_HEADER_SIZE); + tagVarChar = taosMemoryMalloc(bufSize); + int32_t len = -1; + dataConverToStr(varDataVal(tagVarChar), tagType, tagData, tagLen, &len); + varDataSetLen(tagVarChar, len); + } + } + pColInfoData = taosArrayGet(p->pDataBlock, 5); + colDataAppend(pColInfoData, numOfRows, tagVarChar, + (tagData == NULL) || (tagType == TSDB_DATA_TYPE_JSON && tTagIsJsonNull(tagData))); + taosMemoryFree(tagVarChar); + ++numOfRows; + } + metaReaderClear(&smr); + + if (numOfRows >= pOperator->resultInfo.capacity) { + break; + } + } + + // todo temporarily free the cursor here, the true reason why the free is not valid needs to be found + if (ret != 0) { + metaCloseTbCursor(pInfo->pCur); + pInfo->pCur = NULL; + doSetOperatorCompleted(pOperator); + } + + p->info.rows = numOfRows; + pInfo->pRes->info.rows = numOfRows; + + relocateColumnData(pInfo->pRes, pInfo->scanCols, p->pDataBlock, false); + doFilterResult(pInfo); + + blockDataDestroy(p); + + pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; + return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; +} + +static SSDataBlock* sysTableScanUserTables(SOperatorInfo* pOperator) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SSysTableScanInfo* pInfo = pOperator->info; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + // the retrieve is executed on the mnode, so return tables that belongs to the information schema database. + if (pInfo->readHandle.mnd != NULL) { + buildSysDbTableInfo(pInfo, pOperator->resultInfo.capacity); + + doFilterResult(pInfo); + pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; + + doSetOperatorCompleted(pOperator); + return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; + } else { + if (pInfo->pCur == NULL) { + pInfo->pCur = metaOpenTbCursor(pInfo->readHandle.meta); + } + + blockDataCleanup(pInfo->pRes); + int32_t numOfRows = 0; + + const char* db = NULL; + int32_t vgId = 0; + vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId); + + SName sn = {0}; + char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB); + + tNameGetDbName(&sn, varDataVal(dbname)); + varDataSetLen(dbname, strlen(varDataVal(dbname))); + + SSDataBlock* p = buildInfoSchemaTableMetaBlock(TSDB_INS_TABLE_USER_TABLES); + blockDataEnsureCapacity(p, pOperator->resultInfo.capacity); + + char n[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + + int32_t ret = 0; + while ((ret = metaTbCursorNext(pInfo->pCur)) == 0) { + STR_TO_VARSTR(n, pInfo->pCur->mr.me.name); + + // table name + SColumnInfoData* pColInfoData = taosArrayGet(p->pDataBlock, 0); + colDataAppend(pColInfoData, numOfRows, n, false); + + // database name + pColInfoData = taosArrayGet(p->pDataBlock, 1); + colDataAppend(pColInfoData, numOfRows, dbname, false); + + // vgId + pColInfoData = taosArrayGet(p->pDataBlock, 6); + colDataAppend(pColInfoData, numOfRows, (char*)&vgId, false); + + int32_t tableType = pInfo->pCur->mr.me.type; + if (tableType == TSDB_CHILD_TABLE) { + // create time + int64_t ts = pInfo->pCur->mr.me.ctbEntry.ctime; + pColInfoData = taosArrayGet(p->pDataBlock, 2); + colDataAppend(pColInfoData, numOfRows, (char*)&ts, false); + + SMetaReader mr = {0}; + metaReaderInit(&mr, pInfo->readHandle.meta, 0); + + uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid; + int32_t code = metaGetTableEntryByUid(&mr, suid); + if (code != TSDB_CODE_SUCCESS) { + qError("failed to get super table meta, uid:0x%" PRIx64 ", code:%s, %s", suid, tstrerror(terrno), + GET_TASKID(pTaskInfo)); + metaReaderClear(&mr); + metaCloseTbCursor(pInfo->pCur); + pInfo->pCur = NULL; + longjmp(pTaskInfo->env, terrno); + } + + // number of columns + pColInfoData = taosArrayGet(p->pDataBlock, 3); + colDataAppend(pColInfoData, numOfRows, (char*)&mr.me.stbEntry.schemaRow.nCols, false); + + // super table name + STR_TO_VARSTR(n, mr.me.name); + pColInfoData = taosArrayGet(p->pDataBlock, 4); + colDataAppend(pColInfoData, numOfRows, n, false); + metaReaderClear(&mr); + + // table comment + pColInfoData = taosArrayGet(p->pDataBlock, 8); + if (pInfo->pCur->mr.me.ctbEntry.commentLen > 0) { + char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(comment, pInfo->pCur->mr.me.ctbEntry.comment); + colDataAppend(pColInfoData, numOfRows, comment, false); + } else if (pInfo->pCur->mr.me.ctbEntry.commentLen == 0) { + char comment[VARSTR_HEADER_SIZE + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(comment, ""); + colDataAppend(pColInfoData, numOfRows, comment, false); + } else { + colDataAppendNULL(pColInfoData, numOfRows); + } + + // uid + pColInfoData = taosArrayGet(p->pDataBlock, 5); + colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.uid, false); + + // ttl + pColInfoData = taosArrayGet(p->pDataBlock, 7); + colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ctbEntry.ttlDays, false); + + STR_TO_VARSTR(n, "CHILD_TABLE"); + } else if (tableType == TSDB_NORMAL_TABLE) { + // create time + pColInfoData = taosArrayGet(p->pDataBlock, 2); + colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.ctime, false); + + // number of columns + pColInfoData = taosArrayGet(p->pDataBlock, 3); + colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.schemaRow.nCols, false); + + // super table name + pColInfoData = taosArrayGet(p->pDataBlock, 4); + colDataAppendNULL(pColInfoData, numOfRows); + + // table comment + pColInfoData = taosArrayGet(p->pDataBlock, 8); + if (pInfo->pCur->mr.me.ntbEntry.commentLen > 0) { + char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(comment, pInfo->pCur->mr.me.ntbEntry.comment); + colDataAppend(pColInfoData, numOfRows, comment, false); + } else if (pInfo->pCur->mr.me.ntbEntry.commentLen == 0) { + char comment[VARSTR_HEADER_SIZE + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(comment, ""); + colDataAppend(pColInfoData, numOfRows, comment, false); + } else { + colDataAppendNULL(pColInfoData, numOfRows); + } + + // uid + pColInfoData = taosArrayGet(p->pDataBlock, 5); + colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.uid, false); + + // ttl + pColInfoData = taosArrayGet(p->pDataBlock, 7); + colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.ttlDays, false); + + STR_TO_VARSTR(n, "NORMAL_TABLE"); + } + + pColInfoData = taosArrayGet(p->pDataBlock, 9); + colDataAppend(pColInfoData, numOfRows, n, false); + + if (++numOfRows >= pOperator->resultInfo.capacity) { + break; + } + } + + // todo temporarily free the cursor here, the true reason why the free is not valid needs to be found + if (ret != 0) { + metaCloseTbCursor(pInfo->pCur); + pInfo->pCur = NULL; + doSetOperatorCompleted(pOperator); + } + + p->info.rows = numOfRows; + pInfo->pRes->info.rows = numOfRows; + + relocateColumnData(pInfo->pRes, pInfo->scanCols, p->pDataBlock, false); + doFilterResult(pInfo); + + blockDataDestroy(p); + + pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; + return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; + } +} + static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { // build message and send to mnode to fetch the content of system tables. SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SSysTableScanInfo* pInfo = pOperator->info; - // retrieve local table list info from vnode const char* name = tNameGetTableName(&pInfo->name); if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, TSDB_TABLE_FNAME_LEN) == 0) { - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - // the retrieve is executed on the mnode, so return tables that belongs to the information schema database. - if (pInfo->readHandle.mnd != NULL) { - buildSysDbTableInfo(pInfo, pOperator->resultInfo.capacity); - - doFilterResult(pInfo); - pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; - - doSetOperatorCompleted(pOperator); - return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; - } else { - if (pInfo->pCur == NULL) { - pInfo->pCur = metaOpenTbCursor(pInfo->readHandle.meta); - } - - blockDataCleanup(pInfo->pRes); - int32_t numOfRows = 0; - - const char* db = NULL; - int32_t vgId = 0; - vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId); - - SName sn = {0}; - char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; - tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB); - - tNameGetDbName(&sn, varDataVal(dbname)); - varDataSetLen(dbname, strlen(varDataVal(dbname))); - - SSDataBlock* p = buildSysTableMetaBlock(); - blockDataEnsureCapacity(p, pOperator->resultInfo.capacity); - - char n[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - - int32_t ret = 0; - while ((ret = metaTbCursorNext(pInfo->pCur)) == 0) { - STR_TO_VARSTR(n, pInfo->pCur->mr.me.name); - - // table name - SColumnInfoData* pColInfoData = taosArrayGet(p->pDataBlock, 0); - colDataAppend(pColInfoData, numOfRows, n, false); - - // database name - pColInfoData = taosArrayGet(p->pDataBlock, 1); - colDataAppend(pColInfoData, numOfRows, dbname, false); - - // vgId - pColInfoData = taosArrayGet(p->pDataBlock, 6); - colDataAppend(pColInfoData, numOfRows, (char*)&vgId, false); - - int32_t tableType = pInfo->pCur->mr.me.type; - if (tableType == TSDB_CHILD_TABLE) { - // create time - int64_t ts = pInfo->pCur->mr.me.ctbEntry.ctime; - pColInfoData = taosArrayGet(p->pDataBlock, 2); - colDataAppend(pColInfoData, numOfRows, (char*)&ts, false); - - SMetaReader mr = {0}; - metaReaderInit(&mr, pInfo->readHandle.meta, 0); - - uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid; - int32_t code = metaGetTableEntryByUid(&mr, suid); - if (code != TSDB_CODE_SUCCESS) { - qError("failed to get super table meta, uid:0x%" PRIx64 ", code:%s, %s", suid, tstrerror(terrno), - GET_TASKID(pTaskInfo)); - metaReaderClear(&mr); - metaCloseTbCursor(pInfo->pCur); - pInfo->pCur = NULL; - longjmp(pTaskInfo->env, terrno); - } - - // number of columns - pColInfoData = taosArrayGet(p->pDataBlock, 3); - colDataAppend(pColInfoData, numOfRows, (char*)&mr.me.stbEntry.schemaRow.nCols, false); - - // super table name - STR_TO_VARSTR(n, mr.me.name); - pColInfoData = taosArrayGet(p->pDataBlock, 4); - colDataAppend(pColInfoData, numOfRows, n, false); - metaReaderClear(&mr); - - // table comment - pColInfoData = taosArrayGet(p->pDataBlock, 8); - if (pInfo->pCur->mr.me.ctbEntry.commentLen > 0) { - char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(comment, pInfo->pCur->mr.me.ctbEntry.comment); - colDataAppend(pColInfoData, numOfRows, comment, false); - } else if (pInfo->pCur->mr.me.ctbEntry.commentLen == 0) { - char comment[VARSTR_HEADER_SIZE + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(comment, ""); - colDataAppend(pColInfoData, numOfRows, comment, false); - } else { - colDataAppendNULL(pColInfoData, numOfRows); - } - - // uid - pColInfoData = taosArrayGet(p->pDataBlock, 5); - colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.uid, false); - - // ttl - pColInfoData = taosArrayGet(p->pDataBlock, 7); - colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ctbEntry.ttlDays, false); - - STR_TO_VARSTR(n, "CHILD_TABLE"); - } else if (tableType == TSDB_NORMAL_TABLE) { - // create time - pColInfoData = taosArrayGet(p->pDataBlock, 2); - colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.ctime, false); - - // number of columns - pColInfoData = taosArrayGet(p->pDataBlock, 3); - colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.schemaRow.nCols, false); - - // super table name - pColInfoData = taosArrayGet(p->pDataBlock, 4); - colDataAppendNULL(pColInfoData, numOfRows); - - // table comment - pColInfoData = taosArrayGet(p->pDataBlock, 8); - if (pInfo->pCur->mr.me.ntbEntry.commentLen > 0) { - char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(comment, pInfo->pCur->mr.me.ntbEntry.comment); - colDataAppend(pColInfoData, numOfRows, comment, false); - } else if (pInfo->pCur->mr.me.ntbEntry.commentLen == 0) { - char comment[VARSTR_HEADER_SIZE + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(comment, ""); - colDataAppend(pColInfoData, numOfRows, comment, false); - } else { - colDataAppendNULL(pColInfoData, numOfRows); - } - - // uid - pColInfoData = taosArrayGet(p->pDataBlock, 5); - colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.uid, false); - - // ttl - pColInfoData = taosArrayGet(p->pDataBlock, 7); - colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.ttlDays, false); - - STR_TO_VARSTR(n, "NORMAL_TABLE"); - } - - pColInfoData = taosArrayGet(p->pDataBlock, 9); - colDataAppend(pColInfoData, numOfRows, n, false); - - if (++numOfRows >= pOperator->resultInfo.capacity) { - break; - } - } - - // todo temporarily free the cursor here, the true reason why the free is not valid needs to be found - if (ret != 0) { - metaCloseTbCursor(pInfo->pCur); - pInfo->pCur = NULL; - doSetOperatorCompleted(pOperator); - } - - p->info.rows = numOfRows; - pInfo->pRes->info.rows = numOfRows; - - relocateColumnData(pInfo->pRes, pInfo->scanCols, p->pDataBlock, false); - doFilterResult(pInfo); - - blockDataDestroy(p); - - pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; - return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; - } + return sysTableScanUserTables(pOperator); + } else if (strncasecmp(name, TSDB_INS_TABLE_USER_TAGS, TSDB_TABLE_FNAME_LEN) == 0) { + return sysTableScanUserTags(pOperator); } else { // load the meta from mnode of the given epset if (pOperator->status == OP_EXEC_DONE) { return NULL; @@ -2057,7 +2215,7 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { } int32_t buildSysDbTableInfo(const SSysTableScanInfo* pInfo, int32_t capacity) { - SSDataBlock* p = buildSysTableMetaBlock(); + SSDataBlock* p = buildInfoSchemaTableMetaBlock(TSDB_INS_TABLE_USER_TABLES); blockDataEnsureCapacity(p, capacity); size_t size = 0; @@ -2146,7 +2304,8 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan tNameAssign(&pInfo->name, &pScanNode->tableName); const char* name = tNameGetTableName(&pInfo->name); - if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, TSDB_TABLE_FNAME_LEN) == 0) { + if (strncasecmp(name, TSDB_INS_TABLE_USER_TABLES, TSDB_TABLE_FNAME_LEN) == 0 || + strncasecmp(name, TSDB_INS_TABLE_USER_TAGS, TSDB_TABLE_FNAME_LEN) == 0) { pInfo->readHandle = *(SReadHandle*)readHandle; blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); } else { @@ -2940,4 +3099,3 @@ _error: taosMemoryFree(pOperator); return NULL; } - diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 9795907404..a3b79d9597 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -22,31 +22,31 @@ static int32_t getExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain static void destroyOrderOperatorInfo(void* param, int32_t numOfOutput); -SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* pSortPhyNode, - SExecTaskInfo* pTaskInfo) { +// todo add limit/offset impl +SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* pSortNode, SExecTaskInfo* pTaskInfo) { SSortOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSortOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL /* || rowSize > 100 * 1024 * 1024*/) { goto _error; } - SDataBlockDescNode* pDescNode = pSortPhyNode->node.pOutputDataBlockDesc; + SDataBlockDescNode* pDescNode = pSortNode->node.pOutputDataBlockDesc; int32_t numOfCols = 0; SSDataBlock* pResBlock = createResDataBlock(pDescNode); - SExprInfo* pExprInfo = createExprInfo(pSortPhyNode->pExprs, NULL, &numOfCols); + SExprInfo* pExprInfo = createExprInfo(pSortNode->pExprs, NULL, &numOfCols); int32_t numOfOutputCols = 0; SArray* pColMatchColInfo = - extractColMatchInfo(pSortPhyNode->pTargets, pDescNode, &numOfOutputCols, COL_MATCH_FROM_SLOT_ID); + extractColMatchInfo(pSortNode->pTargets, pDescNode, &numOfOutputCols, COL_MATCH_FROM_SLOT_ID); pOperator->exprSupp.pCtx = createSqlFunctionCtx(pExprInfo, numOfCols, &pOperator->exprSupp.rowEntryInfoOffset); pInfo->binfo.pRes = pResBlock; initResultSizeInfo(pOperator, 1024); - pInfo->pSortInfo = createSortInfo(pSortPhyNode->pSortKeys); - pInfo->pCondition = pSortPhyNode->node.pConditions; + pInfo->pSortInfo = createSortInfo(pSortNode->pSortKeys); + pInfo->pCondition = pSortNode->node.pConditions; pInfo->pColMatchInfo = pColMatchColInfo; pOperator->name = "SortOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SORT; @@ -234,9 +234,9 @@ void destroyOrderOperatorInfo(void* param, int32_t numOfOutput) { SSortOperatorInfo* pInfo = (SSortOperatorInfo*)param; pInfo->binfo.pRes = blockDataDestroy(pInfo->binfo.pRes); + tsortDestroySortHandle(pInfo->pSortHandle); taosArrayDestroy(pInfo->pSortInfo); taosArrayDestroy(pInfo->pColMatchInfo); - taosMemoryFreeClear(param); } @@ -539,14 +539,12 @@ int32_t doOpenMultiwayMergeOperator(SOperatorInfo* pOperator) { } pInfo->startTs = taosGetTimestampUs(); - int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_MULTISOURCE_MERGE, pInfo->bufPageSize, numOfBufPage, pInfo->pInputBlock, pTaskInfo->id.str); tsortSetFetchRawDataFp(pInfo->pSortHandle, loadNextDataBlock, NULL, NULL); - tsortSetCompareGroupId(pInfo->pSortHandle, pInfo->groupSort); for (int32_t i = 0; i < pOperator->numOfDownstream; ++i) { @@ -556,7 +554,6 @@ int32_t doOpenMultiwayMergeOperator(SOperatorInfo* pOperator) { } int32_t code = tsortOpen(pInfo->pSortHandle); - if (code != TSDB_CODE_SUCCESS) { longjmp(pTaskInfo->env, terrno); } @@ -674,6 +671,7 @@ void destroyMultiwayMergeOperatorInfo(void* param, int32_t numOfOutput) { pInfo->binfo.pRes = blockDataDestroy(pInfo->binfo.pRes); pInfo->pInputBlock = blockDataDestroy(pInfo->pInputBlock); + tsortDestroySortHandle(pInfo->pSortHandle); taosArrayDestroy(pInfo->pSortInfo); taosArrayDestroy(pInfo->pColMatchInfo); diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 71eb221c64..0feb4430bc 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1353,13 +1353,13 @@ static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, if (chIds && pPullDataMap) { SArray* chAy = *(SArray**)chIds; int32_t size = taosArrayGetSize(chAy); - qDebug("window %" PRId64 " wait child size:%d", win.skey, size); + qDebug("===stream===window %" PRId64 " wait child size:%d", win.skey, size); for (int32_t i = 0; i < size; i++) { - qDebug("window %" PRId64 " wait child id:%d", win.skey, *(int32_t*)taosArrayGet(chAy, i)); + qDebug("===stream===window %" PRId64 " wait child id:%d", win.skey, *(int32_t*)taosArrayGet(chAy, i)); } continue; } else if (pPullDataMap) { - qDebug("close window %" PRId64, win.skey); + qDebug("===stream===close window %" PRId64, win.skey); } SResultRowPosition* pPos = (SResultRowPosition*)pIte; if (pSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { @@ -1513,12 +1513,25 @@ static void destroyStateWindowOperatorInfo(void* param, int32_t numOfOutput) { taosMemoryFreeClear(param); } +static void freeItem(void* param) { + SGroupKeys *pKey = (SGroupKeys*) param; + taosMemoryFree(pKey->pData); +} + void destroyIntervalOperatorInfo(void* param, int32_t numOfOutput) { SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)param; cleanupBasicInfo(&pInfo->binfo); cleanupAggSup(&pInfo->aggSup); - taosArrayDestroy(pInfo->pRecycledPages); + pInfo->pRecycledPages = taosArrayDestroy(pInfo->pRecycledPages); + pInfo->pInterpCols = taosArrayDestroy(pInfo->pInterpCols); + taosArrayDestroyEx(pInfo->pPrevValues, freeItem); + pInfo->pPrevValues = NULL; + pInfo->pDelWins = taosArrayDestroy(pInfo->pDelWins); + pInfo->pDelRes = blockDataDestroy(pInfo->pDelRes); + + cleanupGroupResInfo(&pInfo->groupResInfo); + colDataDestroy(&pInfo->twAggSup.timeWindowData); taosMemoryFreeClear(param); } @@ -2482,7 +2495,9 @@ static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBloc SPullWindowInfo pull = {.window = nextWin, .groupId = tableGroupId}; // add pull data request taosArrayPush(pInfo->pPullWins, &pull); - addPullWindow(pInfo->pPullDataMap, &winRes, taosArrayGetSize(pInfo->pChildren)); + int32_t size = taosArrayGetSize(pInfo->pChildren); + addPullWindow(pInfo->pPullDataMap, &winRes, size); + qDebug("===stream===prepare retrive %" PRId64 ", size:%d", winRes.ts, size); } else { int32_t index = -1; SArray* chArray = NULL; @@ -2492,14 +2507,14 @@ static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBloc chId = getChildIndex(pSDataBlock); index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ); } - if (index != -1 && pSDataBlock->info.type == STREAM_PULL_DATA) { - qDebug("======delete child id %d", chId); - taosArrayRemove(chArray, index); - if (taosArrayGetSize(chArray) == 0) { - // pull data is over - taosHashRemove(pInfo->pPullDataMap, &winRes, sizeof(SWinRes)); - } - } + // if (index != -1 && pSDataBlock->info.type == STREAM_PULL_DATA) { + // qDebug("===stream===delete child id %d", chId); + // taosArrayRemove(chArray, index); + // if (taosArrayGetSize(chArray) == 0) { + // // pull data is over + // taosHashRemove(pInfo->pPullDataMap, &winRes, sizeof(SWinRes)); + // } + // } if (index == -1 || pSDataBlock->info.type == STREAM_PULL_DATA) { ignore = false; } @@ -2623,6 +2638,7 @@ void processPullOver(SSDataBlock* pBlock, SHashObj* pMap) { SArray* chArray = *(SArray**)chIds; int32_t index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ); if (index != -1) { + qDebug("===stream===window %" PRId64 " delete child id %d", winRes.ts, chId); taosArrayRemove(chArray, index); if (taosArrayGetSize(chArray) == 0) { // pull data is over @@ -2641,7 +2657,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { SExprSupp* pSup = &pOperator->exprSupp; - qDebug("interval status %d %s", pOperator->status, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); + qDebug("interval status %d %s", pOperator->status, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); if (pOperator->status == OP_EXEC_DONE) { return NULL; @@ -2657,18 +2673,18 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { } return NULL; } - printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); + printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); return pInfo->binfo.pRes; } else { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); if (pInfo->binfo.pRes->info.rows != 0) { - printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); + printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); return pInfo->binfo.pRes; } if (pInfo->pUpdateRes->info.rows != 0 && pInfo->returnUpdate) { pInfo->returnUpdate = false; ASSERT(!IS_FINAL_OP(pInfo)); - printDataBlock(pInfo->pUpdateRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); + printDataBlock(pInfo->pUpdateRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); // process the rest of the data return pInfo->pUpdateRes; } @@ -2676,13 +2692,13 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { if (pInfo->pPullDataRes->info.rows != 0) { // process the rest of the data ASSERT(IS_FINAL_OP(pInfo)); - printDataBlock(pInfo->pPullDataRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); + printDataBlock(pInfo->pPullDataRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); return pInfo->pPullDataRes; } doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes); if (pInfo->pDelRes->info.rows != 0) { // process the rest of the data - printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); + printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); return pInfo->pDelRes; } } @@ -2693,10 +2709,10 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { clearSpecialDataBlock(pInfo->pUpdateRes); removeDeleteResults(pUpdated, pInfo->pDelWins); pOperator->status = OP_RES_TO_RETURN; - qDebug("%s return data", IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); + qDebug("%s return data", IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); break; } - printDataBlock(pBlock, IS_FINAL_OP(pInfo) ? "interval Final recv" : "interval Semi recv"); + printDataBlock(pBlock, IS_FINAL_OP(pInfo) ? "interval Final recv" : "interval Semi recv"); maxTs = TMAX(maxTs, pBlock->info.window.ekey); if (pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_PULL_DATA || @@ -2771,6 +2787,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { SStreamFinalIntervalOperatorInfo* pTmpInfo = pChildOp->info; pTmpInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE; taosArrayPush(pInfo->pChildren, &pChildOp); + qDebug("===stream===add child, id:%d", chIndex); } SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, chIndex); SStreamFinalIntervalOperatorInfo* pChInfo = pChildOp->info; @@ -2795,14 +2812,14 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); if (pInfo->binfo.pRes->info.rows != 0) { - printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); + printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); return pInfo->binfo.pRes; } if (pInfo->pUpdateRes->info.rows != 0 && pInfo->returnUpdate) { pInfo->returnUpdate = false; ASSERT(!IS_FINAL_OP(pInfo)); - printDataBlock(pInfo->pUpdateRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); + printDataBlock(pInfo->pUpdateRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); // process the rest of the data return pInfo->pUpdateRes; } @@ -2811,14 +2828,14 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { if (pInfo->pPullDataRes->info.rows != 0) { // process the rest of the data ASSERT(IS_FINAL_OP(pInfo)); - printDataBlock(pInfo->pPullDataRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); + printDataBlock(pInfo->pPullDataRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); return pInfo->pPullDataRes; } doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes); if (pInfo->pDelRes->info.rows != 0) { // process the rest of the data - printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); + printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); return pInfo->pDelRes; } // ASSERT(false); diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 9b15cc7adb..4f525441b9 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -91,7 +91,7 @@ SSortHandle* tsortCreateSortHandle(SArray* pSortInfo, int32_t type, int32_t page tsortSetComparFp(pSortHandle, msortComparFn); if (idstr != NULL) { - pSortHandle->idStr = strdup(idstr); + pSortHandle->idStr = strdup(idstr); } return pSortHandle; @@ -109,6 +109,10 @@ static int32_t sortComparClearup(SMsortComparParam* cmpParam) { } void tsortDestroySortHandle(SSortHandle* pSortHandle) { + if (pSortHandle == NULL) { + return; + } + tsortClose(pSortHandle); if (pSortHandle->pMergeTree != NULL) { tMergeTreeDestroy(pSortHandle->pMergeTree); @@ -119,7 +123,6 @@ void tsortDestroySortHandle(SSortHandle* pSortHandle) { blockDataDestroy(pSortHandle->pDataBlock); for (size_t i = 0; i < taosArrayGetSize(pSortHandle->pOrderedSource); i++){ SSortSource** pSource = taosArrayGet(pSortHandle->pOrderedSource, i); - blockDataDestroy((*pSource)->src.pBlock); taosMemoryFreeClear(*pSource); } taosArrayDestroy(pSortHandle->pOrderedSource); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 185f529963..72eabafa16 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1132,9 +1132,9 @@ static bool validateStateOper(const SValueNode* pVal) { if (TSDB_DATA_TYPE_BINARY != pVal->node.resType.type) { return false; } - return (0 == strcasecmp(varDataVal(pVal->datum.p), "GT") || 0 == strcasecmp(varDataVal(pVal->datum.p), "GE") || - 0 == strcasecmp(varDataVal(pVal->datum.p), "LT") || 0 == strcasecmp(varDataVal(pVal->datum.p), "LE") || - 0 == strcasecmp(varDataVal(pVal->datum.p), "EQ") || 0 == strcasecmp(varDataVal(pVal->datum.p), "NE")); + return (0 == strncasecmp(varDataVal(pVal->datum.p), "GT", 2) || 0 == strncasecmp(varDataVal(pVal->datum.p), "GE", 2) || + 0 == strncasecmp(varDataVal(pVal->datum.p), "LT", 2) || 0 == strncasecmp(varDataVal(pVal->datum.p), "LE", 2) || + 0 == strncasecmp(varDataVal(pVal->datum.p), "EQ", 2) || 0 == strncasecmp(varDataVal(pVal->datum.p), "NE", 2)); } static int32_t translateStateCount(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { @@ -2331,6 +2331,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getHistogramFuncEnv, .initFunc = histogramFunctionSetup, .processFunc = histogramFunction, + .sprocessFunc = histogramScalarFunction, .finalizeFunc = histogramFinalize, .invertFunc = NULL, .combineFunc = histogramCombine, @@ -2408,6 +2409,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getDiffFuncEnv, .initFunc = diffFunctionSetup, .processFunc = diffFunction, + .sprocessFunc = diffScalarFunction, .finalizeFunc = functionFinalize }, { @@ -2418,6 +2420,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getStateFuncEnv, .initFunc = functionSetup, .processFunc = stateCountFunction, + .sprocessFunc = stateCountScalarFunction, .finalizeFunc = NULL }, { @@ -2428,6 +2431,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .getEnvFunc = getStateFuncEnv, .initFunc = functionSetup, .processFunc = stateDurationFunction, + .sprocessFunc = stateDurationScalarFunction, .finalizeFunc = NULL }, { diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 12b796c5ca..4f829f2df1 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -2723,6 +2723,9 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { int32_t blockDataOrder = (startKey <= endKey) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; + // please ref. to the comment in lastRowFunction for the reason why disabling the opt version of last/first function. + // we will use this opt implementation in an new version that is only available in scan subplan +#if 0 if (blockDataOrder == TSDB_ORDER_ASC) { // filter according to current result firstly if (pResInfo->numOfRes > 0) { @@ -2770,6 +2773,22 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { } } } +#else + for (int32_t i = pInput->startRowIndex; i < pInput->startRowIndex + pInput->numOfRows; ++i) { + if (pInputCol->hasNull && colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) { + continue; + } + + numOfElems++; + + char* data = colDataGetData(pInputCol, i); + TSKEY cts = getRowPTs(pInput->pPTS, i); + if (pResInfo->numOfRes == 0 || pInfo->ts > cts) { + doSaveCurrentVal(pCtx, i, cts, pInputCol->info.type, data); + pResInfo->numOfRes = 1; + } + } +#endif SET_VAL(pResInfo, numOfElems, 1); return TSDB_CODE_SUCCESS; @@ -2801,6 +2820,8 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { int32_t blockDataOrder = (startKey <= endKey) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; + // please ref. to the comment in lastRowFunction for the reason why disabling the opt version of last/first function. +#if 0 if (blockDataOrder == TSDB_ORDER_ASC) { for (int32_t i = pInput->numOfRows + pInput->startRowIndex - 1; i >= pInput->startRowIndex; --i) { if (pInputCol->hasNull && colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) { @@ -2833,6 +2854,22 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { break; } } +#else + for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { + if (pInputCol->hasNull && colDataIsNull(pInputCol, pInput->totalRows, i, pColAgg)) { + continue; + } + + numOfElems++; + + char* data = colDataGetData(pInputCol, i); + TSKEY cts = getRowPTs(pInput->pPTS, i); + if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { + doSaveCurrentVal(pCtx, i, cts, type, data); + pResInfo->numOfRes = 1; + } + } +#endif SET_VAL(pResInfo, numOfElems, 1); return TSDB_CODE_SUCCESS; @@ -2988,6 +3025,9 @@ int32_t lastRowFunction(SqlFunctionCtx* pCtx) { int32_t blockDataOrder = (startKey <= endKey) ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; +#if 0 + // the optimized version only function if all tuples in one block are monotonious increasing or descreasing. + // this is NOT always works if project operator exists in downstream. if (blockDataOrder == TSDB_ORDER_ASC) { for (int32_t i = pInput->numOfRows + pInput->startRowIndex - 1; i >= pInput->startRowIndex; --i) { char* data = colDataGetData(pInputCol, i); @@ -2997,6 +3037,7 @@ int32_t lastRowFunction(SqlFunctionCtx* pCtx) { if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { doSaveLastrow(pCtx, data, i, cts, pInfo); } + break; } } else { // descending order @@ -3011,7 +3052,19 @@ int32_t lastRowFunction(SqlFunctionCtx* pCtx) { break; } } +#else + for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { + char* data = colDataGetData(pInputCol, i); + TSKEY cts = getRowPTs(pInput->pPTS, i); + numOfElems++; + if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { + doSaveLastrow(pCtx, data, i, cts, pInfo); + pResInfo->numOfRes = 1; + } + } + +#endif SET_VAL(pResInfo, numOfElems, 1); return TSDB_CODE_SUCCESS; } @@ -4416,17 +4469,17 @@ bool getStateFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { static int8_t getStateOpType(char* opStr) { int8_t opType; - if (strcasecmp(opStr, "LT") == 0) { + if (strncasecmp(opStr, "LT", 2) == 0) { opType = STATE_OPER_LT; - } else if (strcasecmp(opStr, "GT") == 0) { + } else if (strncasecmp(opStr, "GT", 2) == 0) { opType = STATE_OPER_GT; - } else if (strcasecmp(opStr, "LE") == 0) { + } else if (strncasecmp(opStr, "LE", 2) == 0) { opType = STATE_OPER_LE; - } else if (strcasecmp(opStr, "GE") == 0) { + } else if (strncasecmp(opStr, "GE", 2) == 0) { opType = STATE_OPER_GE; - } else if (strcasecmp(opStr, "NE") == 0) { + } else if (strncasecmp(opStr, "NE", 2) == 0) { opType = STATE_OPER_NE; - } else if (strcasecmp(opStr, "EQ") == 0) { + } else if (strncasecmp(opStr, "EQ", 2) == 0) { opType = STATE_OPER_EQ; } else { opType = STATE_OPER_INVALID; @@ -5926,6 +5979,7 @@ int32_t cachedLastRowFunction(SqlFunctionCtx* pCtx) { TSKEY cts = getRowPTs(pInput->pPTS, i); if (pResInfo->numOfRes == 0 || pInfo->ts < cts) { doSaveLastrow(pCtx, data, i, cts, pInfo); + pResInfo->numOfRes = 1; } } diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index c16c3e3937..6ad552576c 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -855,6 +855,7 @@ int32_t convertDataBlockToScalarParm(SSDataBlock *input, SScalarParam *output) { memcpy(output->columnData, taosArrayGet(input->pDataBlock, 0), sizeof(SColumnInfoData)); + output->colAlloced = true; return 0; } diff --git a/source/libs/index/src/indexFstDfa.c b/source/libs/index/src/indexFstDfa.c index 046ed0f4f4..3a36010b42 100644 --- a/source/libs/index/src/indexFstDfa.c +++ b/source/libs/index/src/indexFstDfa.c @@ -19,8 +19,8 @@ const static uint32_t STATE_LIMIT = 1000; static int dfaInstsEqual(const void *a, const void *b, size_t size) { - SArray *ar = (SArray *)a; - SArray *br = (SArray *)b; + SArray *ar = *(SArray **)a; + SArray *br = *(SArray **)b; size_t al = ar != NULL ? taosArrayGetSize(ar) : 0; size_t bl = br != NULL ? taosArrayGetSize(br) : 0; if (al != bl) { @@ -71,8 +71,8 @@ FstDfa *dfaBuilderBuild(FstDfaBuilder *builder) { dfaAdd(builder->dfa, cur, 0); - SArray *states = taosArrayInit(0, sizeof(uint32_t)); uint32_t result; + SArray *states = taosArrayInit(0, sizeof(uint32_t)); if (dfaBuilderCacheState(builder, cur, &result)) { taosArrayPush(states, &result); } @@ -146,10 +146,9 @@ bool dfaBuilderCacheState(FstDfaBuilder *builder, FstSparseSet *set, uint32_t *r *result = *v; taosArrayDestroy(tinsts); } else { - DfaState st; - st.insts = tinsts; - st.isMatch = isMatch; + DfaState st = {.insts = tinsts, .isMatch = isMatch}; taosArrayPush(builder->dfa->states, &st); + int32_t sz = taosArrayGetSize(builder->dfa->states) - 1; taosHashPut(builder->cache, &tinsts, sizeof(POINTER_BYTES), &sz, sizeof(sz)); *result = sz; diff --git a/source/libs/index/src/indexFstFile.c b/source/libs/index/src/indexFstFile.c index 6036a06eaa..4f278c7af6 100644 --- a/source/libs/index/src/indexFstFile.c +++ b/source/libs/index/src/indexFstFile.c @@ -85,11 +85,12 @@ static int idxFileCtxDoReadFrom(IFileCtx* ctx, uint8_t* buf, int len, int32_t of blk->blockId = blkId; blk->nread = taosPReadFile(ctx->file.pFile, blk->buf, kBlockSize, blkId * kBlockSize); assert(blk->nread <= kBlockSize); - nread = TMIN(blkLeft, len); if (blk->nread < kBlockSize && blk->nread < len) { break; } + + nread = TMIN(blkLeft, len); memcpy(buf + total, blk->buf + blkOffset, nread); LRUStatus s = taosLRUCacheInsert(ctx->lru, key, strlen(key), blk, cacheMemSize, deleteDataBlockFromLRU, NULL, diff --git a/source/libs/index/src/indexFstSparse.c b/source/libs/index/src/indexFstSparse.c index 60eb7afd90..ebc0cb3637 100644 --- a/source/libs/index/src/indexFstSparse.c +++ b/source/libs/index/src/indexFstSparse.c @@ -78,8 +78,8 @@ bool sparSetContains(FstSparseSet *ss, int32_t ip) { if (ip >= ss->cap || ip < 0) { return false; } - int32_t i = ss->sparse[ip]; + int32_t i = ss->sparse[ip]; if (i >= 0 && i < ss->cap && i < ss->size && ss->dense[i] == ip) { return true; } else { diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 7265e7ee78..1dc3db033b 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -952,6 +952,7 @@ void nodesDestroyNode(SNode* pNode) { case QUERY_NODE_PHYSICAL_PLAN_QUERY_INSERT: { SQueryInserterNode* pSink = (SQueryInserterNode*)pNode; destroyDataSinkNode((SDataSinkNode*)pSink); + nodesDestroyList(pSink->pCols); break; } case QUERY_NODE_PHYSICAL_PLAN_DELETE: { @@ -1543,7 +1544,7 @@ typedef struct SCollectFuncsCxt { int32_t errCode; FFuncClassifier classifier; SNodeList* pFuncs; - SHashObj* pAliasName; + SHashObj* pFuncsSet; } SCollectFuncsCxt; static EDealRes collectFuncs(SNode* pNode, void* pContext) { @@ -1551,28 +1552,40 @@ static EDealRes collectFuncs(SNode* pNode, void* pContext) { if (QUERY_NODE_FUNCTION == nodeType(pNode) && pCxt->classifier(((SFunctionNode*)pNode)->funcId) && !(((SExprNode*)pNode)->orderAlias)) { SExprNode* pExpr = (SExprNode*)pNode; - if (NULL == taosHashGet(pCxt->pAliasName, pExpr->aliasName, strlen(pExpr->aliasName))) { + if (NULL == taosHashGet(pCxt->pFuncsSet, &pExpr, POINTER_BYTES)) { pCxt->errCode = nodesListStrictAppend(pCxt->pFuncs, nodesCloneNode(pNode)); - taosHashPut(pCxt->pAliasName, pExpr->aliasName, strlen(pExpr->aliasName), &pExpr, POINTER_BYTES); + taosHashPut(pCxt->pFuncsSet, &pExpr, POINTER_BYTES, &pExpr, POINTER_BYTES); } return (TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR); } return DEAL_RES_CONTINUE; } +static uint32_t funcNodeHash(const char* pKey, uint32_t len) { + SExprNode* pExpr = *(SExprNode**)pKey; + return MurmurHash3_32(pExpr->aliasName, strlen(pExpr->aliasName)); +} + +static int32_t funcNodeEqual(const void* pLeft, const void* pRight, size_t len) { + if (0 != strcmp((*(const SExprNode**)pLeft)->aliasName, (*(const SExprNode**)pRight)->aliasName)) { + return 1; + } + return nodesEqualNode(*(const SNode**)pLeft, *(const SNode**)pRight) ? 0 : 1; +} + int32_t nodesCollectFuncs(SSelectStmt* pSelect, ESqlClause clause, FFuncClassifier classifier, SNodeList** pFuncs) { if (NULL == pSelect || NULL == pFuncs) { return TSDB_CODE_FAILED; } - SCollectFuncsCxt cxt = { - .errCode = TSDB_CODE_SUCCESS, - .classifier = classifier, - .pFuncs = (NULL == *pFuncs ? nodesMakeList() : *pFuncs), - .pAliasName = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), false, false)}; - if (NULL == cxt.pFuncs) { + SCollectFuncsCxt cxt = {.errCode = TSDB_CODE_SUCCESS, + .classifier = classifier, + .pFuncs = (NULL == *pFuncs ? nodesMakeList() : *pFuncs), + .pFuncsSet = taosHashInit(4, funcNodeHash, false, false)}; + if (NULL == cxt.pFuncs || NULL == cxt.pFuncsSet) { return TSDB_CODE_OUT_OF_MEMORY; } + taosHashSetEqualFp(cxt.pFuncsSet, funcNodeEqual); *pFuncs = NULL; nodesWalkSelectStmt(pSelect, clause, collectFuncs, &cxt); if (TSDB_CODE_SUCCESS == cxt.errCode) { @@ -1584,7 +1597,7 @@ int32_t nodesCollectFuncs(SSelectStmt* pSelect, ESqlClause clause, FFuncClassifi } else { nodesDestroyList(cxt.pFuncs); } - taosHashCleanup(cxt.pAliasName); + taosHashCleanup(cxt.pFuncsSet); return cxt.errCode; } diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index d24da2f83c..fd518f89af 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -371,6 +371,19 @@ static int32_t collectMetaKeyFromShowTables(SCollectMetaKeyCxt* pCxt, SShowStmt* return code; } +static int32_t collectMetaKeyFromShowTags(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) { + int32_t code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, + TSDB_INS_TABLE_USER_TAGS, pCxt->pMetaCache); + if (TSDB_CODE_SUCCESS == code) { + if (NULL != pStmt->pDbName) { + code = reserveDbVgInfoInCache(pCxt->pParseCxt->acctId, ((SValueNode*)pStmt->pDbName)->literal, pCxt->pMetaCache); + } else { + code = reserveDbVgInfoInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, pCxt->pMetaCache); + } + } + return code; +} + static int32_t collectMetaKeyFromShowUsers(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) { return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_USER_USERS, pCxt->pMetaCache); @@ -537,6 +550,8 @@ static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt) { return collectMetaKeyFromShowStreams(pCxt, (SShowStmt*)pStmt); case QUERY_NODE_SHOW_TABLES_STMT: return collectMetaKeyFromShowTables(pCxt, (SShowStmt*)pStmt); + case QUERY_NODE_SHOW_TAGS_STMT: + return collectMetaKeyFromShowTags(pCxt, (SShowStmt*)pStmt); case QUERY_NODE_SHOW_USERS_STMT: return collectMetaKeyFromShowUsers(pCxt, (SShowStmt*)pStmt); case QUERY_NODE_SHOW_LICENCE_STMT: diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 8998971c74..19126b28f1 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -868,7 +868,8 @@ static EDealRes translateNormalValue(STranslateContext* pCxt, SValueNode* pVal, } case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARBINARY: { - if (strict && (pVal->node.resType.bytes > targetDt.bytes - VARSTR_HEADER_SIZE)) { + if (strict && (!IS_VAR_DATA_TYPE(pVal->node.resType.type) || + pVal->node.resType.bytes > targetDt.bytes - VARSTR_HEADER_SIZE)) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); } pVal->datum.p = taosMemoryCalloc(1, targetDt.bytes + 1); @@ -888,6 +889,9 @@ static EDealRes translateNormalValue(STranslateContext* pCxt, SValueNode* pVal, break; } case TSDB_DATA_TYPE_NCHAR: { + if (strict && !IS_VAR_DATA_TYPE(pVal->node.resType.type)) { + return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal); + } pVal->datum.p = taosMemoryCalloc(1, targetDt.bytes + 1); if (NULL == pVal->datum.p) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_OUT_OF_MEMORY); @@ -1168,7 +1172,7 @@ static int32_t translateRepeatScanFunc(STranslateContext* pCxt, SFunctionNode* p return TSDB_CODE_SUCCESS; } if (isSelectStmt(pCxt->pCurrStmt)) { - //select percentile() without from clause is also valid + // select percentile() without from clause is also valid if (NULL == ((SSelectStmt*)pCxt->pCurrStmt)->pFromTable) { return TSDB_CODE_SUCCESS; } @@ -1681,7 +1685,8 @@ static int32_t dnodeToVgroupsInfo(SArray* pDnodes, SVgroupsInfo** pVgsInfo) { static bool sysTableFromVnode(const char* pTable) { return (0 == strcmp(pTable, TSDB_INS_TABLE_USER_TABLES)) || - (0 == strcmp(pTable, TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED)); + (0 == strcmp(pTable, TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED) || + (0 == strcmp(pTable, TSDB_INS_TABLE_USER_TAGS))); } static bool sysTableFromDnode(const char* pTable) { return 0 == strcmp(pTable, TSDB_INS_TABLE_DNODE_VARIABLES); } @@ -1697,7 +1702,7 @@ static int32_t setVnodeSysTableVgroupList(STranslateContext* pCxt, SName* pName, code = getDBVgInfoImpl(pCxt, pName, &vgroupList); } - if (TSDB_CODE_SUCCESS == code) { + if (TSDB_CODE_SUCCESS == code && 0 == strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_USER_TABLES)) { code = addMnodeToVgroupList(&pCxt->pParseCxt->mgmtEpSet, &vgroupList); } @@ -1786,7 +1791,8 @@ static bool isSingleTable(SRealTableNode* pRealTable) { int8_t tableType = pRealTable->pMeta->tableType; if (TSDB_SYSTEM_TABLE == tableType) { return 0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_USER_TABLES) && - 0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED); + 0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED) && + 0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_USER_TAGS); } return (TSDB_CHILD_TABLE == tableType || TSDB_NORMAL_TABLE == tableType); } @@ -5059,6 +5065,8 @@ static const char* getSysTableName(ENodeType type) { return TSDB_INS_TABLE_USER_DATABASES; case QUERY_NODE_SHOW_TABLES_STMT: return TSDB_INS_TABLE_USER_TABLES; + case QUERY_NODE_SHOW_TAGS_STMT: + return TSDB_INS_TABLE_USER_TAGS; case QUERY_NODE_SHOW_STABLES_STMT: return TSDB_INS_TABLE_USER_STABLES; case QUERY_NODE_SHOW_USERS_STMT: diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index e0b020ce21..4903beddbe 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -282,7 +282,7 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect pScan->hasNormalCols = true; } - if (TSDB_CODE_SUCCESS == code) { + if (TSDB_CODE_SUCCESS == code && SCAN_TYPE_SYSTEM_TABLE != pScan->scanType) { code = addPrimaryKeyCol(pScan->tableId, &pScan->pScanCols); } diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index cdbe74bf04..4473c7c8b6 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1050,8 +1050,11 @@ static int32_t sortPriKeyOptApply(SOptimizeContext* pCxt, SLogicSubplan* pLogicS } } - int32_t code = - replaceLogicNode(pLogicSubplan, (SLogicNode*)pSort, (SLogicNode*)nodesListGetNode(pSort->node.pChildren, 0)); + SLogicNode* pChild = (SLogicNode*)nodesListGetNode(pSort->node.pChildren, 0); + if (NULL == pSort->node.pParent) { + TSWAP(pSort->node.pTargets, pChild->pTargets); + } + int32_t code = replaceLogicNode(pLogicSubplan, (SLogicNode*)pSort, pChild); if (TSDB_CODE_SUCCESS == code) { NODES_CLEAR_LIST(pSort->node.pChildren); nodesDestroyNode((SNode*)pSort); @@ -1982,11 +1985,15 @@ static int32_t rewriteUniqueOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLog } static bool lastRowScanOptMayBeOptimized(SLogicNode* pNode) { - if (QUERY_NODE_LOGIC_PLAN_AGG != nodeType(pNode) || !(((SAggLogicNode*)pNode)->hasLastRow) || - NULL != ((SAggLogicNode*)pNode)->pGroupKeys || 1 != LIST_LENGTH(pNode->pChildren) || - QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(nodesListGetNode(pNode->pChildren, 0)) || - NULL != ((SScanLogicNode*)nodesListGetNode(pNode->pChildren, 0))->node.pConditions || - 0 == ((SScanLogicNode*)nodesListGetNode(pNode->pChildren, 0))->cacheLastMode) { + if (QUERY_NODE_LOGIC_PLAN_AGG != nodeType(pNode) || 1 != LIST_LENGTH(pNode->pChildren) || + QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(nodesListGetNode(pNode->pChildren, 0))) { + return false; + } + + SAggLogicNode* pAgg = (SAggLogicNode*)pNode; + SScanLogicNode* pScan = (SScanLogicNode*)nodesListGetNode(pNode->pChildren, 0); + if (!pAgg->hasLastRow || NULL != pAgg->pGroupKeys || NULL != pScan->node.pConditions || 0 == pScan->cacheLastMode || + IS_TSWINDOW_SPECIFIED(pScan->scanRange)) { return false; } diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 7b4a9c97ad..8250fb462f 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -570,7 +570,8 @@ static int32_t createSystemTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pScan->showRewrite = pScanLogicNode->showRewrite; pScan->accountId = pCxt->pPlanCxt->acctId; if (0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_USER_TABLES) || - 0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED)) { + 0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED) || + 0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_USER_TAGS)) { vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode); } else { pSubplan->execNode.nodeId = MNODE_HANDLE; diff --git a/source/libs/planner/test/planJoinTest.cpp b/source/libs/planner/test/planJoinTest.cpp index 8c321e75be..66ef4d3f19 100644 --- a/source/libs/planner/test/planJoinTest.cpp +++ b/source/libs/planner/test/planJoinTest.cpp @@ -45,6 +45,12 @@ TEST_F(PlanJoinTest, withWhere) { "WHERE t1.c1 > t2.c1 AND t1.c2 = 'abc' AND t2.c2 = 'qwe'"); } +TEST_F(PlanJoinTest, withAggAndOrderBy) { + useDb("root", "test"); + + run("SELECT t1.ts, TOP(t2.c1, 10) FROM st1s1 t1 JOIN st1s2 t2 ON t1.ts = t2.ts ORDER BY t2.ts"); +} + TEST_F(PlanJoinTest, multiJoin) { useDb("root", "test"); diff --git a/source/libs/planner/test/planOptimizeTest.cpp b/source/libs/planner/test/planOptimizeTest.cpp index 75b4aa0dc1..770ac94e5b 100644 --- a/source/libs/planner/test/planOptimizeTest.cpp +++ b/source/libs/planner/test/planOptimizeTest.cpp @@ -84,6 +84,12 @@ TEST_F(PlanOptimizeTest, eliminateProjection) { // run("select 1-abs(c1) from (select unique(c1) c1 from st1s3) order by 1 nulls first"); } +TEST_F(PlanOptimizeTest, mergeProjects) { + useDb("root", "test"); + + run("SELECT * FROM (SELECT * FROM t1 WHERE c1 > 10 ORDER BY ts) ORDER BY ts"); +} + TEST_F(PlanOptimizeTest, pushDownProjectCond) { useDb("root", "test"); run("select 1-abs(c1) from (select unique(c1) c1 from st1s3) where 1-c1>5 order by 1 nulls first"); diff --git a/source/libs/qworker/inc/qwInt.h b/source/libs/qworker/inc/qwInt.h index d8d7c5a0ea..c8e5204e91 100644 --- a/source/libs/qworker/inc/qwInt.h +++ b/source/libs/qworker/inc/qwInt.h @@ -75,7 +75,9 @@ typedef struct SQWDebug { bool lockEnable; bool statusEnable; bool dumpEnable; - bool tmp; + bool sleepSimulate; + bool deadSimulate; + bool redirectSimulate; } SQWDebug; extern SQWDebug gQWDebug; @@ -130,12 +132,11 @@ typedef struct SQWTaskCtx { int8_t taskType; int8_t explain; int8_t needFetch; - int32_t queryType; + int32_t msgType; int32_t fetchType; int32_t execId; bool queryRsped; - bool queryFetched; bool queryEnd; bool queryContinue; bool queryInQueue; @@ -228,6 +229,7 @@ typedef struct SQWorkerMgmt { #define QW_SET_EVENT_PROCESSED(ctx, event) atomic_store_8(&(ctx)->events[event], QW_EVENT_PROCESSED) #define QW_GET_PHASE(ctx) atomic_load_8(&(ctx)->phase) +#define QW_SET_PHASE(ctx, _value) do { if ((_value) != QW_PHASE_PRE_FETCH && (_value) != QW_PHASE_POST_FETCH) { atomic_store_8(&(ctx)->phase, _value); } } while (0) #define QW_SET_RSP_CODE(ctx, code) atomic_store_32(&(ctx)->rspCode, code) #define QW_UPDATE_RSP_CODE(ctx, code) atomic_val_compare_exchange_32(&(ctx)->rspCode, 0, code) @@ -362,7 +364,7 @@ int32_t qwAcquireTaskCtx(QW_FPARAMS_DEF, SQWTaskCtx **ctx); int32_t qwGetTaskCtx(QW_FPARAMS_DEF, SQWTaskCtx **ctx); int32_t qwAddAcquireTaskCtx(QW_FPARAMS_DEF, SQWTaskCtx **ctx); void qwReleaseTaskCtx(SQWorker *mgmt, void *ctx); -int32_t qwKillTaskHandle(QW_FPARAMS_DEF, SQWTaskCtx *ctx); +int32_t qwKillTaskHandle(SQWTaskCtx *ctx); int32_t qwUpdateTaskStatus(QW_FPARAMS_DEF, int8_t status); int32_t qwDropTask(QW_FPARAMS_DEF); void qwSaveTbVersionInfo(qTaskInfo_t pTaskInfo, SQWTaskCtx *ctx); @@ -372,13 +374,15 @@ int32_t qwUpdateTimeInQueue(SQWorker *mgmt, int64_t ts, EQueueType type); int64_t qwGetTimeInQueue(SQWorker *mgmt, EQueueType type); void qwClearExpiredSch(SQWorker *mgmt, SArray* pExpiredSch); int32_t qwAcquireScheduler(SQWorker *mgmt, uint64_t sId, int32_t rwType, SQWSchStatus **sch); -void qwFreeTaskCtx(QW_FPARAMS_DEF, SQWTaskCtx *ctx); +void qwFreeTaskCtx(SQWTaskCtx *ctx); void qwDbgDumpMgmtInfo(SQWorker *mgmt); int32_t qwDbgValidateStatus(QW_FPARAMS_DEF, int8_t oriStatus, int8_t newStatus, bool *ignore); int32_t qwDbgBuildAndSendRedirectRsp(int32_t rspType, SRpcHandleInfo *pConn, int32_t code, SEpSet *pEpSet); int32_t qwAddTaskCtx(QW_FPARAMS_DEF); -int32_t qwDbgResponseRedirect(SQWMsg *qwMsg, SQWTaskCtx *ctx); +void qwDbgSimulateRedirect(SQWMsg *qwMsg, SQWTaskCtx *ctx, bool *rsped); +void qwDbgSimulateSleep(void); +void qwDbgSimulateDead(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *rsped); #ifdef __cplusplus diff --git a/source/libs/qworker/inc/qwMsg.h b/source/libs/qworker/inc/qwMsg.h index acb7004a51..5378934343 100644 --- a/source/libs/qworker/inc/qwMsg.h +++ b/source/libs/qworker/inc/qwMsg.h @@ -40,11 +40,13 @@ void qwBuildFetchRsp(void *msg, SOutputData *input, int32_t len, bool qComple int32_t qwBuildAndSendCQueryMsg(QW_FPARAMS_DEF, SRpcHandleInfo *pConn); int32_t qwBuildAndSendQueryRsp(int32_t rspType, SRpcHandleInfo *pConn, int32_t code, SQWTaskCtx *ctx); int32_t qwBuildAndSendExplainRsp(SRpcHandleInfo *pConn, SExplainExecInfo *execInfo, int32_t num); +int32_t qwBuildAndSendErrorRsp(int32_t rspType, SRpcHandleInfo *pConn, int32_t code); void qwFreeFetchRsp(void *msg); int32_t qwMallocFetchRsp(int32_t length, SRetrieveTableRsp **rsp); int32_t qwBuildAndSendHbRsp(SRpcHandleInfo *pConn, SSchedulerHbRsp *rsp, int32_t code); int32_t qwRegisterQueryBrokenLinkArg(QW_FPARAMS_DEF, SRpcHandleInfo *pConn); int32_t qwRegisterHbBrokenLinkArg(SQWorker *mgmt, uint64_t sId, SRpcHandleInfo *pConn); +int32_t qwBuildAndSendDropMsg(QW_FPARAMS_DEF, SRpcHandleInfo *pConn); #ifdef __cplusplus } diff --git a/source/libs/qworker/src/qwDbg.c b/source/libs/qworker/src/qwDbg.c index fa63cf2c3a..98d7825b2c 100644 --- a/source/libs/qworker/src/qwDbg.c +++ b/source/libs/qworker/src/qwDbg.c @@ -9,7 +9,7 @@ #include "tmsg.h" #include "tname.h" -SQWDebug gQWDebug = {.statusEnable = true, .dumpEnable = false, .tmp = false}; +SQWDebug gQWDebug = {.statusEnable = true, .dumpEnable = false, .redirectSimulate = false, .deadSimulate = false, .sleepSimulate = false}; int32_t qwDbgValidateStatus(QW_FPARAMS_DEF, int8_t oriStatus, int8_t newStatus, bool *ignore) { if (!gQWDebug.statusEnable) { @@ -147,8 +147,17 @@ int32_t qwDbgBuildAndSendRedirectRsp(int32_t rspType, SRpcHandleInfo *pConn, int return TSDB_CODE_SUCCESS; } -int32_t qwDbgResponseRedirect(SQWMsg *qwMsg, SQWTaskCtx *ctx) { - if (gQWDebug.tmp) { +void qwDbgSimulateRedirect(SQWMsg *qwMsg, SQWTaskCtx *ctx, bool *rsped) { + static int32_t ignoreTime = 0; + if (*rsped) { + return; + } + + if (gQWDebug.redirectSimulate) { + if (++ignoreTime <= 10) { + return; + } + if (TDMT_SCH_QUERY == qwMsg->msgType && (0 == taosRand() % 3)) { SEpSet epSet = {0}; epSet.inUse = 1; @@ -162,42 +171,94 @@ int32_t qwDbgResponseRedirect(SQWMsg *qwMsg, SQWTaskCtx *ctx) { ctx->phase = QW_PHASE_POST_QUERY; qwDbgBuildAndSendRedirectRsp(qwMsg->msgType + 1, &qwMsg->connInfo, TSDB_CODE_RPC_REDIRECT, &epSet); - return TSDB_CODE_SUCCESS; + *rsped = true; + return; } if (TDMT_SCH_MERGE_QUERY == qwMsg->msgType && (0 == taosRand() % 3)) { - ctx->phase = QW_PHASE_POST_QUERY; + QW_SET_PHASE(ctx, QW_PHASE_POST_QUERY); qwDbgBuildAndSendRedirectRsp(qwMsg->msgType + 1, &qwMsg->connInfo, TSDB_CODE_RPC_REDIRECT, NULL); - return TSDB_CODE_SUCCESS; + *rsped = true; + return; + } + + if ((TDMT_SCH_FETCH == qwMsg->msgType) && (0 == taosRand() % 9)) { + qwDbgBuildAndSendRedirectRsp(qwMsg->msgType + 1, &qwMsg->connInfo, TSDB_CODE_RPC_REDIRECT, NULL); + *rsped = true; + return; } } - - return TSDB_CODE_SUCCESS; } +void qwDbgSimulateSleep(void) { + if (!gQWDebug.sleepSimulate) { + return; + } + + static int32_t ignoreTime = 0; + if (++ignoreTime > 10) { + taosSsleep(taosRand() % 20); + } +} + +void qwDbgSimulateDead(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *rsped) { + if (!gQWDebug.deadSimulate) { + return; + } + + if (*rsped) { + return; + } + + static int32_t ignoreTime = 0; + + if (++ignoreTime > 10 && 0 == taosRand() % 9) { + SRpcHandleInfo *pConn = ((ctx->msgType == TDMT_SCH_FETCH || ctx->msgType == TDMT_SCH_MERGE_FETCH) ? &ctx->dataConnInfo : &ctx->ctrlConnInfo); + qwBuildAndSendErrorRsp(ctx->msgType + 1, pConn, TSDB_CODE_RPC_BROKEN_LINK); + + qwBuildAndSendDropMsg(QW_FPARAMS(), pConn); + *rsped = true; + + return; + } +} + + int32_t qwDbgEnableDebug(char *option) { if (0 == strcasecmp(option, "lock")) { gQWDebug.lockEnable = true; - qDebug("qw lock debug enabled"); + qError("qw lock debug enabled"); return TSDB_CODE_SUCCESS; } if (0 == strcasecmp(option, "status")) { gQWDebug.statusEnable = true; - qDebug("qw status debug enabled"); + qError("qw status debug enabled"); return TSDB_CODE_SUCCESS; } if (0 == strcasecmp(option, "dump")) { gQWDebug.dumpEnable = true; - qDebug("qw dump debug enabled"); + qError("qw dump debug enabled"); return TSDB_CODE_SUCCESS; } - if (0 == strcasecmp(option, "tmp")) { - gQWDebug.tmp = true; - qDebug("qw tmp debug enabled"); + if (0 == strcasecmp(option, "sleep")) { + gQWDebug.sleepSimulate = true; + qError("qw sleep debug enabled"); + return TSDB_CODE_SUCCESS; + } + + if (0 == strcasecmp(option, "dead")) { + gQWDebug.sleepSimulate = true; + qError("qw dead debug enabled"); + return TSDB_CODE_SUCCESS; + } + + if (0 == strcasecmp(option, "redirect")) { + gQWDebug.redirectSimulate = true; + qError("qw redirect debug enabled"); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c index 93268e1bcc..b8d9957c30 100644 --- a/source/libs/qworker/src/qwMsg.c +++ b/source/libs/qworker/src/qwMsg.c @@ -43,6 +43,20 @@ void qwFreeFetchRsp(void *msg) { } } +int32_t qwBuildAndSendErrorRsp(int32_t rspType, SRpcHandleInfo *pConn, int32_t code) { + SRpcMsg rpcRsp = { + .msgType = rspType, + .pCont = NULL, + .contLen = 0, + .code = code, + .info = *pConn, + }; + + tmsgSendRsp(&rpcRsp); + + return TSDB_CODE_SUCCESS; +} + int32_t qwBuildAndSendQueryRsp(int32_t rspType, SRpcHandleInfo *pConn, int32_t code, SQWTaskCtx *ctx) { STbVerInfo* tbInfo = ctx ? &ctx->tbInfo : NULL; int64_t affectedRows = ctx ? ctx->affectedRows : 0; @@ -184,7 +198,6 @@ int32_t qwBuildAndSendDropMsg(QW_FPARAMS_DEF, SRpcHandleInfo *pConn) { int32_t code = tmsgPutToQueue(&mgmt->msgCb, FETCH_QUEUE, &pNewMsg); if (TSDB_CODE_SUCCESS != code) { QW_SCH_TASK_ELOG("put drop task msg to queue failed, vgId:%d, code:%s", mgmt->nodeId, tstrerror(code)); - rpcFreeCont(req); QW_ERR_RET(code); } @@ -374,8 +387,7 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int qwMsg.msgInfo.needFetch = msg->needFetch; char * sql = strndup(msg->msg, msg->sqlLen); - QW_SCH_TASK_DLOG("processQuery start, node:%p, type:%s, handle:%p, sql:%s", node, TMSG_INFO(pMsg->msgType), pMsg->info.handle, sql); - + QW_SCH_TASK_DLOG("processQuery start, node:%p, type:%s, handle:%p, SQL:%s", node, TMSG_INFO(pMsg->msgType), pMsg->info.handle, sql); QW_ERR_RET(qwProcessQuery(QW_FPARAMS(), &qwMsg, sql)); QW_SCH_TASK_DLOG("processQuery end, node:%p", node); diff --git a/source/libs/qworker/src/qwUtil.c b/source/libs/qworker/src/qwUtil.c index 9c49cbcb1f..b56cb29628 100644 --- a/source/libs/qworker/src/qwUtil.c +++ b/source/libs/qworker/src/qwUtil.c @@ -270,7 +270,7 @@ int32_t qwAddAcquireTaskCtx(QW_FPARAMS_DEF, SQWTaskCtx **ctx) { return qwAddTask void qwReleaseTaskCtx(SQWorker *mgmt, void *ctx) { taosHashRelease(mgmt->ctxHash, ctx); } -void qwFreeTaskHandle(QW_FPARAMS_DEF, qTaskInfo_t *taskHandle) { +void qwFreeTaskHandle(qTaskInfo_t *taskHandle) { // Note: free/kill may in RC qTaskInfo_t otaskHandle = atomic_load_ptr(taskHandle); if (otaskHandle && atomic_val_compare_exchange_ptr(taskHandle, otaskHandle, NULL)) { @@ -278,7 +278,7 @@ void qwFreeTaskHandle(QW_FPARAMS_DEF, qTaskInfo_t *taskHandle) { } } -int32_t qwKillTaskHandle(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { +int32_t qwKillTaskHandle(SQWTaskCtx *ctx) { int32_t code = 0; // Note: free/kill may in RC qTaskInfo_t taskHandle = atomic_load_ptr(&ctx->taskHandle); @@ -290,7 +290,7 @@ int32_t qwKillTaskHandle(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { QW_RET(code); } -void qwFreeTaskCtx(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { +void qwFreeTaskCtx(SQWTaskCtx *ctx) { if (ctx->ctrlConnInfo.handle) { tmsgReleaseHandle(&ctx->ctrlConnInfo, TAOS_CONN_SERVER); } @@ -300,7 +300,7 @@ void qwFreeTaskCtx(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { // NO need to release dataConnInfo - qwFreeTaskHandle(QW_FPARAMS(), &ctx->taskHandle); + qwFreeTaskHandle(&ctx->taskHandle); if (ctx->sinkHandle) { dsDestroyDataSinker(ctx->sinkHandle); @@ -336,7 +336,7 @@ int32_t qwDropTaskCtx(QW_FPARAMS_DEF) { QW_ERR_RET(TSDB_CODE_QRY_TASK_CTX_NOT_EXIST); } - qwFreeTaskCtx(QW_FPARAMS(), &octx); + qwFreeTaskCtx(&octx); QW_TASK_DLOG_E("task ctx dropped"); @@ -463,13 +463,21 @@ void qwDestroyImpl(void *pMgmt) { mgmt->hbTimer = NULL; taosTmrCleanUp(mgmt->timer); - // TODO STOP ALL QUERY - - // TODO FREE ALL + uint64_t qId, tId; + int32_t eId; + void *pIter = taosHashIterate(mgmt->ctxHash, NULL); + while (pIter) { + SQWTaskCtx *ctx = (SQWTaskCtx *)pIter; + void *key = taosHashGetKey(pIter, NULL); + QW_GET_QTID(key, qId, tId, eId); + qwFreeTaskCtx(ctx); + QW_TASK_DLOG_E("task ctx freed"); + pIter = taosHashIterate(mgmt->ctxHash, pIter); + } taosHashCleanup(mgmt->ctxHash); - void *pIter = taosHashIterate(mgmt->schHash, NULL); + pIter = taosHashIterate(mgmt->schHash, NULL); while (pIter) { SQWSchStatus *sch = (SQWSchStatus *)pIter; qwDestroySchStatus(sch); diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 1b58dc2824..e99695e962 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -83,6 +83,7 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryEnd) { // if *taskHandle is NULL, it's killed right now if (taskHandle) { + qwDbgSimulateSleep(); code = qExecTask(taskHandle, &pRes, &useconds); if (code) { if (code != TSDB_CODE_OPS_NOT_SUPPORT) { @@ -293,11 +294,7 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu QW_LOCK(QW_WRITE, &ctx->lock); - if (QW_PHASE_PRE_FETCH == phase) { - atomic_store_8((int8_t *)&ctx->queryFetched, true); - } else { - atomic_store_8(&ctx->phase, phase); - } + QW_SET_PHASE(ctx, phase); if (atomic_load_8((int8_t *)&ctx->queryEnd)) { QW_TASK_ELOG_E("query already end"); @@ -370,6 +367,7 @@ int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inpu } _return: + if (ctx) { QW_UPDATE_RSP_CODE(ctx, code); @@ -390,7 +388,6 @@ int32_t qwHandlePostPhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inp int32_t code = 0; SQWTaskCtx *ctx = NULL; SRpcHandleInfo connInfo = {0}; - SRpcHandleInfo *rspConnection = NULL; QW_TASK_DLOG("start to handle event at phase %s", qwPhaseStr(phase)); @@ -403,13 +400,6 @@ int32_t qwHandlePostPhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *inp QW_ERR_JRET(TSDB_CODE_QRY_TASK_DROPPED); } - if (QW_PHASE_POST_QUERY == phase) { - connInfo = ctx->ctrlConnInfo; - rspConnection = &connInfo; - - ctx->queryRsped = true; - } - if (QW_EVENT_RECEIVED(ctx, QW_EVENT_DROP)) { if (QW_PHASE_POST_FETCH == phase) { QW_TASK_WLOG("drop received at wrong phase %s", qwPhaseStr(phase)); @@ -437,17 +427,23 @@ _return: qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_PART_SUCC); } - if (rspConnection) { - qwBuildAndSendQueryRsp(input->msgType + 1, rspConnection, code, ctx); - QW_TASK_DLOG("query msg rsped, handle:%p, code:%x - %s", rspConnection->handle, code, tstrerror(code)); + if (QW_PHASE_POST_QUERY == phase && ctx) { + ctx->queryRsped = true; + + bool rsped = false; + SQWMsg qwMsg = {.msgType = ctx->msgType, .connInfo = ctx->ctrlConnInfo}; + qwDbgSimulateRedirect(&qwMsg, ctx, &rsped); + qwDbgSimulateDead(QW_FPARAMS(), ctx, &rsped); + if (!rsped) { + qwBuildAndSendQueryRsp(input->msgType + 1, &ctx->ctrlConnInfo, code, ctx); + QW_TASK_DLOG("query msg rsped, handle:%p, code:%x - %s", ctx->ctrlConnInfo.handle, code, tstrerror(code)); + } } if (ctx) { QW_UPDATE_RSP_CODE(ctx, code); - if (QW_PHASE_POST_FETCH != phase) { - atomic_store_8(&ctx->phase, phase); - } + QW_SET_PHASE(ctx, phase); QW_UNLOCK(QW_WRITE, &ctx->lock); qwReleaseTaskCtx(mgmt, ctx); @@ -488,8 +484,6 @@ int32_t qwPreprocessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { QW_ERR_JRET(qwAddTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_INIT)); - qwDbgResponseRedirect(qwMsg, ctx); - _return: if (ctx) { @@ -517,7 +511,7 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, const char* sql) { ctx->taskType = qwMsg->msgInfo.taskType; ctx->explain = qwMsg->msgInfo.explain; ctx->needFetch = qwMsg->msgInfo.needFetch; - ctx->queryType = qwMsg->msgType; + ctx->msgType = qwMsg->msgType; //QW_TASK_DLOGL("subplan json string, len:%d, %s", qwMsg->msgLen, qwMsg->msg); @@ -636,8 +630,8 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { QW_LOCK(QW_WRITE, &ctx->lock); if (queryEnd || code || 0 == atomic_load_8((int8_t *)&ctx->queryContinue)) { - // Note: if necessary, fetch need to put cquery to queue again - atomic_store_8(&ctx->phase, 0); + // Note: query is not running anymore + QW_SET_PHASE(ctx, 0); QW_UNLOCK(QW_WRITE, &ctx->lock); break; } @@ -662,14 +656,13 @@ int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg) { QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), &ctx)); - ctx->queryType = qwMsg->msgType; + ctx->msgType = qwMsg->msgType; + ctx->dataConnInfo = qwMsg->connInfo; SOutputData sOutput = {0}; QW_ERR_JRET(qwGetQueryResFromSink(QW_FPARAMS(), ctx, &dataLen, &rsp, &sOutput)); if (NULL == rsp) { - ctx->dataConnInfo = qwMsg->connInfo; - QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_FETCH); } else { bool qComplete = (DS_BUF_EMPTY == sOutput.bufStatus && sOutput.queryEnd); @@ -714,9 +707,16 @@ _return: } if (code || rsp) { - qwBuildAndSendFetchRsp(qwMsg->msgType + 1, &qwMsg->connInfo, rsp, dataLen, code); - QW_TASK_DLOG("%s send, handle:%p, code:%x - %s, dataLen:%d", TMSG_INFO(qwMsg->msgType + 1), qwMsg->connInfo.handle, code, tstrerror(code), - dataLen); + bool rsped = false; + if (ctx) { + qwDbgSimulateRedirect(qwMsg, ctx, &rsped); + qwDbgSimulateDead(QW_FPARAMS(), ctx, &rsped); + } + if (!rsped) { + qwBuildAndSendFetchRsp(qwMsg->msgType + 1, &qwMsg->connInfo, rsp, dataLen, code); + QW_TASK_DLOG("%s send, handle:%p, code:%x - %s, dataLen:%d", TMSG_INFO(qwMsg->msgType + 1), qwMsg->connInfo.handle, code, tstrerror(code), + dataLen); + } } QW_RET(TSDB_CODE_SUCCESS); @@ -724,7 +724,7 @@ _return: int32_t qwProcessDrop(QW_FPARAMS_DEF, SQWMsg *qwMsg) { int32_t code = 0; - bool rsped = false; + bool dropped = false; SQWTaskCtx *ctx = NULL; bool locked = false; @@ -740,18 +740,14 @@ int32_t qwProcessDrop(QW_FPARAMS_DEF, SQWMsg *qwMsg) { } if (QW_QUERY_RUNNING(ctx)) { - QW_ERR_JRET(qwKillTaskHandle(QW_FPARAMS(), ctx)); + QW_ERR_JRET(qwKillTaskHandle(ctx)); qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_DROP); - } else if (ctx->phase > 0) { - QW_ERR_JRET(qwDropTask(QW_FPARAMS())); - rsped = true; } else { - // task not started + QW_ERR_JRET(qwDropTask(QW_FPARAMS())); + dropped = true; } - if (!rsped) { - ctx->ctrlConnInfo = qwMsg->connInfo; - + if (!dropped) { QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_DROP); } @@ -954,7 +950,7 @@ int32_t qwProcessDelete(QW_FPARAMS_DEF, SQWMsg *qwMsg, SDeleteRes *pRes) { _return: - qwFreeTaskCtx(QW_FPARAMS(), &ctx); + qwFreeTaskCtx(&ctx); QW_RET(TSDB_CODE_SUCCESS); } diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 484d95cb5a..14406a26ed 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -55,7 +55,7 @@ int32_t sclCreateColumnInfoData(SDataType* pType, int32_t numOfRows, SScalarPara } pParam->columnData = pColumnData; - pParam->type = SHOULD_FREE_COLDATA; + pParam->colAlloced = true; return TSDB_CODE_SUCCESS; } @@ -166,6 +166,10 @@ void sclFreeRes(SHashObj *res) { } void sclFreeParam(SScalarParam *param) { + if (!param->colAlloced) { + return; + } + if (param->columnData != NULL) { colDataDestroy(param->columnData); taosMemoryFreeClear(param->columnData); @@ -173,6 +177,7 @@ void sclFreeParam(SScalarParam *param) { if (param->pHashFilter != NULL) { taosHashCleanup(param->pHashFilter); + param->pHashFilter = NULL; } } @@ -191,6 +196,19 @@ int32_t sclCopyValueNodeValue(SValueNode *pNode, void **res) { return TSDB_CODE_SUCCESS; } +void sclFreeParamList(SScalarParam *param, int32_t paramNum) { + if (NULL == param) { + return; + } + + for (int32_t i = 0; i < paramNum; ++i) { + SScalarParam* p = param + i; + sclFreeParam(p); + } + + taosMemoryFree(param); +} + int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t *rowNum) { switch (nodeType(node)) { case QUERY_NODE_LEFT_VALUE: { @@ -225,11 +243,14 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t SCL_ERR_RET(scalarGenerateSetFromList((void **)¶m->pHashFilter, node, type)); param->hashValueType = type; + param->colAlloced = true; if (taosHashPut(ctx->pRes, &node, POINTER_BYTES, param, sizeof(*param))) { taosHashCleanup(param->pHashFilter); + param->pHashFilter = NULL; sclError("taosHashPut nodeList failed, size:%d", (int32_t)sizeof(*param)); return TSDB_CODE_QRY_OUT_OF_MEMORY; } + param->colAlloced = false; break; } case QUERY_NODE_COLUMN: { @@ -274,6 +295,7 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t SCL_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } *param = *res; + param->colAlloced = false; break; } default: @@ -455,11 +477,7 @@ int32_t sclExecFunction(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *outp _return: - for (int32_t i = 0; i < paramNum; ++i) { -// sclFreeParamNoData(params + i); - } - - taosMemoryFreeClear(params); + sclFreeParamList(params, paramNum); SCL_RET(code); } @@ -533,11 +551,7 @@ int32_t sclExecLogic(SLogicConditionNode *node, SScalarCtx *ctx, SScalarParam *o _return: - for (int32_t i = 0; i < paramNum; ++i) { -// sclFreeParamNoData(params + i); - } - - taosMemoryFreeClear(params); + sclFreeParamList(params, paramNum); SCL_RET(code); } @@ -573,14 +587,8 @@ int32_t sclExecOperator(SOperatorNode *node, SScalarCtx *ctx, SScalarParam *outp code = terrno; _return: - for (int32_t i = 0; i < paramNum; ++i) { - if (params[i].type == SHOULD_FREE_COLDATA) { - colDataDestroy(params[i].columnData); - taosMemoryFreeClear(params[i].columnData); - } - } - taosMemoryFreeClear(params); + sclFreeParamList(params, paramNum); SCL_RET(code); } @@ -871,7 +879,6 @@ EDealRes sclWalkFunction(SNode* pNode, SScalarCtx *ctx) { return DEAL_RES_ERROR; } - output.type = DELEGATED_MGMT_COLDATA; if (taosHashPut(ctx->pRes, &pNode, POINTER_BYTES, &output, sizeof(output))) { ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; return DEAL_RES_ERROR; @@ -906,7 +913,6 @@ EDealRes sclWalkOperator(SNode* pNode, SScalarCtx *ctx) { return DEAL_RES_ERROR; } - output.type = DELEGATED_MGMT_COLDATA; if (taosHashPut(ctx->pRes, &pNode, POINTER_BYTES, &output, sizeof(output))) { ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; return DEAL_RES_ERROR; diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index b94379c7b5..cadeaa0164 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -2408,6 +2408,10 @@ int32_t irateScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam return nonCalcScalarFunction(pInput, inputNum, pOutput); } +int32_t diffScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + return nonCalcScalarFunction(pInput, inputNum, pOutput); +} + int32_t twaScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { return avgScalarFunction(pInput, inputNum, pOutput); } @@ -2423,3 +2427,392 @@ int32_t hllScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam * int32_t csumScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { return sumScalarFunction(pInput, inputNum, pOutput); } + +typedef enum { + STATE_OPER_INVALID = 0, + STATE_OPER_LT, + STATE_OPER_GT, + STATE_OPER_LE, + STATE_OPER_GE, + STATE_OPER_NE, + STATE_OPER_EQ, +} EStateOperType; + +#define STATE_COMP(_op, _lval, _rval, _rtype) STATE_COMP_IMPL(_op, _lval, GET_STATE_VAL(_rval, _rtype)) + +#define GET_STATE_VAL(_val, _type) ((_type == TSDB_DATA_TYPE_BIGINT) ? (*(int64_t *)_val) : (*(double *)_val)) + +#define STATE_COMP_IMPL(_op, _lval, _rval) \ + do { \ + switch (_op) { \ + case STATE_OPER_LT: \ + return ((_lval) < (_rval)); \ + break; \ + case STATE_OPER_GT: \ + return ((_lval) > (_rval)); \ + break; \ + case STATE_OPER_LE: \ + return ((_lval) <= (_rval)); \ + break; \ + case STATE_OPER_GE: \ + return ((_lval) >= (_rval)); \ + break; \ + case STATE_OPER_NE: \ + return ((_lval) != (_rval)); \ + break; \ + case STATE_OPER_EQ: \ + return ((_lval) == (_rval)); \ + break; \ + default: \ + break; \ + } \ + } while (0) + +static int8_t getStateOpType(char* opStr) { + int8_t opType; + if (strncasecmp(opStr, "LT", 2) == 0) { + opType = STATE_OPER_LT; + } else if (strncasecmp(opStr, "GT", 2) == 0) { + opType = STATE_OPER_GT; + } else if (strncasecmp(opStr, "LE", 2) == 0) { + opType = STATE_OPER_LE; + } else if (strncasecmp(opStr, "GE", 2) == 0) { + opType = STATE_OPER_GE; + } else if (strncasecmp(opStr, "NE", 2) == 0) { + opType = STATE_OPER_NE; + } else if (strncasecmp(opStr, "EQ", 2) == 0) { + opType = STATE_OPER_EQ; + } else { + opType = STATE_OPER_INVALID; + } + + return opType; +} + +static bool checkStateOp(int8_t op, SColumnInfoData* pCol, int32_t index, SScalarParam *pCondParam) { + char* data = colDataGetData(pCol, index); + char* param = pCondParam->columnData->pData; + int32_t paramType = GET_PARAM_TYPE(pCondParam); + switch (pCol->info.type) { + case TSDB_DATA_TYPE_TINYINT: { + int8_t v = *(int8_t*)data; + STATE_COMP(op, v, param, paramType); + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + uint8_t v = *(uint8_t*)data; + STATE_COMP(op, v, param, paramType); + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + int16_t v = *(int16_t*)data; + STATE_COMP(op, v, param, paramType); + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + uint16_t v = *(uint16_t*)data; + STATE_COMP(op, v, param, paramType); + break; + } + case TSDB_DATA_TYPE_INT: { + int32_t v = *(int32_t*)data; + STATE_COMP(op, v, param, paramType); + break; + } + case TSDB_DATA_TYPE_UINT: { + uint32_t v = *(uint32_t*)data; + STATE_COMP(op, v, param, paramType); + break; + } + case TSDB_DATA_TYPE_BIGINT: { + int64_t v = *(int64_t*)data; + STATE_COMP(op, v, param, paramType); + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + uint64_t v = *(uint64_t*)data; + STATE_COMP(op, v, param, paramType); + break; + } + case TSDB_DATA_TYPE_FLOAT: { + float v = *(float*)data; + STATE_COMP(op, v, param, paramType); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + double v = *(double*)data; + STATE_COMP(op, v, param, paramType); + break; + } + default: { + ASSERT(0); + } + } + return false; +} + +int32_t stateCountScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + SColumnInfoData *pInputData = pInput->columnData; + SColumnInfoData *pOutputData = pOutput->columnData; + + int8_t op = getStateOpType(varDataVal(pInput[1].columnData->pData)); + int64_t count = 0; + + for (int32_t i = 0; i < pInput->numOfRows; ++i) { + if (colDataIsNull_s(pInputData, i)) { + colDataAppendNULL(pOutputData, i); + continue; + } + + bool ret = checkStateOp(op, pInputData, i, &pInput[2]); + int64_t out = -1; + if (ret) { + out = ++count; + } else { + count = 0; + } + colDataAppend(pOutputData, i, (char*)&out, false); + } + + pOutput->numOfRows = pInput->numOfRows; + return TSDB_CODE_SUCCESS; +} + +int32_t stateDurationScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + SColumnInfoData *pInputData = pInput->columnData; + SColumnInfoData *pOutputData = pOutput->columnData; + + int8_t op = getStateOpType(varDataVal(pInput[1].columnData->pData)); + + for (int32_t i = 0; i < pInput->numOfRows; ++i) { + if (colDataIsNull_s(pInputData, i)) { + colDataAppendNULL(pOutputData, i); + continue; + } + + bool ret = checkStateOp(op, pInputData, i, &pInput[2]); + int64_t out = -1; + if (ret) { + out = 0; + } + colDataAppend(pOutputData, i, (char*)&out, false); + } + + pOutput->numOfRows = pInput->numOfRows; + return TSDB_CODE_SUCCESS; +} + +typedef enum { UNKNOWN_BIN = 0, USER_INPUT_BIN, LINEAR_BIN, LOG_BIN } EHistoBinType; + +static int8_t getHistogramBinType(char* binTypeStr) { + int8_t binType; + if (strcasecmp(binTypeStr, "user_input") == 0) { + binType = USER_INPUT_BIN; + } else if (strcasecmp(binTypeStr, "linear_bin") == 0) { + binType = LINEAR_BIN; + } else if (strcasecmp(binTypeStr, "log_bin") == 0) { + binType = LOG_BIN; + } else { + binType = UNKNOWN_BIN; + } + + return binType; +} + +typedef struct SHistoFuncBin { + double lower; + double upper; + int64_t count; + double percentage; +} SHistoFuncBin; + +static bool getHistogramBinDesc(SHistoFuncBin** bins, int32_t* binNum, char* binDescStr, int8_t binType, bool normalized) { + cJSON* binDesc = cJSON_Parse(binDescStr); + int32_t numOfBins; + double* intervals; + if (cJSON_IsObject(binDesc)) { /* linaer/log bins */ + int32_t numOfParams = cJSON_GetArraySize(binDesc); + int32_t startIndex; + if (numOfParams != 4) { + return false; + } + + cJSON* start = cJSON_GetObjectItem(binDesc, "start"); + cJSON* factor = cJSON_GetObjectItem(binDesc, "factor"); + cJSON* width = cJSON_GetObjectItem(binDesc, "width"); + cJSON* count = cJSON_GetObjectItem(binDesc, "count"); + cJSON* infinity = cJSON_GetObjectItem(binDesc, "infinity"); + + if (!cJSON_IsNumber(start) || !cJSON_IsNumber(count) || !cJSON_IsBool(infinity)) { + return false; + } + + if (count->valueint <= 0 || count->valueint > 1000) { // limit count to 1000 + return false; + } + + if (isinf(start->valuedouble) || (width != NULL && isinf(width->valuedouble)) || + (factor != NULL && isinf(factor->valuedouble)) || (count != NULL && isinf(count->valuedouble))) { + return false; + } + + int32_t counter = (int32_t)count->valueint; + if (infinity->valueint == false) { + startIndex = 0; + numOfBins = counter + 1; + } else { + startIndex = 1; + numOfBins = counter + 3; + } + + intervals = taosMemoryCalloc(numOfBins, sizeof(double)); + if (cJSON_IsNumber(width) && factor == NULL && binType == LINEAR_BIN) { + // linear bin process + if (width->valuedouble == 0) { + taosMemoryFree(intervals); + return false; + } + for (int i = 0; i < counter + 1; ++i) { + intervals[startIndex] = start->valuedouble + i * width->valuedouble; + if (isinf(intervals[startIndex])) { + taosMemoryFree(intervals); + return false; + } + startIndex++; + } + } else if (cJSON_IsNumber(factor) && width == NULL && binType == LOG_BIN) { + // log bin process + if (start->valuedouble == 0) { + taosMemoryFree(intervals); + return false; + } + if (factor->valuedouble < 0 || factor->valuedouble == 0 || factor->valuedouble == 1) { + taosMemoryFree(intervals); + return false; + } + for (int i = 0; i < counter + 1; ++i) { + intervals[startIndex] = start->valuedouble * pow(factor->valuedouble, i * 1.0); + if (isinf(intervals[startIndex])) { + taosMemoryFree(intervals); + return false; + } + startIndex++; + } + } else { + taosMemoryFree(intervals); + return false; + } + + if (infinity->valueint == true) { + intervals[0] = -INFINITY; + intervals[numOfBins - 1] = INFINITY; + // in case of desc bin orders, -inf/inf should be swapped + ASSERT(numOfBins >= 4); + if (intervals[1] > intervals[numOfBins - 2]) { + TSWAP(intervals[0], intervals[numOfBins - 1]); + } + } + } else if (cJSON_IsArray(binDesc)) { /* user input bins */ + if (binType != USER_INPUT_BIN) { + return false; + } + numOfBins = cJSON_GetArraySize(binDesc); + intervals = taosMemoryCalloc(numOfBins, sizeof(double)); + cJSON* bin = binDesc->child; + if (bin == NULL) { + taosMemoryFree(intervals); + return false; + } + int i = 0; + while (bin) { + intervals[i] = bin->valuedouble; + if (!cJSON_IsNumber(bin)) { + taosMemoryFree(intervals); + return false; + } + if (i != 0 && intervals[i] <= intervals[i - 1]) { + taosMemoryFree(intervals); + return false; + } + bin = bin->next; + i++; + } + } else { + return false; + } + + *binNum = numOfBins - 1; + *bins = taosMemoryCalloc(numOfBins, sizeof(SHistoFuncBin)); + for (int32_t i = 0; i < *binNum; ++i) { + (*bins)[i].lower = intervals[i] < intervals[i + 1] ? intervals[i] : intervals[i + 1]; + (*bins)[i].upper = intervals[i + 1] > intervals[i] ? intervals[i + 1] : intervals[i]; + (*bins)[i].count = 0; + } + + taosMemoryFree(intervals); + return true; +} + +int32_t histogramScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { + SColumnInfoData *pInputData = pInput->columnData; + SColumnInfoData *pOutputData = pOutput->columnData; + + SHistoFuncBin *bins; + int32_t numOfBins = 0; + int32_t totalCount = 0; + + int8_t binType = getHistogramBinType(varDataVal(pInput[1].columnData->pData)); + char* binDesc = varDataVal(pInput[2].columnData->pData); + int64_t normalized = *(int64_t *)(pInput[3].columnData->pData); + + int32_t type = GET_PARAM_TYPE(pInput); + if (!getHistogramBinDesc(&bins, &numOfBins, binDesc, binType, (bool)normalized)) { + return TSDB_CODE_FAILED; + } + + for (int32_t i = 0; i < pInput->numOfRows; ++i) { + if (colDataIsNull_s(pInputData, i)) { + continue; + } + + char* data = colDataGetData(pInputData, i); + double v; + GET_TYPED_DATA(v, double, type, data); + + for (int32_t k = 0; k < numOfBins; ++k) { + if (v > bins[k].lower && v <= bins[k].upper) { + bins[k].count++; + totalCount++; + break; + } + } + } + + if (normalized) { + for (int32_t k = 0; k < numOfBins; ++k) { + if (totalCount != 0) { + bins[k].percentage = bins[k].count / (double)totalCount; + } else { + bins[k].percentage = 0; + } + } + } + + for (int32_t k = 0; k < numOfBins; ++k) { + int32_t len; + char buf[512] = {0}; + if (!normalized) { + len = sprintf(varDataVal(buf), "{\"lower_bin\":%g, \"upper_bin\":%g, \"count\":%" PRId64 "}", + bins[k].lower, bins[k].upper, bins[k].count); + } else { + len = sprintf(varDataVal(buf), "{\"lower_bin\":%g, \"upper_bin\":%g, \"count\":%lf}", + bins[k].lower, bins[k].upper, bins[k].percentage); + } + varDataSetLen(buf, len); + colDataAppend(pOutputData, k, buf, false); + } + + taosMemoryFree(bins); + pOutput->numOfRows = numOfBins; + return TSDB_CODE_SUCCESS; +} diff --git a/source/libs/scalar/test/filter/filterTests.cpp b/source/libs/scalar/test/filter/filterTests.cpp index d8c64eef55..4c4d03fb37 100644 --- a/source/libs/scalar/test/filter/filterTests.cpp +++ b/source/libs/scalar/test/filter/filterTests.cpp @@ -207,7 +207,7 @@ void flttMakeListNode(SNode **pNode, SNodeList *list, int32_t resType) { void initScalarParam(SScalarParam* pParam) { memset(pParam, 0, sizeof(SScalarParam)); - pParam->type = SHOULD_FREE_COLDATA; + pParam->colAlloced = true; } } diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index bc0270635d..65b45cc612 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -55,13 +55,11 @@ typedef enum { #define SCHEDULE_DEFAULT_MAX_TASK_NUM 1000 #define SCHEDULE_DEFAULT_MAX_NODE_TABLE_NUM 200 // unit is TSDB_TABLE_NUM_UNIT #define SCHEDULE_DEFAULT_POLICY SCH_LOAD_SEQ +#define SCHEDULE_DEFAULT_MAX_NODE_NUM 20 #define SCH_DEFAULT_TASK_TIMEOUT_USEC 10000000 #define SCH_MAX_TASK_TIMEOUT_USEC 60000000 -#define SCH_MAX_CANDIDATE_EP_NUM TSDB_MAX_REPLICA - - - +#define SCH_DEFAULT_MAX_RETRY_NUM 6 typedef struct SSchDebug { bool lockEnable; @@ -211,6 +209,7 @@ typedef struct SSchTask { int32_t maxExecTimes; // task max exec times int32_t maxRetryTimes; // task max retry times int32_t retryTimes; // task retry times + bool waitRetry; // wait for retry int32_t execId; // task current execute index SSchLevel *level; // level SRWLatch planLock; // task update plan lock @@ -274,7 +273,8 @@ typedef struct SSchJob { int32_t errCode; SRWLatch resLock; SExecResult execRes; - void *resData; //TODO free it or not + void *fetchRes; //TODO free it or not + bool fetched; int32_t resNumOfRows; SSchResInfo userRes; const char *sql; @@ -326,7 +326,7 @@ extern SSchedulerMgmt schMgmt; #define SCH_IS_EXPLAIN_JOB(_job) (EXPLAIN_MODE_ANALYZE == (_job)->attr.explainMode) #define SCH_NETWORK_ERR(_code) ((_code) == TSDB_CODE_RPC_BROKEN_LINK || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL) #define SCH_MERGE_TASK_NETWORK_ERR(_task, _code, _len) (SCH_NETWORK_ERR(_code) && (((_len) > 0) || (!SCH_IS_DATA_BIND_TASK(_task)))) -#define SCH_REDIRECT_MSGTYPE(_msgType) ((_msgType) == TDMT_SCH_QUERY || (_msgType) == TDMT_SCH_MERGE_QUERY || (_msgType) == TDMT_SCH_FETCH || (_msgType) == TDMT_SCH_MERGE_FETCH) +#define SCH_REDIRECT_MSGTYPE(_msgType) ((_msgType) == TDMT_SCH_LINK_BROKEN || (_msgType) == TDMT_SCH_QUERY || (_msgType) == TDMT_SCH_MERGE_QUERY || (_msgType) == TDMT_SCH_FETCH || (_msgType) == TDMT_SCH_MERGE_FETCH) #define SCH_TASK_NEED_REDIRECT(_task, _msgType, _code, _rspLen) (SCH_REDIRECT_MSGTYPE(_msgType) && (NEED_SCHEDULER_REDIRECT_ERROR(_code) || SCH_MERGE_TASK_NETWORK_ERR((_task), (_code), (_rspLen)))) #define SCH_NEED_RETRY(_msgType, _code) ((SCH_NETWORK_ERR(_code) && SCH_REDIRECT_MSGTYPE(_msgType)) || (_code) == TSDB_CODE_SCH_TIMEOUT_ERROR) @@ -368,6 +368,8 @@ extern SSchedulerMgmt schMgmt; qError("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_TASK_ID(pTask), SCH_TASK_EID(pTask),__VA_ARGS__) #define SCH_TASK_DLOG(param, ...) \ qDebug("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_TASK_ID(pTask), SCH_TASK_EID(pTask),__VA_ARGS__) +#define SCH_TASK_TLOG(param, ...) \ + qTrace("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_TASK_ID(pTask), SCH_TASK_EID(pTask),__VA_ARGS__) #define SCH_TASK_DLOGL(param, ...) \ qDebugL("QID:0x%" PRIx64 ",TID:0x%" PRIx64 ",EID:%d " param, pJob->queryId, SCH_TASK_ID(pTask), SCH_TASK_EID(pTask),__VA_ARGS__) #define SCH_TASK_WLOG(param, ...) \ @@ -441,7 +443,7 @@ void schFreeRpcCtx(SRpcCtx *pCtx); int32_t schGetCallbackFp(int32_t msgType, __async_send_cb_fn_t *fp); bool schJobNeedToStop(SSchJob *pJob, int8_t *pStatus); int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask); -int32_t schSaveJobQueryRes(SSchJob *pJob, SQueryTableRsp *rsp); +int32_t schSaveJobExecRes(SSchJob *pJob, SQueryTableRsp *rsp); int32_t schProcessOnExplainDone(SSchJob *pJob, SSchTask *pTask, SRetrieveTableRsp *pRsp); void schProcessOnDataFetched(SSchJob *job); int32_t schGetTaskInJob(SSchJob *pJob, uint64_t taskId, SSchTask **pTask); @@ -492,7 +494,7 @@ int32_t schSwitchTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask); void schDirectPostJobRes(SSchedulerReq* pReq, int32_t errCode); int32_t schHandleJobFailure(SSchJob *pJob, int32_t errCode); int32_t schHandleJobDrop(SSchJob *pJob, int32_t errCode); -bool schChkCurrentOp(SSchJob *pJob, int32_t op, bool sync); +bool schChkCurrentOp(SSchJob *pJob, int32_t op, int8_t sync); extern SSchDebug gSCHDebug; diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index 1b1268baf1..13a369fac9 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -110,7 +110,7 @@ int32_t schUpdateJobStatus(SSchJob *pJob, int8_t newStatus) { break; case JOB_TASK_STATUS_PART_SUCC: if (newStatus != JOB_TASK_STATUS_FAIL && newStatus != JOB_TASK_STATUS_SUCC && - newStatus != JOB_TASK_STATUS_DROP) { + newStatus != JOB_TASK_STATUS_DROP && newStatus != JOB_TASK_STATUS_EXEC) { SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } @@ -389,13 +389,18 @@ int32_t schDumpJobExecRes(SSchJob* pJob, SExecResult* pRes) { int32_t schDumpJobFetchRes(SSchJob* pJob, void** pData) { int32_t code = 0; - if (pJob->resData && ((SRetrieveTableRsp *)pJob->resData)->completed) { - SCH_ERR_RET(schSwitchJobStatus(pJob, JOB_TASK_STATUS_SUCC, NULL)); + + SCH_LOCK(SCH_WRITE, &pJob->resLock); + + pJob->fetched = true; + + if (pJob->fetchRes && ((SRetrieveTableRsp *)pJob->fetchRes)->completed) { + SCH_ERR_JRET(schSwitchJobStatus(pJob, JOB_TASK_STATUS_SUCC, NULL)); } while (true) { - *pData = atomic_load_ptr(&pJob->resData); - if (*pData != atomic_val_compare_exchange_ptr(&pJob->resData, *pData, NULL)) { + *pData = atomic_load_ptr(&pJob->fetchRes); + if (*pData != atomic_val_compare_exchange_ptr(&pJob->fetchRes, *pData, NULL)) { continue; } @@ -414,7 +419,11 @@ int32_t schDumpJobFetchRes(SSchJob* pJob, void** pData) { SCH_JOB_DLOG("fetch done, totalRows:%d", pJob->resNumOfRows); - return TSDB_CODE_SUCCESS; +_return: + + SCH_UNLOCK(SCH_WRITE, &pJob->resLock); + + return code; } int32_t schNotifyUserExecRes(SSchJob* pJob) { @@ -512,8 +521,12 @@ int32_t schHandleJobDrop(SSchJob *pJob, int32_t errCode) { } -int32_t schProcessOnJobPartialSuccess(SSchJob *pJob) { - schPostJobRes(pJob, SCH_OP_EXEC); +int32_t schProcessOnJobPartialSuccess(SSchJob *pJob) { + if (schChkCurrentOp(pJob, SCH_OP_FETCH, -1)) { + SCH_ERR_RET(schLaunchFetchTask(pJob)); + } else { + schPostJobRes(pJob, 0); + } return TSDB_CODE_SUCCESS; } @@ -526,7 +539,7 @@ int32_t schProcessOnExplainDone(SSchJob *pJob, SSchTask *pTask, SRetrieveTableRs SCH_TASK_DLOG("got explain rsp, rows:%d, complete:%d", htonl(pRsp->numOfRows), pRsp->completed); atomic_store_32(&pJob->resNumOfRows, htonl(pRsp->numOfRows)); - atomic_store_ptr(&pJob->resData, pRsp); + atomic_store_ptr(&pJob->fetchRes, pRsp); SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_SUCC); @@ -561,7 +574,7 @@ int32_t schLaunchJobLowerLevel(SSchJob *pJob, SSchTask *pTask) { return TSDB_CODE_SUCCESS; } -int32_t schSaveJobQueryRes(SSchJob *pJob, SQueryTableRsp *rsp) { +int32_t schSaveJobExecRes(SSchJob *pJob, SQueryTableRsp *rsp) { if (rsp->tbFName[0]) { SCH_LOCK(SCH_WRITE, &pJob->resLock); @@ -600,7 +613,7 @@ int32_t schGetTaskInJob(SSchJob *pJob, uint64_t taskId, SSchTask **pTask) { int32_t schLaunchJob(SSchJob *pJob) { if (EXPLAIN_MODE_STATIC == pJob->attr.explainMode) { - SCH_ERR_RET(qExecStaticExplain(pJob->pDag, (SRetrieveTableRsp **)&pJob->resData)); + SCH_ERR_RET(qExecStaticExplain(pJob->pDag, (SRetrieveTableRsp **)&pJob->fetchRes)); SCH_ERR_RET(schSwitchJobStatus(pJob, JOB_TASK_STATUS_PART_SUCC, NULL)); } else { SSchLevel *level = taosArrayGet(pJob->levels, pJob->levelIdx); @@ -661,7 +674,7 @@ void schFreeJobImpl(void *job) { qDestroyQueryPlan(pJob->pDag); taosMemoryFreeClear(pJob->userRes.execRes); - taosMemoryFreeClear(pJob->resData); + taosMemoryFreeClear(pJob->fetchRes); taosMemoryFree(pJob); int32_t jobNum = atomic_sub_fetch_32(&schMgmt.jobNum, 1); @@ -795,9 +808,14 @@ void schDirectPostJobRes(SSchedulerReq* pReq, int32_t errCode) { } } -bool schChkCurrentOp(SSchJob *pJob, int32_t op, bool sync) { +bool schChkCurrentOp(SSchJob *pJob, int32_t op, int8_t sync) { + bool r = false; SCH_LOCK(SCH_READ, &pJob->opStatus.lock); - bool r = (pJob->opStatus.op == op) && (pJob->opStatus.syncReq == sync); + if (sync >= 0) { + r = (pJob->opStatus.op == op) && (pJob->opStatus.syncReq == sync); + } else { + r = (pJob->opStatus.op == op); + } SCH_UNLOCK(SCH_READ, &pJob->opStatus.lock); return r; diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index d195c22c37..6983bbf013 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -256,7 +256,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa SCH_ERR_JRET(rsp->code); - SCH_ERR_JRET(schSaveJobQueryRes(pJob, rsp)); + SCH_ERR_JRET(schSaveJobExecRes(pJob, rsp)); atomic_add_fetch_32(&pJob->resNumOfRows, rsp->affectedRows); @@ -277,8 +277,8 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); } - if (pJob->resData) { - SCH_TASK_ELOG("explain result is already generated, res:%p", pJob->resData); + if (pJob->fetchRes) { + SCH_TASK_ELOG("explain result is already generated, res:%p", pJob->fetchRes); SCH_ERR_JRET(TSDB_CODE_SCH_STATUS_ERROR); } @@ -325,13 +325,13 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t execId, SDa return TSDB_CODE_SUCCESS; } - if (pJob->resData) { - SCH_TASK_ELOG("got fetch rsp while res already exists, res:%p", pJob->resData); + if (pJob->fetchRes) { + SCH_TASK_ELOG("got fetch rsp while res already exists, res:%p", pJob->fetchRes); taosMemoryFreeClear(rsp); SCH_ERR_JRET(TSDB_CODE_SCH_STATUS_ERROR); } - atomic_store_ptr(&pJob->resData, rsp); + atomic_store_ptr(&pJob->fetchRes, rsp); atomic_add_fetch_32(&pJob->resNumOfRows, htonl(rsp->numOfRows)); if (rsp->completed) { @@ -1010,6 +1010,7 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, memcpy(pMsg->msg + len, pTask->msg, pTask->msgLen); persistHandle = true; + SCH_SET_TASK_HANDLE(pTask, rpcAllocHandle()); break; } case TDMT_SCH_FETCH: diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index 9483ecd6eb..282e81bb5d 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -47,10 +47,10 @@ void schFreeTask(SSchJob *pJob, SSchTask *pTask) { void schInitTaskRetryTimes(SSchJob *pJob, SSchTask *pTask, SSchLevel *pLevel) { if (SCH_IS_DATA_BIND_TASK(pTask) || (!SCH_IS_QUERY_JOB(pJob)) || (SCH_ALL != schMgmt.cfg.schPolicy)) { - pTask->maxRetryTimes = SCH_MAX_CANDIDATE_EP_NUM; + pTask->maxRetryTimes = SCH_DEFAULT_MAX_RETRY_NUM; } else { int32_t nodeNum = taosArrayGetSize(pJob->nodeList); - pTask->maxRetryTimes = TMAX(nodeNum, SCH_MAX_CANDIDATE_EP_NUM); + pTask->maxRetryTimes = TMAX(nodeNum, SCH_DEFAULT_MAX_RETRY_NUM); } pTask->maxExecTimes = pTask->maxRetryTimes * (pLevel->level + 1); @@ -64,11 +64,11 @@ int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel * pTask->execId = -1; pTask->timeoutUsec = SCH_DEFAULT_TASK_TIMEOUT_USEC; pTask->taskId = schGenTaskId(); - pTask->execNodes = - taosHashInit(SCH_MAX_CANDIDATE_EP_NUM, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); schInitTaskRetryTimes(pJob, pTask, pLevel); + pTask->execNodes = + taosHashInit(pTask->maxExecTimes, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); pTask->profile.execTime = taosArrayInit(pTask->maxExecTimes, sizeof(int64_t)); if (NULL == pTask->execNodes || NULL == pTask->profile.execTime) { SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -125,8 +125,8 @@ int32_t schDropTaskExecNode(SSchJob *pJob, SSchTask *pTask, void *handle, int32_ SCH_TASK_DLOG("execId %d removed from execNodeList", execId); } - if (execId != pTask->execId) { // ignore it - SCH_TASK_DLOG("execId %d is not current execId %d", execId, pTask->execId); + if ((execId != pTask->execId) || pTask->waitRetry) { // ignore it + SCH_TASK_DLOG("execId %d is already not current execId %d, waitRetry %d", execId, pTask->execId, pTask->waitRetry); SCH_ERR_RET(TSDB_CODE_SCH_IGNORE_ERROR); } @@ -138,7 +138,17 @@ int32_t schUpdateTaskExecNode(SSchJob *pJob, SSchTask *pTask, void *handle, int3 return TSDB_CODE_SUCCESS; } + if ((execId != pTask->execId) || pTask->waitRetry) { // ignore it + SCH_TASK_DLOG("handle not updated since execId %d is already not current execId %d, waitRetry %d", execId, pTask->execId, pTask->waitRetry); + return TSDB_CODE_SUCCESS; + } + SSchNodeInfo *nodeInfo = taosHashGet(pTask->execNodes, &execId, sizeof(execId)); + if (NULL == nodeInfo) { // ignore it + SCH_TASK_DLOG("handle not updated since execId %d already not exist, current execId %d, waitRetry %d", execId, pTask->execId, pTask->waitRetry); + return TSDB_CODE_SUCCESS; + } + nodeInfo->handle = handle; SCH_TASK_DLOG("handle updated to %p for execId %d", handle, execId); @@ -335,6 +345,7 @@ int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32 return TSDB_CODE_SUCCESS; } + pTask->waitRetry = true; schDropTaskOnExecNode(pJob, pTask); taosHashClear(pTask->execNodes); SCH_ERR_JRET(schRemoveTaskFromExecList(pJob, pTask)); @@ -394,6 +405,18 @@ _return: int32_t schHandleRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32_t rspCode) { int32_t code = 0; + if (JOB_TASK_STATUS_PART_SUCC == pJob->status) { + SCH_LOCK(SCH_WRITE, &pJob->resLock); + if (pJob->fetched) { + SCH_UNLOCK(SCH_WRITE, &pJob->resLock); + SCH_TASK_ELOG("already fetched while got error %s", tstrerror(rspCode)); + SCH_ERR_RET(rspCode); + } + SCH_UNLOCK(SCH_WRITE, &pJob->resLock); + + schUpdateJobStatus(pJob, JOB_TASK_STATUS_EXEC); + } + if (SCH_IS_DATA_BIND_TASK(pTask)) { if (NULL == pData->pEpSet) { SCH_TASK_ELOG("no epset updated while got error %s", tstrerror(rspCode)); @@ -591,7 +614,7 @@ int32_t schSetAddrsFromNodeList(SSchJob *pJob, SSchTask *pTask) { if (pJob->nodeList) { nodeNum = taosArrayGetSize(pJob->nodeList); - for (int32_t i = 0; i < nodeNum && addNum < SCH_MAX_CANDIDATE_EP_NUM; ++i) { + for (int32_t i = 0; i < nodeNum; ++i) { SQueryNodeLoad *nload = taosArrayGet(pJob->nodeList, i); SQueryNodeAddr *naddr = &nload->addr; @@ -600,8 +623,8 @@ int32_t schSetAddrsFromNodeList(SSchJob *pJob, SSchTask *pTask) { SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - SCH_TASK_DLOG("set %dth candidate addr, id %d, fqdn:%s, port:%d", i, naddr->nodeId, SCH_GET_CUR_EP(naddr)->fqdn, - SCH_GET_CUR_EP(naddr)->port); + SCH_TASK_TLOG("set %dth candidate addr, id %d, inUse:%d/%d, fqdn:%s, port:%d", i, naddr->nodeId, naddr->epSet.inUse, naddr->epSet.numOfEps, + SCH_GET_CUR_EP(naddr)->fqdn, SCH_GET_CUR_EP(naddr)->port); ++addNum; } @@ -621,9 +644,9 @@ int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) { } pTask->candidateIdx = 0; - pTask->candidateAddrs = taosArrayInit(SCH_MAX_CANDIDATE_EP_NUM, sizeof(SQueryNodeAddr)); + pTask->candidateAddrs = taosArrayInit(SCHEDULE_DEFAULT_MAX_NODE_NUM, sizeof(SQueryNodeAddr)); if (NULL == pTask->candidateAddrs) { - SCH_TASK_ELOG("taosArrayInit %d condidate addrs failed", SCH_MAX_CANDIDATE_EP_NUM); + SCH_TASK_ELOG("taosArrayInit %d condidate addrs failed", SCHEDULE_DEFAULT_MAX_NODE_NUM); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -790,6 +813,7 @@ int32_t schLaunchTaskImpl(SSchJob *pJob, SSchTask *pTask) { atomic_add_fetch_32(&pTask->level->taskLaunchedNum, 1); pTask->execId++; pTask->retryTimes++; + pTask->waitRetry = false; SCH_TASK_DLOG("start to launch task, execId %d, retry %d", pTask->execId, pTask->retryTimes); @@ -885,9 +909,9 @@ void schDropTaskInHashList(SSchJob *pJob, SHashObj *list) { int32_t schLaunchFetchTask(SSchJob *pJob) { int32_t code = 0; - void *resData = atomic_load_ptr(&pJob->resData); - if (resData) { - SCH_JOB_DLOG("res already fetched, res:%p", resData); + void *fetchRes = atomic_load_ptr(&pJob->fetchRes); + if (fetchRes) { + SCH_JOB_DLOG("res already fetched, res:%p", fetchRes); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 2a96db3bfc..99a06575a9 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -143,6 +143,9 @@ int32_t streamTaskEnqueueRetrieve(SStreamTask* pTask, SStreamRetrieveReq* pReq, // enqueue if (pData != NULL) { + qDebug("task %d(child %d) recv retrieve req from task %d, reqId %ld", pTask->taskId, pTask->selfChildId, + pReq->srcTaskId, pReq->reqId); + pData->type = STREAM_INPUT__DATA_RETRIEVE; pData->srcVgId = 0; // decode diff --git a/source/libs/stream/src/streamData.c b/source/libs/stream/src/streamData.c index dbf6350c93..6be15222db 100644 --- a/source/libs/stream/src/streamData.c +++ b/source/libs/stream/src/streamData.c @@ -57,7 +57,9 @@ int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock pDataBlock->info.type = pRetrieve->streamBlockType; + pData->reqId = pReq->reqId; pData->blocks = pArray; + return 0; } diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index e2faf28abe..5dec33d0fb 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -65,6 +65,7 @@ int32_t tDecodeStreamDispatchReq(SDecoder* pDecoder, SStreamDispatchReq* pReq) { int32_t tEncodeStreamRetrieveReq(SEncoder* pEncoder, const SStreamRetrieveReq* pReq) { if (tStartEncode(pEncoder) < 0) return -1; if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1; + if (tEncodeI64(pEncoder, pReq->reqId) < 0) return -1; if (tEncodeI32(pEncoder, pReq->dstNodeId) < 0) return -1; if (tEncodeI32(pEncoder, pReq->dstTaskId) < 0) return -1; if (tEncodeI32(pEncoder, pReq->srcNodeId) < 0) return -1; @@ -77,6 +78,7 @@ int32_t tEncodeStreamRetrieveReq(SEncoder* pEncoder, const SStreamRetrieveReq* p int32_t tDecodeStreamRetrieveReq(SDecoder* pDecoder, SStreamRetrieveReq* pReq) { if (tStartDecode(pDecoder) < 0) return -1; if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1; + if (tDecodeI64(pDecoder, &pReq->reqId) < 0) return -1; if (tDecodeI32(pDecoder, &pReq->dstNodeId) < 0) return -1; if (tDecodeI32(pDecoder, &pReq->dstTaskId) < 0) return -1; if (tDecodeI32(pDecoder, &pReq->srcNodeId) < 0) return -1; @@ -121,6 +123,7 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) int32_t sz = taosArrayGetSize(pTask->childEpInfo); ASSERT(sz > 0); for (int32_t i = 0; i < sz; i++) { + req.reqId = tGenIdPI64(); SStreamChildEpInfo* pEpInfo = taosArrayGetP(pTask->childEpInfo, i); req.dstNodeId = pEpInfo->nodeId; req.dstTaskId = pEpInfo->taskId; @@ -154,6 +157,9 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) ASSERT(0); return -1; } + + qDebug("task %d(child %d) send retrieve req to task %d at node %d, reqId %ld", pTask->taskId, pTask->selfChildId, + pEpInfo->taskId, pEpInfo->nodeId, req.reqId); } return 0; FAIL: diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 9d22da8662..f6eb9e32f2 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -54,6 +54,9 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes) block.info.type = STREAM_PULL_OVER; block.info.childId = pTask->selfChildId; taosArrayPush(pRes, &block); + + qDebug("task %d(child %d) processed retrieve, reqId %ld", pTask->taskId, pTask->selfChildId, + pRetrieveBlock->reqId); } break; } diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 1f26033f63..0cbd7d36b2 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -223,6 +223,7 @@ void syncNodeVoteForSelf(SSyncNode* pSyncNode); // snapshot -------------- bool syncNodeHasSnapshot(SSyncNode* pSyncNode); +void syncNodeMaybeUpdateCommitBySnapshot(SSyncNode* pSyncNode); SyncIndex syncNodeGetLastIndex(SSyncNode* pSyncNode); SyncTerm syncNodeGetLastTerm(SSyncNode* pSyncNode); diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index dca80f6826..9678b335fd 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -711,6 +711,9 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc syncNodeEventLog(ths, logBuf); } while (0); + // maybe update commit index by snapshot + syncNodeMaybeUpdateCommitBySnapshot(ths); + // prepare response msg SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(ths->vgId); pReply->srcId = ths->myRaftId; @@ -718,7 +721,7 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc pReply->term = ths->pRaftStore->currentTerm; pReply->privateTerm = ths->pNewNodeReceiver->privateTerm; pReply->success = false; - pReply->matchIndex = SYNC_INDEX_INVALID; + pReply->matchIndex = ths->commitIndex; // msg event log do { diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index c18c2b4d38..5137922522 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -147,6 +147,15 @@ static void syncNodeStartSnapshotOnce(SSyncNode* ths, SyncIndex beginIndex, Sync int32_t syncNodeOnAppendEntriesReplySnapshot2Cb(SSyncNode* ths, SyncAppendEntriesReply* pMsg) { int32_t ret = 0; + // print log + do { + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries-reply, term:%lu, match:%ld, success:%d", pMsg->term, + pMsg->matchIndex, pMsg->success); + syncNodeEventLog(ths, logBuf); + + } while (0); + // if already drop replica, do not process if (!syncNodeInRaftGroup(ths, &(pMsg->srcId)) && !ths->pRaftCfg->isStandBy) { syncNodeEventLog(ths, "recv sync-append-entries-reply, maybe replica already dropped"); @@ -238,7 +247,14 @@ int32_t syncNodeOnAppendEntriesReplySnapshot2Cb(SSyncNode* ths, SyncAppendEntrie SSnapshot oldSnapshot; ths->pFsm->FpGetSnapshotInfo(ths->pFsm, &oldSnapshot); SyncTerm newSnapshotTerm = oldSnapshot.lastApplyTerm; - syncNodeStartSnapshotOnce(ths, SYNC_INDEX_BEGIN, nextIndex, newSnapshotTerm, pMsg); + + SyncIndex endIndex; + if (ths->pLogStore->syncLogExist(ths->pLogStore, nextIndex + 1)) { + endIndex = nextIndex; + } else { + endIndex = oldSnapshot.lastApplyIndex; + } + syncNodeStartSnapshotOnce(ths, pMsg->matchIndex + 1, endIndex, newSnapshotTerm, pMsg); // get sender SSyncSnapshotSender* pSender = syncNodeGetSnapshotSender(ths, &(pMsg->srcId)); @@ -256,6 +272,11 @@ int32_t syncNodeOnAppendEntriesReplySnapshot2Cb(SSyncNode* ths, SyncAppendEntrie } syncIndexMgrSetIndex(ths->pNextIndex, &(pMsg->srcId), nextIndex); + SyncIndex oldMatchIndex = syncIndexMgrGetIndex(ths->pMatchIndex, &(pMsg->srcId)); + if (pMsg->matchIndex > oldMatchIndex) { + syncIndexMgrSetIndex(ths->pMatchIndex, &(pMsg->srcId), pMsg->matchIndex); + } + // event log, update next-index do { char host[64]; diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index b6c01f2923..bb7454ea6f 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1083,6 +1083,17 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) { return pSyncNode; } +void syncNodeMaybeUpdateCommitBySnapshot(SSyncNode* pSyncNode) { + if (pSyncNode->pFsm != NULL && pSyncNode->pFsm->FpGetSnapshotInfo != NULL) { + SSnapshot snapshot; + int32_t code = pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot); + ASSERT(code == 0); + if (snapshot.lastApplyIndex > pSyncNode->commitIndex) { + pSyncNode->commitIndex = snapshot.lastApplyIndex; + } + } +} + void syncNodeStart(SSyncNode* pSyncNode) { // start raft if (pSyncNode->replicaNum == 1) { diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 87cc5685f3..924a4df90d 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -374,14 +374,14 @@ cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender) { char *snapshotSender2Str(SSyncSnapshotSender *pSender) { cJSON *pJson = snapshotSender2Json(pSender); - char * serialized = cJSON_Print(pJson); + char *serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } char *snapshotSender2SimpleStr(SSyncSnapshotSender *pSender, char *event) { int32_t len = 256; - char * s = taosMemoryMalloc(len); + char *s = taosMemoryMalloc(len); SRaftId destId = pSender->pSyncNode->replicasId[pSender->replicaIndex]; char host[64]; @@ -434,8 +434,8 @@ void snapshotReceiverDestroy(SSyncSnapshotReceiver *pReceiver) { if (pReceiver != NULL) { // close writer if (pReceiver->pWriter != NULL) { - int32_t ret = - pReceiver->pSyncNode->pFsm->FpSnapshotStopWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, false); + int32_t ret = pReceiver->pSyncNode->pFsm->FpSnapshotStopWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, + false, &(pReceiver->snapshot)); ASSERT(ret == 0); pReceiver->pWriter = NULL; } @@ -483,8 +483,8 @@ static void snapshotReceiverDoStart(SSyncSnapshotReceiver *pReceiver, SyncSnapsh static void snapshotReceiverForceStop(SSyncSnapshotReceiver *pReceiver) { // force close, abandon incomplete data if (pReceiver->pWriter != NULL) { - int32_t ret = - pReceiver->pSyncNode->pFsm->FpSnapshotStopWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, false); + int32_t ret = pReceiver->pSyncNode->pFsm->FpSnapshotStopWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, false, + &(pReceiver->snapshot)); ASSERT(ret == 0); pReceiver->pWriter = NULL; } @@ -524,8 +524,8 @@ int32_t snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncSnapshotSend // FpSnapshotStopWrite should not be called, assert writer == NULL int32_t snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver) { if (pReceiver->pWriter != NULL) { - int32_t ret = - pReceiver->pSyncNode->pFsm->FpSnapshotStopWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, false); + int32_t ret = pReceiver->pSyncNode->pFsm->FpSnapshotStopWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, false, + &(pReceiver->snapshot)); ASSERT(ret == 0); pReceiver->pWriter = NULL; } @@ -574,7 +574,8 @@ static int32_t snapshotReceiverFinish(SSyncSnapshotReceiver *pReceiver, SyncSnap } // stop writer, apply data - code = pReceiver->pSyncNode->pFsm->FpSnapshotStopWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, true); + code = pReceiver->pSyncNode->pFsm->FpSnapshotStopWrite(pReceiver->pSyncNode->pFsm, pReceiver->pWriter, true, + &(pReceiver->snapshot)); if (code != 0) { syncNodeErrorLog(pReceiver->pSyncNode, "snapshot stop writer true error"); ASSERT(0); @@ -646,7 +647,7 @@ cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver) { cJSON_AddStringToObject(pFromId, "addr", u64buf); { uint64_t u64 = pReceiver->fromId.addr; - cJSON * pTmp = pFromId; + cJSON *pTmp = pFromId; char host[128] = {0}; uint16_t port; syncUtilU642Addr(u64, host, sizeof(host), &port); @@ -679,14 +680,14 @@ cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver) { char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver) { cJSON *pJson = snapshotReceiver2Json(pReceiver); - char * serialized = cJSON_Print(pJson); + char *serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } char *snapshotReceiver2SimpleStr(SSyncSnapshotReceiver *pReceiver, char *event) { int32_t len = 256; - char * s = taosMemoryMalloc(len); + char *s = taosMemoryMalloc(len); SRaftId fromId = pReceiver->fromId; char host[128]; diff --git a/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp b/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp index 339ebe90e7..8c6c68bbf8 100644 --- a/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp +++ b/source/libs/sync/test/syncConfigChangeSnapshotTest.cpp @@ -125,7 +125,7 @@ int32_t SnapshotStartWrite(struct SSyncFSM* pFsm, void* pParam, void** ppWriter) return 0; } -int32_t SnapshotStopWrite(struct SSyncFSM* pFsm, void* pWriter, bool isApply) { +int32_t SnapshotStopWrite(struct SSyncFSM* pFsm, void* pWriter, bool isApply, SSnapshot *pSnapshot) { char logBuf[256] = {0}; snprintf(logBuf, sizeof(logBuf), "==callback== ==SnapshotStopWrite== pFsm:%p, pWriter:%p, isApply:%d", pFsm, pWriter, isApply); diff --git a/source/libs/sync/test/syncSnapshotReceiverTest.cpp b/source/libs/sync/test/syncSnapshotReceiverTest.cpp index e5d93ddff4..b744843b1e 100644 --- a/source/libs/sync/test/syncSnapshotReceiverTest.cpp +++ b/source/libs/sync/test/syncSnapshotReceiverTest.cpp @@ -30,7 +30,7 @@ int32_t SnapshotStopRead(struct SSyncFSM* pFsm, void* pReader) { return 0; } int32_t SnapshotDoRead(struct SSyncFSM* pFsm, void* pReader, void** ppBuf, int32_t* len) { return 0; } int32_t SnapshotStartWrite(struct SSyncFSM* pFsm, void* pParam, void** ppWriter) { return 0; } -int32_t SnapshotStopWrite(struct SSyncFSM* pFsm, void* pWriter, bool isApply) { return 0; } +int32_t SnapshotStopWrite(struct SSyncFSM* pFsm, void* pWriter, bool isApply, SSnapshot *pSnapshot) { return 0; } int32_t SnapshotDoWrite(struct SSyncFSM* pFsm, void* pWriter, void* pBuf, int32_t len) { return 0; } SSyncSnapshotReceiver* createReceiver() { diff --git a/source/libs/sync/test/syncTestTool.cpp b/source/libs/sync/test/syncTestTool.cpp index f35c6f8a2f..b0a561cb89 100644 --- a/source/libs/sync/test/syncTestTool.cpp +++ b/source/libs/sync/test/syncTestTool.cpp @@ -126,7 +126,7 @@ int32_t SnapshotStartWrite(struct SSyncFSM* pFsm, void* pParam, void** ppWriter) return 0; } -int32_t SnapshotStopWrite(struct SSyncFSM* pFsm, void* pWriter, bool isApply) { +int32_t SnapshotStopWrite(struct SSyncFSM* pFsm, void* pWriter, bool isApply, SSnapshot *pSnapshot) { if (isApply) { gSnapshotLastApplyIndex = gFinishLastApplyIndex; gSnapshotLastApplyTerm = gFinishLastApplyTerm; diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 3768eef7c9..cca451ceb7 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -67,7 +67,8 @@ static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell, int dropOfp, TXN * static int tdbBtcMoveDownward(SBTC *pBtc); static int tdbBtcMoveUpward(SBTC *pBtc); -int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, tdb_cmpr_fn_t kcmpr, SBTree **ppBt) { +int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, char const *tbname, SPgno pgno, tdb_cmpr_fn_t kcmpr, + SBTree **ppBt) { SBTree *pBt; int ret; @@ -99,13 +100,54 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, tdb_cmpr_fn_t kcmpr, SB // pBt->minLeaf pBt->minLeaf = pBt->minLocal; + // if pgno == 0 fetch new btree root leaf page + if (pgno == 0) { + // fetch page & insert into main db + // allocate a new child page + SPage *pPage; + TXN txn; + tdbTxnOpen(&txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, 0); + + pPager->inTran = 1; + + SBtreeInitPageArg zArg; + zArg.flags = 0x1 | 0x2; // root leaf node; + zArg.pBt = pBt; + ret = tdbPagerFetchPage(pPager, &pgno, &pPage, tdbBtreeInitPage, &zArg, &txn); + if (ret < 0) { + return -1; + } + + // TODO: Need to zero the page + + ret = tdbPagerWrite(pPager, pPage); + if (ret < 0) { + return -1; + } + + if (strcmp(TDB_MAINDB_NAME, tbname)) { + ret = tdbTbInsert(pPager->pEnv->pMainDb, tbname, strlen(tbname) + 1, &pgno, sizeof(SPgno), &txn); + if (ret < 0) { + return -1; + } + } + + // tdbUnrefPage(pPage); + tdbPCacheRelease(pPager->pCache, pPage, &txn); + tdbCommit(pPager->pEnv, &txn); + tdbTxnClose(&txn); + } + + ASSERT(pgno != 0); + pBt->root = pgno; + /* // TODO: pBt->root ret = tdbBtreeOpenImpl(pBt); if (ret < 0) { tdbOsFree(pBt); return -1; } - + */ *ppBt = pBt; return 0; } @@ -338,7 +380,6 @@ static int tdbBtreeOpenImpl(SBTree *pBt) { ASSERT(pgno != 0); pBt->root = pgno; - return 0; } diff --git a/source/libs/tdb/src/db/tdbDb.c b/source/libs/tdb/src/db/tdbDb.c index 186217878b..298992a560 100644 --- a/source/libs/tdb/src/db/tdbDb.c +++ b/source/libs/tdb/src/db/tdbDb.c @@ -64,6 +64,14 @@ int32_t tdbOpen(const char *dbname, int32_t szPage, int32_t pages, TDB **ppDb) { mkdir(dbname, 0755); +#ifdef USE_MAINDB + // open main db + ret = tdbTbOpen(TDB_MAINDB_NAME, -1, sizeof(SPgno), NULL, pDb, &pDb->pMainDb); + if (ret < 0) { + return -1; + } +#endif + *ppDb = pDb; return 0; } @@ -72,6 +80,10 @@ int tdbClose(TDB *pDb) { SPager *pPager; if (pDb) { +#ifdef USE_MAINDB + if (pDb->pMainDb) tdbTbClose(pDb->pMainDb); +#endif + for (pPager = pDb->pgrList; pPager; pPager = pDb->pgrList) { pDb->pgrList = pPager->pNext; tdbPagerClose(pPager); @@ -179,4 +191,4 @@ void tdbEnvRemovePager(TDB *pDb, SPager *pPager) { if (pDb->nPgrHash > 8 && pDb->nPager < pDb->nPgrHash / 2) { // TODO } -} \ No newline at end of file +} diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index 60eaf467ae..e7765ed667 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -174,6 +174,13 @@ int tdbPagerWrite(SPager *pPager, SPage *pPage) { for (ppPage = &pPager->pDirty; (*ppPage) && TDB_PAGE_PGNO(*ppPage) < TDB_PAGE_PGNO(pPage); ppPage = &((*ppPage)->pDirtyNext)) { } + + if (*ppPage && TDB_PAGE_PGNO(*ppPage) == TDB_PAGE_PGNO(pPage)) { + tdbUnrefPage(pPage); + + return 0; + } + ASSERT(*ppPage == NULL || TDB_PAGE_PGNO(*ppPage) > TDB_PAGE_PGNO(pPage)); pPage->pDirtyNext = *ppPage; *ppPage = pPage; @@ -467,7 +474,7 @@ int tdbPagerRestore(SPager *pPager, SBTree *pBt) { } TXN txn; - tdbTxnOpen(&txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, 0); + tdbTxnOpen(&txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED); SBtreeInitPageArg iArg; iArg.pBt = pBt; iArg.flags = 0; @@ -487,6 +494,7 @@ int tdbPagerRestore(SPager *pPager, SBTree *pBt) { return -1; } + /* ret = tdbPagerFetchPage(pPager, &pgno, &pPage, tdbBtreeInitPage, &iArg, &txn); if (ret < 0) { return -1; @@ -499,6 +507,18 @@ int tdbPagerRestore(SPager *pPager, SBTree *pBt) { } tdbPCacheRelease(pPager->pCache, pPage, &txn); + */ + i64 offset = pPager->pageSize * (pgno - 1); + if (tdbOsLSeek(pPager->fd, offset, SEEK_SET) < 0) { + ASSERT(0); + return -1; + } + + ret = tdbOsWrite(pPager->fd, pageBuf, pPager->pageSize); + if (ret < 0) { + ASSERT(0); + return -1; + } } tdbOsFSync(pPager->fd); diff --git a/source/libs/tdb/src/db/tdbTable.c b/source/libs/tdb/src/db/tdbTable.c index f1bfb4eabc..380baefb46 100644 --- a/source/libs/tdb/src/db/tdbTable.c +++ b/source/libs/tdb/src/db/tdbTable.c @@ -16,7 +16,7 @@ #include "tdbInt.h" struct STTB { - TDB * pEnv; + TDB *pEnv; SBTree *pBt; }; @@ -25,12 +25,16 @@ struct STBC { }; int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprFn, TDB *pEnv, TTB **ppTb) { - TTB * pTb; + TTB *pTb; SPager *pPager; int ret; char fFullName[TDB_FILENAME_LEN]; - SPage * pPage; + SPage *pPage; SPgno pgno; + void *pKey = NULL; + int nKey = 0; + void *pData = NULL; + int nData = 0; *ppTb = NULL; @@ -42,6 +46,48 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF // pTb->pEnv pTb->pEnv = pEnv; +#ifdef USE_MAINDB + snprintf(fFullName, TDB_FILENAME_LEN, "%s/%s", pEnv->dbName, TDB_MAINDB_NAME); + + if (strcmp(TDB_MAINDB_NAME, tbname)) { + pPager = tdbEnvGetPager(pEnv, fFullName); + if (!pPager) { + return -1; + } + + ret = tdbTbGet(pPager->pEnv->pMainDb, tbname, strlen(tbname) + 1, &pData, &nData); + if (ret < 0) { + // new pgno & insert into main db + pgno = 0; + } else { + pgno = *(SPgno *)pData; + + tdbFree(pKey); + tdbFree(pData); + } + + } else { + pPager = tdbEnvGetPager(pEnv, fFullName); + if (pPager == NULL) { + ret = tdbPagerOpen(pEnv->pCache, fFullName, &pPager); + if (ret < 0) { + return -1; + } + + tdbEnvAddPager(pEnv, pPager); + + pPager->pEnv = pEnv; + } + + if (pPager->dbOrigSize > 0) { + pgno = 1; + } else { + pgno = 0; + } + } + +#else + pPager = tdbEnvGetPager(pEnv, tbname); if (pPager == NULL) { snprintf(fFullName, TDB_FILENAME_LEN, "%s/%s", pEnv->dbName, tbname); @@ -53,10 +99,12 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF tdbEnvAddPager(pEnv, pPager); } +#endif + ASSERT(pPager != NULL); // pTb->pBt - ret = tdbBtreeOpen(keyLen, valLen, pPager, keyCmprFn, &(pTb->pBt)); + ret = tdbBtreeOpen(keyLen, valLen, pPager, tbname, pgno, keyCmprFn, &(pTb->pBt)); if (ret < 0) { return -1; } diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h index 2713a12af3..1f38cea038 100644 --- a/source/libs/tdb/src/inc/tdbInt.h +++ b/source/libs/tdb/src/inc/tdbInt.h @@ -150,7 +150,8 @@ struct SBTC { }; // SBTree -int tdbBtreeOpen(int keyLen, int valLen, SPager *pFile, tdb_cmpr_fn_t kcmpr, SBTree **ppBt); +int tdbBtreeOpen(int keyLen, int valLen, SPager *pFile, char const *tbname, SPgno pgno, tdb_cmpr_fn_t kcmpr, + SBTree **ppBt); int tdbBtreeClose(SBTree *pBt); int tdbBtreeInsert(SBTree *pBt, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn); int tdbBtreeDelete(SBTree *pBt, const void *pKey, int kLen, TXN *pTxn); @@ -356,6 +357,12 @@ static inline SCell *tdbPageGetCell(SPage *pPage, int idx) { return pCell; } +#define USE_MAINDB + +#ifdef USE_MAINDB +#define TDB_MAINDB_NAME "main.tdb" +#endif + struct STDB { char *dbName; char *jnName; @@ -365,6 +372,9 @@ struct STDB { int nPager; int nPgrHash; SPager **pgrHash; +#ifdef USE_MAINDB + TTB *pMainDb; +#endif }; struct SPager { @@ -381,6 +391,9 @@ struct SPager { u8 inTran; SPager *pNext; // used by TDB SPager *pHashNext; // used by TDB +#ifdef USE_MAINDB + TDB *pEnv; +#endif }; #ifdef __cplusplus diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 8a16b20a6f..725f3b32cf 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -170,7 +170,7 @@ void rpcSetDefaultAddr(void* thandle, const char* ip, const char* fqdn) { transSetDefaultAddr(thandle, ip, fqdn); } -int64_t rpcAllocHandle() { return transAllocHandle(); } +void* rpcAllocHandle() { return (void*)transAllocHandle(); } int32_t rpcInit() { transInit(); diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 03e869e078..be3111e870 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1,4 +1,5 @@ /** Copyright (c) 2019 TAOS Data, Inc. + * * This program is free software: you can use, redistribute, and/or modify * it under the terms of the GNU Affero General Public License, version 3 @@ -809,7 +810,7 @@ SCliConn* cliGetConn(SCliMsg* pMsg, SCliThrd* pThrd, bool* ignore) { conn = exh->handle; if (conn == NULL) { conn = getConnFromPool(pThrd->pool, EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet)); - *ignore = (conn && 0 == specifyConnRef(conn, true, refId)) ? false : true; + if (conn != NULL) specifyConnRef(conn, true, refId); } transReleaseExHandle(transGetRefMgt(), refId); } @@ -849,14 +850,20 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) { bool ignore = false; SCliConn* conn = cliGetConn(pMsg, pThrd, &ignore); if (ignore == true) { + tError("ignore msg"); return; } + if (conn != NULL) { transCtxMerge(&conn->ctx, &pCtx->appCtx); transQueuePush(&conn->cliMsgs, pMsg); cliSend(conn); } else { conn = cliCreateConn(pThrd); + + int64_t refId = (int64_t)pMsg->msg.info.handle; + if (refId != 0) specifyConnRef(conn, true, refId); + transCtxMerge(&conn->ctx, &pCtx->appCtx); transQueuePush(&conn->cliMsgs, pMsg); @@ -1206,7 +1213,13 @@ SCliThrd* transGetWorkThrd(STrans* trans, int64_t handle, bool* validHandle) { if (idx < 0) return NULL; return ((SCliObj*)trans->tcphandle)->pThreadObj[idx]; } - return transGetWorkThrdFromHandle(handle, validHandle); + SCliThrd* pThrd = transGetWorkThrdFromHandle(handle, validHandle); + if (*validHandle == true && pThrd == NULL) { + int idx = cliRBChoseIdx(trans); + if (idx < 0) return NULL; + pThrd = ((SCliObj*)trans->tcphandle)->pThreadObj[idx]; + } + return pThrd; } void transReleaseCliHandle(void* handle) { int idx = -1; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index cc5c9eb651..b323887d93 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -336,8 +336,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_VND_TABLE_NOT_EXIST, "Table does not exists TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_TABLE_ACTION, "Invalid table action") TAOS_DEFINE_ERROR(TSDB_CODE_VND_COL_ALREADY_EXISTS, "Table column already exists") TAOS_DEFINE_ERROR(TSDB_CODE_VND_TABLE_COL_NOT_EXISTS, "Table column not exists") -TAOS_DEFINE_ERROR(TSDB_CODE_VND_READ_END, "Read end") - // tsdb TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_ID, "Invalid table ID") diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index 585594e035..4961355f06 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -79,22 +79,39 @@ class TDSql: self.queryResult = None tdLog.info("sql:%s, expect error occured" % (sql)) - def query(self, sql, row_tag=None): + def query(self, sql, row_tag=None,queyTimes=10): self.sql = sql - try: - self.cursor.execute(sql) - self.queryResult = self.cursor.fetchall() - self.queryRows = len(self.queryResult) - self.queryCols = len(self.cursor.description) - except Exception as e: - caller = inspect.getframeinfo(inspect.stack()[1][0]) - args = (caller.filename, caller.lineno, sql, repr(e)) - tdLog.notice("%s(%d) failed: sql:%s, %s" % args) - traceback.print_exc() - raise Exception(repr(e)) - if row_tag: - return self.queryResult - return self.queryRows + i=1 + while i <= queyTimes: + try: + self.cursor.execute(sql) + self.queryResult = self.cursor.fetchall() + self.queryRows = len(self.queryResult) + self.queryCols = len(self.cursor.description) + if row_tag: + return self.queryResult + return self.queryRows + except Exception as e: + i+=1 + tdLog.notice("Try to query again, query times: %d "%i) + pass + else: + try: + tdLog.notice("Try the last query ") + self.cursor.execute(sql) + self.queryResult = self.cursor.fetchall() + self.queryRows = len(self.queryResult) + self.queryCols = len(self.cursor.description) + if row_tag: + return self.queryResult + return self.queryRows + except Exception as e: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, sql, repr(e)) + tdLog.notice("%s(%d) failed: sql:%s, %s" % args) + traceback.print_exc() + raise Exception(repr(e)) + def is_err_sql(self, sql): err_flag = True @@ -266,16 +283,27 @@ class TDSql: time.sleep(1) continue - def execute(self, sql): + def execute(self, sql,queyTimes=10): self.sql = sql - try: - self.affectedRows = self.cursor.execute(sql) - except Exception as e: - caller = inspect.getframeinfo(inspect.stack()[1][0]) - args = (caller.filename, caller.lineno, sql, repr(e)) - tdLog.notice("%s(%d) failed: sql:%s, %s" % args) - raise Exception(repr(e)) - return self.affectedRows + i=1 + while i <= queyTimes: + try: + self.affectedRows = self.cursor.execute(sql) + return self.affectedRows + except Exception as e: + i+=1 + tdLog.notice("Try to execute sql again, query times: %d "%i) + pass + else: + try: + tdLog.notice("Try the last execute sql ") + self.affectedRows = self.cursor.execute(sql) + return self.affectedRows + except Exception as e: + caller = inspect.getframeinfo(inspect.stack()[1][0]) + args = (caller.filename, caller.lineno, sql, repr(e)) + tdLog.notice("%s(%d) failed: sql:%s, %s" % args) + raise Exception(repr(e)) def checkAffectedRows(self, expectAffectedRows): if self.affectedRows != expectAffectedRows: diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 01896ec161..0bdbd644a0 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -266,26 +266,27 @@ ./test.sh -f tsim/tmq/snapshot1.sim # --- stable +./test.sh -f tsim/stable/alter_comment.sim +./test.sh -f tsim/stable/alter_count.sim +./test.sh -f tsim/stable/alter_import.sim +./test.sh -f tsim/stable/alter_insert1.sim +./test.sh -f tsim/stable/alter_insert2.sim +./test.sh -f tsim/stable/alter_metrics.sim +./test.sh -f tsim/stable/column_add.sim +./test.sh -f tsim/stable/column_drop.sim +./test.sh -f tsim/stable/column_modify.sim ./test.sh -f tsim/stable/disk.sim ./test.sh -f tsim/stable/dnode3.sim ./test.sh -f tsim/stable/metrics.sim ./test.sh -f tsim/stable/refcount.sim ./test.sh -f tsim/stable/show.sim -./test.sh -f tsim/stable/values.sim -./test.sh -f tsim/stable/vnode3.sim -./test.sh -f tsim/stable/column_add.sim -./test.sh -f tsim/stable/column_drop.sim -./test.sh -f tsim/stable/column_modify.sim ./test.sh -f tsim/stable/tag_add.sim ./test.sh -f tsim/stable/tag_drop.sim +./test.sh -f tsim/stable/tag_filter.sim ./test.sh -f tsim/stable/tag_modify.sim ./test.sh -f tsim/stable/tag_rename.sim -./test.sh -f tsim/stable/alter_comment.sim -./test.sh -f tsim/stable/alter_count.sim -./test.sh -f tsim/stable/alter_insert1.sim -./test.sh -f tsim/stable/alter_insert2.sim -./test.sh -f tsim/stable/alter_import.sim -./test.sh -f tsim/stable/tag_filter.sim +./test.sh -f tsim/stable/values.sim +./test.sh -f tsim/stable/vnode3.sim # --- for multi process mode ./test.sh -f tsim/user/basic.sim -m @@ -309,6 +310,8 @@ ./test.sh -f tsim/valgrind/checkError1.sim ./test.sh -f tsim/valgrind/checkError2.sim ./test.sh -f tsim/valgrind/checkError3.sim +./test.sh -f tsim/valgrind/checkError4.sim +./test.sh -f tsim/valgrind/checkError5.sim # --- vnode # unsupport ./test.sh -f tsim/vnode/replica3_basic.sim @@ -366,7 +369,7 @@ ./test.sh -f tsim/compute/diff2.sim ./test.sh -f tsim/compute/first.sim ./test.sh -f tsim/compute/interval.sim -# jira ./test.sh -f tsim/compute/last_row.sim +./test.sh -f tsim/compute/last_row.sim ./test.sh -f tsim/compute/last.sim ./test.sh -f tsim/compute/leastsquare.sim ./test.sh -f tsim/compute/max.sim @@ -415,7 +418,7 @@ ./test.sh -f tsim/tag/4.sim ./test.sh -f tsim/tag/5.sim # jira ./test.sh -f tsim/tag/6.sim -# jira ./test.sh -f tsim/tag/add.sim +./test.sh -f tsim/tag/add.sim ./test.sh -f tsim/tag/bigint.sim ./test.sh -f tsim/tag/binary_binary.sim ./test.sh -f tsim/tag/binary.sim diff --git a/tests/script/tsim/compute/last_row.sim b/tests/script/tsim/compute/last_row.sim index 590fada86a..70fb6626cd 100644 --- a/tests/script/tsim/compute/last_row.sim +++ b/tests/script/tsim/compute/last_row.sim @@ -65,8 +65,6 @@ if $data00 != 19 then return -1 endi - - print =============== step7 sql select last_row(tbcol) from $mt print ===> $data00 diff --git a/tests/script/tsim/show/basic.sim b/tests/script/tsim/show/basic.sim index d0643bc7f2..4d646f39e3 100644 --- a/tests/script/tsim/show/basic.sim +++ b/tests/script/tsim/show/basic.sim @@ -99,7 +99,7 @@ if $rows != 1 then endi #sql select * from information_schema.`streams` sql select * from information_schema.user_tables -if $rows != 30 then +if $rows != 31 then return -1 endi #sql select * from information_schema.user_table_distributed @@ -197,7 +197,7 @@ if $rows != 1 then endi #sql select * from performance_schema.`streams` sql select * from information_schema.user_tables -if $rows != 30 then +if $rows != 31 then return -1 endi #sql select * from information_schema.user_table_distributed diff --git a/tests/script/tsim/stable/alter_count.sim b/tests/script/tsim/stable/alter_count.sim index decad53f64..83ea4b14fa 100644 --- a/tests/script/tsim/stable/alter_count.sim +++ b/tests/script/tsim/stable/alter_count.sim @@ -249,9 +249,7 @@ endi print ============== step18 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 3000 system sh/exec.sh -n dnode1 -s start -sleep 3000 #sql select count(g) from tb #if $data00 != 12 then diff --git a/tests/script/tsim/stable/alter_insert1.sim b/tests/script/tsim/stable/alter_insert1.sim index 82781f2fe5..bcea0b48c4 100644 --- a/tests/script/tsim/stable/alter_insert1.sim +++ b/tests/script/tsim/stable/alter_insert1.sim @@ -934,11 +934,8 @@ if $data79 != null then endi print ======== step9 - system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 3000 system sh/exec.sh -n dnode1 -s start -sleep 3000 sql select * from tb order by ts asc if $rows != 8 then diff --git a/tests/script/tsim/stable/alter_insert2.sim b/tests/script/tsim/stable/alter_insert2.sim index a30175f398..faae24d32a 100644 --- a/tests/script/tsim/stable/alter_insert2.sim +++ b/tests/script/tsim/stable/alter_insert2.sim @@ -604,9 +604,7 @@ sql_error alter table tb drop column a print ======== step9 system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 3000 system sh/exec.sh -n dnode1 -s start -sleep 3000 sql select * from tb order by ts desc if $rows != 7 then diff --git a/tests/script/tsim/stable/disk.sim b/tests/script/tsim/stable/disk.sim index ff734b4234..aeb1f1d91f 100644 --- a/tests/script/tsim/stable/disk.sim +++ b/tests/script/tsim/stable/disk.sim @@ -77,12 +77,11 @@ if $data00 != $rowNum then endi print =============== step3 -# TODO : where condition -# sql select count(tbcol) from $tb where ts <= 1519833840000 -# print ===> $data00 -# if $data00 != 5 then -# return -1 -# endi +sql select count(tbcol) from $tb where ts <= 1519833840000 +print ===> $data00 +if $data00 != 5 then + return -1 +endi print =============== step4 sql select count(tbcol) as b from $tb @@ -105,89 +104,82 @@ if $data00 != $rowNum then endi print =============== step6 -# TODO -# sql select count(tbcol) as b from $tb where ts <= 1519833840000 interval(1m) -# print ===> $data00 -# if $data00 != 1 then -# return -1 -# endi -# if $rows != 5 then -# return -1 -# endi +sql select count(tbcol) as b from $tb where ts <= 1519833840000 interval(1m) +print ===> $data00 +if $data00 != 1 then + return -1 +endi +if $rows != 5 then + return -1 +endi print =============== step7 -# TODO -# sql select count(*) from $mt -# print select count(*) from $mt ===> $data00 -# if $data00 != $totalNum then -# return -1 -# endi +sql select count(*) from $mt +print select count(*) from $mt ===> $data00 +if $data00 != $totalNum then + return -1 +endi -# TODO -# print ==========> block opt will cause this crash, table scan need to fix this during plan gen ===============> -#sql select count(tbcol) from $mt -#print ===> $data00 -#if $data00 != $totalNum then -# return -1 -#endi +print ==========> block opt will cause this crash, table scan need to fix this during plan gen ===============> +sql select count(tbcol) from $mt +print ===> $data00 +if $data00 != $totalNum then + return -1 +endi print =============== step8 -# TODO -# sql select count(tbcol) as c from $mt where ts <= 1519833840000 -# print ===> $data00 -# if $data00 != 50 then -# return -1 -# endi -# -# sql select count(tbcol) as c from $mt where tgcol < 5 -# print ===> $data00 -# if $data00 != 100 then -# return -1 -# endi -# -# sql select count(tbcol) as c from $mt where tgcol < 5 and ts <= 1519833840000 -# print ===> $data00 -# if $data00 != 25 then -# return -1 -# endi +sql select count(tbcol) as c from $mt where ts <= 1519833840000 +print ===> $data00 +if $data00 != 50 then + return -1 +endi + +sql select count(tbcol) as c from $mt where tgcol < 5 +print ===> $data00 +if $data00 != 100 then + return -1 +endi + +sql select count(tbcol) as c from $mt where tgcol < 5 and ts <= 1519833840000 +print ===> $data00 +if $data00 != 25 then + return -1 +endi print =============== step9 -# TODO : count from stable -# sql select count(tbcol) as b from $mt interval(1m) -# print select count(tbcol) as b from $mt interval(1m) ===> $data01 -# if $data01 != 10 then -# return -1 -# endi +sql select count(tbcol) as b from $mt interval(1m) +print select count(tbcol) as b from $mt interval(1m) ===> $data01 +if $data00 != 10 then + return -1 +endi -# sql select count(tbcol) as b from $mt interval(1d) -# print ===> $data02 -# if $data01 != 200 then -# return -1 -# endi +sql select count(tbcol) as b from $mt interval(1d) +print ===> $data02 +if $data00 != 200 then + return -1 +endi print =============== step10 -# TODO -# print select count(tbcol) as b from $mt group by tgcol -# sql select count(tbcol) as b from $mt group by tgcol -# print ===> $data00 -# if $data00 != $rowNum then -# return -1 -# endi +print select count(tbcol) as b from $mt group by tgcol +sql select count(tbcol) as b from $mt group by tgcol +print ===> $data00 +if $data00 != $rowNum then + return -1 +endi + +if $rows != $tbNum then + return -1 +endi -# if $rows != $tbNum then -# return -1 -# endi -# print =============== step11 -# TODO : where condition -# sql select count(tbcol) as b from $mt where ts <= 1519833840000 interval(1m) group by tgcol -# print ===> $data01 -# if $data01 != 1 then -# return -1 -# endi -# if $rows != 50 then -# return -1 -# endi +sql select count(tbcol) as b from $mt where ts <= 1519833840000 partition by tgcol interval(1m) +print ===> $data01 +if $data00 != 1 then + return -1 +endi +if $rows != 50 then + return -1 +endi print =============== clear sql drop database $db diff --git a/tests/script/tsim/stable/dnode3.sim b/tests/script/tsim/stable/dnode3.sim index 03e8df26b7..9e728a12ab 100644 --- a/tests/script/tsim/stable/dnode3.sim +++ b/tests/script/tsim/stable/dnode3.sim @@ -69,8 +69,6 @@ if $rows != 2 then return -1 endi -sleep 100 - print =============== step2 $i = 1 $tb = $tbPrefix . $i @@ -88,12 +86,11 @@ if $data00 != $rowNum then endi print =============== step3 -# TODO : where condition -# sql select count(tbcol) from $tb where ts <= 1519833840000 -# print ===> $data00 -# if $data00 != 5 then -# return -1 -# endi +sql select count(tbcol) from $tb where ts <= 1519833840000 +print ===> $data00 +if $data00 != 5 then + return -1 +endi print =============== step4 sql select count(tbcol) as b from $tb @@ -116,82 +113,81 @@ if $data00 != $rowNum then endi print =============== step6 -# sql select count(tbcol) as b from $tb where ts <= 1519833840000 interval(1m) -# print ===> $data00 -# if $data00 != 1 then -# return -1 -# endi -# if $rows != 5 then -# return -1 -# endi +sql select count(tbcol) as b from $tb where ts <= 1519833840000 interval(1m) +print ===> $data00 +if $data00 != 1 then + return -1 +endi +if $rows != 5 then + return -1 +endi print =============== step7 -# print select count(*) from $mt -# sql select count(*) from $mt -# print ===> $data00 -# if $data00 != $totalNum then -# return -1 -# endi -# -# sql select count(tbcol) from $mt -# print ===> $data00 -# if $data00 != $totalNum then -# return -1 -# endi +print select count(*) from $mt +sql select count(*) from $mt +print ===> $data00 +if $data00 != $totalNum then + return -1 +endi + +sql select count(tbcol) from $mt +print ===> $data00 +if $data00 != $totalNum then + return -1 +endi print =============== step8 -# sql select count(tbcol) as c from $mt where ts <= 1519833840000 -# print ===> $data00 -# if $data00 != 50 then -# return -1 -# endi -# -# sql select count(tbcol) as c from $mt where tgcol < 5 -# print ===> $data00 -# if $data00 != 100 then -# return -1 -# endi -# -# sql select count(tbcol) as c from $mt where tgcol < 5 and ts <= 1519833840000 -# print ===> $data00 -# if $data00 != 25 then -# return -1 -# endi +sql select count(tbcol) as c from $mt where ts <= 1519833840000 +print ===> $data00 +if $data00 != 50 then + return -1 +endi + +sql select count(tbcol) as c from $mt where tgcol < 5 +print ===> $data00 +if $data00 != 100 then + return -1 +endi + +sql select count(tbcol) as c from $mt where tgcol < 5 and ts <= 1519833840000 +print ===> $data00 +if $data00 != 25 then + return -1 +endi print =============== step9 -# TODO : group by in stable -# sql select count(tbcol) as b from $mt interval(1m) -# print ===> $data00 -# if $data00 != 10 then -# return -1 -# endi -# -# sql select count(tbcol) as b from $mt interval(1d) -# print ===> $data00 -# if $data00 != 200 then -# return -1 -# endi +sql select count(tbcol) as b from $mt interval(1m) +print ===> $data00 +if $data00 != 10 then + return -1 +endi + +sql select count(tbcol) as b from $mt interval(1d) +print ===> $data00 +if $data00 != 200 then + return -1 +endi print =============== step10 -# sql select count(tbcol) as b from $mt group by tgcol -# print ===> $data00 -# if $data00 != $rowNum then -# return -1 -# endi -# -# if $rows != $tbNum then -# return -1 -# endi +sql select count(tbcol) as b from $mt group by tgcol +print ===> $data00 +if $data00 != $rowNum then + return -1 +endi + +if $rows != $tbNum then + return -1 +endi print =============== step11 -# sql select count(tbcol) as b from $mt where ts <= 1519833840000 interval(1m) group by tgcol -# print ===> $data00 -# if $data00 != 1 then -# return -1 -# endi -# if $rows != 50 then -# return -1 -# endi +sql select count(tbcol) as b from $mt where ts <= 1519833840000 partition by tgcol interval(1m) +print ===> $data00 +if $data00 != 1 then + return -1 +endi +if $rows != 50 then + return -1 +endi print =============== clear sql drop database $db diff --git a/tests/script/tsim/stable/metrics.sim b/tests/script/tsim/stable/metrics.sim index c652670d7f..a1c370d40a 100644 --- a/tests/script/tsim/stable/metrics.sim +++ b/tests/script/tsim/stable/metrics.sim @@ -24,17 +24,15 @@ if $rows != 1 then return -1 endi -#TODO OPEN THIS WHEN STABLE DELETE WORKS -#print =============== step2 -#sql drop table $mt -#sql show stables -#if $rows != 0 then -# return -1 -#endi +print =============== step2 +sql drop table $mt +sql show stables +if $rows != 0 then + return -1 +endi -#print =============== step3 -#sql create table $mt (ts timestamp, speed int) TAGS(sp int) -#TODO OPEN THIS WHEN STABLE DELETE WORKS +print =============== step3 +sql create table $mt (ts timestamp, speed int) TAGS(sp int) sql show stables if $rows != 1 then @@ -46,11 +44,10 @@ endi if $data04 != 1 then return -1 endi -# TODO : select * from stable -# sql select * from $mt -# if $rows != 0 then -# return -1 -# endi +sql select * from $mt +if $rows != 0 then + return -1 +endi print =============== step4 $i = 0 @@ -67,9 +64,9 @@ sql show tables if $rows != 3 then return -1 endi -# if $data03 != $mt then -# return -1 -# endi +if $data04 != $mt then + return -1 +endi sql show stables if $rows != 1 then @@ -78,9 +75,9 @@ endi if $data00 != $mt then return -1 endi -# if $data04 != 3 then -# return -1 -# endi +if $data03 != 2 then + return -1 +endi print =============== step5 $i = 0 @@ -95,28 +92,28 @@ sql insert into $tb values (now + 1m , 1 ) print =============== step6 -# sql select * from $mt -# print select * from $mt ==> $rows $data00 -# if $rows != 3 then -# return -1 -# endi +sql select * from $mt +print select * from $mt ==> $rows $data00 +if $rows != 3 then + return -1 +endi print =============== step7 -# sql select * from $mt where sp = 1 -# print select * from $mt where sp = 1 ==> $rows $data00 -# if $rows != 1 then -# return -1 -# endi +sql select * from $mt where sp = 1 +print select * from $mt where sp = 1 ==> $rows $data00 +if $rows != 1 then + return -1 +endi print =============== step8 sql drop table $mt print =============== step9 -#sql show tables -#if $rows != 0 then -# return -1 -#endi +sql show tables +if $rows != 0 then + return -1 +endi sql show stables if $rows != 0 then diff --git a/tests/script/tsim/stable/refcount.sim b/tests/script/tsim/stable/refcount.sim index d77c8e0890..8f4f09cbb3 100644 --- a/tests/script/tsim/stable/refcount.sim +++ b/tests/script/tsim/stable/refcount.sim @@ -11,7 +11,7 @@ sql create table d1.t2 (ts timestamp, i int); sql create table d1.t3 (ts timestamp, i int); sql insert into d1.t1 values(now, 1); sql insert into d1.t2 values(now, 1); -# sql drop table d1.t1; +sql drop table d1.t1; sql drop database d1; sql show databases; @@ -27,14 +27,14 @@ sql create table d2.t2 (ts timestamp, i int); sql create table d2.t3 (ts timestamp, i int); sql insert into d2.t1 values(now, 1); sql insert into d2.t2 values(now, 1); -# sql drop table d2.t1; -# sql drop table d2.t2; -# sql drop table d2.t3; -# -# sql show d2.tables; -# if $rows != 0 then -# return -1 -# endi +sql drop table d2.t1; +sql drop table d2.t2; +sql drop table d2.t3; + +sql show d2.tables; +if $rows != 0 then + return -1 +endi sql show d2.vgroups; if $rows != 2 then @@ -56,14 +56,14 @@ sql create table d3.t1 using d3.st tags(1); sql create table d3.t2 using d3.st tags(1); sql create table d3.t3 using d3.st tags(1); sql insert into d3.t1 values(now, 1); -# sql drop table d3.t1; -# sql drop table d3.t2; -# sql drop table d3.t3; -# -# sql show d3.tables; -# if $rows != 0 then -# return -1 -# endi +sql drop table d3.t1; +sql drop table d3.t2; +sql drop table d3.t3; + +sql show d3.tables; +if $rows != 0 then + return -1 +endi sql show d3.vgroups; if $rows != 2 then @@ -85,13 +85,13 @@ sql create table d4.t1 using d4.st tags(1); sql create table d4.t2 using d4.st tags(1); sql create table d4.t3 using d4.st tags(1); sql insert into d4.t1 values(now, 1); -# sql drop table d4.t1; +sql drop table d4.t1; sql drop table d4.st; -# -# sql show d4.tables; -# if $rows != 0 then -# return -1 -# endi + +sql show d4.tables; +if $rows != 0 then + return -1 +endi sql show d4.stables; if $rows != 0 then @@ -113,7 +113,7 @@ sql create table d5.t1 using d5.st tags(1); sql create table d5.t2 using d5.st tags(1); sql create table d5.t3 using d5.st tags(1); sql insert into d5.t1 values(now, 1); -# sql drop table d5.t1; +sql drop table d5.t1; sql drop database d5; diff --git a/tests/script/tsim/stable/tag_filter.sim b/tests/script/tsim/stable/tag_filter.sim index 1f400eb803..f44142fbbf 100644 --- a/tests/script/tsim/stable/tag_filter.sim +++ b/tests/script/tsim/stable/tag_filter.sim @@ -31,7 +31,6 @@ if $rows != 1 then return -1 endi - sql select * from db.stb where t1 < 1 if $rows != 0 then return -=1 diff --git a/tests/script/tsim/stable/values.sim b/tests/script/tsim/stable/values.sim index 88eca28a12..d3da101e27 100644 --- a/tests/script/tsim/stable/values.sim +++ b/tests/script/tsim/stable/values.sim @@ -12,22 +12,22 @@ sql create table vdb0.vtb01 using vdb0.mt tags( 0 ) sql create database vdb1 sql create table vdb1.mt (ts timestamp, tbcol int) TAGS(tgcol int) -# sql_error create table vdb1.vtb10 using vdb0.mt tags( 1 ) -# sql_error create table vdb1.vtb11 using vdb0.mt tags( 1 ) +sql_error create table vdb1.vtb10 using vdb0.mt tags( 1 ) +sql_error create table vdb1.vtb11 using vdb0.mt tags( 1 ) sql create table vdb1.vtb10 using vdb1.mt tags( 1 ) sql create table vdb1.vtb11 using vdb1.mt tags( 1 ) sql create database vdb2 sql create table vdb2.mt (ts timestamp, tbcol int) TAGS(tgcol int) -# sql_error create table vdb2.vtb20 using vdb0.mt tags( 2 ) -# sql_error create table vdb2.vtb21 using vdb0.mt tags( 2 ) +sql_error create table vdb2.vtb20 using vdb0.mt tags( 2 ) +sql_error create table vdb2.vtb21 using vdb0.mt tags( 2 ) sql create table vdb2.vtb20 using vdb2.mt tags( 2 ) sql create table vdb2.vtb21 using vdb2.mt tags( 2 ) sql create database vdb3 sql create table vdb3.mt (ts timestamp, tbcol int) TAGS(tgcol int) -# sql_error create table vdb3.vtb20 using vdb0.mt tags( 2 ) -# sql_error create table vdb3.vtb21 using vdb0.mt tags( 2 ) +sql_error create table vdb3.vtb20 using vdb0.mt tags( 2 ) +sql_error create table vdb3.vtb21 using vdb0.mt tags( 2 ) sql create table vdb3.vtb30 using vdb3.mt tags( 3 ) sql create table vdb3.vtb31 using vdb3.mt tags( 3 ) @@ -40,7 +40,7 @@ sql insert into vdb2.vtb20 values (1519833600000 , 12) (1519833600001, 22) (1519 sql insert into vdb2.vtb21 values (1519833600000 , 12) (1519833600001, 22) (1519833600002, 32) sql insert into vdb3.vtb30 values (1519833600000 , 13) (1519833600001, 23) (1519833600002, 33) sql insert into vdb3.vtb31 values (1519833600000 , 13) (1519833600001, 23) (1519833600002, 33) -# sql select * from vdb0.mt +sql select * from vdb0.mt sql select ts from vdb0.mt if $rows != 6 then @@ -56,8 +56,7 @@ sql insert into vdb2.vtb20 values (1519833600003 , 42) (1519833600005, 52) (1519 sql insert into vdb2.vtb21 values (1519833600003 , 42) (1519833600005, 52) (1519833600004, 62) sql insert into vdb3.vtb30 values (1519833600003 , 43) (1519833600005, 53) (1519833600004, 63) sql insert into vdb3.vtb31 values (1519833600003 , 43) (1519833600005, 53) (1519833600004, 63) -# TODO : select * from stable -# sql select * from vdb0.mt +sql select * from vdb0.mt sql select ts from vdb0.mt if $rows != 12 then @@ -65,50 +64,49 @@ if $rows != 12 then endi print =============== step4 -# TODO : insert into diffrent table -# sql insert into vdb0.vtb00 values(1519833600006, 60) (1519833600007, 70) vdb0.vtb01 values(1519833600006, 60) (1519833600007, 70) -# sql insert into vdb1.vtb10 values(1519833600006, 61) (1519833600007, 71) vdb1.vtb11 values(1519833600006, 61) (1519833600007, 71) -# sql insert into vdb2.vtb20 values(1519833600006, 62) (1519833600007, 72) vdb2.vtb21 values(1519833600006, 62) (1519833600007, 72) -# sql insert into vdb3.vtb30 values(1519833600006, 63) (1519833600007, 73) vdb3.vtb31 values(1519833600006, 63) (1519833600007, 73) -# # sql select * from vdb0.mt -# sql select ts from vdb0.mt -# -# if $rows != 16 then -# return -1 -# endi +sql insert into vdb0.vtb00 values(1519833600006, 60) (1519833600007, 70) vdb0.vtb01 values(1519833600006, 60) (1519833600007, 70) +sql insert into vdb1.vtb10 values(1519833600006, 61) (1519833600007, 71) vdb1.vtb11 values(1519833600006, 61) (1519833600007, 71) +sql insert into vdb2.vtb20 values(1519833600006, 62) (1519833600007, 72) vdb2.vtb21 values(1519833600006, 62) (1519833600007, 72) +sql insert into vdb3.vtb30 values(1519833600006, 63) (1519833600007, 73) vdb3.vtb31 values(1519833600006, 63) (1519833600007, 73) +sql select * from vdb0.mt +sql select ts from vdb0.mt + +if $rows != 16 then + return -1 +endi print =============== step5 -# sql insert into vdb0.vtb00 values(1519833600008, 80) (1519833600007, 70) vdb0.vtb01 values(1519833600006, 80) (1519833600007, 70) -# sql insert into vdb1.vtb10 values(1519833600008, 81) (1519833600007, 71) vdb1.vtb11 values(1519833600006, 81) (1519833600007, 71) -# sql insert into vdb2.vtb20 values(1519833600008, 82) (1519833600007, 72) vdb2.vtb21 values(1519833600006, 82) (1519833600007, 72) -# sql insert into vdb3.vtb30 values(1519833600008, 83) (1519833600007, 73) vdb3.vtb31 values(1519833600006, 83) (1519833600007, 73) -# # sql select * from vdb0.mt -# sql select ts from vdb0.mt -# -# if $rows != 17 then -# return -1 -# endi +sql insert into vdb0.vtb00 values(1519833600008, 80) (1519833600007, 70) vdb0.vtb01 values(1519833600006, 80) (1519833600007, 70) +sql insert into vdb1.vtb10 values(1519833600008, 81) (1519833600007, 71) vdb1.vtb11 values(1519833600006, 81) (1519833600007, 71) +sql insert into vdb2.vtb20 values(1519833600008, 82) (1519833600007, 72) vdb2.vtb21 values(1519833600006, 82) (1519833600007, 72) +sql insert into vdb3.vtb30 values(1519833600008, 83) (1519833600007, 73) vdb3.vtb31 values(1519833600006, 83) (1519833600007, 73) +sql select * from vdb0.mt +sql select ts from vdb0.mt + +if $rows != 17 then + return -1 +endi print =============== step6 -# sql insert into vdb0.vtb00 values(1519833600009, 90) (1519833600010, 100) vdb1.vtb10 values(1519833600009, 90) (1519833600010, 100) vdb2.vtb20 values(1519833600009, 90) (1519833600010, 100) vdb3.vtb30 values(1519833600009, 90) (1519833600010, 100) -# sql insert into vdb0.vtb01 values(1519833600009, 90) (1519833600010, 100) vdb1.vtb11 values(1519833600009, 90) (1519833600010, 100) vdb2.vtb21 values(1519833600009, 90) (1519833600010, 100) vdb3.vtb31 values(1519833600009, 90) (1519833600010, 100) -# -# # sql select * from vdb0.mt -# sql select ts from vdb0.mt -# -# if $rows != 21 then -# return -1 -# endi +sql insert into vdb0.vtb00 values(1519833600009, 90) (1519833600010, 100) vdb1.vtb10 values(1519833600009, 90) (1519833600010, 100) vdb2.vtb20 values(1519833600009, 90) (1519833600010, 100) vdb3.vtb30 values(1519833600009, 90) (1519833600010, 100) +sql insert into vdb0.vtb01 values(1519833600009, 90) (1519833600010, 100) vdb1.vtb11 values(1519833600009, 90) (1519833600010, 100) vdb2.vtb21 values(1519833600009, 90) (1519833600010, 100) vdb3.vtb31 values(1519833600009, 90) (1519833600010, 100) + +sql select * from vdb0.mt +sql select ts from vdb0.mt + +if $rows != 21 then + return -1 +endi print =============== step7 -# sql insert into vdb0.vtb00 values(1519833600012, 120) (1519833600011, 110) vdb1.vtb10 values(1519833600012, 120) (1519833600011, 110) vdb2.vtb20 values(1519833600012, 120) (1519833600011, 110) vdb3.vtb30 values(1519833600012, 120) (1519833600011, 110) -# sql insert into vdb0.vtb01 values(1519833600012, 120) (1519833600011, 110) vdb1.vtb11 values(1519833600012, 120) (1519833600011, 110) vdb2.vtb21 values(1519833600012, 120) (1519833600011, 110) vdb3.vtb31 values(1519833600012, 120) (1519833600011, 110) -# -# # sql select * from vdb0.mt -# sql select ts from vdb0.mt -# -# if $rows != 25 then -# return -1 -# endi +sql insert into vdb0.vtb00 values(1519833600012, 120) (1519833600011, 110) vdb1.vtb10 values(1519833600012, 120) (1519833600011, 110) vdb2.vtb20 values(1519833600012, 120) (1519833600011, 110) vdb3.vtb30 values(1519833600012, 120) (1519833600011, 110) +sql insert into vdb0.vtb01 values(1519833600012, 120) (1519833600011, 110) vdb1.vtb11 values(1519833600012, 120) (1519833600011, 110) vdb2.vtb21 values(1519833600012, 120) (1519833600011, 110) vdb3.vtb31 values(1519833600012, 120) (1519833600011, 110) + +sql select * from vdb0.mt +sql select ts from vdb0.mt + +if $rows != 25 then + return -1 +endi system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/stable/vnode3.sim b/tests/script/tsim/stable/vnode3.sim index 186d0f5eea..584578b211 100644 --- a/tests/script/tsim/stable/vnode3.sim +++ b/tests/script/tsim/stable/vnode3.sim @@ -60,12 +60,11 @@ if $data00 != $rowNum then endi print =============== step3 -# TODO : where condition -# sql select count(tbcol) from $tb where ts <= 1519833840000 -# print ===> $data00 -# if $data00 != 5 then -# return -1 -# endi +sql select count(tbcol) from $tb where ts <= 1519833840000 +print ===> $data00 +if $data00 != 5 then + return -1 +endi print =============== step4 sql select count(tbcol) as b from $tb @@ -88,81 +87,80 @@ if $data00 != $rowNum then endi print =============== step6 -# sql select count(tbcol) as b from $tb where ts <= 1519833840000 interval(1m) -# print ===> $data00 -# if $data00 != 1 then -# return -1 -# endi -# if $rows != 5 then -# return -1 -#endi +sql select count(tbcol) as b from $tb where ts <= 1519833840000 interval(1m) +print ===> $data00 +if $data00 != 1 then + return -1 +endi +if $rows != 5 then + return -1 +endi print =============== step7 -# TODO : count(*) err -# sql select count(*) from $mt -# print ===> $data00 -# if $data00 != $totalNum then -# return -1 -# endi -# -# sql select count(tbcol) from $mt -# print ===> $data00 -# if $data00 != $totalNum then -# return -1 -# endi +sql select count(*) from $mt +print ===> $data00 +if $data00 != $totalNum then + return -1 +endi + +sql select count(tbcol) from $mt +print ===> $data00 +if $data00 != $totalNum then + return -1 +endi print =============== step8 -# sql select count(tbcol) as c from $mt where ts <= 1519833840000 -# print ===> $data00 -# if $data00 != 50 then -# return -1 -# endi +sql select count(tbcol) as c from $mt where ts <= 1519833840000 +print ===> $data00 +if $data00 != 50 then + return -1 +endi -# sql select count(tbcol) as c from $mt where tgcol < 5 -# print ===> $data00 -# if $data00 != 100 then -# return -1 -# endi +sql select count(tbcol) as c from $mt where tgcol < 5 +print ===> $data00 +if $data00 != 100 then + return -1 +endi -# sql select count(tbcol) as c from $mt where tgcol < 5 and ts <= 1519833840000 -# print ===> $data00 -# if $data00 != 25 then -# return -1 -# endi +sql select count(tbcol) as c from $mt where tgcol < 5 and ts <= 1519833840000 +print ===> $data00 +if $data00 != 25 then + return -1 +endi print =============== step9 -# sql select count(tbcol) as b from $mt interval(1m) -# print ===> $data00 -# if $data00 != 10 then -# return -1 -# endi -# -# sql select count(tbcol) as b from $mt interval(1d) -# print ===> $data00 -# if $data00 != 200 then -# return -1 -# endi +sql select count(tbcol) as b from $mt interval(1m) +print ===> $data00 +if $data00 != 10 then + return -1 +endi + +sql select count(tbcol) as b from $mt interval(1d) +print ===> $data00 +if $data00 != 200 then + return -1 +endi print =============== step10 -# sql select count(tbcol) as b from $mt group by tgcol -# print ===> $data00 -# if $data00 != $rowNum then -# return -1 -# endi -# -# if $rows != $tbNum then -# return -1 -# endi +sql select count(tbcol) as b from $mt group by tgcol +print ===> $data00 +if $data00 != $rowNum then + return -1 +endi + +if $rows != $tbNum then + return -1 +endi print =============== step11 -# sql select count(tbcol) as b from $mt where ts <= 1519833840000 interval(1m) group by tgcol -# print ===> $data01 -# if $data01 != 1 then -# return -1 -# endi -# if $rows != 50 then -# return -1 -# endi +sql select count(tbcol) as b from $mt where ts <= 1519833840000 partition by tgcol interval(1m) +print ===> $data00 +if $data00 != 1 then + return -1 +endi +if $rows != 50 then + return -1 +endi print =============== clear sql drop database $db diff --git a/tests/script/tsim/sync/3Replica5VgElect3mnodedrop.sim b/tests/script/tsim/sync/3Replica5VgElect3mnodedrop.sim index d84679add1..960f579f55 100644 --- a/tests/script/tsim/sync/3Replica5VgElect3mnodedrop.sim +++ b/tests/script/tsim/sync/3Replica5VgElect3mnodedrop.sim @@ -91,21 +91,21 @@ if $rows != $vgroups then return -1 endi -if $data[0][4] == LEADER then - if $data[0][6] == FOLLOWER then - if $data[0][8] == FOLLOWER then +if $data[0][4] == leader then + if $data[0][6] == follower then + if $data[0][8] == follower then print ---- vgroup $data[0][0] leader locate on dnode $data[0][3] endi endi -elif $data[0][6] == LEADER then - if $data[0][4] == FOLLOWER then - if $data[0][8] == FOLLOWER then +elif $data[0][6] == leader then + if $data[0][4] == follower then + if $data[0][8] == follower then print ---- vgroup $data[0][0] leader locate on dnode $data[0][5] endi endi -elif $data[0][8] == LEADER then - if $data[0][4] == FOLLOWER then - if $data[0][6] == FOLLOWER then +elif $data[0][8] == leader then + if $data[0][4] == follower then + if $data[0][6] == follower then print ---- vgroup $data[0][0] leader locate on dnode $data[0][7] endi endi @@ -113,21 +113,21 @@ else goto check_vg_ready endi -if $data[1][4] == LEADER then - if $data[1][6] == FOLLOWER then - if $data[1][8] == FOLLOWER then +if $data[1][4] == leader then + if $data[1][6] == follower then + if $data[1][8] == follower then print ---- vgroup $data[1][0] leader locate on dnode $data[1][3] endi endi -elif $data[1][6] == LEADER then - if $data[1][4] == FOLLOWER then - if $data[1][8] == FOLLOWER then +elif $data[1][6] == leader then + if $data[1][4] == follower then + if $data[1][8] == follower then print ---- vgroup $data[1][0] leader locate on dnode $data[1][5] endi endi -elif $data[1][8] == LEADER then - if $data[1][4] == FOLLOWER then - if $data[1][6] == FOLLOWER then +elif $data[1][8] == leader then + if $data[1][4] == follower then + if $data[1][6] == follower then print ---- vgroup $data[1][0] leader locate on dnode $data[1][7] endi endi @@ -135,21 +135,21 @@ else goto check_vg_ready endi -if $data[2][4] == LEADER then - if $data[2][6] == FOLLOWER then - if $data[2][8] == FOLLOWER then +if $data[2][4] == leader then + if $data[2][6] == follower then + if $data[2][8] == follower then print ---- vgroup $data[2][0] leader locate on dnode $data[2][3] endi endi -elif $data[2][6] == LEADER then - if $data[2][4] == FOLLOWER then - if $data[2][8] == FOLLOWER then +elif $data[2][6] == leader then + if $data[2][4] == follower then + if $data[2][8] == follower then print ---- vgroup $data[2][0] leader locate on dnode $data[2][5] endi endi -elif $data[2][8] == LEADER then - if $data[2][4] == FOLLOWER then - if $data[2][6] == FOLLOWER then +elif $data[2][8] == leader then + if $data[2][4] == follower then + if $data[2][6] == follower then print ---- vgroup $data[2][0] leader locate on dnode $data[2][7] endi endi @@ -157,21 +157,21 @@ else goto check_vg_ready endi -if $data[3][4] == LEADER then - if $data[3][6] == FOLLOWER then - if $data[3][8] == FOLLOWER then +if $data[3][4] == leader then + if $data[3][6] == follower then + if $data[3][8] == follower then print ---- vgroup $data[3][0] leader locate on dnode $data[3][3] endi endi -elif $data[3][6] == LEADER then - if $data[3][4] == FOLLOWER then - if $data[3][8] == FOLLOWER then +elif $data[3][6] == leader then + if $data[3][4] == follower then + if $data[3][8] == follower then print ---- vgroup $data[3][0] leader locate on dnode $data[3][5] endi endi -elif $data[3][8] == LEADER then - if $data[3][4] == FOLLOWER then - if $data[3][6] == FOLLOWER then +elif $data[3][8] == leader then + if $data[3][4] == follower then + if $data[3][6] == follower then print ---- vgroup $data[3][0] leader locate on dnode $data[3][7] endi endi @@ -179,21 +179,21 @@ else goto check_vg_ready endi -if $data[4][4] == LEADER then - if $data[4][6] == FOLLOWER then - if $data[4][8] == FOLLOWER then +if $data[4][4] == leader then + if $data[4][6] == follower then + if $data[4][8] == follower then print ---- vgroup $data[4][0] leader locate on dnode $data[4][3] endi endi -elif $data[4][6] == LEADER then - if $data[4][4] == FOLLOWER then - if $data[4][8] == FOLLOWER then +elif $data[4][6] == leader then + if $data[4][4] == follower then + if $data[4][8] == follower then print ---- vgroup $data[4][0] leader locate on dnode $data[4][5] endi endi -elif $data[4][8] == LEADER then - if $data[4][4] == FOLLOWER then - if $data[4][6] == FOLLOWER then +elif $data[4][8] == leader then + if $data[4][4] == follower then + if $data[4][6] == follower then print ---- vgroup $data[4][0] leader locate on dnode $data[4][7] endi endi @@ -286,13 +286,13 @@ if $data[0][0] != 1 then return -1 endi -if $data[0][2] != LEADER then +if $data[0][2] != leader then goto check_mnode_ready_2 endi -if $data[1][2] != FOLLOWER then +if $data[1][2] != follower then goto check_mnode_ready_2 endi -if $data[2][2] != FOLLOWER then +if $data[2][2] != follower then goto check_mnode_ready_2 endi @@ -318,21 +318,21 @@ if $rows != $vgroups then return -1 endi -if $data[0][4] == LEADER then - if $data[0][6] == FOLLOWER then - if $data[0][8] == FOLLOWER then +if $data[0][4] == leader then + if $data[0][6] == follower then + if $data[0][8] == follower then print ---- vgroup $data[0][0] leader locate on dnode $data[0][3] endi endi -elif $data[0][6] == LEADER then - if $data[0][4] == FOLLOWER then - if $data[0][8] == FOLLOWER then +elif $data[0][6] == leader then + if $data[0][4] == follower then + if $data[0][8] == follower then print ---- vgroup $data[0][0] leader locate on dnode $data[0][5] endi endi -elif $data[0][8] == LEADER then - if $data[0][4] == FOLLOWER then - if $data[0][6] == FOLLOWER then +elif $data[0][8] == leader then + if $data[0][4] == follower then + if $data[0][6] == follower then print ---- vgroup $data[0][0] leader locate on dnode $data[0][7] endi endi @@ -340,21 +340,21 @@ else goto check_vg_ready1 endi -if $data[1][4] == LEADER then - if $data[1][6] == FOLLOWER then - if $data[1][8] == FOLLOWER then +if $data[1][4] == leader then + if $data[1][6] == follower then + if $data[1][8] == follower then print ---- vgroup $data[1][0] leader locate on dnode $data[1][3] endi endi -elif $data[1][6] == LEADER then - if $data[1][4] == FOLLOWER then - if $data[1][8] == FOLLOWER then +elif $data[1][6] == leader then + if $data[1][4] == follower then + if $data[1][8] == follower then print ---- vgroup $data[1][0] leader locate on dnode $data[1][5] endi endi -elif $data[1][8] == LEADER then - if $data[1][4] == FOLLOWER then - if $data[1][6] == FOLLOWER then +elif $data[1][8] == leader then + if $data[1][4] == follower then + if $data[1][6] == follower then print ---- vgroup $data[1][0] leader locate on dnode $data[1][7] endi endi @@ -362,21 +362,21 @@ else goto check_vg_ready1 endi -if $data[2][4] == LEADER then - if $data[2][6] == FOLLOWER then - if $data[2][8] == FOLLOWER then +if $data[2][4] == leader then + if $data[2][6] == follower then + if $data[2][8] == follower then print ---- vgroup $data[2][0] leader locate on dnode $data[2][3] endi endi -elif $data[2][6] == LEADER then - if $data[2][4] == FOLLOWER then - if $data[2][8] == FOLLOWER then +elif $data[2][6] == leader then + if $data[2][4] == follower then + if $data[2][8] == follower then print ---- vgroup $data[2][0] leader locate on dnode $data[2][5] endi endi -elif $data[2][8] == LEADER then - if $data[2][4] == FOLLOWER then - if $data[2][6] == FOLLOWER then +elif $data[2][8] == leader then + if $data[2][4] == follower then + if $data[2][6] == follower then print ---- vgroup $data[2][0] leader locate on dnode $data[2][7] endi endi @@ -384,21 +384,21 @@ else goto check_vg_ready1 endi -if $data[3][4] == LEADER then - if $data[3][6] == FOLLOWER then - if $data[3][8] == FOLLOWER then +if $data[3][4] == leader then + if $data[3][6] == follower then + if $data[3][8] == follower then print ---- vgroup $data[3][0] leader locate on dnode $data[3][3] endi endi -elif $data[3][6] == LEADER then - if $data[3][4] == FOLLOWER then - if $data[3][8] == FOLLOWER then +elif $data[3][6] == leader then + if $data[3][4] == follower then + if $data[3][8] == follower then print ---- vgroup $data[3][0] leader locate on dnode $data[3][5] endi endi -elif $data[3][8] == LEADER then - if $data[3][4] == FOLLOWER then - if $data[3][6] == FOLLOWER then +elif $data[3][8] == leader then + if $data[3][4] == follower then + if $data[3][6] == follower then print ---- vgroup $data[3][0] leader locate on dnode $data[3][7] endi endi @@ -406,21 +406,21 @@ else goto check_vg_ready1 endi -if $data[4][4] == LEADER then - if $data[4][6] == FOLLOWER then - if $data[4][8] == FOLLOWER then +if $data[4][4] == leader then + if $data[4][6] == follower then + if $data[4][8] == follower then print ---- vgroup $data[4][0] leader locate on dnode $data[4][3] endi endi -elif $data[4][6] == LEADER then - if $data[4][4] == FOLLOWER then - if $data[4][8] == FOLLOWER then +elif $data[4][6] == leader then + if $data[4][4] == follower then + if $data[4][8] == follower then print ---- vgroup $data[4][0] leader locate on dnode $data[4][5] endi endi -elif $data[4][8] == LEADER then - if $data[4][4] == FOLLOWER then - if $data[4][6] == FOLLOWER then +elif $data[4][8] == leader then + if $data[4][4] == follower then + if $data[4][6] == follower then print ---- vgroup $data[4][0] leader locate on dnode $data[4][7] endi endi @@ -539,27 +539,27 @@ if $data[0][0] != 1 then return -1 endi -if $data[0][2] == LEADER then - if $data[1][2] != FOLLOWER then +if $data[0][2] == leader then + if $data[1][2] != follower then goto check_mnode_ready_3 endi - if $data[2][2] != FOLLOWER then + if $data[2][2] != follower then goto check_mnode_ready_3 endi endi -if $data[1][2] == LEADER then - if $data[0][2] != FOLLOWER then +if $data[1][2] == leader then + if $data[0][2] != follower then goto check_mnode_ready_3 endi - if $data[2][2] != FOLLOWER then + if $data[2][2] != follower then goto check_mnode_ready_3 endi endi -if $data[2][2] == LEADER then - if $data[1][2] != FOLLOWER then +if $data[2][2] == leader then + if $data[1][2] != follower then goto check_mnode_ready_3 endi - if $data[0][2] != FOLLOWER then + if $data[0][2] != follower then goto check_mnode_ready_3 endi endi diff --git a/tests/script/tsim/sync/vnodesnapshot-test.sim b/tests/script/tsim/sync/vnodesnapshot-test.sim index 8589078b50..2f0eccf02e 100644 --- a/tests/script/tsim/sync/vnodesnapshot-test.sim +++ b/tests/script/tsim/sync/vnodesnapshot-test.sim @@ -170,84 +170,3 @@ if $rows != 100 then return -1 endi -system sh/exec.sh -n dnode1 -s stop -x SIGINT -system sh/exec.sh -n dnode2 -s stop -x SIGINT -system sh/exec.sh -n dnode3 -s stop -x SIGINT -system sh/exec.sh -n dnode4 -s stop -x SIGINT -######################################################## - - -######################################################## -print ===> start dnode1 dnode3 dnode4 -system sh/exec.sh -n dnode1 -s start -#system sh/exec.sh -n dnode2 -s start -system sh/exec.sh -n dnode3 -s start -system sh/exec.sh -n dnode4 -s start - -sleep 7000 - -print =============== query data -sql connect -sql use db -sql select * from ct1 -print rows: $rows -print $data00 $data01 $data02 -if $rows != 100 then - return -1 -endi - -system sh/exec.sh -n dnode1 -s stop -x SIGINT -#system sh/exec.sh -n dnode2 -s stop -x SIGINT -system sh/exec.sh -n dnode3 -s stop -x SIGINT -system sh/exec.sh -n dnode4 -s stop -x SIGINT -######################################################## - - -######################################################## -print ===> start dnode1 dnode2 dnode4 -system sh/exec.sh -n dnode1 -s start -system sh/exec.sh -n dnode2 -s start -#system sh/exec.sh -n dnode3 -s start -system sh/exec.sh -n dnode4 -s start - -sleep 3000 - -print =============== query data -sql select * from ct1 -print rows: $rows -print $data00 $data01 $data02 -if $rows != 100 then - return -1 -endi - -system sh/exec.sh -n dnode1 -s stop -x SIGINT -system sh/exec.sh -n dnode2 -s stop -x SIGINT -#system sh/exec.sh -n dnode3 -s stop -x SIGINT -system sh/exec.sh -n dnode4 -s stop -x SIGINT -######################################################## - - -######################################################## -print ===> start dnode1 dnode2 dnode3 -system sh/exec.sh -n dnode1 -s start -system sh/exec.sh -n dnode2 -s start -system sh/exec.sh -n dnode3 -s start -#system sh/exec.sh -n dnode4 -s start - -sleep 3000 - -print =============== query data -sql select * from ct1 -print rows: $rows -print $data00 $data01 $data02 -if $rows != 100 then - return -1 -endi - -system sh/exec.sh -n dnode1 -s stop -x SIGINT -system sh/exec.sh -n dnode2 -s stop -x SIGINT -system sh/exec.sh -n dnode3 -s stop -x SIGINT -#system sh/exec.sh -n dnode4 -s stop -x SIGINT -######################################################## - - diff --git a/tests/script/tsim/tag/5.sim b/tests/script/tsim/tag/5.sim index 92695ddfcf..319d9c7bc0 100644 --- a/tests/script/tsim/tag/5.sim +++ b/tests/script/tsim/tag/5.sim @@ -24,7 +24,7 @@ sql create table $mt (ts timestamp, tbcol int) TAGS(tgcol1 tinyint, tgcol2 int, $i = 0 while $i < 5 $tb = $tbPrefix . $i - sql create table $tb using $mt tags( 0, 0, 0, 0, 0 ) + sql create table $tb using $mt tags( 0, 0, 0, 0, '0' ) $x = 0 while $x < $rowNum $ms = $x . m @@ -35,7 +35,7 @@ while $i < 5 endw while $i < 10 $tb = $tbPrefix . $i - sql create table $tb using $mt tags( 1, 1, 1, 1, 1 ) + sql create table $tb using $mt tags( 1, 1, 1, 1, '1' ) $x = 0 while $x < $rowNum $ms = $x . m diff --git a/tests/script/tsim/tag/add.sim b/tests/script/tsim/tag/add.sim index 423224b1d6..a22d337eff 100644 --- a/tests/script/tsim/tag/add.sim +++ b/tests/script/tsim/tag/add.sim @@ -250,8 +250,8 @@ sql alter table $mt add tag tgcol6 binary(10) sql reset query cache sql alter table $tb set tag tgcol4=false -sql alter table $tb set tag tgcol5=5 -sql alter table $tb set tag tgcol6=6 +sql alter table $tb set tag tgcol5='5' +sql alter table $tb set tag tgcol6='6' sql reset query cache sql select * from $mt where tgcol5 = '5' @@ -390,8 +390,8 @@ sql alter table $mt add tag tgcol5 binary(17) sql alter table $mt add tag tgcol6 bool sql reset query cache sql alter table $tb set tag tgcol4=4 -sql alter table $tb set tag tgcol5=5 -sql alter table $tb set tag tgcol6=1 +sql alter table $tb set tag tgcol5='5' +sql alter table $tb set tag tgcol6='1' sql reset query cache sql select * from $mt where tgcol5 = '5' @@ -409,7 +409,7 @@ endi if $data03 != 5 then return -1 endi -if $data04 != 1 then +if $data04 != 0 then return -1 endi @@ -425,7 +425,7 @@ $i = 9 $mt = $mtPrefix . $i $tb = $tbPrefix . $i sql create table $mt (ts timestamp, tbcol int) TAGS(tgcol1 double, tgcol2 binary(10), tgcol3 binary(10)) -sql create table $tb using $mt tags( 1, 2, '3' ) +sql create table $tb using $mt tags( 1, '2', '3' ) sql insert into $tb values(now, 1) sql select * from $mt where tgcol2 = '2' if $rows != 1 then @@ -520,7 +520,7 @@ sql alter table $mt add tag tgcol4 binary(10) sql alter table $mt add tag tgcol5 bool sql reset query cache -sql alter table $tb set tag tgcol4=4 +sql alter table $tb set tag tgcol4='4' sql alter table $tb set tag tgcol5=false sql reset query cache @@ -598,9 +598,9 @@ sql alter table $mt add tag tgcol7 bigint sql alter table $mt add tag tgcol8 smallint sql reset query cache -sql alter table $tb set tag tgcol4=4 +sql alter table $tb set tag tgcol4='4' sql alter table $tb set tag tgcol5=5 -sql alter table $tb set tag tgcol6=6 +sql alter table $tb set tag tgcol6='6' sql alter table $tb set tag tgcol7=7 sql alter table $tb set tag tgcol8=8 sql reset query cache @@ -687,11 +687,11 @@ sql alter table $mt add tag tgcol5 bigint sql reset query cache sql alter table $tb set tag tgcol1=false -sql alter table $tb set tag tgcol2=5 +sql alter table $tb set tag tgcol2='5' sql alter table $tb set tag tgcol3=4 -sql alter table $tb set tag tgcol4=3 +sql alter table $tb set tag tgcol4='3' sql alter table $tb set tag tgcol5=2 -sql alter table $tb set tag tgcol6=1 +sql alter table $tb set tag tgcol6='1' sql reset query cache sql select * from $mt where tgcol4 = '3' @@ -783,8 +783,8 @@ sql alter table $mt add tag tgcol4 int sql alter table $mt add tag tgcol6 bigint sql reset query cache -sql alter table $tb set tag tgcol1=7 -sql alter table $tb set tag tgcol2=8 +sql alter table $tb set tag tgcol1='7' +sql alter table $tb set tag tgcol2='8' sql alter table $tb set tag tgcol3=9 sql alter table $tb set tag tgcol4=10 sql alter table $tb set tag tgcol5=11 diff --git a/tests/script/tsim/tag/change.sim b/tests/script/tsim/tag/change.sim index f58e387217..236ad8ea67 100644 --- a/tests/script/tsim/tag/change.sim +++ b/tests/script/tsim/tag/change.sim @@ -40,19 +40,19 @@ if $data03 != 2 then return -1 endi -sql alter table $mt change tag tagcx tgcol3 -x step21 +sql alter table $mt rename tag tagcx tgcol3 -x step21 return -1 step21: -sql alter table $mt change tag tgcol1 tgcol2 -x step22 +sql alter table $mt rename tag tgcol1 tgcol2 -x step22 return -1 step22: -#sql alter table $mt change tag tgcol1 xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx -x step20 +#sql alter table $mt rename tag tgcol1 xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx -x step20 # return -1 #step20: -sql alter table $mt change tag tgcol1 tgcol3 -sql alter table $mt change tag tgcol2 tgcol4 -sql alter table $mt change tag tgcol4 tgcol3 -x step23 +sql alter table $mt rename tag tgcol1 tgcol3 +sql alter table $mt rename tag tgcol2 tgcol4 +sql alter table $mt rename tag tgcol4 tgcol3 -x step23 return -1 step23: @@ -77,8 +77,8 @@ if $data03 != 2 then return -1 endi -sql alter table $mt change tag tgcol1 tgcol3 -sql alter table $mt change tag tgcol2 tgcol4 +sql alter table $mt rename tag tgcol1 tgcol3 +sql alter table $mt rename tag tgcol2 tgcol4 print =============== step4 $i = 4 @@ -101,8 +101,8 @@ if $data03 != 2.00000 then return -1 endi -sql alter table $mt change tag tgcol1 tgcol3 -sql alter table $mt change tag tgcol2 tgcol4 +sql alter table $mt rename tag tgcol1 tgcol3 +sql alter table $mt rename tag tgcol2 tgcol4 print =============== step5 $i = 5 @@ -125,8 +125,8 @@ if $data03 != 2 then return -1 endi -sql alter table $mt change tag tgcol1 tgcol3 -sql alter table $mt change tag tgcol2 tgcol4 +sql alter table $mt rename tag tgcol1 tgcol3 +sql alter table $mt rename tag tgcol2 tgcol4 print =============== step6 $i = 6 @@ -163,15 +163,14 @@ endi sql alter table $mt drop tag tgcol3 sql reset query cache -sql alter table $mt change tag tgcol4 tgcol3 -sql alter table $mt change tag tgcol1 tgcol7 -sql alter table $mt change tag tgcol2 tgcol8 +sql alter table $mt rename tag tgcol4 tgcol3 +sql alter table $mt rename tag tgcol1 tgcol7 +sql alter table $mt rename tag tgcol2 tgcol8 sql reset query cache -sql alter table $mt change tag tgcol3 tgcol9 -sql alter table $mt change tag tgcol5 tgcol10 -sql alter table $mt change tag tgcol6 tgcol11 +sql alter table $mt rename tag tgcol3 tgcol9 +sql alter table $mt rename tag tgcol5 tgcol10 +sql alter table $mt rename tag tgcol6 tgcol11 -sleep 3000 sql reset query cache print =============== step2 diff --git a/tests/script/tsim/valgrind/basic1.sim b/tests/script/tsim/valgrind/basic1.sim index 3e39f35fa7..f3d418cfd1 100644 --- a/tests/script/tsim/valgrind/basic1.sim +++ b/tests/script/tsim/valgrind/basic1.sim @@ -1,50 +1,99 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c debugflag -v 131 system sh/exec.sh -n dnode1 -s start -v sql connect -print ======== step1 -sql drop database if exists db1; -sql create database db1 vgroups 3; -sql use db1; -sql create stable st1 (ts timestamp, f1 int, f2 binary(200)) tags(t1 int); -sql create table tb1 using st1 tags(1); -sql insert into tb1 values ('2022-07-07 10:01:01', 11, "aaa"); -sql insert into tb1 values ('2022-07-07 11:01:02', 12, "bbb"); -sql create table tb2 using st1 tags(2); -sql insert into tb2 values ('2022-07-07 10:02:01', 21, "aaa"); -sql insert into tb2 values ('2022-07-07 11:02:02', 22, "bbb"); -sql create table tb3 using st1 tags(3); -sql insert into tb3 values ('2022-07-07 10:03:01', 31, "aaa"); -sql insert into tb3 values ('2022-07-07 11:03:02', 32, "bbb"); -sql create table tb4 using st1 tags(4); +print =============== step1: create drop show dnodes +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ---> dnode not ready! + return -1 + endi +sql show dnodes +print ---> $data00 $data01 $data02 $data03 $data04 $data05 +if $rows != 1 then + return -1 +endi +if $data(1)[4] != ready then + goto step1 +endi -sql insert into tb4 select * from tb1; +print =============== step2: create db +sql create database db +sql use db +sql create table db.stb (ts timestamp, c1 int, c2 binary(4)) tags(t1 int, t2 float, t3 binary(16)) comment "abd" +sql create table db.c1 using db.stb tags(101, 102, "103") -goto _OVER +print =============== step3: alter stb +sql_error alter table db.stb add column ts int +sql alter table db.stb add column c3 int +sql alter table db.stb add column c4 bigint +sql alter table db.stb add column c5 binary(12) +sql alter table db.stb drop column c1 +sql alter table db.stb drop column c4 +sql alter table db.stb MODIFY column c2 binary(32) +sql alter table db.stb add tag t4 bigint +sql alter table db.stb add tag c1 int +sql alter table db.stb add tag t5 binary(12) +sql alter table db.stb drop tag c1 +sql alter table db.stb drop tag t5 +sql alter table db.stb MODIFY tag t3 binary(32) +sql alter table db.stb rename tag t1 tx +sql alter table db.stb comment 'abcde' ; +sql drop table db.stb -sql select * from tb4; -if $rows != 2 then - return -1 -endi -sql insert into tb4 select ts,f1,f2 from st1; -sql select * from tb4; -if $rows != 6 then - return -1 -endi -sql create table tba (ts timestamp, f1 binary(10), f2 bigint, f3 double); -sql_error insert into tba select * from tb1; -sql insert into tba (ts,f2,f1) select * from tb1; -sql select * from tba; -if $rows != 2 then - return -1 -endi -sql create table tbb (ts timestamp, f1 binary(10), f2 bigint, f3 double); -sql insert into tbb (f2,f1,ts) select f1+1,f2,ts+3 from tb2; -sql select * from tbb; -if $rows != 2 then - return -1 -endi +print =============== step4: alter tb +sql create table tb (ts timestamp, a int) +sql insert into tb values(now-28d, -28) +sql select count(a) from tb +sql alter table tb add column b smallint +sql insert into tb values(now-25d, -25, 0) +sql select count(b) from tb +sql alter table tb add column c tinyint +sql insert into tb values(now-22d, -22, 3, 0) +sql select count(c) from tb +sql alter table tb add column d int +sql insert into tb values(now-19d, -19, 6, 0, 0) +sql select count(d) from tb +sql alter table tb add column e bigint +sql alter table tb add column f float +sql alter table tb add column g double +sql alter table tb add column h binary(10) +sql select count(a), count(b), count(c), count(d), count(e), count(f), count(g), count(h) from tb +sql select * from tb order by ts desc + +print =============== step5: alter stb and insert data +sql create table stb (ts timestamp, c1 int, c2 binary(4)) tags(t1 int, t2 float, t3 binary(16)) comment "abd" +sql show db.stables +sql describe stb +sql_error alter table stb add column ts int + +sql create table db.ctb using db.stb tags(101, 102, "103") +sql insert into db.ctb values(now, 1, "2") +sql show db.tables +sql select * from db.stb +sql select * from tb + +sql alter table stb add column c3 int +sql describe stb +sql select * from db.stb +sql select * from tb +sql insert into db.ctb values(now+1s, 1, 2, 3) +sql select * from db.stb + +sql alter table db.stb add column c4 bigint +sql select * from db.stb +sql insert into db.ctb values(now+2s, 1, 2, 3, 4) + +sql alter table db.stb drop column c1 +sql reset query cache +sql select * from tb +sql insert into db.ctb values(now+3s, 2, 3, 4) +sql select * from db.stb _OVER: system sh/exec.sh -n dnode1 -s stop -x SIGINT @@ -53,10 +102,10 @@ $null= system_content sh/checkValgrind.sh -n dnode1 print cmd return result ----> [ $system_content ] -if $system_content > 1 then +if $system_content > 0 then return -1 endi if $system_content == $null then return -1 -endi \ No newline at end of file +endi diff --git a/tests/script/tsim/valgrind/checkError1.sim b/tests/script/tsim/valgrind/checkError1.sim index fe4f4654b1..76a29ee62f 100644 --- a/tests/script/tsim/valgrind/checkError1.sim +++ b/tests/script/tsim/valgrind/checkError1.sim @@ -1,28 +1,13 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 system sh/cfg.sh -n dnode1 -c debugflag -v 131 +system sh/cfg.sh -n dnode2 -c debugflag -v 131 system sh/exec.sh -n dnode1 -s start -v +system sh/exec.sh -n dnode2 -s start -v sql connect -print =============== step1: show dnodes -$x = 0 -step1: - $x = $x + 1 - sleep 1000 - if $x == 10 then - print ---> dnode not ready! - return -1 - endi -sql show dnodes -print ---> $data00 $data01 $data02 $data03 $data04 $data05 -if $rows != 1 then - return -1 -endi -if $data(1)[4] != ready then - goto step1 -endi - -print =============== step2: create alter drop show user +print =============== step1: create alter drop show user sql create user u1 pass 'taosdata' sql show users sql alter user u1 sysinfo 1 @@ -31,11 +16,34 @@ sql alter user u1 pass 'taosdata' sql drop user u1 sql_error alter user u2 sysinfo 0 -print =============== step3: create drop dnode +print =============== step2 create drop dnode sql create dnode $hostname port 7200 -sql drop dnode 2 +sql create dnode $hostname port 7300 +sql drop dnode 3 sql alter dnode 1 'debugflag 131' +print =============== step3: show dnodes +$x = 0 +step3: + $x = $x + 1 + sleep 1000 + if $x == 60 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +if $rows != 2 then + return -1 +endi +if $data(1)[4] != ready then + goto step3 +endi +if $data(2)[4] != ready then + goto step3 +endi + print =============== create database, stable, table sql create database db vgroups 3 sql use db @@ -45,7 +53,7 @@ sql create table tba (ts timestamp, c1 binary(10), c2 nchar(10)); print =============== run show xxxx sql show dnodes -if $rows != 1 then +if $rows != 2 then return -1 endi @@ -76,7 +84,7 @@ endi print =============== run select * from information_schema.xxxx sql select * from information_schema.`dnodes` -if $rows != 1 then +if $rows != 2 then return -1 endi @@ -96,7 +104,7 @@ if $rows != 1 then endi sql select * from information_schema.user_tables -if $rows != 30 then +if $rows != 31 then return -1 endi @@ -127,16 +135,27 @@ endi print =============== stop system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT print =============== check $null= system_content sh/checkValgrind.sh -n dnode1 print cmd return result ----> [ $system_content ] -if $system_content > 1 then +if $system_content > 0 then return -1 endi if $system_content == $null then return -1 endi + +system_content sh/checkValgrind.sh -n dnode2 +print cmd return result ----> [ $system_content ] +if $system_content > 4 then + return -1 +endi + +if $system_content == $null then + return -1 +endi \ No newline at end of file diff --git a/tests/script/tsim/valgrind/checkError2.sim b/tests/script/tsim/valgrind/checkError2.sim index 9746fe9ecf..e81d702d82 100644 --- a/tests/script/tsim/valgrind/checkError2.sim +++ b/tests/script/tsim/valgrind/checkError2.sim @@ -60,8 +60,8 @@ sql select c1, c2, c3 from ct1 sql select ts, c1, c2, c3 from stb sql select * from ct1 where ts < now -1d and ts > now +1d sql select * from stb where ts < now -1d and ts > now +1d -#sql select * from ct1 where ts < now -1d and ts > now +1d order by ts desc -#sql select * from stb where ts < now -1d and ts > now +1d order by ts desc +sql select * from ct1 where ts < now -1d and ts > now +1d order by ts desc +sql select * from stb where ts < now -1d and ts > now +1d order by ts desc print =============== step7: count sql select count(*) from ct1; @@ -78,18 +78,18 @@ sql select max(c1), max(c2), max(c3) from ct1 sql select sum(c1), sum(c2), sum(c3) from ct1 print =============== step9: insert select -#sql create table ct4 using stb tags(4000); -#sql insert into ct4 select * from ct1; -#sql select * from ct4; -#sql insert into ct4 select ts,c1,c2,c3 from stb; +sql create table ct4 using stb tags(4000); +sql insert into ct4 select * from ct1; +sql select * from ct4; +sql insert into ct4 select ts,c1,c2,c3 from stb; -#sql create table tb1 (ts timestamp, c1 int, c2 float, c3 double); -#sql insert into tb1 (ts, c1, c2, c3) select * from ct1; -#sql select * from tb1; +sql create table tb1 (ts timestamp, c1 int, c2 float, c3 double); +sql insert into tb1 (ts, c1, c2, c3) select * from ct1; +sql select * from tb1; -#sql create table tb2 (ts timestamp, f1 binary(10), c1 int, c2 double); -#sql insert into tb2 (c2, c1, ts) select c2+1, c1, ts+3 from ct2; -#sql select * from tb2; +sql create table tb2 (ts timestamp, f1 binary(10), c1 int, c2 double); +sql insert into tb2 (c2, c1, ts) select c2+1, c1, ts+3 from ct2; +sql select * from tb2; _OVER: system sh/exec.sh -n dnode1 -s stop -x SIGINT @@ -98,7 +98,7 @@ $null= system_content sh/checkValgrind.sh -n dnode1 print cmd return result ----> [ $system_content ] -if $system_content > 1 then +if $system_content > 0 then return -1 endi diff --git a/tests/script/tsim/valgrind/checkError3.sim b/tests/script/tsim/valgrind/checkError3.sim index 86926509bb..52ef01785e 100644 --- a/tests/script/tsim/valgrind/checkError3.sim +++ b/tests/script/tsim/valgrind/checkError3.sim @@ -90,7 +90,7 @@ $null= system_content sh/checkValgrind.sh -n dnode1 print cmd return result ----> [ $system_content ] -if $system_content > 1 then +if $system_content > 0 then return -1 endi diff --git a/tests/script/tsim/valgrind/checkError4.sim b/tests/script/tsim/valgrind/checkError4.sim new file mode 100644 index 0000000000..745ff40517 --- /dev/null +++ b/tests/script/tsim/valgrind/checkError4.sim @@ -0,0 +1,94 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c debugflag -v 131 +system sh/exec.sh -n dnode1 -s start -v +sql connect + +print =============== step1: create drop show dnodes +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ---> dnode not ready! + return -1 + endi +sql show dnodes +print ---> $data00 $data01 $data02 $data03 $data04 $data05 +if $rows != 1 then + return -1 +endi +if $data(1)[4] != ready then + goto step1 +endi + +print =============== step2: create db +sql create database d1 vgroups 2 buffer 3 +sql create database d2 vgroups 2 buffer 3 +sql show databases; +sql show d1.vgroups; + +print =============== step3: create show stable +sql create table if not exists d1.stb1 (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned) +sql create table if not exists d1.stb2 (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned) +sql create table if not exists d2.stb1 (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned) +sql show d1.stables +sql show d2.stables + +print =============== step4: create show table +sql create table d1.ct1 using d1.stb1 tags(1000) +sql create table d1.ct2 using d1.stb1 tags(2000) +sql create table d1.ct3 using d1.stb2 tags(3000) +sql create table d2.ct1 using d2.stb1 tags(1000) +sql create table d2.ct2 using d2.stb1 tags(2000) +sql show d1.tables +sql show d2.tables + +print =============== step5: insert data +sql insert into d1.ct1 values(now+0s, 10, 2.0, 3.0) +sql insert into d2.ct1 values(now+1s, 11, 2.1, 3.1)(now+2s, -12, -2.2, -3.2)(now+3s, -13, -2.3, -3.3) +sql insert into d1.ct2 values(now+0s, 10, 2.0, 3.0) +sql insert into d2.ct2 values(now+1s, 11, 2.1, 3.1)(now+2s, -12, -2.2, -3.2)(now+3s, -13, -2.3, -3.3) +sql insert into d1.ct3 values('2021-01-01 00:00:00.000', 10, 2.0, 3.0) + +print =============== step6: create db +sql drop table d1.ct1 +sql drop table d1.stb2 +sql drop database d1 +sql drop database d2 + +print =============== step7: repeat +sql create database d1 vgroups 2 buffer 3 +sql create database d2 vgroups 2 buffer 3 +sql create table if not exists d1.stb1 (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned) +sql create table if not exists d1.stb2 (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned) +sql create table if not exists d2.stb1 (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned) +sql create table d1.ct1 using d1.stb1 tags(1000) +sql create table d1.ct2 using d1.stb1 tags(2000) +sql create table d1.ct3 using d1.stb2 tags(3000) +sql create table d2.ct1 using d2.stb1 tags(1000) +sql create table d2.ct2 using d2.stb1 tags(2000) +sql insert into d1.ct1 values(now+0s, 10, 2.0, 3.0) +sql insert into d2.ct1 values(now+1s, 11, 2.1, 3.1)(now+2s, -12, -2.2, -3.2)(now+3s, -13, -2.3, -3.3) +sql insert into d1.ct2 values(now+0s, 10, 2.0, 3.0) +sql insert into d2.ct2 values(now+1s, 11, 2.1, 3.1)(now+2s, -12, -2.2, -3.2)(now+3s, -13, -2.3, -3.3) +sql insert into d1.ct3 values('2021-01-01 00:00:00.000', 10, 2.0, 3.0) +sql drop table d1.ct1 +sql drop table d1.stb2 +sql drop database d1 +sql drop database d2 + +_OVER: +system sh/exec.sh -n dnode1 -s stop -x SIGINT +print =============== check +$null= + +system_content sh/checkValgrind.sh -n dnode1 +print cmd return result ----> [ $system_content ] +if $system_content > 0 then + return -1 +endi + +if $system_content == $null then + return -1 +endi diff --git a/tests/script/tsim/valgrind/checkError5.sim b/tests/script/tsim/valgrind/checkError5.sim new file mode 100644 index 0000000000..61964d1c42 --- /dev/null +++ b/tests/script/tsim/valgrind/checkError5.sim @@ -0,0 +1,82 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c debugflag -v 131 +system sh/exec.sh -n dnode1 -s start -v +sql connect + +print =============== step1: create drop show dnodes +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ---> dnode not ready! + return -1 + endi +sql show dnodes +print ---> $data00 $data01 $data02 $data03 $data04 $data05 +if $rows != 1 then + return -1 +endi +if $data(1)[4] != ready then + goto step1 +endi + +print =============== step2: create db +sql create database db +sql use db +sql create table db.stb (ts timestamp, c1 int, c2 binary(4)) tags(t1 int, t2 float, t3 binary(16)) comment "abd" + +print =============== step3: alter stb +sql_error alter table db.stb add column ts int +sql alter table db.stb add column c3 int +sql alter table db.stb add column c4 bigint +sql alter table db.stb add column c5 binary(12) +sql alter table db.stb drop column c1 +sql alter table db.stb drop column c4 +sql alter table db.stb MODIFY column c2 binary(32) +sql alter table db.stb add tag t4 bigint +sql alter table db.stb add tag c1 int +sql alter table db.stb add tag t5 binary(12) +sql alter table db.stb drop tag c1 +sql alter table db.stb drop tag t5 +sql alter table db.stb MODIFY tag t3 binary(32) +sql alter table db.stb rename tag t1 tx + +sql alter table db.stb comment 'abcde' ; +goto _OVER + +print =============== step4: alter tb +sql create table tb (ts timestamp, a int) +sql insert into tb values(now-28d, -28) +sql select count(a) from tb +sql alter table tb add column b smallint +sql insert into tb values(now-25d, -25, 0) +sql select count(b) from tb +sql alter table tb add column c tinyint +sql insert into tb values(now-22d, -22, 3, 0) +sql select count(c) from tb +sql alter table tb add column d int +sql insert into tb values(now-19d, -19, 6, 0, 0) +sql select count(d) from tb +sql alter table tb add column e bigint +sql alter table tb add column f float +sql alter table tb add column g double +sql alter table tb add column h binary(10) +sql select count(a), count(b), count(c), count(d), count(e), count(f), count(g), count(h) from tb +sql select * from tb order by ts desc + +_OVER: +system sh/exec.sh -n dnode1 -s stop -x SIGINT +print =============== check +$null= + +system_content sh/checkValgrind.sh -n dnode1 +print cmd return result ----> [ $system_content ] +if $system_content > 0 then + return -1 +endi + +if $system_content == $null then + return -1 +endi diff --git a/tests/script/tsim/valgrind/checkError6.sim b/tests/script/tsim/valgrind/checkError6.sim new file mode 100644 index 0000000000..a9f66647f9 --- /dev/null +++ b/tests/script/tsim/valgrind/checkError6.sim @@ -0,0 +1,77 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c debugflag -v 131 +system sh/exec.sh -n dnode1 -s start -v +sql connect + +print =============== step1: create drop show dnodes +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ---> dnode not ready! + return -1 + endi +sql show dnodes +print ---> $data00 $data01 $data02 $data03 $data04 $data05 +if $rows != 1 then + return -1 +endi +if $data(1)[4] != ready then + goto step1 +endi + +$tbPrefix = tb +$tbNum = 5 +$rowNum = 10 + +print =============== step2: prepare data +sql create database db vgroups 2 +sql use db +sql create table if not exists stb (ts timestamp, tbcol int, tbcol2 float, tbcol3 double) tags (tgcol int unsigned) + +$i = 0 +while $i < $tbNum + $tb = $tbPrefix . $i + sql create table $tb using stb tags( $i ) + $x = 0 + while $x < $rowNum + $cc = $x * 60000 + $ms = 1601481600000 + $cc + sql insert into $tb values ($ms , $x , $x , $x ) + $x = $x + 1 + endw + $i = $i + 1 +endw + +print =============== step3: avg +sql select avg(tbcol) from tb1 +sql select avg(tbcol) from tb1 where ts <= 1601481840000 +sql select avg(tbcol) as b from tb1 +sql select avg(tbcol) as b from tb1 interval(1d) +goto _OVER +sql select avg(tbcol) as b from tb1 where ts <= 1601481840000s interval(1m) +sql select avg(tbcol) as c from stb +sql select avg(tbcol) as c from stb where ts <= 1601481840000 +sql select avg(tbcol) as c from stb where tgcol < 5 and ts <= 1601481840000 +sql select avg(tbcol) as c from stb interval(1m) +sql select avg(tbcol) as c from stb interval(1d) +sql select avg(tbcol) as b from stb where ts <= 1601481840000s interval(1m) +sql select avg(tbcol) as c from stb group by tgcol +sql select avg(tbcol) as b from stb where ts <= 1601481840000s partition by tgcol interval(1m) + +_OVER: +system sh/exec.sh -n dnode1 -s stop -x SIGINT +print =============== check +$null= + +system_content sh/checkValgrind.sh -n dnode1 +print cmd return result ----> [ $system_content ] +if $system_content > 0 then + return -1 +endi + +if $system_content == $null then + return -1 +endi diff --git a/tests/system-test/0-others/udf_cfg1.py b/tests/system-test/0-others/udf_cfg1.py new file mode 100644 index 0000000000..fc9265617d --- /dev/null +++ b/tests/system-test/0-others/udf_cfg1.py @@ -0,0 +1,305 @@ +from distutils.log import error +import taos +import sys +import time +import os +import platform + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +import subprocess + +class TDTestCase: + updatecfgDict = {'debugFlag': 143, "cDebugFlag": 143, "uDebugFlag": 143, "rpcDebugFlag": 143, "tmrDebugFlag": 143, + "jniDebugFlag": 143, "simDebugFlag": 143, "dDebugFlag": 143, "dDebugFlag": 143, "vDebugFlag": 143, "mDebugFlag": 143, "qDebugFlag": 143, + "wDebugFlag": 143, "sDebugFlag": 143, "tsdbDebugFlag": 143, "tqDebugFlag": 143, "fsDebugFlag": 143, "fnDebugFlag": 143 ,"udf":0} + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), logSql) + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files or "taosd.exe" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def prepare_udf_so(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + print(projPath) + + if platform.system().lower() == 'windows': + self.libudf1 = subprocess.Popen('(for /r %s %%i in ("udf1.d*") do @echo %%i)|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + self.libudf2 = subprocess.Popen('(for /r %s %%i in ("udf2.d*") do @echo %%i)|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + if (not tdDnodes.dnodes[0].remoteIP == ""): + tdDnodes.dnodes[0].remote_conn.get(tdDnodes.dnodes[0].config["path"]+'/debug/build/lib/libudf1.so',projPath+"\\debug\\build\\lib\\") + tdDnodes.dnodes[0].remote_conn.get(tdDnodes.dnodes[0].config["path"]+'/debug/build/lib/libudf2.so',projPath+"\\debug\\build\\lib\\") + self.libudf1 = self.libudf1.replace('udf1.dll','libudf1.so') + self.libudf2 = self.libudf2.replace('udf2.dll','libudf2.so') + else: + self.libudf1 = subprocess.Popen('find %s -name "libudf1.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + self.libudf2 = subprocess.Popen('find %s -name "libudf2.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + self.libudf1 = self.libudf1.replace('\r','').replace('\n','') + self.libudf2 = self.libudf2.replace('\r','').replace('\n','') + + + def prepare_data(self): + + tdSql.execute("drop database if exists db ") + tdSql.execute("create database if not exists db duration 300") + tdSql.execute("use db") + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + + for i in range(9): + tdSql.execute( + f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute( + f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + + tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + + tdSql.execute( + f'''insert into t1 values + ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) + ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) + ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a ) + ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a ) + ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a ) + ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) + ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) + ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) + ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) + ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ''' + ) + + tdSql.execute("create table tb (ts timestamp , num1 int , num2 int, num3 double , num4 binary(30))") + tdSql.execute( + f'''insert into tb values + ( '2020-04-21 01:01:01.000', NULL, 1, 1, "binary1" ) + ( '2020-10-21 01:01:01.000', 1, 1, 1.11, "binary1" ) + ( '2020-12-31 01:01:01.000', 2, 22222, 22, "binary1" ) + ( '2021-01-01 01:01:06.000', 3, 33333, 33, "binary1" ) + ( '2021-05-07 01:01:10.000', 4, 44444, 44, "binary1" ) + ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, "binary1" ) + ( '2021-09-30 01:01:16.000', 5, 55555, 55, "binary1" ) + ( '2022-02-01 01:01:20.000', 6, 66666, 66, "binary1" ) + ( '2022-10-28 01:01:26.000', 0, 00000, 00, "binary1" ) + ( '2022-12-01 01:01:30.000', 8, -88888, -88, "binary1" ) + ( '2022-12-31 01:01:36.000', 9, -9999999, -99, "binary1" ) + ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, "binary1" ) + ''' + ) + + # udf functions with join + ts_start = 1652517451000 + tdSql.execute("create stable st (ts timestamp , c1 int , c2 int ,c3 double ,c4 double ) tags(ind int)") + tdSql.execute("create table sub1 using st tags(1)") + tdSql.execute("create table sub2 using st tags(2)") + + for i in range(10): + ts = ts_start + i *1000 + tdSql.execute(" insert into sub1 values({} , {},{},{},{})".format(ts,i ,i*10,i*100.0,i*1000.0)) + tdSql.execute(" insert into sub2 values({} , {},{},{},{})".format(ts,i ,i*10,i*100.0,i*1000.0)) + + + def create_udf_function(self): + + for i in range(5): + # create scalar functions + tdSql.execute("create function udf1 as '%s' outputtype int bufSize 8;"%self.libudf1) + + # create aggregate functions + + tdSql.execute("create aggregate function udf2 as '%s' outputtype double bufSize 8;"%self.libudf2) + + functions = tdSql.getResult("show functions") + function_nums = len(functions) + if function_nums == 2: + tdLog.info("create two udf functions success ") + + # drop functions + + tdSql.execute("drop function udf1") + tdSql.execute("drop function udf2") + + functions = tdSql.getResult("show functions") + for function in functions: + if "udf1" in function[0] or "udf2" in function[0]: + tdLog.info("drop udf functions failed ") + tdLog.exit("drop udf functions failed") + + tdLog.info("drop two udf functions success ") + + # create scalar functions + tdSql.execute("create function udf1 as '%s' outputtype int bufSize 8;"%self.libudf1) + + # create aggregate functions + + tdSql.execute("create aggregate function udf2 as '%s' outputtype double bufSize 8;"%self.libudf2) + + functions = tdSql.getResult("show functions") + function_nums = len(functions) + if function_nums == 2: + tdLog.info("create two udf functions success ") + + def basic_udf_query(self): + + # scalar functions + + tdSql.execute("use db ") + tdSql.error("select num1 , udf1(num1) ,num2 ,udf1(num2),num3 ,udf1(num3),num4 ,udf1(num4) from tb") + tdSql.error("select c1 , udf1(c1) ,c2 ,udf1(c2), c3 ,udf1(c3), c4 ,udf1(c4) from stb1 order by c1") + + # aggregate functions + tdSql.error("select udf2(num1) ,udf2(num2), udf2(num3) from tb") + + # Arithmetic compute + tdSql.error("select udf2(num1)+100 ,udf2(num2)-100, udf2(num3)*100 ,udf2(num3)/100 from tb") + + tdSql.error("select udf2(c1) ,udf2(c6) from stb1 ") + + tdSql.error("select udf2(c1)+100 ,udf2(c6)-100 ,udf2(c1)*100 ,udf2(c6)/100 from stb1 ") + + # # bug for crash when query sub table + tdSql.error("select udf2(c1+100) ,udf2(c6-100) ,udf2(c1*100) ,udf2(c6/100) from ct1") + + tdSql.error("select udf2(c1+100) ,udf2(c6-100) ,udf2(c1*100) ,udf2(c6/100) from stb1 ") + + # regular table with aggregate functions + + tdSql.error("select udf1(num1) , count(num1) from tb;") + tdSql.error("select udf1(num1) , avg(num1) from tb;") + tdSql.error("select udf1(num1) , twa(num1) from tb;") + tdSql.error("select udf1(num1) , irate(num1) from tb;") + tdSql.error("select udf1(num1) , sum(num1) from tb;") + tdSql.error("select udf1(num1) , stddev(num1) from tb;") + tdSql.error("select udf1(num1) , mode(num1) from tb;") + tdSql.error("select udf1(num1) , HYPERLOGLOG(num1) from tb;") + # stable + tdSql.error("select udf1(c1) , count(c1) from stb1;") + tdSql.error("select udf1(c1) , avg(c1) from stb1;") + tdSql.error("select udf1(c1) , twa(c1) from stb1;") + tdSql.error("select udf1(c1) , irate(c1) from stb1;") + tdSql.error("select udf1(c1) , sum(c1) from stb1;") + tdSql.error("select udf1(c1) , stddev(c1) from stb1;") + tdSql.error("select udf1(c1) , mode(c1) from stb1;") + tdSql.error("select udf1(c1) , HYPERLOGLOG(c1) from stb1;") + + # regular table with select functions + + tdSql.error("select udf1(num1) , max(num1) from tb;") + + + tdSql.error("select udf1(num1) , min(num1) from tb;") + tdSql.error("select udf1(num1) , first(num1) from tb;") + tdSql.error("select udf1(num1) , last(num1) from tb;") + tdSql.error("select udf1(num1) , top(num1,1) from tb;") + tdSql.error("select udf1(num1) , bottom(num1,1) from tb;") + + + # stable + tdSql.error("select udf1(c1) , max(c1) from stb1;") + tdSql.error("select udf1(c1) , min(c1) from stb1;") + tdSql.error("select udf1(c1) , first(c1) from stb1;") + tdSql.error("select udf1(c1) , last(c1) from stb1;") + tdSql.error("select udf1(c1) , top(c1 ,1) from stb1;") + tdSql.error("select udf1(c1) , bottom(c1,1) from stb1;") + + + # regular table with compute functions + + tdSql.error("select udf1(num1) , abs(num1) from tb;") + + # stable with compute functions + tdSql.error("select udf1(c1) , abs(c1) from stb1;") + + # nest query + tdSql.error("select abs(udf1(c1)) , abs(ceil(c1)) from stb1 order by ts;") + tdSql.error("select abs(udf1(c1)) , abs(ceil(c1)) from ct1 order by ts;") + + # bug fix for crash + # order by udf function result + for _ in range(50): + tdSql.error("select udf2(c1) from stb1 group by 1-udf1(c1)") + print(tdSql.queryResult) + + # udf functions with filter + + tdSql.error("select c1 ,udf1(c1) , c6 ,udf1(c6) from stb1 where c1 > 8 order by ts") + tdSql.error("select udf1(sub1.c1), udf1(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") + + tdSql.error("select sub1.c1 , udf1(sub1.c1), sub2.c2 ,udf1(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") + tdSql.error("select udf2(sub1.c1), udf2(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") + tdSql.error("select sub1.c1 , udf2(sub1.c1), sub2.c2 ,udf2(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") + + # udf functions with group by + tdSql.error("select udf1(c1) from ct1 group by c1") + tdSql.error("select udf1(c1) from stb1 group by c1") + tdSql.error("select c1,c2, udf1(c1,c2) from ct1 group by c1,c2") + tdSql.error("select c1,c2, udf1(c1,c2) from stb1 group by c1,c2") + tdSql.error("select udf2(c1) from ct1 group by c1") + tdSql.error("select udf2(c1) from stb1 group by c1") + tdSql.error("select c1,c2, udf2(c1,c6) from ct1 group by c1,c2") + tdSql.error("select c1,c2, udf2(c1,c6) from stb1 group by c1,c2") + tdSql.error("select udf2(c1) from stb1 group by udf1(c1)") + tdSql.error("select udf2(c1) from stb1 group by floor(c1)") + + # udf mix with order by + tdSql.error("select udf2(c1) from stb1 group by floor(c1) order by udf2(c1)") + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + + print(" env is ok for all ") + self.prepare_udf_so() + self.prepare_data() + self.create_udf_function() + self.basic_udf_query() + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/0-others/udf_cfg2.py b/tests/system-test/0-others/udf_cfg2.py new file mode 100644 index 0000000000..07f83b1455 --- /dev/null +++ b/tests/system-test/0-others/udf_cfg2.py @@ -0,0 +1,667 @@ +from distutils.log import error +import taos +import sys +import time +import os +import platform + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +import subprocess + +class TDTestCase: + updatecfgDict = {'debugFlag': 143, "cDebugFlag": 143, "uDebugFlag": 143, "rpcDebugFlag": 143, "tmrDebugFlag": 143, + "jniDebugFlag": 143, "simDebugFlag": 143, "dDebugFlag": 143, "dDebugFlag": 143, "vDebugFlag": 143, "mDebugFlag": 143, "qDebugFlag": 143, + "wDebugFlag": 143, "sDebugFlag": 143, "tsdbDebugFlag": 143, "tqDebugFlag": 143, "fsDebugFlag": 143, "fnDebugFlag": 143 ,"udf":1} + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), logSql) + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files or "taosd.exe" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def prepare_udf_so(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + print(projPath) + + if platform.system().lower() == 'windows': + self.libudf1 = subprocess.Popen('(for /r %s %%i in ("udf1.d*") do @echo %%i)|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + self.libudf2 = subprocess.Popen('(for /r %s %%i in ("udf2.d*") do @echo %%i)|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + if (not tdDnodes.dnodes[0].remoteIP == ""): + tdDnodes.dnodes[0].remote_conn.get(tdDnodes.dnodes[0].config["path"]+'/debug/build/lib/libudf1.so',projPath+"\\debug\\build\\lib\\") + tdDnodes.dnodes[0].remote_conn.get(tdDnodes.dnodes[0].config["path"]+'/debug/build/lib/libudf2.so',projPath+"\\debug\\build\\lib\\") + self.libudf1 = self.libudf1.replace('udf1.dll','libudf1.so') + self.libudf2 = self.libudf2.replace('udf2.dll','libudf2.so') + else: + self.libudf1 = subprocess.Popen('find %s -name "libudf1.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + self.libudf2 = subprocess.Popen('find %s -name "libudf2.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + self.libudf1 = self.libudf1.replace('\r','').replace('\n','') + self.libudf2 = self.libudf2.replace('\r','').replace('\n','') + + + def prepare_data(self): + + tdSql.execute("drop database if exists db ") + tdSql.execute("create database if not exists db duration 300") + tdSql.execute("use db") + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + + for i in range(9): + tdSql.execute( + f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute( + f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + + tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + + tdSql.execute( + f'''insert into t1 values + ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) + ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) + ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a ) + ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a ) + ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a ) + ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) + ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) + ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) + ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) + ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ''' + ) + + tdSql.execute("create table tb (ts timestamp , num1 int , num2 int, num3 double , num4 binary(30))") + tdSql.execute( + f'''insert into tb values + ( '2020-04-21 01:01:01.000', NULL, 1, 1, "binary1" ) + ( '2020-10-21 01:01:01.000', 1, 1, 1.11, "binary1" ) + ( '2020-12-31 01:01:01.000', 2, 22222, 22, "binary1" ) + ( '2021-01-01 01:01:06.000', 3, 33333, 33, "binary1" ) + ( '2021-05-07 01:01:10.000', 4, 44444, 44, "binary1" ) + ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, "binary1" ) + ( '2021-09-30 01:01:16.000', 5, 55555, 55, "binary1" ) + ( '2022-02-01 01:01:20.000', 6, 66666, 66, "binary1" ) + ( '2022-10-28 01:01:26.000', 0, 00000, 00, "binary1" ) + ( '2022-12-01 01:01:30.000', 8, -88888, -88, "binary1" ) + ( '2022-12-31 01:01:36.000', 9, -9999999, -99, "binary1" ) + ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, "binary1" ) + ''' + ) + + # udf functions with join + ts_start = 1652517451000 + tdSql.execute("create stable st (ts timestamp , c1 int , c2 int ,c3 double ,c4 double ) tags(ind int)") + tdSql.execute("create table sub1 using st tags(1)") + tdSql.execute("create table sub2 using st tags(2)") + + for i in range(10): + ts = ts_start + i *1000 + tdSql.execute(" insert into sub1 values({} , {},{},{},{})".format(ts,i ,i*10,i*100.0,i*1000.0)) + tdSql.execute(" insert into sub2 values({} , {},{},{},{})".format(ts,i ,i*10,i*100.0,i*1000.0)) + + + def create_udf_function(self): + + for i in range(5): + # create scalar functions + tdSql.execute("create function udf1 as '%s' outputtype int bufSize 8;"%self.libudf1) + + # create aggregate functions + + tdSql.execute("create aggregate function udf2 as '%s' outputtype double bufSize 8;"%self.libudf2) + + functions = tdSql.getResult("show functions") + function_nums = len(functions) + if function_nums == 2: + tdLog.info("create two udf functions success ") + + # drop functions + + tdSql.execute("drop function udf1") + tdSql.execute("drop function udf2") + + functions = tdSql.getResult("show functions") + for function in functions: + if "udf1" in function[0] or "udf2" in function[0]: + tdLog.info("drop udf functions failed ") + tdLog.exit("drop udf functions failed") + + tdLog.info("drop two udf functions success ") + + # create scalar functions + tdSql.execute("create function udf1 as '%s' outputtype int bufSize 8;"%self.libudf1) + + # create aggregate functions + + tdSql.execute("create aggregate function udf2 as '%s' outputtype double bufSize 8;"%self.libudf2) + + functions = tdSql.getResult("show functions") + function_nums = len(functions) + if function_nums == 2: + tdLog.info("create two udf functions success ") + + def basic_udf_query(self): + + # scalar functions + + tdSql.execute("use db ") + tdSql.query("select num1 , udf1(num1) ,num2 ,udf1(num2),num3 ,udf1(num3),num4 ,udf1(num4) from tb") + tdSql.checkData(0,0,None) + tdSql.checkData(0,1,None) + tdSql.checkData(0,2,1) + tdSql.checkData(0,3,88) + tdSql.checkData(0,4,1.000000000) + tdSql.checkData(0,5,88) + tdSql.checkData(0,6,"binary1") + tdSql.checkData(0,7,88) + + tdSql.checkData(3,0,3) + tdSql.checkData(3,1,88) + tdSql.checkData(3,2,33333) + tdSql.checkData(3,3,88) + tdSql.checkData(3,4,33.000000000) + tdSql.checkData(3,5,88) + tdSql.checkData(3,6,"binary1") + tdSql.checkData(3,7,88) + + tdSql.checkData(11,0,None) + tdSql.checkData(11,1,None) + tdSql.checkData(11,2,None) + tdSql.checkData(11,3,None) + tdSql.checkData(11,4,None) + tdSql.checkData(11,5,None) + tdSql.checkData(11,6,"binary1") + tdSql.checkData(11,7,88) + + tdSql.query("select c1 , udf1(c1) ,c2 ,udf1(c2), c3 ,udf1(c3), c4 ,udf1(c4) from stb1 order by c1") + tdSql.checkData(0,0,None) + tdSql.checkData(0,1,None) + tdSql.checkData(0,2,None) + tdSql.checkData(0,3,None) + tdSql.checkData(0,4,None) + tdSql.checkData(0,5,None) + tdSql.checkData(0,6,None) + tdSql.checkData(0,7,None) + + tdSql.checkData(20,0,8) + tdSql.checkData(20,1,88) + tdSql.checkData(20,2,88888) + tdSql.checkData(20,3,88) + tdSql.checkData(20,4,888) + tdSql.checkData(20,5,88) + tdSql.checkData(20,6,88) + tdSql.checkData(20,7,88) + + + # aggregate functions + tdSql.query("select udf2(num1) ,udf2(num2), udf2(num3) from tb") + tdSql.checkData(0,0,15.362291496) + tdSql.checkData(0,1,10000949.553189287) + tdSql.checkData(0,2,168.633425216) + + # Arithmetic compute + tdSql.query("select udf2(num1)+100 ,udf2(num2)-100, udf2(num3)*100 ,udf2(num3)/100 from tb") + tdSql.checkData(0,0,115.362291496) + tdSql.checkData(0,1,10000849.553189287) + tdSql.checkData(0,2,16863.342521576) + tdSql.checkData(0,3,1.686334252) + + tdSql.query("select udf2(c1) ,udf2(c6) from stb1 ") + tdSql.checkData(0,0,25.514701644) + tdSql.checkData(0,1,265.247614504) + + tdSql.query("select udf2(c1)+100 ,udf2(c6)-100 ,udf2(c1)*100 ,udf2(c6)/100 from stb1 ") + tdSql.checkData(0,0,125.514701644) + tdSql.checkData(0,1,165.247614504) + tdSql.checkData(0,2,2551.470164435) + tdSql.checkData(0,3,2.652476145) + + # # bug for crash when query sub table + tdSql.query("select udf2(c1+100) ,udf2(c6-100) ,udf2(c1*100) ,udf2(c6/100) from ct1") + tdSql.checkData(0,0,378.215547010) + tdSql.checkData(0,1,353.808067460) + tdSql.checkData(0,2,2114.237451187) + tdSql.checkData(0,3,2.125468151) + + tdSql.query("select udf2(c1+100) ,udf2(c6-100) ,udf2(c1*100) ,udf2(c6/100) from stb1 ") + tdSql.checkData(0,0,490.358032462) + tdSql.checkData(0,1,400.460106627) + tdSql.checkData(0,2,2551.470164435) + tdSql.checkData(0,3,2.652476145) + + + # regular table with aggregate functions + + tdSql.error("select udf1(num1) , count(num1) from tb;") + tdSql.error("select udf1(num1) , avg(num1) from tb;") + tdSql.error("select udf1(num1) , twa(num1) from tb;") + tdSql.error("select udf1(num1) , irate(num1) from tb;") + tdSql.error("select udf1(num1) , sum(num1) from tb;") + tdSql.error("select udf1(num1) , stddev(num1) from tb;") + tdSql.error("select udf1(num1) , mode(num1) from tb;") + tdSql.error("select udf1(num1) , HYPERLOGLOG(num1) from tb;") + # stable + tdSql.error("select udf1(c1) , count(c1) from stb1;") + tdSql.error("select udf1(c1) , avg(c1) from stb1;") + tdSql.error("select udf1(c1) , twa(c1) from stb1;") + tdSql.error("select udf1(c1) , irate(c1) from stb1;") + tdSql.error("select udf1(c1) , sum(c1) from stb1;") + tdSql.error("select udf1(c1) , stddev(c1) from stb1;") + tdSql.error("select udf1(c1) , mode(c1) from stb1;") + tdSql.error("select udf1(c1) , HYPERLOGLOG(c1) from stb1;") + + # regular table with select functions + + tdSql.query("select udf1(num1) , max(num1) from tb;") + tdSql.checkRows(1) + tdSql.query("select floor(num1) , max(num1) from tb;") + tdSql.checkRows(1) + tdSql.query("select udf1(num1) , min(num1) from tb;") + tdSql.checkRows(1) + tdSql.query("select ceil(num1) , min(num1) from tb;") + tdSql.checkRows(1) + tdSql.query("select udf1(num1) , first(num1) from tb;") + + tdSql.query("select abs(num1) , first(num1) from tb;") + + tdSql.query("select udf1(num1) , last(num1) from tb;") + + tdSql.query("select round(num1) , last(num1) from tb;") + + tdSql.query("select udf1(num1) , top(num1,1) from tb;") + tdSql.checkRows(1) + tdSql.query("select udf1(num1) , bottom(num1,1) from tb;") + tdSql.checkRows(1) + # tdSql.query("select udf1(num1) , last_row(num1) from tb;") + # tdSql.checkRows(1) + + # tdSql.query("select round(num1) , last_row(num1) from tb;") + # tdSql.checkRows(1) + + + # stable + tdSql.query("select udf1(c1) , max(c1) from stb1;") + tdSql.checkRows(1) + tdSql.query("select abs(c1) , max(c1) from stb1;") + tdSql.checkRows(1) + tdSql.query("select udf1(c1) , min(c1) from stb1;") + tdSql.checkRows(1) + tdSql.query("select floor(c1) , min(c1) from stb1;") + tdSql.checkRows(1) + tdSql.query("select udf1(c1) , first(c1) from stb1;") + + tdSql.query("select udf1(c1) , last(c1) from stb1;") + + tdSql.query("select udf1(c1) , top(c1 ,1) from stb1;") + tdSql.checkRows(1) + tdSql.query("select abs(c1) , top(c1 ,1) from stb1;") + tdSql.checkRows(1) + tdSql.query("select udf1(c1) , bottom(c1,1) from stb1;") + tdSql.checkRows(1) + tdSql.query("select ceil(c1) , bottom(c1,1) from stb1;") + tdSql.checkRows(1) + + # tdSql.query("select udf1(c1) , last_row(c1) from stb1;") + # tdSql.checkRows(1) + # tdSql.query("select ceil(c1) , last_row(c1) from stb1;") + # tdSql.checkRows(1) + + # regular table with compute functions + + tdSql.query("select udf1(num1) , abs(num1) from tb;") + tdSql.checkRows(12) + tdSql.query("select floor(num1) , abs(num1) from tb;") + tdSql.checkRows(12) + + # # bug need fix + + #tdSql.query("select udf1(num1) , csum(num1) from tb;") + #tdSql.checkRows(9) + #tdSql.query("select ceil(num1) , csum(num1) from tb;") + #tdSql.checkRows(9) + #tdSql.query("select udf1(c1) , csum(c1) from stb1;") + #tdSql.checkRows(22) + #tdSql.query("select floor(c1) , csum(c1) from stb1;") + #tdSql.checkRows(22) + + # stable with compute functions + tdSql.query("select udf1(c1) , abs(c1) from stb1;") + tdSql.checkRows(25) + tdSql.query("select abs(c1) , ceil(c1) from stb1;") + tdSql.checkRows(25) + + # nest query + tdSql.query("select abs(udf1(c1)) , abs(ceil(c1)) from stb1 order by ts;") + tdSql.checkRows(25) + tdSql.checkData(0,0,None) + tdSql.checkData(0,1,None) + tdSql.checkData(1,0,88) + tdSql.checkData(1,1,8) + + tdSql.query("select abs(udf1(c1)) , abs(ceil(c1)) from ct1 order by ts;") + tdSql.checkRows(13) + tdSql.checkData(0,0,88) + tdSql.checkData(0,1,8) + tdSql.checkData(1,0,88) + tdSql.checkData(1,1,7) + + # bug fix for crash + # order by udf function result + for _ in range(50): + tdSql.query("select udf2(c1) from stb1 group by 1-udf1(c1)") + print(tdSql.queryResult) + + # udf functions with filter + + tdSql.query("select abs(udf1(c1)) , abs(ceil(c1)) from stb1 where c1 is null order by ts;") + tdSql.checkRows(3) + tdSql.checkData(0,0,None) + tdSql.checkData(0,1,None) + + tdSql.query("select c1 ,udf1(c1) , c6 ,udf1(c6) from stb1 where c1 > 8 order by ts") + tdSql.checkRows(3) + tdSql.checkData(0,0,9) + tdSql.checkData(0,1,88) + tdSql.checkData(0,2,-99.990000000) + tdSql.checkData(0,3,88) + + tdSql.query("select sub1.c1, sub2.c2 from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") + tdSql.checkData(0,0,0) + tdSql.checkData(0,1,0) + tdSql.checkData(1,0,1) + tdSql.checkData(1,1,10) + + tdSql.query("select udf1(sub1.c1), udf1(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") + tdSql.checkData(0,0,88) + tdSql.checkData(0,1,88) + tdSql.checkData(1,0,88) + tdSql.checkData(1,1,88) + + tdSql.query("select sub1.c1 , udf1(sub1.c1), sub2.c2 ,udf1(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") + tdSql.checkData(0,0,0) + tdSql.checkData(0,1,88) + tdSql.checkData(0,2,0) + tdSql.checkData(0,3,88) + tdSql.checkData(1,0,1) + tdSql.checkData(1,1,88) + tdSql.checkData(1,2,10) + tdSql.checkData(1,3,88) + + tdSql.query("select udf2(sub1.c1), udf2(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") + tdSql.checkData(0,0,16.881943016) + tdSql.checkData(0,1,168.819430161) + tdSql.error("select sub1.c1 , udf2(sub1.c1), sub2.c2 ,udf2(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") + + # udf functions with group by + tdSql.query("select udf1(c1) from ct1 group by c1") + tdSql.checkRows(10) + tdSql.query("select udf1(c1) from stb1 group by c1") + tdSql.checkRows(11) + tdSql.query("select c1,c2, udf1(c1,c2) from ct1 group by c1,c2") + tdSql.checkRows(10) + tdSql.query("select c1,c2, udf1(c1,c2) from stb1 group by c1,c2") + tdSql.checkRows(11) + + tdSql.query("select udf2(c1) from ct1 group by c1") + tdSql.checkRows(10) + tdSql.query("select udf2(c1) from stb1 group by c1") + tdSql.checkRows(11) + tdSql.query("select c1,c2, udf2(c1,c6) from ct1 group by c1,c2") + tdSql.checkRows(10) + tdSql.query("select c1,c2, udf2(c1,c6) from stb1 group by c1,c2") + tdSql.checkRows(11) + tdSql.query("select udf2(c1) from stb1 group by udf1(c1)") + tdSql.checkRows(2) + tdSql.query("select udf2(c1) from stb1 group by floor(c1)") + tdSql.checkRows(11) + + # udf mix with order by + tdSql.query("select udf2(c1) from stb1 group by floor(c1) order by udf2(c1)") + tdSql.checkRows(11) + + + def multi_cols_udf(self): + tdSql.query("select num1,num2,num3,udf1(num1,num2,num3) from tb") + tdSql.checkData(0,0,None) + tdSql.checkData(0,1,1) + tdSql.checkData(0,2,1.000000000) + tdSql.checkData(0,3,None) + tdSql.checkData(1,0,1) + tdSql.checkData(1,1,1) + tdSql.checkData(1,2,1.110000000) + tdSql.checkData(1,3,88) + + tdSql.query("select c1,c6,udf1(c1,c6) from stb1 order by ts") + tdSql.checkData(1,0,8) + tdSql.checkData(1,1,88.880000000) + tdSql.checkData(1,2,88) + + tdSql.query("select abs(udf1(c1,c6,c1,c6)) , abs(ceil(c1)) from stb1 where c1 is not null order by ts;") + tdSql.checkRows(22) + + tdSql.query("select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") + tdSql.checkData(0,0,169.661427555) + tdSql.checkData(0,1,169.661427555) + + def try_query_sql(self): + udf1_sqls = [ + "select num1 , udf1(num1) ,num2 ,udf1(num2),num3 ,udf1(num3),num4 ,udf1(num4) from tb" , + "select c1 , udf1(c1) ,c2 ,udf1(c2), c3 ,udf1(c3), c4 ,udf1(c4) from stb1 order by c1" , + "select udf1(num1) , max(num1) from tb;" , + "select udf1(num1) , min(num1) from tb;" , + #"select udf1(num1) , top(num1,1) from tb;" , + #"select udf1(num1) , bottom(num1,1) from tb;" , + "select udf1(c1) , max(c1) from stb1;" , + "select udf1(c1) , min(c1) from stb1;" , + #"select udf1(c1) , top(c1 ,1) from stb1;" , + #"select udf1(c1) , bottom(c1,1) from stb1;" , + "select udf1(num1) , abs(num1) from tb;" , + #"select udf1(num1) , csum(num1) from tb;" , + #"select udf1(c1) , csum(c1) from stb1;" , + "select udf1(c1) , abs(c1) from stb1;" , + "select abs(udf1(c1)) , abs(ceil(c1)) from stb1 order by ts;" , + "select abs(udf1(c1)) , abs(ceil(c1)) from ct1 order by ts;" , + "select abs(udf1(c1)) , abs(ceil(c1)) from stb1 where c1 is null order by ts;" , + "select c1 ,udf1(c1) , c6 ,udf1(c6) from stb1 where c1 > 8 order by ts" , + "select udf1(sub1.c1), udf1(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null" , + "select sub1.c1 , udf1(sub1.c1), sub2.c2 ,udf1(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null" , + "select udf1(c1) from ct1 group by c1" , + "select udf1(c1) from stb1 group by c1" , + "select c1,c2, udf1(c1,c2) from ct1 group by c1,c2" , + "select c1,c2, udf1(c1,c2) from stb1 group by c1,c2" , + "select num1,num2,num3,udf1(num1,num2,num3) from tb" , + "select c1,c6,udf1(c1,c6) from stb1 order by ts" , + "select abs(udf1(c1,c6,c1,c6)) , abs(ceil(c1)) from stb1 where c1 is not null order by ts;" + ] + udf2_sqls = ["select udf2(sub1.c1), udf2(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null" , + "select udf2(c1) from stb1 group by 1-udf1(c1)" , + "select udf2(num1) ,udf2(num2), udf2(num3) from tb" , + "select udf2(num1)+100 ,udf2(num2)-100, udf2(num3)*100 ,udf2(num3)/100 from tb" , + "select udf2(c1) ,udf2(c6) from stb1 " , + "select udf2(c1)+100 ,udf2(c6)-100 ,udf2(c1)*100 ,udf2(c6)/100 from stb1 " , + "select udf2(c1+100) ,udf2(c6-100) ,udf2(c1*100) ,udf2(c6/100) from ct1" , + "select udf2(c1+100) ,udf2(c6-100) ,udf2(c1*100) ,udf2(c6/100) from stb1 " , + "select udf2(c1) from ct1 group by c1" , + "select udf2(c1) from stb1 group by c1" , + "select c1,c2, udf2(c1,c6) from ct1 group by c1,c2" , + "select c1,c2, udf2(c1,c6) from stb1 group by c1,c2" , + "select udf2(c1) from stb1 group by udf1(c1)" , + "select udf2(c1) from stb1 group by floor(c1)" , + "select udf2(c1) from stb1 group by floor(c1) order by udf2(c1)" , + + "select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null" , + "select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null" , + "select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null" , + "select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null"] + + return udf1_sqls ,udf2_sqls + + + + def unexpected_create(self): + + tdLog.info(" create function with out bufsize ") + tdSql.query("drop function udf1 ") + tdSql.query("drop function udf2 ") + + # create function without buffer + tdSql.execute("create function udf1 as '%s' outputtype int"%self.libudf1) + tdSql.execute("create aggregate function udf2 as '%s' outputtype double"%self.libudf2) + udf1_sqls ,udf2_sqls = self.try_query_sql() + + for scalar_sql in udf1_sqls: + tdSql.query(scalar_sql) + for aggregate_sql in udf2_sqls: + tdSql.error(aggregate_sql) + + # create function without aggregate + + tdLog.info(" create function with out aggregate ") + tdSql.query("drop function udf1 ") + tdSql.query("drop function udf2 ") + + # create function without buffer + tdSql.execute("create aggregate function udf1 as '%s' outputtype int bufSize 8 "%self.libudf1) + tdSql.execute("create function udf2 as '%s' outputtype double bufSize 8"%self.libudf2) + udf1_sqls ,udf2_sqls = self.try_query_sql() + + for scalar_sql in udf1_sqls: + tdSql.error(scalar_sql) + for aggregate_sql in udf2_sqls: + tdSql.error(aggregate_sql) + + tdSql.execute(" create function db as '%s' outputtype int bufSize 8 "%self.libudf1) + tdSql.execute(" create aggregate function test as '%s' outputtype int bufSize 8 "%self.libudf1) + tdSql.error(" select db(c1) from stb1 ") + tdSql.error(" select db(c1,c6), db(c6) from stb1 ") + tdSql.error(" select db(num1,num2), db(num1) from tb ") + tdSql.error(" select test(c1) from stb1 ") + tdSql.error(" select test(c1,c6), test(c6) from stb1 ") + tdSql.error(" select test(num1,num2), test(num1) from tb ") + + + + def loop_kill_udfd(self): + + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + + cfgPath = buildPath + "/../sim/dnode1/cfg" + udfdPath = buildPath +'/build/bin/udfd' + + for i in range(3): + + tdLog.info(" loop restart udfd %d_th" % i) + + tdSql.query("select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") + tdSql.checkData(0,0,169.661427555) + tdSql.checkData(0,1,169.661427555) + # stop udfd cmds + get_processID = "ps -ef | grep -w udfd | grep -v grep| grep -v defunct | awk '{print $2}'" + processID = subprocess.check_output(get_processID, shell=True).decode("utf-8") + stop_udfd = " kill -9 %s" % processID + os.system(stop_udfd) + + time.sleep(2) + + tdSql.query("select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") + tdSql.checkData(0,0,169.661427555) + tdSql.checkData(0,1,169.661427555) + + # # start udfd cmds + # start_udfd = "nohup " + udfdPath +'-c' +cfgPath +" > /dev/null 2>&1 &" + # tdLog.info("start udfd : %s " % start_udfd) + + def test_function_name(self): + tdLog.info(" create function name is not build_in functions ") + tdSql.execute(" drop function udf1 ") + tdSql.execute(" drop function udf2 ") + tdSql.error("create function max as '%s' outputtype int bufSize 8"%self.libudf1) + tdSql.error("create aggregate function sum as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create function max as '%s' outputtype int bufSize 8"%self.libudf1) + tdSql.error("create aggregate function sum as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create aggregate function tbname as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create aggregate function function as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create aggregate function stable as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create aggregate function union as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create aggregate function 123 as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create aggregate function 123db as '%s' outputtype double bufSize 8"%self.libudf2) + tdSql.error("create aggregate function mnode as '%s' outputtype double bufSize 8"%self.libudf2) + + def restart_taosd_query_udf(self): + + self.create_udf_function() + + for i in range(5): + tdLog.info(" this is %d_th restart taosd " %i) + tdSql.execute("use db ") + tdSql.query("select count(*) from stb1") + tdSql.checkRows(1) + tdSql.query("select udf2(sub1.c1 ,sub1.c2), udf2(sub2.c2 ,sub2.c1) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") + tdSql.checkData(0,0,169.661427555) + tdSql.checkData(0,1,169.661427555) + tdDnodes.stop(1) + tdDnodes.start(1) + time.sleep(2) + + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + + print(" env is ok for all ") + self.prepare_udf_so() + self.prepare_data() + self.create_udf_function() + self.basic_udf_query() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/db.py b/tests/system-test/2-query/db.py new file mode 100644 index 0000000000..a4d603bada --- /dev/null +++ b/tests/system-test/2-query/db.py @@ -0,0 +1,63 @@ +import taos +import sys +import datetime +import inspect + +from util.log import * +from util.sql import * +from util.cases import * +import random + + +class TDTestCase: + updatecfgDict = {'debugFlag': 143, "cDebugFlag": 143, "uDebugFlag": 143, "rpcDebugFlag": 143, "tmrDebugFlag": 143, + "jniDebugFlag": 143, "simDebugFlag": 143, "dDebugFlag": 143, "dDebugFlag": 143, "vDebugFlag": 143, "mDebugFlag": 143, "qDebugFlag": 143, + "wDebugFlag": 143, "sDebugFlag": 143, "tsdbDebugFlag": 143, "tqDebugFlag": 143, "fsDebugFlag": 143, "udfDebugFlag": 143} + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def case1(self): + tdSql.execute("create database if not exists dbms precision 'ms'") + tdSql.execute("create database if not exists dbus precision 'us'") + tdSql.execute("create database if not exists dbns precision 'ns'") + + tdSql.execute("create table dbms.ntb (ts timestamp, c1 int, c2 bigint)") + tdSql.execute("create table dbus.ntb (ts timestamp, c1 int, c2 bigint)") + tdSql.execute("create table dbns.ntb (ts timestamp, c1 int, c2 bigint)") + + tdSql.execute("insert into dbms.ntb values ('2022-01-01 08:00:00.001', 1, 2)") + tdSql.execute("insert into dbms.ntb values ('2022-01-01 08:00:00.002', 3, 4)") + + tdSql.execute("insert into dbus.ntb values ('2022-01-01 08:00:00.000001', 1, 2)") + tdSql.execute("insert into dbus.ntb values ('2022-01-01 08:00:00.000002', 3, 4)") + + tdSql.execute("insert into dbns.ntb values ('2022-01-01 08:00:00.000000001', 1, 2)") + tdSql.execute("insert into dbns.ntb values ('2022-01-01 08:00:00.000000002', 3, 4)") + + tdSql.query("select count(c1) from dbms.ntb interval(1a)") + tdSql.checkRows(2) + + tdSql.query("select count(c1) from dbus.ntb interval(1u)") + tdSql.checkRows(2) + + tdSql.query("select count(c1) from dbns.ntb interval(1b)") + tdSql.checkRows(2) + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + tdSql.prepare() + + tdLog.printNoPrefix("==========start case1 run ...............") + + self.case1() + + tdLog.printNoPrefix("==========end case1 run ...............") + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/json_tag_large_tables.py b/tests/system-test/2-query/json_tag_large_tables.py index fc41858580..5d7df6ceb8 100644 --- a/tests/system-test/2-query/json_tag_large_tables.py +++ b/tests/system-test/2-query/json_tag_large_tables.py @@ -43,13 +43,48 @@ class TDTestCase: tdSql.execute('create database db vgroups 1') tdSql.execute('use db') print("============== STEP 1 ===== prepare data & validate json string") + + i = 0 + # add 100000 table + tdSql.execute("create table if not exists jsons1(ts timestamp, dataInt int, dataBool bool, dataStr nchar(50), dataStrBin binary(150)) tags(jtag json)") + while i <= 10 0000: + sql = """insert into jsons1_{%d} using jsons1 tags('{"tag1":{%d}}') values(1591060618000, 1, false, 'json1', '你是') (1591060608000, 23, true, '等等', 'json')"""%(i, i) + tdSql.execute(sql) + i = i + 1 + + // do query + i = 0 + while i <= 10 0000: + sql = """select count(*) from jsons1 where jtag->'tag1' = %d"""%(i) + tdSql.query(sql) + if 1 != tdSql.getRows(): + print("err: %s"%(sql)) + + while i <= 10000000 + sql = """insert into jsons1_{%d} using jsons1 tags('{"tag1":{%d}}') values(1591060618000, 1, false, 'json1', '你是') (1591060608000, 23, true, '等等', 'json')"""%(i, i) + tdSql.execute(sql) + i = i + 1 + + i = 0 + # drop super table + tdSql.execute("create table if not exists jsons1(ts timestamp, dataInt int, dataBool bool, dataStr nchar(50), dataStrBin binary(150)) tags(jtag json)") + while i <= 100000: + sql = """insert into jsons1_{%d} using jsons1 tags('{"tag1":{%d}}') values(1591060618000, 1, false, 'json1', '你是') (1591060608000, 23, true, '等等', 'json')"""%(i, i) + tdSql.execute(sql) + i = i + 1 + + tdSql.execute('drop stable jsons1') + + + # drop database i = 0 tdSql.execute("create table if not exists jsons1(ts timestamp, dataInt int, dataBool bool, dataStr nchar(50), dataStrBin binary(150)) tags(jtag json)") while i <= 100000: - f = "insert into jsons1_{} using jsons1 tags('{\"tag1\":\"fff\",\"tag2\":{}, \"tag3\":true}') values(1591060618000, 1, false, 'json1', '你是') (1591060608000, 23, true, '等等', 'json')".format - sql = f(i, i) + sql = """insert into jsons1_{%d} using jsons1 tags('{"tag1":{%d}}') values(1591060618000, 1, false, 'json1', '你是') (1591060608000, 23, true, '等等', 'json')"""%(i, i) tdSql.execute(sql) i = i + 1 + tdSql.execute('drop database db') + # test duplicate key using the first one. elimate empty key #tdSql.execute("CREATE TABLE if not exists jsons1_8 using jsons1 tags('{\"tag1\":null, \"tag1\":true, \"tag1\":45, \"1tag$\":2, \" \":90, \"\":32}')") tdSql.query("select jtag from jsons1_8") tdSql.checkRows(0); diff --git a/tests/system-test/2-query/queryQnode.py b/tests/system-test/2-query/queryQnode.py index 8bdb99deec..b5e2bd3328 100644 --- a/tests/system-test/2-query/queryQnode.py +++ b/tests/system-test/2-query/queryQnode.py @@ -392,6 +392,31 @@ class TDTestCase: tdSql.query("select c0,c1 from stb11_1 where (c0>1000) union all select c0,c1 from stb11_1 where c0>2000;") assert unionallQnode==tdSql.queryResult + queryPolicy=1 + + tdSql.execute('alter local "queryPolicy" "%d"'%queryPolicy) + tdSql.query("show local variables;") + for i in range(tdSql.queryRows): + if tdSql.queryResult[i][0] == "queryPolicy" : + if int(tdSql.queryResult[i][1]) == int(queryPolicy): + tdLog.success('alter queryPolicy to %d successfully'%queryPolicy) + else : + tdLog.debug(tdSql.queryResult) + tdLog.exit("alter queryPolicy to %d failed"%queryPolicy) + tdSql.execute("reset query cache") + + tdSql.execute("use db1;") + tdSql.query("show dnodes;") + dnodeId=tdSql.getData(0,0) + tdSql.query("select max(c1) from stb10;") + assert maxQnode==tdSql.getData(0,0) + tdSql.query("select min(c1) from stb11;") + assert minQnode==tdSql.getData(0,0) + tdSql.query("select c0,c1 from stb11_1 where (c0>1000) union select c0,c1 from stb11_1 where c0>2000;") + assert unionQnode==tdSql.queryResult + tdSql.query("select c0,c1 from stb11_1 where (c0>1000) union all select c0,c1 from stb11_1 where c0>2000;") + assert unionallQnode==tdSql.queryResult + # test case : queryPolicy = 2 def test_case2(self): self.taosBenchCreate("127.0.0.1","no","db1", "stb1", 10, 2, 1*10) diff --git a/tests/system-test/2-query/tsbsQuery.py b/tests/system-test/2-query/tsbsQuery.py index 1e017bb39b..664bc8dc5b 100644 --- a/tests/system-test/2-query/tsbsQuery.py +++ b/tests/system-test/2-query/tsbsQuery.py @@ -40,9 +40,9 @@ class TDTestCase: f"insert into rct{j} values ( {ts+i*10000}, {80+i}, {90+i}, {85+i}, {30+i*10}, {1.2*i}, {221+i*2}, {20+i*0.2}, {1500+i*20}, {150+i*2},{5+i} )" ) tdSql.execute( - f"insert into dct{j} values ( {ts+i*10000}, {1+i*0.1},{1400+i*15}, {1+i},{1500+i*20}, {150+i*2},{5+i} )" + f"insert into dct{j} values ( {ts+i*10000}, {1+i*0.1},{1400+i*15}, {i},{1500+i*20}, {150+i*2},{5+i} )" ) - + tdSql.execute("insert into dct9 (ts,fuel_state) values('2021-07-13 14:06:33.123Z',1.2) ;") # def check_avg(self ,origin_query , check_query): # avg_result = tdSql.getResult(origin_query) # origin_result = tdSql.getResult(check_query) @@ -60,13 +60,15 @@ class TDTestCase: def tsbsIotQuery(self): + tdSql.execute("use db_tsbs") # test interval and partition tdSql.query(" SELECT avg(velocity) as mean_velocity ,name,driver,fleet FROM readings WHERE ts > 1451606400000 AND ts <= 1451606460000 partition BY name,driver,fleet; ") + print(tdSql.queryResult) parRows=tdSql.queryRows tdSql.query(" SELECT avg(velocity) as mean_velocity ,name,driver,fleet FROM readings WHERE ts > 1451606400000 AND ts <= 1451606460000 partition BY name,driver,fleet interval(10m); ") - # tdSql.checkRows(parRows) + tdSql.checkRows(parRows) # test insert into @@ -77,18 +79,53 @@ class TDTestCase: # test paitition interval fill - # tdSql.query("SELECT name,floor(avg(velocity)/10)/floor(avg(velocity)/10) AS mv FROM readings WHERE name!='' AND ts > '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z' partition by name interval(10m) fill(value,0) ;") + tdSql.query("SELECT name,floor(avg(velocity)/10)/floor(avg(velocity)/10) AS mv FROM readings WHERE name!='' AND ts > '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z' partition by name interval(10m) fill(value,0) ;") - # # test partition interval limit - # tdSql.query("SELECT ts,model,floor(2*(sum(nzs)/count(nzs)))/floor(2*(sum(nzs)/count(nzs))) AS broken_down FROM (SELECT ts,model, status/status AS nzs FROM diagnostics WHERE ts >= '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z' ) WHERE ts >= '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z' partition BY model,ts interval(10m) limit 10;") + # test partition interval limit (PRcore-TD-17410) + # tdSql.query("select name,driver from (SELECT name,driver,fleet ,avg(velocity) as mean_velocity FROM readings partition BY name,driver,fleet interval (10m) limit 1);") # tdSql.checkRows(10) # test partition interval Pseudo time-column tdSql.query("SELECT count(ms1)/144 FROM (SELECT _wstart as ts1,model, fleet,avg(status) AS ms1 FROM diagnostics WHERE ts >= '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z' partition by model, fleet interval(10m)) WHERE ts1 >= '2016-01-01T00:00:00Z' AND ts1 < '2016-01-05T00:00:01Z' AND ms1<1;") + # 1 high-load: + # tdSql.query("SELECT ts,name,driver,current_load,load_capacity FROM (SELECT last(ts) as ts,name,driver, current_load,load_capacity FROM diagnostics WHERE fleet = 'South' partition by name,driver) WHERE current_load>= (0.9 * load_capacity) partition by name ORDER BY name desc, ts DESC;") + + # tdSql.query("SELECT ts,name,driver,current_load,load_capacity FROM (SELECT last(ts) as ts,name,driver, current_load,load_capacity FROM diagnostics WHERE fleet = 'South' partition by name,driver) WHERE current_load>= (0.9 * load_capacity) partition by name ORDER BY name ;") + + # 2 stationary-trucks + tdSql.query("select name,driver from (SELECT name,driver,fleet ,avg(velocity) as mean_velocity FROM readings WHERE ts > '2016-01-01T15:07:21Z' AND ts <= '2016-01-01T16:17:21Z' partition BY name,driver,fleet interval(10m) LIMIT 1)") + tdSql.query("select name,driver from (SELECT name,driver,fleet ,avg(velocity) as mean_velocity FROM readings WHERE ts > '2016-01-01T15:07:21Z' AND ts <= '2016-01-01T16:17:21Z' partition BY name,driver,fleet interval(10m) LIMIT 1) WHERE fleet = 'West' AND mean_velocity < 1000 partition BY name") + + # 3 long-driving-sessions + # tdSql.query("SELECT name,driver FROM(SELECT name,driver,count(*) AS ten_min FROM(SELECT _wstart as ts,name,driver,avg(velocity) as mean_velocity FROM readings where ts > '2016-01-01T00:00:34Z' AND ts <= '2016-01-01T04:00:34Z' partition BY name,driver interval(10m)) WHERE mean_velocity > 1 GROUP BY name,driver) WHERE ten_min > 22 ;") + + + #4 long-daily-sessions + tdSql.query("SELECT name,driver FROM(SELECT name,driver,count(*) AS ten_min FROM(SELECT name,driver,avg(velocity) as mean_velocity FROM readings WHERE fleet ='West' AND ts > '2016-01-01T12:31:37Z' AND ts <= '2016-01-05T12:31:37Z' partition BY name,driver interval(10m) ) WHERE mean_velocity > 1 GROUP BY name,driver) WHERE ten_min > 60") + + # 5. avg-daily-driving-duration + tdSql.query("select _wstart as ts,fleet,name,driver,count(mv)/6 as hours_driven from ( select _wstart as ts,fleet,name,driver,avg(velocity) as mv from readings where ts > '2016-01-01T00:00:00Z' and ts < '2016-01-05T00:00:01Z' partition by fleet,name,driver interval(10m)) where ts > '2016-01-01T00:00:00Z' and ts < '2016-01-05T00:00:01Z' partition by fleet,name,driver interval(1d) ;") + + + # 6. avg-daily-driving-session + #taosc core dumped + tdSql.execute("create table random_measure2_1 (ts timestamp,ela float, name binary(40))") + tdSql.query("SELECT ts,diff(mv) AS difka FROM (SELECT ts,name,floor(avg(velocity)/10)/floor(avg(velocity)/10) AS mv FROM readings WHERE name!='' AND ts > '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z' partition by name,ts interval(10m) fill(value,0)) GROUP BY name,ts;") + tdSql.query("SELECT _wstart,name,floor(avg(velocity)/10)/floor(avg(velocity)/10) AS mv FROM readings WHERE name!='' AND ts > '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z' partition by name interval(10m) fill(value,0)") + + # 7. avg-load + tdSql.query("SELECT fleet, model,avg(ml) AS mean_load_percentage FROM (SELECT fleet, model,current_load/load_capacity AS ml FROM diagnostics partition BY name, fleet, model) partition BY fleet, model order by fleet ;") + + # 8. daily-activity + tdSql.query(" SELECT model,ms1 FROM (SELECT _wstart as ts1,model, fleet,avg(status) AS ms1 FROM diagnostics WHERE ts >= '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z' partition by model, fleet interval(10m) fill(value,0)) WHERE ts1 >= '2016-01-01T00:00:00Z' AND ts1 < '2016-01-05T00:00:01Z' AND ms1<1;") + tdSql.query("SELECT _wstart,model,fleet,count(ms1)/144 FROM (SELECT _wstart as ts1,model, fleet,avg(status) AS ms1 FROM diagnostics WHERE ts >= '2016-01-01T00:00:00Z' AND ts < '2016-01-05T00:00:01Z' partition by model, fleet interval(10m) fill(value,0)) WHERE ts1 >= '2016-01-01T00:00:00Z' AND ts1 < '2016-01-05T00:00:01Z' AND ms1<1 partition by model, fleet interval(1d) ;") + + #it's already supported: + # last-loc + tdSql.query("") - # test def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdLog.printNoPrefix("==========step1:create database and table,insert data ==============") self.prepareData() diff --git a/tests/system-test/2-query/unique.py b/tests/system-test/2-query/unique.py index fce0d21f4d..3e6e14be38 100644 --- a/tests/system-test/2-query/unique.py +++ b/tests/system-test/2-query/unique.py @@ -426,7 +426,7 @@ class TDTestCase: tdSql.query(" select unique(t1+c1) from stb1 ") tdSql.checkRows(13) tdSql.query(" select unique(t1+c1) from stb1 partition by tbname ") - tdSql.checkRows(13) + tdSql.checkRows(20) tdSql.query(" select unique(t1) from stb1 partition by tbname ") tdSql.checkRows(2) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 8bafe3c966..ddbbefe739 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -11,6 +11,9 @@ python3 ./test.py -f 0-others/udfTest.py python3 ./test.py -f 0-others/udf_create.py python3 ./test.py -f 0-others/udf_restart_taosd.py python3 ./test.py -f 0-others/cachelast.py +python3 ./test.py -f 0-others/udf_cfg1.py +python3 ./test.py -f 0-others/udf_cfg2.py + python3 ./test.py -f 0-others/sysinfo.py python3 ./test.py -f 0-others/user_control.py python3 ./test.py -f 0-others/fsync.py @@ -29,6 +32,7 @@ python3 ./test.py -f 1-insert/block_wise.py python3 ./test.py -f 1-insert/create_retentions.py python3 ./test.py -f 1-insert/table_param_ttl.py +python3 ./test.py -f 2-query/db.py python3 ./test.py -f 2-query/between.py python3 ./test.py -f 2-query/distinct.py python3 ./test.py -f 2-query/varchar.py diff --git a/tests/test/c/tmqDemo.c b/tests/test/c/tmqDemo.c index 02fd3c1396..61c50fb0e8 100644 --- a/tests/test/c/tmqDemo.c +++ b/tests/test/c/tmqDemo.c @@ -388,13 +388,11 @@ void perf_loop(tmq_t* tmq, tmq_list_t* topics, int32_t totalMsgs, int64_t walLog } /*taosSsleep(3);*/ int32_t batchCnt = 0; - int32_t skipLogNum = 0; int64_t startTime = taosGetTimestampUs(); while (running) { TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 3000); if (tmqmessage) { batchCnt++; - /*skipLogNum += tmqGetSkipLogNum(tmqmessage);*/ if (0 != g_stConfInfo.showMsgFlag) { /*msg_process(tmqmessage);*/ } @@ -412,7 +410,7 @@ void perf_loop(tmq_t* tmq, tmq_list_t* topics, int32_t totalMsgs, int64_t walLog } if (0 == g_stConfInfo.simCase) { - printf("consume result: msgs: %d, skip log cnt: %d, time used:%.3f second\n", batchCnt, skipLogNum, consumeTime); + printf("consume result: msgs: %d, time used:%.3f second\n", batchCnt, consumeTime); } else { printf("{consume success: %d}", totalMsgs); } diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c index b4f86d52b5..5459e3f159 100644 --- a/tests/test/c/tmqSim.c +++ b/tests/test/c/tmqSim.c @@ -20,6 +20,7 @@ #include #include #include +#include #include "taos.h" #include "taosdef.h" @@ -35,6 +36,8 @@ #define MAX_ROW_STR_LEN (16 * 1024) #define MAX_CONSUMER_THREAD_CNT (16) #define MAX_VGROUP_CNT (32) +#define SEND_TIME_UNIT 10 // ms +#define MAX_SQL_LEN 1048576 typedef enum { NOTIFY_CMD_START_CONSUM, @@ -42,6 +45,12 @@ typedef enum { NOTIFY_CMD_ID_BUTT, } NOTIFY_CMD_ID; +typedef enum enumQUERY_TYPE { + NO_INSERT_TYPE, + INSERT_TYPE, + QUERY_TYPE_BUT +} QUERY_TYPE; + typedef struct { TdThread thread; int32_t consumerId; @@ -58,6 +67,7 @@ typedef struct { int64_t consumeMsgCnt; int64_t consumeRowCnt; + int64_t consumeLen; int32_t checkresult; char topicString[1024]; @@ -77,14 +87,19 @@ typedef struct { int32_t rowsOfPerVgroups[MAX_VGROUP_CNT][2]; // [i][0]: vgroup id, [i][1]: rows of consume int64_t ts; - TAOS* taos; + TAOS* taos; + + // below parameters is used by omb test + int32_t producerRate; // unit: msgs/s + int64_t totalProduceMsgs; + int64_t totalMsgsLen; } SThreadInfo; typedef struct { // input from argvs char cdbName[32]; - char dbName[32]; + char dbName[64]; int32_t showMsgFlag; int32_t showRowFlag; int32_t saveRowFlag; @@ -93,11 +108,22 @@ typedef struct { int32_t useSnapshot; int64_t nowTime; SThreadInfo stThreads[MAX_CONSUMER_THREAD_CNT]; + + SThreadInfo stProdThreads[MAX_CONSUMER_THREAD_CNT]; + + // below parameters is used by omb test + char topic[64]; + int32_t producers; + int32_t producerRate; + int32_t runDurationMinutes; + int32_t batchSize; + int32_t payloadLen; } SConfInfo; static SConfInfo g_stConfInfo; TdFilePtr g_fp = NULL; static int running = 1; +char* g_payload = NULL; // char* g_pRowValue = NULL; // TdFilePtr g_fp = NULL; @@ -117,7 +143,29 @@ static void printHelp() { printf("%s%s\n", indent, "-s"); printf("%s%s%s%d\n", indent, indent, "saveRowFlag, default is ", g_stConfInfo.saveRowFlag); printf("%s%s\n", indent, "-y"); - printf("%s%s%s%d\n", indent, indent, "consume delay, default is s", g_stConfInfo.consumeDelay); + printf("%s%s%s%ds\n", indent, indent, "consume delay, default is ", g_stConfInfo.consumeDelay); + printf("%s%s\n", indent, "-e"); + printf("%s%s%s%d\n", indent, indent, "snapshot, default is ", g_stConfInfo.useSnapshot); + + printf("%s%s\n", indent, "-t"); + printf("%s%s%s\n", indent, indent, "topic name, default is null"); + + printf("%s%s\n", indent, "-x"); + printf("%s%s%s\n", indent, indent, "consume thread number, default is 1"); + + + printf("%s%s\n", indent, "-l"); + printf("%s%s%s\n", indent, indent, "run duration unit is minutes, default is ", g_stConfInfo.runDurationMinutes); + printf("%s%s\n", indent, "-p"); + printf("%s%s%s\n", indent, indent, "producer thread number, default is 0"); + printf("%s%s\n", indent, "-b"); + printf("%s%s%s\n", indent, indent, "batch size, default is 1"); + printf("%s%s\n", indent, "-i"); + printf("%s%s%s\n", indent, indent, "produce rate unit is msgs /s, default is 100000"); + printf("%s%s\n", indent, "-n"); + printf("%s%s%s\n", indent, indent, "payload len unit is byte, default is 1000"); + + exit(EXIT_SUCCESS); } @@ -144,7 +192,11 @@ void initLogFile() { pid_t process_id = getpid(); - sprintf(filename, "%s/../log/tmqlog-%d-%s.txt", configDir, process_id, getCurrentTimeString(tmpString)); + if (0 != strlen(g_stConfInfo.topic)) { + sprintf(filename, "/tmp/tmqlog-%d-%s.txt", process_id, getCurrentTimeString(tmpString)); + } else { + sprintf(filename, "%s/../log/tmqlog-%d-%s.txt", configDir, process_id, getCurrentTimeString(tmpString)); + } #ifdef WINDOWS for (int i = 2; i < sizeof(filename); i++) { if (filename[i] == ':') filename[i] = '-'; @@ -199,6 +251,9 @@ void parseArgument(int32_t argc, char* argv[]) { g_stConfInfo.showRowFlag = 0; g_stConfInfo.saveRowFlag = 0; g_stConfInfo.consumeDelay = 5; + g_stConfInfo.numOfThread = 1; + g_stConfInfo.batchSize = 1; + g_stConfInfo.producers = 0; g_stConfInfo.nowTime = taosGetTimestampMs(); @@ -222,12 +277,38 @@ void parseArgument(int32_t argc, char* argv[]) { g_stConfInfo.consumeDelay = atol(argv[++i]); } else if (strcmp(argv[i], "-e") == 0) { g_stConfInfo.useSnapshot = atol(argv[++i]); + } else if (strcmp(argv[i], "-t") == 0) { + char tmpBuf[56]; + strcpy(tmpBuf, argv[++i]); + sprintf(g_stConfInfo.topic, "`%s`", tmpBuf); + } else if (strcmp(argv[i], "-x") == 0) { + g_stConfInfo.numOfThread = atol(argv[++i]); + } else if (strcmp(argv[i], "-l") == 0) { + g_stConfInfo.runDurationMinutes = atol(argv[++i]); + } else if (strcmp(argv[i], "-p") == 0) { + g_stConfInfo.producers = atol(argv[++i]); + } else if (strcmp(argv[i], "-b") == 0) { + g_stConfInfo.batchSize = atol(argv[++i]); + } else if (strcmp(argv[i], "-i") == 0) { + g_stConfInfo.producerRate = atol(argv[++i]); + } else if (strcmp(argv[i], "-n") == 0) { + g_stConfInfo.payloadLen = atol(argv[++i]); } else { pError("%s unknow para: %s %s", GREEN, argv[++i], NC); exit(-1); } } + g_payload = taosMemoryCalloc(g_stConfInfo.payloadLen + 1, 1); + if (NULL == g_payload) { + pPrint("%s failed to malloc for payload %s", GREEN, NC); + exit(-1); + } + + for (int32_t i = 0; i < g_stConfInfo.payloadLen; i++) { + strcpy(&g_payload[i], "a"); + } + initLogFile(); taosFprintfFile(g_fp, "====parseArgument() success\n"); @@ -240,6 +321,11 @@ void parseArgument(int32_t argc, char* argv[]) { pPrint("%s showMsgFlag:%d %s", GREEN, g_stConfInfo.showMsgFlag, NC); pPrint("%s showRowFlag:%d %s", GREEN, g_stConfInfo.showRowFlag, NC); pPrint("%s saveRowFlag:%d %s", GREEN, g_stConfInfo.saveRowFlag, NC); + + pPrint("%s snapshot:%d %s", GREEN, g_stConfInfo.useSnapshot, NC); + + pPrint("%s omb topic:%s %s", GREEN, g_stConfInfo.topic, NC); + pPrint("%s numOfThread:%d %s", GREEN, g_stConfInfo.numOfThread, NC); #endif } @@ -909,8 +995,439 @@ int32_t getConsumeInfo() { return 0; } + +static int32_t omb_data_msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIndex, int64_t* lenOfRows) { + char buf[16*1024]; + int32_t totalRows = 0; + int32_t totalLen = 0; + + // printf("topic: %s\n", tmq_get_topic_name(msg)); + //int32_t vgroupId = tmq_get_vgroup_id(msg); + //const char* dbName = tmq_get_db_name(msg); + + //taosFprintfFile(g_fp, "consumerId: %d, msg index:%" PRId64 "\n", pInfo->consumerId, msgIndex); + //taosFprintfFile(g_fp, "dbName: %s, topic: %s, vgroupId: %d\n", dbName != NULL ? dbName : "invalid table", + // tmq_get_topic_name(msg), vgroupId); + + while (1) { + TAOS_ROW row = taos_fetch_row(msg); + + if (row == NULL) break; + + TAOS_FIELD* fields = taos_fetch_fields(msg); + int32_t numOfFields = taos_field_count(msg); + //int32_t* length = taos_fetch_lengths(msg); + //int32_t precision = taos_result_precision(msg); + //const char* tbName = tmq_get_table_name(msg); + + taos_print_row(buf, row, fields, numOfFields); + totalLen += strlen(buf); + totalRows++; + } + + *lenOfRows = totalLen; + return totalRows; +} + +void omb_loop_consume(SThreadInfo* pInfo) { + int32_t code; + + int32_t once_flag = 0; + + int64_t totalMsgs = 0; + int64_t totalRows = 0; + + char tmpString[128]; + taosFprintfFile(g_fp, "%s consumer id %d start to loop pull msg\n", getCurrentTimeString(tmpString), + pInfo->consumerId); + printf("%s consumer id %d start to loop pull msg\n", getCurrentTimeString(tmpString), + pInfo->consumerId); + + pInfo->ts = taosGetTimestampMs(); + + int64_t lastTotalMsgs = 0; + uint64_t lastPrintTime = taosGetTimestampMs(); + uint64_t startTs = taosGetTimestampMs(); + + int64_t totalLenOfMsg = 0; + int64_t lastTotalLenOfMsg = 0; + int32_t consumeDelay = g_stConfInfo.consumeDelay == -1 ? -1 : (g_stConfInfo.consumeDelay * 1000); + while (running) { + TAOS_RES* tmqMsg = tmq_consumer_poll(pInfo->tmq, consumeDelay); + if (tmqMsg) { + int64_t lenOfMsg = 0; + totalRows += omb_data_msg_process(tmqMsg, pInfo, totalMsgs, &lenOfMsg); + totalLenOfMsg += lenOfMsg; + taos_free_result(tmqMsg); + totalMsgs++; + int64_t currentPrintTime = taosGetTimestampMs(); + if (currentPrintTime - lastPrintTime > 10 * 1000) { + int64_t currentLenOfMsg = totalLenOfMsg - lastTotalLenOfMsg; + int64_t deltaTime = currentPrintTime - lastPrintTime; + printf("consumer id %d has currently cons total rows: %" PRId64 ", msgs: %" PRId64 ", rate: %.3f msgs/s, %.1f MB/s\n", + pInfo->consumerId, totalRows, totalMsgs, + (totalMsgs - lastTotalMsgs) * 1000.0 / deltaTime, + currentLenOfMsg*1000.0/(1024*1024)/deltaTime); + + taosFprintfFile( + g_fp, "consumer id %d has currently poll total msgs: %" PRId64 ", period cons rate: %.3f msgs/s, %.1f MB/s\n", + pInfo->consumerId, totalMsgs, (totalMsgs - lastTotalMsgs) * 1000.0 / deltaTime, currentLenOfMsg*1000.0/deltaTime); + lastPrintTime = currentPrintTime; + lastTotalMsgs = totalMsgs; + lastTotalLenOfMsg = totalLenOfMsg; + } + } else { + char tmpString[128]; + taosFprintfFile(g_fp, "%s no poll more msg when time over, break consume\n", getCurrentTimeString(tmpString)); + printf("%s no poll more msg when time over, break consume\n", getCurrentTimeString(tmpString)); + int64_t currentPrintTime = taosGetTimestampMs(); + int64_t currentLenOfMsg = totalLenOfMsg - lastTotalLenOfMsg; + int64_t deltaTime = currentPrintTime - lastPrintTime; + printf("consumer id %d has currently cons total rows: %" PRId64 ", msgs: %" PRId64 ", rate: %.3f msgs/s, %.1f MB/s\n", + pInfo->consumerId, totalRows, totalMsgs, + (totalMsgs - lastTotalMsgs) * 1000.0 / deltaTime, + currentLenOfMsg*1000.0/(1024*1024)/deltaTime); + break; + } + } + + pInfo->consumeMsgCnt = totalMsgs; + pInfo->consumeRowCnt = totalRows; + pInfo->consumeLen = totalLenOfMsg; + +} + + +void* ombConsumeThreadFunc(void* param) { + SThreadInfo* pInfo = (SThreadInfo*)param; + + //################### set key ######################## + tmq_conf_t* conf = tmq_conf_new(); + // tmq_conf_set(conf, "td.connect.ip", "localhost"); + // tmq_conf_set(conf, "td.connect.port", "6030"); + tmq_conf_set(conf, "td.connect.user", "root"); + tmq_conf_set(conf, "td.connect.pass", "taosdata"); + // tmq_conf_set(conf, "td.connect.db", g_stConfInfo.dbName); + tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, pInfo); + tmq_conf_set(conf, "group.id", "ombCgrp"); + // tmq_conf_set(conf, "msg.with.table.name", "true"); + // tmq_conf_set(conf, "client.id", "c-001"); + // tmq_conf_set(conf, "enable.auto.commit", "true"); + tmq_conf_set(conf, "enable.auto.commit", "false"); + // tmq_conf_set(conf, "auto.commit.interval.ms", "1000"); + // tmq_conf_set(conf, "auto.offset.reset", "none"); + // tmq_conf_set(conf, "auto.offset.reset", "earliest"); + tmq_conf_set(conf, "auto.offset.reset", "earliest"); + // + if (g_stConfInfo.useSnapshot) { + tmq_conf_set(conf, "experimental.snapshot.enable", "true"); + } + + pInfo->tmq = tmq_consumer_new(conf, NULL, 0); + + tmq_conf_destroy(conf); + + //################### set topic ########################## + pInfo->topicList = tmq_list_new(); + tmq_list_append(pInfo->topicList, g_stConfInfo.topic); + + if ((NULL == pInfo->tmq) || (NULL == pInfo->topicList)) { + taosFprintfFile(g_fp, "create consumer fail! tmq is null or topicList is null\n"); + assert(0); + return NULL; + } + + int32_t err = tmq_subscribe(pInfo->tmq, pInfo->topicList); + if (err != 0) { + pError("tmq_subscribe() fail, reason: %s\n", tmq_err2str(err)); + taosFprintfFile(g_fp, "tmq_subscribe() fail! reason: %s\n", tmq_err2str(err)); + assert(0); + return NULL; + } + + tmq_list_destroy(pInfo->topicList); + pInfo->topicList = NULL; + + omb_loop_consume(pInfo); + + err = tmq_unsubscribe(pInfo->tmq); + if (err != 0) { + pError("tmq_unsubscribe() fail, reason: %s\n", tmq_err2str(err)); + taosFprintfFile(g_fp, "tmq_unsubscribe()! reason: %s\n", tmq_err2str(err)); + } + + err = tmq_consumer_close(pInfo->tmq); + if (err != 0) { + pError("tmq_consumer_close() fail, reason: %s\n", tmq_err2str(err)); + taosFprintfFile(g_fp, "tmq_consumer_close()! reason: %s\n", tmq_err2str(err)); + } + pInfo->tmq = NULL; + + return NULL; +} + + + +static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type) { + TAOS_RES *res = taos_query(taos, command); + int32_t code = taos_errno(res); + + if (code != 0) { + pPrint("%s Failed to execute <%s>, reason: %s %s", GREEN, command, taos_errstr(res), NC); + taos_free_result(res); + return -1; + } + + if (INSERT_TYPE == type) { + int affectedRows = taos_affected_rows(res); + taos_free_result(res); + return affectedRows; + } + + taos_free_result(res); + return 0; +} + +void* ombProduceThreadFunc(void* param) { + SThreadInfo* pInfo = (SThreadInfo*)param; + + pInfo->taos = taos_connect(NULL, "root", "taosdata", NULL, 0); + if (pInfo->taos == NULL) { + printf("taos_connect() fail\n"); + return NULL; + } + + int64_t affectedRowsTotal = 0; + int64_t sendMsgs = 0; + + uint32_t totalSendLoopTimes = g_stConfInfo.runDurationMinutes * 60 * 1000 / SEND_TIME_UNIT; // send some msgs per 10ms + uint32_t batchPerTblTimes = pInfo->producerRate / 100 / g_stConfInfo.batchSize; + uint32_t remainder = (pInfo->producerRate / 100) % g_stConfInfo.batchSize; + if (remainder) { + batchPerTblTimes += 1; + } + + char* sqlBuf = taosMemoryMalloc(MAX_SQL_LEN); + if (NULL == sqlBuf) { + printf("malloc fail for sqlBuf\n"); + return NULL; + } + + printf("Produce Info: totalSendLoopTimes: %d, batchPerTblTimes: %d, producerRate: %d\n", totalSendLoopTimes, batchPerTblTimes, pInfo->producerRate); + + char ctbName[64] = {0}; + sprintf(ctbName, "%s.ctb%d", g_stConfInfo.dbName, pInfo->consumerId); + + int64_t lastPrintTime = taosGetTimestampUs(); + int64_t totalMsgLen = 0; + //int64_t timeStamp = taosGetTimestampUs(); + while (totalSendLoopTimes) { + int64_t startTs = taosGetTimestampUs(); + for (int i = 0; i < batchPerTblTimes; ++i) { + uint32_t msgsOfSql = g_stConfInfo.batchSize; + if ((i == batchPerTblTimes - 1) && (0 != remainder)) { + msgsOfSql = remainder; + } + int len = 0; + len += snprintf(sqlBuf+len, MAX_SQL_LEN - len, "insert into %s values ", ctbName); + for (int j = 0; j < msgsOfSql; j++) { + int64_t timeStamp = taosGetTimestampNs(); + len += snprintf(sqlBuf+len, MAX_SQL_LEN - len, "(%" PRId64 ", \"%s\")", timeStamp, g_payload); + sendMsgs++; + pInfo->totalProduceMsgs++; + } + + totalMsgLen += len; + pInfo->totalMsgsLen += len; + + int64_t affectedRows = queryDbExec(pInfo->taos, sqlBuf, INSERT_TYPE); + if (affectedRows < 0) { + return NULL; + } + + affectedRowsTotal += affectedRows; + + //printf("Produce Info: affectedRows: %" PRId64 "\n", affectedRows); + } + totalSendLoopTimes -= 1; + + // calc spent time + int64_t currentTs = taosGetTimestampUs(); + int64_t delta = currentTs - startTs; + if (delta < SEND_TIME_UNIT * 1000) { + int64_t sleepLen = (int32_t)(SEND_TIME_UNIT * 1000 - delta); + //printf("sleep %" PRId64 " us, use time: %" PRId64 " us\n", sleepLen, delta); + taosUsleep((int32_t)sleepLen); + } + + currentTs = taosGetTimestampUs(); + delta = currentTs - lastPrintTime; + if (delta > 10 * 1000 * 1000) { + printf("producer[%d] info: %" PRId64 " msgs, %" PRId64 " Byte, %" PRId64 " us, totalSendLoopTimes: %d\n", + pInfo->consumerId, sendMsgs, totalMsgLen, delta, totalSendLoopTimes); + printf("producer[%d] rate: %1.f msgs/s, %1.f KB/s\n", + pInfo->consumerId, + sendMsgs * 1000.0 * 1000 / delta, + (totalMsgLen / 1024.0) / (delta / (1000*1000))); + lastPrintTime = currentTs; + sendMsgs = 0; + totalMsgLen = 0; + } + } + + printf("affectedRowsTotal: %"PRId64"\n", affectedRowsTotal); + return NULL; +} + + +void printProduceInfo(int64_t start) { + int64_t totalMsgs = 0; + int64_t totalLenOfMsgs = 0; + for (int i = 0; i < g_stConfInfo.producers; i++) { + totalMsgs += g_stConfInfo.stProdThreads[i].totalProduceMsgs; + totalLenOfMsgs += g_stConfInfo.stProdThreads[i].totalMsgsLen; + } + + int64_t end = taosGetTimestampUs(); + + int64_t t = end - start; + if (0 == t) t = 1; + + double tInMs = (double)t / 1000000.0; + printf("Spent %.3f seconds to prod %" PRIu64 " msgs, %" PRIu64 " Byte\n\n", tInMs, totalMsgs, totalLenOfMsgs); + + + printf("Spent %.3f seconds to prod %" PRIu64 " msgs with %d producer(s), throughput: %.3f msgs/s, %.1f MB/s\n\n", + tInMs, totalMsgs, g_stConfInfo.producers, + (double)totalMsgs / tInMs, + (double)totalLenOfMsgs/(1024.0*1024)/tInMs); + return; +} + + +void startOmbConsume() { + TdThreadAttr thattr; + taosThreadAttrInit(&thattr); + taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE); + + if (0 != g_stConfInfo.producers) { + TAOS* taos = taos_connect(NULL, "root", "taosdata", NULL, 0); + if (taos == NULL) { + taosFprintfFile(g_fp, "taos_connect() fail, can not notify and save consume result to main scripte\n"); + ASSERT(0); + return ; + } + + char stbName[16] = "stb"; + char ctbPrefix[16] = "ctb"; + + char sql[256] = {0}; + sprintf(sql, "drop database if exists %s", g_stConfInfo.dbName); + printf("SQL: %s\n", sql); + queryDbExec(taos, sql, NO_INSERT_TYPE); + + sprintf(sql, "create database if not exists %s precision 'ns' vgroups %d", g_stConfInfo.dbName, g_stConfInfo.producers); + printf("SQL: %s\n", sql); + queryDbExec(taos, sql, NO_INSERT_TYPE); + + sprintf(sql, "create stable %s.%s (ts timestamp, payload binary(%d)) tags (t bigint) ", g_stConfInfo.dbName, stbName, g_stConfInfo.payloadLen); + printf("SQL: %s\n", sql); + queryDbExec(taos, sql, NO_INSERT_TYPE); + + for (int i = 0; i < g_stConfInfo.producers; i++) { + sprintf(sql, "create table %s.%s%d using %s.stb tags(%d) ", g_stConfInfo.dbName, ctbPrefix, i, g_stConfInfo.dbName, i); + printf("SQL: %s\n", sql); + queryDbExec(taos, sql, NO_INSERT_TYPE); + } + + // create topic + sprintf(sql, "create topic %s as stable %s.%s", g_stConfInfo.topic, g_stConfInfo.dbName, stbName); + printf("SQL: %s\n", sql); + queryDbExec(taos, sql, NO_INSERT_TYPE); + + + int32_t producerRate = ceil(g_stConfInfo.producerRate / g_stConfInfo.producers); + + printf("==== create %d produce thread ====\n", g_stConfInfo.producers); + for (int32_t i = 0; i < g_stConfInfo.producers; ++i) { + g_stConfInfo.stProdThreads[i].consumerId = i; + g_stConfInfo.stProdThreads[i].producerRate = producerRate; + taosThreadCreate(&(g_stConfInfo.stProdThreads[i].thread), &thattr, ombProduceThreadFunc, + (void*)(&(g_stConfInfo.stProdThreads[i]))); + } + + if (0 == g_stConfInfo.numOfThread) { + int64_t start = taosGetTimestampUs(); + for (int32_t i = 0; i < g_stConfInfo.producers; i++) { + taosThreadJoin(g_stConfInfo.stProdThreads[i].thread, NULL); + taosThreadClear(&g_stConfInfo.stProdThreads[i].thread); + } + + printProduceInfo(start); + + taosFprintfFile(g_fp, "==== close tmqlog ====\n"); + taosCloseFile(&g_fp); + return; + } + + } + + // pthread_create one thread to consume + taosFprintfFile(g_fp, "==== create %d consume thread ====\n", g_stConfInfo.numOfThread); + for (int32_t i = 0; i < g_stConfInfo.numOfThread; ++i) { + g_stConfInfo.stThreads[i].consumerId = i; + taosThreadCreate(&(g_stConfInfo.stThreads[i].thread), &thattr, ombConsumeThreadFunc, + (void*)(&(g_stConfInfo.stThreads[i]))); + } + + int64_t start = taosGetTimestampUs(); + + for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { + taosThreadJoin(g_stConfInfo.stThreads[i].thread, NULL); + taosThreadClear(&g_stConfInfo.stThreads[i].thread); + } + + int64_t end = taosGetTimestampUs(); + + int64_t totalRows = 0; + int64_t totalMsgs = 0; + int64_t totalLenOfMsgs = 0; + for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { + totalMsgs += g_stConfInfo.stThreads[i].consumeMsgCnt; + totalLenOfMsgs += g_stConfInfo.stThreads[i].consumeLen; + totalRows += g_stConfInfo.stThreads[i].consumeRowCnt; + } + + int64_t t = end - start; + if (0 == t) t = 1; + + double tInMs = (double)t / 1000000.0; + taosFprintfFile(g_fp, + "Spent %.3f seconds to poll msgs: %" PRIu64 " with %d thread(s), throughput: %.3f msgs/s, %.1f MB/s\n\n", + tInMs, totalMsgs, g_stConfInfo.numOfThread, + (double)(totalMsgs / tInMs), + (double)totalLenOfMsgs/(1024*1024)/tInMs); + + printf("Spent %.3f seconds to cons rows: %" PRIu64 " msgs: %" PRIu64 " with %d thread(s), throughput: %.3f msgs/s, %.1f MB/s\n\n", + tInMs, totalRows, totalMsgs, g_stConfInfo.numOfThread, + (double)(totalMsgs / tInMs), + (double)totalLenOfMsgs/(1024*1024)/tInMs); + + taosFprintfFile(g_fp, "==== close tmqlog ====\n"); + taosCloseFile(&g_fp); + + return; +} + + int main(int32_t argc, char* argv[]) { parseArgument(argc, argv); + + if (0 != strlen(g_stConfInfo.topic)) { + startOmbConsume(); + return 0; + } + getConsumeInfo(); saveConfigToLogFile(); diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index d6b7f18fb9..4cfa46bd3c 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -748,6 +748,13 @@ int32_t shellDumpResult(TAOS_RES *tres, char *fname, int32_t *error_no, bool ver void shellReadHistory() { SShellHistory *pHistory = &shell.history; + int64_t file_size; + if (taosStatFile(pHistory->file, &file_size, NULL) != 0) { + return; + } else if (file_size > SHELL_MAX_COMMAND_SIZE) { + taosRemoveFile(pHistory->file); + return; + } TdFilePtr pFile = taosOpenFile(pHistory->file, TD_FILE_READ | TD_FILE_STREAM); if (pFile == NULL) return; diff --git a/tools/taos-tools b/tools/taos-tools index d807c3ffa6..b7b922268c 160000 --- a/tools/taos-tools +++ b/tools/taos-tools @@ -1 +1 @@ -Subproject commit d807c3ffa6f750f7765e102917d1328cadf21c13 +Subproject commit b7b922268c4a06d9db77ffdfde0726f3d9900b72