diff --git a/docs/zh/06-advanced/06-TDgpt/04-forecast/index.md b/docs/zh/06-advanced/06-TDgpt/04-forecast/index.md index 8b41cae3eb..560e6cdcc0 100644 --- a/docs/zh/06-advanced/06-TDgpt/04-forecast/index.md +++ b/docs/zh/06-advanced/06-TDgpt/04-forecast/index.md @@ -141,5 +141,5 @@ gen_figure = true 如果设置了 `gen_figure` 为 true，分析结果中还会有绘制的分析预测结果图（如下图所示）。 -预测对比结果 +预测对比结果 diff --git a/docs/zh/06-advanced/06-TDgpt/05-anomaly-detection/index.md b/docs/zh/06-advanced/06-TDgpt/05-anomaly-detection/index.md index 7eff427518..4802fa2e4f 100644 --- a/docs/zh/06-advanced/06-TDgpt/05-anomaly-detection/index.md +++ b/docs/zh/06-advanced/06-TDgpt/05-anomaly-detection/index.md @@ -102,5 +102,5 @@ lof={"algo":"auto", "n_neighbor": 3} 如果设置了 `gen_figure` 为 `true`，比较程序会自动将每个参与比较的算法分析结果采用图片方式呈现出来（如下图所示为 ksigma 的异常检测结果标注）。 -异常检测标注图 +异常检测标注图 diff --git a/docs/zh/08-operation/09-backup.md b/docs/zh/08-operation/09-backup.md index 5b02b4fa55..fbc2e612e1 100644 --- a/docs/zh/08-operation/09-backup.md +++ b/docs/zh/08-operation/09-backup.md @@ -61,7 +61,7 @@ TDengine Enterprise 的备份和恢复功能包括以下几个概念： ## 2.2. 数据备份 -通过浏览器访问 taosExplorer 服务，访问地址通常为 TDengine 集群所在 IP 地址的端口 6060，如 http://localhost:6060。 在 +通过浏览器访问 taosExplorer 服务，访问地址通常为 TDengine 集群所在 IP 地址的端口 6060，如 `http://localhost:6060`。 在 taosExplorer 服务页面中，进入“系统管理 - 备份”页面，在“备份计划”标签页下，点击“创建备份计划”，填写备份计划的相关信息。 需要填写的信息包括： diff --git a/docs/zh/08-operation/10-disaster.md b/docs/zh/08-operation/10-disaster.md index b274e1373b..96f9822725 100644 --- a/docs/zh/08-operation/10-disaster.md +++ b/docs/zh/08-operation/10-disaster.md @@ -23,7 +23,7 @@ TDengine 支持 WAL 机制，实现数据的容错能力，保证数据的高可 - 第 1 步，在集群 A 中创建一个数据库 db1，并向该数据库持续写入数据。 -- 第 2 步， 通过 Web 浏览器访问集群 A 的 taosExplorer 服务， 访问地址通常 为TDengine 集群所在 IP 地址的端口 6060，如 http://localhost:6060。 +- 第 2 步， 通过 Web 浏览器访问集群 A 的 taosExplorer 服务， 访问地址通常 为TDengine 集群所在 IP 地址的端口 6060，如 `http://localhost:6060`。 - 第 3 步，访问 TDengine 集群 B，创建一个与集群 A 中数据库 db1 参数配置相同的数据库 db2。 diff --git a/docs/zh/10-third-party/05-bi/03-powerbi.md b/docs/zh/10-third-party/05-bi/03-powerbi.md index bd51b5591d..b864fa2835 100644 --- a/docs/zh/10-third-party/05-bi/03-powerbi.md +++ b/docs/zh/10-third-party/05-bi/03-powerbi.md @@ -23,12 +23,12 @@ Power BI是由Microsoft提供的一种商业分析工具。通过配置使用ODB 第3步，在“选择您想为其安装数据源的驱动程序”列表中选择“TDengine”，点击“完成”按钮，进入TDengine ODBC数据源配置页面。填写如下必要信息。 - DSN：数据源名称，必填，比如“MyTDengine”。 - 连接类型：勾选“WebSocket”复选框。 - - URL：ODBC 数据源 URL，必填，比如“http://127.0.0.1:6041”。 + - URL：ODBC 数据源 URL，必填，比如`http://127.0.0.1:6041`。 - 数据库：表示需要连接的数据库，可选，比如“test”。 - 用户名：输入用户名，如果不填，默认为“root”。 - 密码：输入用户密码，如果不填，默认为“taosdata”。 -第4步，点击“测试连接”按钮，测试连接情况，如果成功连接，则会提示“成功连接到http://127.0.0.1:6041”。 +第4步，点击“测试连接”按钮，测试连接情况，如果成功连接，则会提示“成功连接到`http://127.0.0.1:6041`”。 第5步，点击“确定”按钮，即可保存配置并退出。 ## 导入TDengine数据到Power BI diff --git a/docs/zh/14-reference/03-taos-sql/14-stream.md b/docs/zh/14-reference/03-taos-sql/14-stream.md index e255102f76..666cf97ea7 100644 --- a/docs/zh/14-reference/03-taos-sql/14-stream.md +++ b/docs/zh/14-reference/03-taos-sql/14-stream.md @@ -30,11 +30,11 @@ subquery: SELECT select_list window_clause ``` -支持会话窗口、状态窗口、滑动窗口、事件窗口和计数窗口，其中，状态窗口、事件窗口和计数窗口搭配超级表时必须与partition by tbname一起使用。对于数据源表是复合主键的流，不支持状态窗口、事件窗口、计数窗口的计算。 +支持会话窗口、状态窗口、滑动窗口、事件窗口和计数窗口。其中，状态窗口、事件窗口和计数窗口搭配超级表时必须与 partition by tbname 一起使用。对于数据源表是复合主键的流，不支持状态窗口、事件窗口、计数窗口的计算。 -stb_name 是保存计算结果的超级表的表名，如果该超级表不存在，会自动创建；如果已存在，则检查列的schema信息。详见 写入已存在的超级表。 +stb_name 是保存计算结果的超级表的表名，如果该超级表不存在，会自动创建；如果已存在，则检查列的schema信息。详见 [写入已存在的超级表](#写入已存在的超级表) 。 -TAGS 子句定义了流计算中创建TAG的规则，可以为每个partition对应的子表生成自定义的TAG值，详见 自定义TAG +TAGS 子句定义了流计算中创建TAG的规则，可以为每个 partition 对应的子表生成自定义的TAG值，详见 [自定义 TAG](#自定义 TAG) ```sql create_definition: col_name column_definition @@ -42,7 +42,7 @@ column_definition: type_name [COMMENT 'string_value'] ``` -subtable 子句定义了流式计算中创建的子表的命名规则，详见 流式计算的 partition 部分。 +subtable 子句定义了流式计算中创建的子表的命名规则，详见 [流式计算的 partition](#流式计算的 partition)。 ```sql window_clause: { @@ -54,23 +54,25 @@ window_clause: { } ``` -其中，SESSION 是会话窗口，tol_val 是时间间隔的最大范围。在 tol_val 时间间隔范围内的数据都属于同一个窗口，如果连续的两条数据的时间超过 tol_val，则自动开启下一个窗口。该窗口的 _wend 等于最后一条数据的时间加上 tol_val。 +其中： -STATE_WINDOW 是状态窗口，col 用来标识状态量，相同的状态量数值则归属于同一个状态窗口，col 数值改变后则当前窗口结束，自动开启下一个窗口。 +- SESSION 是会话窗口，tol_val 是时间间隔的最大范围。在 tol_val 时间间隔范围内的数据都属于同一个窗口，如果连续的两条数据的时间超过 tol_val，则自动开启下一个窗口。该窗口的 _wend 等于最后一条数据的时间加上 tol_val。 -INTERVAL 是时间窗口，又可分为滑动时间窗口和翻转时间窗口。INTERVAL 子句用于指定窗口相等时间周期，SLIDING 字句用于指定窗口向前滑动的时间。当 interval_val 与 sliding_val 相等的时候，时间窗口即为翻转时间窗口，否则为滑动时间窗口，注意：sliding_val 必须小于等于 interval_val。 +- STATE_WINDOW 是状态窗口，col 用来标识状态量，相同的状态量数值则归属于同一个状态窗口，col 数值改变后则当前窗口结束，自动开启下一个窗口。 -EVENT_WINDOW 是事件窗口，根据开始条件和结束条件来划定窗口。当 start_trigger_condition 满足时则窗口开始，直到 end_trigger_condition 满足时窗口关闭。 start_trigger_condition 和 end_trigger_condition 可以是任意 TDengine 支持的条件表达式，且可以包含不同的列。 +- INTERVAL 是时间窗口，又可分为滑动时间窗口和翻转时间窗口。INTERVAL 子句用于指定窗口相等时间周期，SLIDING 字句用于指定窗口向前滑动的时间。当 interval_val 与 sliding_val 相等的时候，时间窗口即为翻转时间窗口，否则为滑动时间窗口，注意：sliding_val 必须小于等于 interval_val。 -COUNT_WINDOW 是计数窗口,按固定的数据行数来划分窗口。 count_val 是常量，是正整数，必须大于等于2，小于2147483648。 count_val 表示每个 COUNT_WINDOW 包含的最大数据行数，总数据行数不能整除 count_val 时，最后一个窗口的行数会小于 count_val 。 sliding_val 是常量，表示窗口滑动的数量，类似于 INTERVAL 的 SLIDING 。 +- EVENT_WINDOW 是事件窗口，根据开始条件和结束条件来划定窗口。当 start_trigger_condition 满足时则窗口开始，直到 end_trigger_condition 满足时窗口关闭。 start_trigger_condition 和 end_trigger_condition 可以是任意 TDengine 支持的条件表达式，且可以包含不同的列。 + +- COUNT_WINDOW 是计数窗口，按固定的数据行数来划分窗口。 count_val 是常量，是正整数，必须大于等于2，小于2147483648。 count_val 表示每个 COUNT_WINDOW 包含的最大数据行数，总数据行数不能整除 count_val 时，最后一个窗口的行数会小于 count_val 。 sliding_val 是常量，表示窗口滑动的数量，类似于 INTERVAL 的 SLIDING 。 窗口的定义与时序数据特色查询中的定义完全相同，详见 [TDengine 特色查询](../distinguished) 例如，如下语句创建流式计算。第一个流计算，自动创建名为 avg_vol 的超级表，以一分钟为时间窗口、30 秒为前向增量统计这些电表的平均电压，并将来自 meters 表的数据的计算结果写入 avg_vol 表，不同 partition 的数据会分别创建子表并写入不同子表。 -第二个流计算，自动创建名为 streamt0 的超级表，将数据按时间戳的顺序，以 voltage < 0 作为窗口的开始条件，voltage > 9作为窗口的结束条件，划分窗口做聚合运算，并将来自 meters 表的数据的计算结果写入 streamt0 表，不同 partition 的数据会分别创建子表并写入不同子表。 +第二个流计算，自动创建名为 streamt0 的超级表，将数据按时间戳的顺序，以 voltage < 0 作为窗口的开始条件，voltage > 9 作为窗口的结束条件，划分窗口做聚合运算，并将来自 meters 表的数据的计算结果写入 streamt0 表，不同 partition 的数据会分别创建子表并写入不同子表。 -第三个流计算，自动创建名为 streamt1 的超级表，将数据按时间戳的顺序，以10条数据为一组，划分窗口做聚合运算，并将来自 meters 表的数据的计算结果写入 streamt1 表，不同 partition 的数据会分别创建子表并写入不同子表。 +第三个流计算，自动创建名为 streamt1 的超级表，将数据按时间戳的顺序，以 10 条数据为一组，划分窗口做聚合运算，并将来自 meters 表的数据的计算结果写入 streamt1 表，不同 partition 的数据会分别创建子表并写入不同子表。 ```sql CREATE STREAM avg_vol_s INTO avg_vol AS @@ -83,7 +85,7 @@ CREATE STREAM streams1 IGNORE EXPIRED 1 WATERMARK 100s INTO streamt1 AS SELECT _wstart, count(*), avg(voltage) from meters PARTITION BY tbname COUNT_WINDOW(10); ``` -notification_definition 子句定义了窗口计算过程中，在窗口打开/关闭等指定事件发生时，需要向哪些地址发送通知。详见 [流式计算的事件通知](#流失计算的事件通知) +notification_definition 子句定义了窗口计算过程中，在窗口打开/关闭等指定事件发生时，需要向哪些地址发送通知。详见 [流式计算的事件通知](#流式计算的事件通知) ## 流式计算的 partition @@ -99,19 +101,19 @@ notification_definition 子句定义了窗口计算过程中，在窗口打开/ CREATE STREAM avg_vol_s INTO avg_vol SUBTABLE(CONCAT('new-', tname)) AS SELECT _wstart, count(*), avg(voltage) FROM meters PARTITION BY tbname tname INTERVAL(1m); ``` -PARTITION 子句中，为 tbname 定义了一个别名 tname, 在PARTITION 子句中的别名可以用于 SUBTABLE 子句中的表达式计算，在上述示例中，流新创建的子表将以前缀 'new-' 连接原表名作为表名(从3.2.3.0开始，为了避免 SUBTABLE 中的表达式无法区分各个子表，即误将多个相同时间线写入一个子表，在指定的子表名后面加上 _stableName_groupId)。 +PARTITION 子句中，为 tbname 定义了一个别名 tname, 在 PARTITION 子句中的别名可以用于 SUBTABLE 子句中的表达式计算，在上述示例中，流新创建的子表将以前缀 'new-' 连接原表名作为表名（从 3.2.3.0 版本开始，为了避免 SUBTABLE 中的表达式无法区分各个子表，即误将多个相同时间线写入一个子表，在指定的子表名后面加上 _stableName_groupId）。 注意，子表名的长度若超过 TDengine 的限制，将被截断。若要生成的子表名已经存在于另一超级表，由于 TDengine 的子表名是唯一的，因此对应新子表的创建以及数据的写入将会失败。 ## 流式计算读取历史数据 -正常情况下，流式计算不会处理创建前已经写入源表中的数据，若要处理已经写入的数据，可以在创建流时设置 fill_history 1 选项，这样创建的流式计算会自动处理创建前、创建中、创建后写入的数据。流计算处理历史数据的最大窗口数是2000万，超过限制会报错。例如： +正常情况下，流式计算不会处理创建前已经写入源表中的数据，若要处理已经写入的数据，可以在创建流时设置 fill_history 1 选项，这样创建的流式计算会自动处理创建前、创建中、创建后写入的数据。流计算处理历史数据的最大窗口数是 2000万，超过限制会报错。例如： ```sql create stream if not exists s1 fill_history 1 into st1 as select count(*) from t1 interval(10s) ``` -结合 fill_history 1 选项，可以实现只处理特定历史时间范围的数据，例如：只处理某历史时刻（2020年1月30日）之后的数据 +结合 fill_history 1 选项，可以实现只处理特定历史时间范围的数据，例如：只处理某历史时刻（2020 年 1 月 30 日）之后的数据 ```sql create stream if not exists s1 fill_history 1 into st1 as select count(*) from t1 where ts > '2020-01-30' interval(10s) @@ -156,7 +158,7 @@ SELECT * from information_schema.`ins_streams`; 2. WINDOW_CLOSE：窗口关闭时触发（窗口关闭由事件时间决定，可配合 watermark 使用） 3. MAX_DELAY time：若窗口关闭，则触发计算。若窗口未关闭，且未关闭时长超过 max delay 指定的时间，则触发计算。 -4. FORCE_WINDOW_CLOSE：以操作系统当前时间为准，只计算当前关闭窗口的结果，并推送出去。窗口只会在被关闭的时刻计算一次，后续不会再重复计算。该模式当前只支持 INTERVAL 窗口（不支持滑动）；FILL_HISTORY 必须为 0，IGNORE EXPIRED 必须为 1，IGNORE UPDATE 必须为 1；FILL 只支持 PREV 、NULL、NONE、VALUE。 +4. FORCE_WINDOW_CLOSE：以操作系统当前时间为准，只计算当前关闭窗口的结果，并推送出去。窗口只会在被关闭的时刻计算一次，后续不会再重复计算。该模式当前只支持 INTERVAL 窗口（不支持滑动）；FILL_HISTORY 必须为 0，IGNORE EXPIRED 必须为 1，IGNORE UPDATE 必须为 1；FILL 只支持 PREV、NULL、NONE、VALUE。 由于窗口关闭是由事件时间决定的，如事件流中断、或持续延迟，则事件时间无法更新，可能导致无法得到最新的计算结果。 @@ -217,33 +219,33 @@ TDengine 对于修改数据提供两种处理方式，由 IGNORE UPDATE 选项 ## 写入已存在的超级表 ```sql -[field1_name,...] +[field1_name, ...] ``` -在本页文档顶部的 [field1_name,...] 是用来指定 stb_name 的列与 subquery 输出结果的对应关系的。如果 stb_name 的列与 subquery 输出结果的位置、数量全部匹配，则不需要显示指定对应关系。如果 stb_name 的列与 subquery 输出结果的数据类型不匹配，会把 subquery 输出结果的类型转换成对应的 stb_name 的列的类型。创建流计算时不能指定 stb_name 的列和 TAG 的数据类型，否则会报错。 +在本页文档顶部的 [field1_name, ...] 是用来指定 stb_name 的列与 subquery 输出结果的对应关系的。如果 stb_name 的列与 subquery 输出结果的位置、数量全部匹配，则不需要显示指定对应关系。如果 stb_name 的列与 subquery 输出结果的数据类型不匹配，会把 subquery 输出结果的类型转换成对应的 stb_name 的列的类型。创建流计算时不能指定 stb_name 的列和 TAG 的数据类型，否则会报错。 对于已经存在的超级表，检查列的schema信息 1. 检查列的 schema 信息是否匹配，对于不匹配的，则自动进行类型转换，当前只有数据长度大于 4096byte 时才报错，其余场景都能进行类型转换。 2. 检查列的个数是否相同，如果不同，需要显示的指定超级表与 subquery 的列的对应关系，否则报错；如果相同，可以指定对应关系，也可以不指定，不指定则按位置顺序对应。 -## 自定义TAG +## 自定义 TAG -用户可以为每个 partition 对应的子表生成自定义的TAG值。 +用户可以为每个 partition 对应的子表生成自定义的 TAG 值。 ```sql -CREATE STREAM streams2 trigger at_once INTO st1 TAGS(cc varchar(100)) as select _wstart, count(*) c1 from st partition by concat("tag-", tbname) as cc interval(10s)); +CREATE STREAM streams2 trigger at_once INTO st1 TAGS(cc varchar(100)) as select _wstart, count(*) c1 from st partition by concat("tag-", tbname) as cc interval(10s)); ``` -PARTITION 子句中，为 concat("tag-", tbname)定义了一个别名cc, 对应超级表st1的自定义TAG的名字。在上述示例中，流新创建的子表的TAG将以前缀 'new-' 连接原表名作为TAG的值。 +PARTITION 子句中，为 concat("tag-", tbname) 定义了一个别名 cc，对应超级表 st1 的自定义 TAG 的名字。在上述示例中，流新创建的子表的 TAG 将以前缀 'new-' 连接原表名作为 TAG 的值。 会对TAG信息进行如下检查 -1. 检查tag的schema信息是否匹配，对于不匹配的，则自动进行数据类型转换，当前只有数据长度大于4096byte时才报错，其余场景都能进行类型转换。 -2. 检查tag的个数是否相同，如果不同，需要显示的指定超级表与subquery的tag的对应关系，否则报错；如果相同，可以指定对应关系，也可以不指定，不指定则按位置顺序对应。 +1. 检查 tag 的 schema 信息是否匹配，对于不匹配的，则自动进行数据类型转换，当前只有数据长度大于 4096byte 时才报错，其余场景都能进行类型转换。 +2. 检查 tag 的个数是否相同，如果不同，需要显示的指定超级表与 subquery 的 tag 的对应关系，否则报错；如果相同，可以指定对应关系，也可以不指定，不指定则按位置顺序对应。 ## 清理中间状态 ``` -DELETE_MARK time +DELETE_MARK time ``` -DELETE_MARK用于删除缓存的窗口状态，也就是删除流计算的中间结果。如果不设置，默认值是10年 +DELETE_MARK 用于删除缓存的窗口状态，也就是删除流计算的中间结果。如果不设置，默认值是 10 年 T = 最新事件时间 - DELETE_MARK ## 流式计算支持的函数 @@ -274,15 +276,15 @@ T = 最新事件时间 - DELETE_MARK ## 暂停、恢复流计算 1.流计算暂停计算任务 PAUSE STREAM [IF EXISTS] stream_name; -没有指定IF EXISTS，如果该stream不存在，则报错；如果存在，则暂停流计算。指定了IF EXISTS，如果该stream不存在，则返回成功；如果存在，则暂停流计算 +没有指定 IF EXISTS，如果该 stream 不存在，则报错；如果存在，则暂停流计算。指定了 IF EXISTS，如果该 stream 不存在，则返回成功；如果存在，则暂停流计算。 2.流计算恢复计算任务 RESUME STREAM [IF EXISTS] [IGNORE UNTREATED] stream_name; -没有指定IF EXISTS，如果该stream不存在，则报错，如果存在，则恢复流计算；指定了IF EXISTS，如果stream不存在，则返回成功；如果存在，则恢复流计算。如果指定IGNORE UNTREATED，则恢复流计算时，忽略流计算暂停期间写入的数据。 +没有指定 IF EXISTS，如果该 stream 不存在，则报错，如果存在，则恢复流计算；指定了 IF EXISTS，如果 stream 不存在，则返回成功；如果存在，则恢复流计算。如果指定 IGNORE UNTREATED，则恢复流计算时，忽略流计算暂停期间写入的数据。 ## 状态数据备份与同步 -流计算的中间结果成为计算的状态数据，需要在流计算整个生命周期中进行持久化保存。为了确保流计算中间状态能够在集群环境下在不同的节点间可靠地同步和迁移，至3.3.2.1 版本开始，需要在运行环境中部署 rsync 软件，还需要增加以下的步骤： -1. 在配置文件中配置 snode 的地址（IP+端口）和状态数据备份目录（该目录系 snode 所在的物理节点的目录）。 +流计算的中间结果成为计算的状态数据，需要在流计算整个生命周期中进行持久化保存。为了确保流计算中间状态能够在集群环境下在不同的节点间可靠地同步和迁移，从 3.3.2.1 版本开始，需要在运行环境中部署 rsync 软件，还需要增加以下的步骤： +1. 在配置文件中配置 snode 的地址（IP + 端口）和状态数据备份目录（该目录系 snode 所在的物理节点的目录）。 2. 然后创建 snode。 完成上述两个步骤以后才能创建流。 如果没有创建 snode 并正确配置 snode 的地址，流计算过程中将无法生成检查点（checkpoint），并可能导致后续的计算结果产生错误。 @@ -293,7 +295,7 @@ RESUME STREAM [IF EXISTS] [IGNORE UNTREATED] stream_name; ## 创建 snode 的方式 -使用以下命令创建 snode（stream node）， snode 是流计算中有状态的计算节点，可用于部署聚合任务，同时负责备份不同的流计算任务生成的检查点数据。 +使用以下命令创建 snode（stream node），snode 是流计算中有状态的计算节点，可用于部署聚合任务，同时负责备份不同的流计算任务生成的检查点数据。 ```sql CREATE SNODE ON DNODE [id] ``` @@ -304,7 +306,7 @@ CREATE SNODE ON DNODE [id] ### 使用说明 -流式计算支持在窗口打开/关闭时，向外部系统发送相关的事件通知。用户通过 notification_definition 来指定需要通知的事件，以及用于接收通知消息的目标地址。 +流式计算支持在窗口打开/关闭时，向外部系统发送相关的事件通知。用户通过 `notification_definition` 来指定需要通知的事件，以及用于接收通知消息的目标地址。 ```sql notification_definition: @@ -321,14 +323,14 @@ notification_options: { ``` 上述语法中的相关规则含义如下： -1. `url`: 指定通知的目标地址，必须包括协议、IP 或域名、端口号，并允许包含路径、参数。目前仅支持 websocket 协议。例如：'ws://localhost:8080'，'ws://localhost:8080/notify'，'wss://localhost:8080/notify?key=foo'。 -1. `event_type`: 定义需要通知的事件，支持的事件类型有: - 1. 'WINDOW_OPEN'：窗口打开事件，所有类型的窗口打开时都会触发 - 1. 'WINDOW_CLOSE'：窗口关闭事件，所有类型的窗口关闭时都会触发 -1. `NOTIFY_HISTORY`: 控制是否在计算历史数据时触发通知，默认值为0，即不触发 -1. `ON_FAILURE`: 向通知地址发送通知失败时(比如网络不佳场景)是否允许丢弃部分事件，默认值为 `PAUSE` +1. `url`：指定通知的目标地址，必须包括协议、IP 或域名、端口号，并允许包含路径、参数。目前仅支持 websocket 协议。例如：`ws://localhost:8080`、`ws://localhost:8080/notify`、`wss://localhost:8080/notify?key=foo`。 +1. `event_type`：定义需要通知的事件，支持的事件类型有： + 1. WINDOW_OPEN：窗口打开事件，所有类型的窗口打开时都会触发。 + 1. WINDOW_CLOSE：窗口关闭事件，所有类型的窗口关闭时都会触发。 +1. `NOTIFY_HISTORY`：控制是否在计算历史数据时触发通知，默认值为 0，即不触发。 +1. `ON_FAILURE`：向通知地址发送通知失败时(比如网络不佳场景)是否允许丢弃部分事件，默认值为 `PAUSE`。 1. PAUSE 表示发送通知失败时暂停流计算任务。taosd 会重试发送通知，直到发送成功后，任务自动恢复运行。 - 1. DROP 表示发送通知失败时直接丢弃事件信息，流计算任务继续运行，不受影响 + 1. DROP 表示发送通知失败时直接丢弃事件信息，流计算任务继续运行，不受影响。 比如，以下示例创建一个流，计算电表电流的每分钟平均值，并在窗口打开、关闭时向两个通知地址发送通知，计算历史数据时不发送通知，不允许在通知发送失败时丢弃通知： @@ -432,89 +434,89 @@ CREATE STREAM avg_current_stream FILL_HISTORY 1 ### 根级字段说明 -1. "messageId": 字符串类型，是通知消息的唯一标识符，确保整条消息可以被追踪和去重。 -1. "timestamp": 长整型时间戳，表示通知消息生成的时间，精确到毫秒，即: '00:00, Jan 1 1970 UTC' 以来的毫秒数。 -1. "streams": 对象数组，包含多个流任务的事件信息。(详细信息见下节) +1. messageId：字符串类型，是通知消息的唯一标识符，确保整条消息可以被追踪和去重。 +1. timestamp：长整型时间戳，表示通知消息生成的时间，精确到毫秒，即: '00:00, Jan 1 1970 UTC' 以来的毫秒数。 +1. streams：对象数组，包含多个流任务的事件信息。(详细信息见下节) ### stream 对象的字段说明 -1. "streamName": 字符串类型，流任务的名称，用于标识事件所属的流。 -1. "events": 对象数组，该流任务下的事件列表，包含一个或多个事件对象。(详细信息见下节) +1. streamName：字符串类型，流任务的名称，用于标识事件所属的流。 +1. events：对象数组，该流任务下的事件列表，包含一个或多个事件对象。(详细信息见下节) ### event 对象的字段说明 #### 通用字段 这部分是所有 event 对象所共有的字段。 -1. "tableName": 字符串类型，是对应目标子表的表名。 -1. "eventType": 字符串类型，表示事件类型 ("WINDOW_OPEN", "WINDOW_CLOSE" 或 "WINDOW_INVALIDATION")。 -1. "eventTime": 长整型时间戳，表示事件生成时间，精确到毫秒，即: '00:00, Jan 1 1970 UTC' 以来的毫秒数。 -1. "windowId": 字符串类型，窗口的唯一标识符，确保打开和关闭事件的 ID 一致，便于外部系统将两者关联。如果 taosd 发生故障重启，部分事件可能会重复发送，会保证同一窗口的 windowId 保持不变。 -1. "windowType": 字符串类型，表示窗口类型 ("Time", "State", "Session", "Event", "Count")。 +1. tableName：字符串类型，是对应目标子表的表名。 +1. eventType：字符串类型，表示事件类型，支持 WINDOW_OPEN、WINDOW_CLOSE、WINDOW_INVALIDATION 三种类型。 +1. eventTime：长整型时间戳，表示事件生成时间，精确到毫秒，即：'00:00, Jan 1 1970 UTC' 以来的毫秒数。 +1. windowId：字符串类型，窗口的唯一标识符，确保打开和关闭事件的 ID 一致，便于外部系统将两者关联。如果 taosd 发生故障重启，部分事件可能会重复发送，会保证同一窗口的 windowId 保持不变。 +1. windowType：字符串类型，表示窗口类型，支持 Time、State、Session、Event、Count 五种类型。 #### 时间窗口相关字段 -这部分是 "windowType" 为"Time" 时 event 对象才有的字段。 -1. 如果 "eventType" 为 "WINDOW_OPEN"，则包含如下字段： - 1. "windowStart": 长整型时间戳，表示窗口的开始时间，精度与结果表的时间精度一致。 -1. 如果 "eventType" 为 "WINDOW_CLOSE"，则包含如下字段： - 1. "windowStart": 长整型时间戳，表示窗口的开始时间，精度与结果表的时间精度一致。 - 1. "windowEnd": 长整型时间戳，表示窗口的结束时间，精度与结果表的时间精度一致。 - 1. "result": 计算结果，为键值对形式，包含窗口计算的结果列列名及其对应的值。 +这部分是 windowType 为 Time 时 event 对象才有的字段。 +1. 如果 eventType 为 WINDOW_OPEN，则包含如下字段： + 1. windowStart：长整型时间戳，表示窗口的开始时间，精度与结果表的时间精度一致。 +1. 如果 eventType 为 WINDOW_CLOSE，则包含如下字段： + 1. windowStart：长整型时间戳，表示窗口的开始时间，精度与结果表的时间精度一致。 + 1. windowEnd：长整型时间戳，表示窗口的结束时间，精度与结果表的时间精度一致。 + 1. result：计算结果，为键值对形式，包含窗口计算的结果列列名及其对应的值。 #### 状态窗口相关字段 -这部分是 "windowType" 为"State" 时 event 对象才有的字段。 -1. 如果 "eventType" 为 "WINDOW_OPEN"，则包含如下字段： - 1. "windowStart": 长整型时间戳，表示窗口的开始时间，精度与结果表的时间精度一致。 - 1. "prevState": 与状态列的类型相同，表示上一个窗口的状态值。如果没有上一个窗口(即: 现在是第一个窗口)，则为 NULL。 - 1. "curState": 与状态列的类型相同，表示当前窗口的状态值。 -1. 如果 "eventType" 为 "WINDOW_CLOSE"，则包含如下字段： - 1. "windowStart": 长整型时间戳，表示窗口的开始时间，精度与结果表的时间精度一致。 - 1. "windowEnd": 长整型时间戳，表示窗口的结束时间，精度与结果表的时间精度一致。 - 1. "curState": 与状态列的类型相同，表示当前窗口的状态值。 - 1. "nextState": 与状态列的类型相同，表示下一个窗口的状态值。 - 1. "result": 计算结果，为键值对形式，包含窗口计算的结果列列名及其对应的值。 +这部分是 windowType 为 State 时 event 对象才有的字段。 +1. 如果 eventType 为 WINDOW_OPEN，则包含如下字段： + 1. windowStart：长整型时间戳，表示窗口的开始时间，精度与结果表的时间精度一致。 + 1. prevState：与状态列的类型相同，表示上一个窗口的状态值。如果没有上一个窗口(即：现在是第一个窗口)，则为 NULL。 + 1. curState：与状态列的类型相同，表示当前窗口的状态值。 +1. 如果 eventType 为 WINDOW_CLOSE，则包含如下字段： + 1. windowStart：长整型时间戳，表示窗口的开始时间，精度与结果表的时间精度一致。 + 1. windowEnd：长整型时间戳，表示窗口的结束时间，精度与结果表的时间精度一致。 + 1. curState：与状态列的类型相同，表示当前窗口的状态值。 + 1. nextState：与状态列的类型相同，表示下一个窗口的状态值。 + 1. result：计算结果，为键值对形式，包含窗口计算的结果列列名及其对应的值。 #### 会话窗口相关字段 -这部分是 "windowType" 为"Session" 时 event 对象才有的字段。 -1. 如果 "eventType" 为 "WINDOW_OPEN"，则包含如下字段： - 1. "windowStart": 长整型时间戳，表示窗口的开始时间，精度与结果表的时间精度一致。 -1. 如果 "eventType" 为 "WINDOW_CLOSE"，则包含如下字段： - 1. "windowStart": 长整型时间戳，表示窗口的开始时间，精度与结果表的时间精度一致。 - 1. "windowEnd": 长整型时间戳，表示窗口的结束时间，精度与结果表的时间精度一致。 - 1. "result": 计算结果，为键值对形式，包含窗口计算的结果列列名及其对应的值。 +这部分是 windowType 为 Session 时 event 对象才有的字段。 +1. 如果 eventType 为 WINDOW_OPEN，则包含如下字段： + 1. windowStart：长整型时间戳，表示窗口的开始时间，精度与结果表的时间精度一致。 +1. 如果 eventType 为 WINDOW_CLOSE，则包含如下字段： + 1. windowStart：长整型时间戳，表示窗口的开始时间，精度与结果表的时间精度一致。 + 1. windowEnd：长整型时间戳，表示窗口的结束时间，精度与结果表的时间精度一致。 + 1. result：计算结果，为键值对形式，包含窗口计算的结果列列名及其对应的值。 #### 事件窗口相关字段 -这部分是 "windowType" 为"Event" 时 event 对象才有的字段。 -1. 如果 "eventType" 为 "WINDOW_OPEN"，则包含如下字段： - 1. "windowStart": 长整型时间戳，表示窗口的开始时间，精度与结果表的时间精度一致。 - 1. "triggerCondition": 触发窗口开始的条件信息，包括以下字段： - 1. "conditionIndex": 整型，表示满足的触发窗口开始的条件的索引，从0开始编号。 - 1. "fieldValue": 键值对形式，包含条件列列名及其对应的值。 -1. 如果 "eventType" 为 "WINDOW_CLOSE"，则包含如下字段： - 1. "windowStart": 长整型时间戳，表示窗口的开始时间，精度与结果表的时间精度一致。 - 1. "windowEnd": 长整型时间戳，表示窗口的结束时间，精度与结果表的时间精度一致。 - 1. "triggerCondition": 触发窗口关闭的条件信息，包括以下字段： - 1. "conditionIndex": 整型，表示满足的触发窗口关闭的条件的索引，从0开始编号。 - 1. "fieldValue": 键值对形式，包含条件列列名及其对应的值。 - 1. "result": 计算结果，为键值对形式，包含窗口计算的结果列列名及其对应的值。 +这部分是 windowType 为 Event 时 event 对象才有的字段。 +1. 如果 eventType 为 WINDOW_OPEN，则包含如下字段： + 1. windowStart：长整型时间戳，表示窗口的开始时间，精度与结果表的时间精度一致。 + 1. triggerCondition：触发窗口开始的条件信息，包括以下字段： + 1. conditionIndex：整型，表示满足的触发窗口开始的条件的索引，从0开始编号。 + 1. fieldValue：键值对形式，包含条件列列名及其对应的值。 +1. 如果 eventType 为 WINDOW_CLOSE，则包含如下字段： + 1. windowStart：长整型时间戳，表示窗口的开始时间，精度与结果表的时间精度一致。 + 1. windowEnd：长整型时间戳，表示窗口的结束时间，精度与结果表的时间精度一致。 + 1. triggerCondition：触发窗口关闭的条件信息，包括以下字段： + 1. conditionIndex：整型，表示满足的触发窗口关闭的条件的索引，从0开始编号。 + 1. fieldValue：键值对形式，包含条件列列名及其对应的值。 + 1. result：计算结果，为键值对形式，包含窗口计算的结果列列名及其对应的值。 #### 计数窗口相关字段 -这部分是 "windowType" 为"Count" 时 event 对象才有的字段。 -1. 如果 "eventType" 为 "WINDOW_OPEN"，则包含如下字段： - 1. "windowStart": 长整型时间戳，表示窗口的开始时间，精度与结果表的时间精度一致。 -1. 如果 "eventType" 为 "WINDOW_CLOSE"，则包含如下字段： - 1. "windowStart": 长整型时间戳，表示窗口的开始时间，精度与结果表的时间精度一致。 - 1. "windowEnd": 长整型时间戳，表示窗口的结束时间，精度与结果表的时间精度一致。 - 1. "result": 计算结果，为键值对形式，包含窗口计算的结果列列名及其对应的值。 +这部分是 windowType 为 Count 时 event 对象才有的字段。 +1. 如果 eventType 为 WINDOW_OPEN，则包含如下字段： + 1. windowStart：长整型时间戳，表示窗口的开始时间，精度与结果表的时间精度一致。 +1. 如果 eventType 为 WINDOW_CLOSE，则包含如下字段： + 1. windowStart：长整型时间戳，表示窗口的开始时间，精度与结果表的时间精度一致。 + 1. windowEnd：长整型时间戳，表示窗口的结束时间，精度与结果表的时间精度一致。 + 1. result：计算结果，为键值对形式，包含窗口计算的结果列列名及其对应的值。 #### 窗口失效相关字段 -因为流计算过程中会遇到数据乱序、更新、删除等情况，可能造成已生成的窗口被删除，或者结果需要重新计算。此时会向通知地址发送一条 "WINDOW_INVALIDATION" 的通知，说明哪些窗口已经被删除。 -这部分是 "eventType" 为 "WINDOW_INVALIDATION" 时，event 对象才有的字段。 -1. "windowStart": 长整型时间戳，表示窗口的开始时间，精度与结果表的时间精度一致。 -1. "windowEnd": 长整型时间戳，表示窗口的结束时间，精度与结果表的时间精度一致。 +因为流计算过程中会遇到数据乱序、更新、删除等情况，可能造成已生成的窗口被删除，或者结果需要重新计算。此时会向通知地址发送一条 WINDOW_INVALIDATION 的通知，说明哪些窗口已经被删除。 +这部分是 eventType 为 WINDOW_INVALIDATION 时，event 对象才有的字段。 +1. windowStart：长整型时间戳，表示窗口的开始时间，精度与结果表的时间精度一致。 +1. windowEnd: 长整型时间戳，表示窗口的结束时间，精度与结果表的时间精度一致。 diff --git a/docs/zh/14-reference/03-taos-sql/20-keywords.md b/docs/zh/14-reference/03-taos-sql/20-keywords.md index 0a1bdc4ce8..9c023e0f92 100644 --- a/docs/zh/14-reference/03-taos-sql/20-keywords.md +++ b/docs/zh/14-reference/03-taos-sql/20-keywords.md @@ -37,6 +37,7 @@ description: TDengine 保留关键字的详细列表 | ASOF | | | AT_ONCE | | | ATTACH | | +| AUTO | 3.3.5.0 及后续版本 | ### B |关键字|说明| @@ -82,6 +83,9 @@ description: TDengine 保留关键字的详细列表 | COMP | | | COMPACT | | | COMPACTS | | +| COMPACT_INTERVAL | 3.3.5.0 及后续版本 | +| COMPACT_TIME_OFFSET | 3.3.5.0 及后续版本 | +| COMPACT_TIME_RANGE | 3.3.5.0 及后续版本 | | CONCAT | | | CONFLICT | | | CONNECTION | | @@ -111,6 +115,7 @@ description: TDengine 保留关键字的详细列表 | DESC | | | DESCRIBE | | | DETACH | | +| DISK_INFO | 3.3.5.0 及后续版本 | | DISTINCT | | | DISTRIBUTED | | | DIVIDE | | @@ -201,6 +206,7 @@ description: TDengine 保留关键字的详细列表 | INTO | | | IPTOKEN | | | IROWTS | | +| IROWTS_ORIGIN | 3.3.5.0 及后续版本 | | IS | | | IS_IMPORT | | | ISFILLED | | @@ -269,6 +275,8 @@ description: TDengine 保留关键字的详细列表 | NONE | | | NORMAL | | | NOT | | +| NOTIFY | 3.3.6.0 及后续版本 | +| NOTIFY_HISTORY | 3.3.6.0 及后续版本 | | NOTNULL | | | NOW | | | NULL | | @@ -282,6 +290,7 @@ description: TDengine 保留关键字的详细列表 | OFFSET | | | ON | | | ONLY | | +| ON_FAILURE | 3.3.6.0 及后续版本 | | OR | | | ORDER | | | OUTER | | @@ -329,6 +338,7 @@ description: TDengine 保留关键字的详细列表 | RATIO | | | READ | | | RECURSIVE | | +| REGEXP | 3.3.6.0 及后续版本 | | REDISTRIBUTE | | | REM | | | REPLACE | | @@ -475,7 +485,7 @@ description: TDengine 保留关键字的详细列表 | WINDOW_OFFSET | | | WITH | | | WRITE | | -| WSTART | | +| WSTART | | ### \_ diff --git a/include/client/taos.h b/include/client/taos.h index 17f97d3d3d..433779f811 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -247,11 +247,13 @@ typedef struct TAOS_STMT2_BINDV { DLL_EXPORT TAOS_STMT2 *taos_stmt2_init(TAOS *taos, TAOS_STMT2_OPTION *option); DLL_EXPORT int taos_stmt2_prepare(TAOS_STMT2 *stmt, const char *sql, unsigned long length); DLL_EXPORT int taos_stmt2_bind_param(TAOS_STMT2 *stmt, TAOS_STMT2_BINDV *bindv, int32_t col_idx); -DLL_EXPORT int taos_stmt2_exec(TAOS_STMT2 *stmt, int *affected_rows); -DLL_EXPORT int taos_stmt2_close(TAOS_STMT2 *stmt); -DLL_EXPORT int taos_stmt2_is_insert(TAOS_STMT2 *stmt, int *insert); -DLL_EXPORT int taos_stmt2_get_fields(TAOS_STMT2 *stmt, int *count, TAOS_FIELD_ALL **fields); -DLL_EXPORT void taos_stmt2_free_fields(TAOS_STMT2 *stmt, TAOS_FIELD_ALL *fields); +DLL_EXPORT int taos_stmt2_bind_param_a(TAOS_STMT2 *stmt, TAOS_STMT2_BINDV *bindv, int32_t col_idx, __taos_async_fn_t fp, + void *param); +DLL_EXPORT int taos_stmt2_exec(TAOS_STMT2 *stmt, int *affected_rows); +DLL_EXPORT int taos_stmt2_close(TAOS_STMT2 *stmt); +DLL_EXPORT int taos_stmt2_is_insert(TAOS_STMT2 *stmt, int *insert); +DLL_EXPORT int taos_stmt2_get_fields(TAOS_STMT2 *stmt, int *count, TAOS_FIELD_ALL **fields); +DLL_EXPORT void taos_stmt2_free_fields(TAOS_STMT2 *stmt, TAOS_FIELD_ALL *fields); DLL_EXPORT TAOS_RES *taos_stmt2_result(TAOS_STMT2 *stmt); DLL_EXPORT char *taos_stmt2_error(TAOS_STMT2 *stmt); @@ -360,7 +362,8 @@ typedef enum tmq_res_t { TMQ_RES_INVALID = -1, TMQ_RES_DATA = 1, TMQ_RES_TABLE_META = 2, - TMQ_RES_METADATA = 3 + TMQ_RES_METADATA = 3, + TMQ_RES_RAWDATA = 4 } tmq_res_t; typedef struct tmq_topic_assignment { diff --git a/include/common/tcommon.h b/include/common/tcommon.h index c30f2ab4ec..bd5bdb927d 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -121,10 +121,11 @@ enum { TMQ_MSG_TYPE__POLL_DATA_META_RSP, TMQ_MSG_TYPE__WALINFO_RSP, TMQ_MSG_TYPE__POLL_BATCH_META_RSP, + TMQ_MSG_TYPE__POLL_RAW_DATA_RSP, }; static char* tmqMsgTypeStr[] = { - "data", "meta", "ask ep", "meta data", "wal info", "batch meta" + "data", "meta", "ask ep", "meta data", "wal info", "batch meta", "raw data" }; enum { diff --git a/include/common/tglobal.h b/include/common/tglobal.h index bb3f0964de..1b59f1de8a 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -296,6 +296,8 @@ extern int32_t tsMaxStreamBackendCache; extern int32_t tsPQSortMemThreshold; extern bool tsStreamCoverage; extern int8_t tsS3EpNum; +extern int32_t tsStreamNotifyMessageSize; +extern int32_t tsStreamNotifyFrameSize; extern bool tsExperimental; // #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 6f78ec32a5..85e05d614c 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1355,10 +1355,10 @@ typedef struct { int8_t encryptAlgorithm; char dnodeListStr[TSDB_DNODE_LIST_LEN]; // 1. add auto-compact parameters - int32_t compactInterval; // minutes - int32_t compactStartTime; // minutes - int32_t compactEndTime; // minutes - int8_t compactTimeOffset; // hour + int32_t compactInterval; // minutes + int32_t compactStartTime; // minutes + int32_t compactEndTime; // minutes + int8_t compactTimeOffset; // hour } SCreateDbReq; int32_t tSerializeSCreateDbReq(void* buf, int32_t bufLen, SCreateDbReq* pReq); @@ -1777,6 +1777,8 @@ typedef struct { int64_t numOfBatchInsertSuccessReqs; int32_t numOfCachedTables; int32_t learnerProgress; // use one reservered + int64_t syncAppliedIndex; + int64_t syncCommitIndex; } SVnodeLoad; typedef struct { @@ -2312,6 +2314,10 @@ typedef struct SSysTableSchema { int32_t tSerializeSRetrieveTableReq(void* buf, int32_t bufLen, SRetrieveTableReq* pReq); int32_t tDeserializeSRetrieveTableReq(void* buf, int32_t bufLen, SRetrieveTableReq* pReq); +#define RETRIEVE_TABLE_RSP_VERSION 0 +#define RETRIEVE_TABLE_RSP_TMQ_VERSION 1 +#define RETRIEVE_TABLE_RSP_TMQ_RAW_VERSION 2 + typedef struct { int64_t useconds; int8_t completed; // all results are returned to client @@ -3955,8 +3961,8 @@ typedef struct { int8_t igExists; int8_t intervalUnit; int8_t slidingUnit; - int8_t timezone; // int8_t is not enough, timezone is unit of second - int32_t dstVgId; // for stream + int8_t timezone; // int8_t is not enough, timezone is unit of second + int32_t dstVgId; // for stream int64_t interval; int64_t offset; int64_t sliding; @@ -4200,7 +4206,9 @@ typedef struct { STqOffsetVal reqOffset; int8_t enableReplay; int8_t sourceExcluded; + int8_t rawData; int8_t enableBatchMeta; + SHashObj *uidHash; // to find if uid is duplicated } SMqPollReq; int32_t tSerializeSMqPollReq(void* buf, int32_t bufLen, SMqPollReq* pReq); @@ -4274,13 +4282,21 @@ typedef struct { SArray* createTableLen; SArray* createTableReq; }; + struct{ + int32_t len; + void* rawData; + }; }; + void* data; //for free in client, only effected if type is data or metadata. raw data not effected + bool blockDataElementFree; // if true, free blockDataElement in blockData,(true in server, false in client) } SMqDataRsp; int32_t tEncodeMqDataRsp(SEncoder* pEncoder, const SMqDataRsp* pObj); int32_t tDecodeMqDataRsp(SDecoder* pDecoder, SMqDataRsp* pRsp); +int32_t tDecodeMqRawDataRsp(SDecoder* pDecoder, SMqDataRsp* pRsp); void tDeleteMqDataRsp(SMqDataRsp* pRsp); +void tDeleteMqRawDataRsp(SMqDataRsp* pRsp); int32_t tEncodeSTaosxRsp(SEncoder* pEncoder, const SMqDataRsp* pRsp); int32_t tDecodeSTaosxRsp(SDecoder* pDecoder, SMqDataRsp* pRsp); @@ -4530,6 +4546,7 @@ typedef struct { typedef struct { SArray* aSubmitTbData; // SArray + bool raw; } SSubmitReq2; typedef struct { @@ -4538,8 +4555,9 @@ typedef struct { char data[]; // SSubmitReq2 } SSubmitReq2Msg; +int32_t transformRawSSubmitTbData(void* data, int64_t suid, int64_t uid, int32_t sver); int32_t tEncodeSubmitReq(SEncoder* pCoder, const SSubmitReq2* pReq); -int32_t tDecodeSubmitReq(SDecoder* pCoder, SSubmitReq2* pReq); +int32_t tDecodeSubmitReq(SDecoder* pCoder, SSubmitReq2* pReq, SArray* rawList); void tDestroySubmitTbData(SSubmitTbData* pTbData, int32_t flag); void tDestroySubmitReq(SSubmitReq2* pReq, int32_t flag); diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 3e95f1e286..1afec35c3c 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -128,6 +128,7 @@ typedef struct SScanLogicNode { bool paraTablesSort; // for table merge scan bool smallDataTsSort; // disable row id sort for table merge scan bool needSplit; + bool noPseudoRefAfterGrp; // no pseudo columns referenced ater group/partition clause } SScanLogicNode; typedef struct SJoinLogicNode { diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 95f522f504..f4bf5fafd4 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -180,8 +180,11 @@ int32_t smlBindData(SQuery* handle, bool dataFormat, SArray* tags, SArray* colsS STableMeta* pTableMeta, char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl, char* msgBuf, int32_t msgBufLen, void* charsetCxt); int32_t smlBuildOutput(SQuery* handle, SHashObj* pVgHash); +int32_t smlBuildOutputRaw(SQuery* handle, SHashObj* pVgHash); +int rawBlockBindRawData(SHashObj* pVgroupHash, SArray* pVgroupList, STableMeta* pTableMeta, void* data); int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreateTbReq* pCreateTb, void* fields, int numFields, bool needChangeLength, char* errstr, int32_t errstrLen, bool raw); +int32_t checkSchema(SSchema* pColSchema, int8_t* fields, char* errstr, int32_t errstrLen); int32_t rewriteToVnodeModifyOpStmt(SQuery* pQuery, SArray* pBufArray); int32_t serializeVgroupsCreateTableBatch(SHashObj* pVgroupHashmap, SArray** pOut); diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 5b28eadc4f..a17e67279c 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -300,6 +300,7 @@ int32_t cleanupTaskQueue(); int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code); int32_t taosAsyncWait(); int32_t taosAsyncRecover(); +int32_t taosStmt2AsyncBind(__async_exec_fn_t execFn, void* execParam); void destroySendMsgInfo(SMsgSendInfo* pMsgBody); diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index c191e96d0e..643d459791 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -293,6 +293,7 @@ int32_t syncBecomeAssignedLeader(SSyncNode* ths, SRpcMsg* pRpcMsg); int32_t syncUpdateArbTerm(int64_t rid, SyncTerm arbTerm); SSyncState syncGetState(int64_t rid); +void syncGetCommitIndex(int64_t rid, int64_t* syncCommitIndex); int32_t syncGetArbToken(int64_t rid, char* outToken); int32_t syncCheckSynced(int64_t rid); void syncGetRetryEpSet(int64_t rid, SEpSet* pEpSet); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 464dffa937..397118411c 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -1016,6 +1016,8 @@ int32_t taosGetErrSize(); #define TSDB_CODE_TMQ_NO_TABLE_QUALIFIED TAOS_DEF_ERROR_CODE(0, 0x4015) #define TSDB_CODE_TMQ_NO_NEED_REBALANCE TAOS_DEF_ERROR_CODE(0, 0x4016) #define TSDB_CODE_TMQ_INVALID_STATUS TAOS_DEF_ERROR_CODE(0, 0x4017) +#define TSDB_CODE_TMQ_INVALID_DATA TAOS_DEF_ERROR_CODE(0, 0x4018) +#define TSDB_CODE_TMQ_RAW_DATA_SPLIT TAOS_DEF_ERROR_CODE(0, 0x4019) // stream #define TSDB_CODE_STREAM_TASK_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x4100) diff --git a/include/util/tdef.h b/include/util/tdef.h index c6686ad811..41cb147622 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -322,8 +322,8 @@ typedef enum ELogicConditionType { #define TSDB_ARB_GROUP_MEMBER_NUM 2 #define TSDB_ARB_TOKEN_SIZE 32 -#define TSDB_TRANS_STAGE_LEN 12 -#define TSDB_TRANS_TYPE_LEN 16 +#define TSDB_TRANS_STAGE_LEN 12 +#define TSDB_TRANS_TYPE_LEN 16 #define TSDB_TRANS_ERROR_LEN 512 #define TSDB_TRANS_OBJTYPE_LEN 40 #define TSDB_TRANS_RESULT_LEN 100 @@ -357,6 +357,8 @@ typedef enum ELogicConditionType { #define TSDB_MAX_REPLICA 5 #define TSDB_MAX_LEARNER_REPLICA 10 +#define TSDB_SYNC_RESTORE_lEN 20 +#define TSDB_SYNC_APPLY_COMMIT_LEN 41 #define TSDB_SYNC_LOG_BUFFER_SIZE 4096 #define TSDB_SYNC_LOG_BUFFER_RETENTION 256 #define TSDB_SYNC_LOG_BUFFER_THRESHOLD (1024 * 1024 * 5) @@ -672,7 +674,7 @@ typedef enum { TSDB_VERSION_END, } EVersionType; -#define MIN_RESERVE_MEM_SIZE 1024 // MB +#define MIN_RESERVE_MEM_SIZE 1024 // MB #ifdef __cplusplus } diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 2543a1f3ec..c9e0410341 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -45,6 +45,7 @@ enum { RES_TYPE__TMQ_META, RES_TYPE__TMQ_METADATA, RES_TYPE__TMQ_BATCH_META, + RES_TYPE__TMQ_RAWDATA, }; #define SHOW_VARIABLES_RESULT_COLS 5 @@ -55,6 +56,7 @@ enum { #define SHOW_VARIABLES_RESULT_FIELD5_LEN (TSDB_CONFIG_INFO_LEN + VARSTR_HEADER_SIZE) #define TD_RES_QUERY(res) (*(int8_t*)(res) == RES_TYPE__QUERY) +#define TD_RES_TMQ_RAW(res) (*(int8_t*)(res) == RES_TYPE__TMQ_RAWDATA) #define TD_RES_TMQ(res) (*(int8_t*)(res) == RES_TYPE__TMQ) #define TD_RES_TMQ_META(res) (*(int8_t*)(res) == RES_TYPE__TMQ_META) #define TD_RES_TMQ_METADATA(res) (*(int8_t*)(res) == RES_TYPE__TMQ_METADATA) diff --git a/source/client/inc/clientStmt2.h b/source/client/inc/clientStmt2.h index 05a4c849f8..283573803e 100644 --- a/source/client/inc/clientStmt2.h +++ b/source/client/inc/clientStmt2.h @@ -133,6 +133,19 @@ SStmtQNode* tail; uint64_t qRemainNum; } SStmtQueue; */ +typedef struct { + TAOS_STMT2 *stmt; + TAOS_STMT2_BINDV *bindv; + int32_t col_idx; + __taos_async_fn_t fp; + void *param; +} ThreadArgs; + +typedef struct AsyncBindParam { + TdThreadMutex mutex; + TdThreadCond waitCond; + uint8_t asyncBindNum; +} AsyncBindParam; typedef struct { STscObj *taos; @@ -150,12 +163,13 @@ typedef struct { SStmtExecInfo exec; SStmtBindInfo bInfo; - char *db; - int64_t reqid; - int32_t errCode; - tsem_t asyncQuerySem; - bool semWaited; - SStmtStatInfo stat; + char *db; + int64_t reqid; + int32_t errCode; + tsem_t asyncExecSem; + bool execSemWaited; + AsyncBindParam asyncBindParam; + SStmtStatInfo stat; } STscStmt2; /* extern char *gStmtStatusStr[]; @@ -226,6 +240,8 @@ int stmtGetParamNum2(TAOS_STMT2 *stmt, int *nums); int stmtIsInsert2(TAOS_STMT2 *stmt, int *insert); TAOS_RES *stmtUseResult2(TAOS_STMT2 *stmt); const char *stmtErrstr2(TAOS_STMT2 *stmt); +int stmt2AsyncBind(TAOS_STMT2 *stmt, TAOS_STMT2_BINDV *bindv, int32_t col_idx, __taos_async_fn_t fp, void *param); +int stmtAsyncBindThreadFunc(void *args); #ifdef __cplusplus } diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 27e21d9a58..184f73c664 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -502,7 +502,7 @@ void taos_close(TAOS *taos) { } int taos_errno(TAOS_RES *res) { - if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { + if (res == NULL || TD_RES_TMQ_RAW(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { return terrno; } @@ -514,7 +514,7 @@ int taos_errno(TAOS_RES *res) { } const char *taos_errstr(TAOS_RES *res) { - if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { + if (res == NULL || TD_RES_TMQ_RAW(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { return (const char *)tstrerror(terrno); } @@ -554,6 +554,8 @@ void taos_free_result(TAOS_RES *res) { tDeleteMqMetaRsp(&pRsp->metaRsp); } else if (TD_RES_TMQ_BATCH_META(res)) { tDeleteMqBatchMetaRsp(&pRsp->batchMetaRsp); + } else if (TD_RES_TMQ_RAW(res)) { + tDeleteMqRawDataRsp(&pRsp->dataRsp); } taosMemoryFree(pRsp); } @@ -572,7 +574,7 @@ void taos_kill_query(TAOS *taos) { } int taos_field_count(TAOS_RES *res) { - if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { + if (res == NULL || TD_RES_TMQ_RAW(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { return 0; } @@ -583,7 +585,7 @@ int taos_field_count(TAOS_RES *res) { int taos_num_fields(TAOS_RES *res) { return taos_field_count(res); } TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) { - if (taos_num_fields(res) == 0 || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { + if (taos_num_fields(res) == 0 || TD_RES_TMQ_RAW(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { return NULL; } @@ -643,7 +645,7 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { return NULL; } else { tscError("invalid result passed to taos_fetch_row"); - terrno = TSDB_CODE_TSC_INTERNAL_ERROR; + terrno = TSDB_CODE_TMQ_INVALID_DATA; return NULL; } } @@ -764,7 +766,7 @@ int taos_print_row_with_size(char *str, uint32_t size, TAOS_ROW row, TAOS_FIELD } int *taos_fetch_lengths(TAOS_RES *res) { - if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { + if (res == NULL || TD_RES_TMQ_RAW(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { return NULL; } @@ -773,7 +775,7 @@ int *taos_fetch_lengths(TAOS_RES *res) { } TAOS_ROW *taos_result_block(TAOS_RES *res) { - if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { + if (res == NULL || TD_RES_TMQ_RAW(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { terrno = TSDB_CODE_INVALID_PARA; return NULL; } @@ -841,7 +843,7 @@ const char *taos_get_client_info() { return td_version; } // return int32_t int taos_affected_rows(TAOS_RES *res) { - if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res) || + if (res == NULL || TD_RES_TMQ_RAW(res) || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res)) { return 0; } @@ -853,7 +855,7 @@ int taos_affected_rows(TAOS_RES *res) { // return int64_t int64_t taos_affected_rows64(TAOS_RES *res) { - if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res) || + if (res == NULL || TD_RES_TMQ_RAW(res) || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res)) { return 0; } @@ -864,7 +866,7 @@ int64_t taos_affected_rows64(TAOS_RES *res) { } int taos_result_precision(TAOS_RES *res) { - if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { + if (res == NULL || TD_RES_TMQ_RAW(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { return TSDB_TIME_PRECISION_MILLI; } @@ -904,7 +906,7 @@ int taos_select_db(TAOS *taos, const char *db) { } void taos_stop_query(TAOS_RES *res) { - if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res) || + if (res == NULL || TD_RES_TMQ_RAW(res) || TD_RES_TMQ(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res)) { return; } @@ -913,7 +915,7 @@ void taos_stop_query(TAOS_RES *res) { } bool taos_is_null(TAOS_RES *res, int32_t row, int32_t col) { - if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { + if (res == NULL || TD_RES_TMQ_RAW(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { return true; } SReqResultInfo *pResultInfo = tscGetCurResInfo(res); @@ -938,7 +940,7 @@ int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) { } int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows) { - if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { + if (res == NULL || TD_RES_TMQ_RAW(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { return 0; } @@ -973,7 +975,7 @@ int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows) { return 0; } else { tscError("taos_fetch_block_s invalid res type"); - return -1; + return TSDB_CODE_TMQ_INVALID_DATA; } } @@ -981,7 +983,7 @@ int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData) { *numOfRows = 0; *pData = NULL; - if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { + if (res == NULL || TD_RES_TMQ_RAW(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { return 0; } @@ -1018,7 +1020,7 @@ int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData) { } int *taos_get_column_data_offset(TAOS_RES *res, int columnIndex) { - if (res == NULL || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { + if (res == NULL || TD_RES_TMQ_RAW(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { return 0; } @@ -1038,7 +1040,7 @@ int *taos_get_column_data_offset(TAOS_RES *res, int columnIndex) { int taos_is_null_by_column(TAOS_RES *res, int columnIndex, bool result[], int *rows) { if (res == NULL || result == NULL || rows == NULL || *rows <= 0 || columnIndex < 0 || TD_RES_TMQ_META(res) || - TD_RES_TMQ_BATCH_META(res)) { + TD_RES_TMQ_RAW(res) || TD_RES_TMQ_BATCH_META(res)) { return TSDB_CODE_INVALID_PARA; } @@ -2166,11 +2168,16 @@ int taos_stmt2_bind_param(TAOS_STMT2 *stmt, TAOS_STMT2_BINDV *bindv, int32_t col } STscStmt2 *pStmt = (STscStmt2 *)stmt; - if (pStmt->options.asyncExecFn && !pStmt->semWaited) { - if (tsem_wait(&pStmt->asyncQuerySem) != 0) { - tscError("wait async query sem failed"); + if( atomic_load_8((int8_t*)&pStmt->asyncBindParam.asyncBindNum)>1) { + tscError("async bind param is still working, please try again later"); + return TSDB_CODE_TSC_STMT_API_ERROR; + } + + if (pStmt->options.asyncExecFn && !pStmt->execSemWaited) { + if (tsem_wait(&pStmt->asyncExecSem) != 0) { + tscError("wait asyncExecSem failed"); } - pStmt->semWaited = true; + pStmt->execSemWaited = true; } SSHashObj *hashTbnames = tSimpleHashInit(100, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR)); @@ -2243,6 +2250,35 @@ out: return code; } +int taos_stmt2_bind_param_a(TAOS_STMT2 *stmt, TAOS_STMT2_BINDV *bindv, int32_t col_idx, __taos_async_fn_t fp, + void *param) { + if (stmt == NULL || bindv == NULL || fp == NULL) { + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + + STscStmt2 *pStmt = (STscStmt2 *)stmt; + if (atomic_load_8((int8_t *)&pStmt->asyncBindParam.asyncBindNum) > 0) { + tscError("async bind param is still working, please try again later"); + return TSDB_CODE_TSC_STMT_API_ERROR; + } + + ThreadArgs *args = (ThreadArgs *)taosMemoryMalloc(sizeof(ThreadArgs)); + args->stmt = stmt; + args->bindv = bindv; + args->col_idx = col_idx; + args->fp = fp; + args->param = param; + (void)atomic_add_fetch_8(&pStmt->asyncBindParam.asyncBindNum, 1); + int code_s = taosStmt2AsyncBind(stmtAsyncBindThreadFunc, (void *)args); + if (code_s != TSDB_CODE_SUCCESS) { + (void)atomic_sub_fetch_8(&pStmt->asyncBindParam.asyncBindNum, 1); + // terrno = TAOS_SYSTEM_ERROR(errno); + } + + return code_s; +} + int taos_stmt2_exec(TAOS_STMT2 *stmt, int *affected_rows) { if (stmt == NULL) { tscError("NULL parameter for %s", __FUNCTION__); diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index 4ff50692b7..e99d5c2e83 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -1844,9 +1844,10 @@ static void* getRawDataFromRes(void* pRetrieve) { } void* rawData = NULL; // deal with compatibility - if (*(int64_t*)pRetrieve == 0) { + if (*(int64_t*)pRetrieve == RETRIEVE_TABLE_RSP_VERSION) { rawData = ((SRetrieveTableRsp*)pRetrieve)->data; - } else if (*(int64_t*)pRetrieve == 1) { + } else if (*(int64_t*)pRetrieve == RETRIEVE_TABLE_RSP_TMQ_VERSION || + *(int64_t*)pRetrieve == RETRIEVE_TABLE_RSP_TMQ_RAW_VERSION) { rawData = ((SRetrieveTableRspForTmq*)pRetrieve)->data; } return rawData; @@ -1947,7 +1948,10 @@ static int32_t initRawCacheHash() { } static bool needRefreshMeta(void* rawData, STableMeta* pTableMeta, SSchemaWrapper* pSW) { - if (rawData == NULL || pTableMeta == NULL || pSW == NULL) { + if (rawData == NULL || pSW == NULL){ + return false; + } + if (pTableMeta == NULL) { uError("invalid parameter in %s", __func__); return false; } @@ -1965,6 +1969,7 @@ static bool needRefreshMeta(void* rawData, STableMeta* pTableMeta, SSchemaWrappe if (pSW->nCols != pTableMeta->tableInfo.numOfColumns) { return true; } + for (int i = 0; i < pSW->nCols; i++) { int j = 0; for (; j < pTableMeta->tableInfo.numOfColumns; j++) { @@ -1972,7 +1977,7 @@ static bool needRefreshMeta(void* rawData, STableMeta* pTableMeta, SSchemaWrappe char* fieldName = pSW->pSchema[i].name; if (strcmp(pColSchema->name, fieldName) == 0) { - if (*fields != pColSchema->type || *(int32_t*)(fields + sizeof(int8_t)) != pColSchema->bytes) { + if (checkSchema(pColSchema, fields, NULL, 0) != 0){ return true; } break; @@ -2069,7 +2074,7 @@ static int32_t processCacheMeta(SHashObj* pVgHash, SHashObj* pNameHash, SHashObj SVCreateTbReq* pCreateReqDst, SCatalog* pCatalog, SRequestConnInfo* conn, SName* pName, STableMeta** pMeta, SSchemaWrapper* pSW, void* rawData, int32_t retry) { if (pVgHash == NULL || pNameHash == NULL || pMetaHash == NULL || pCatalog == NULL || conn == NULL || pName == NULL || - pMeta == NULL || pSW == NULL || rawData == NULL) { + pMeta == NULL) { uError("invalid parameter in %s", __func__); return TSDB_CODE_INVALID_PARA; } @@ -2168,7 +2173,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, uint32_t dataLen) { void* rawData = getRawDataFromRes(pRetrieve); RAW_NULL_CHECK(rawData); - uDebug(LOG_ID_TAG " write raw data block tbname:%s", LOG_ID_VALUE, tbName); + uTrace(LOG_ID_TAG " write raw data block tbname:%s", LOG_ID_VALUE, tbName); SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}}; tstrncpy(pName.dbname, pRequest->pDb, TSDB_DB_NAME_LEN); tstrncpy(pName.tname, tbName, TSDB_TABLE_NAME_LEN); @@ -2251,7 +2256,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, uint32_t dataLen) void* rawData = getRawDataFromRes(pRetrieve); RAW_NULL_CHECK(rawData); - uDebug(LOG_ID_TAG " write raw data block tbname:%s", LOG_ID_VALUE, tbName); + uTrace(LOG_ID_TAG " write raw data block tbname:%s", LOG_ID_VALUE, tbName); SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}}; tstrncpy(pName.dbname, pRequest->pDb, TSDB_DB_NAME_LEN); tstrncpy(pName.tname, tbName, TSDB_TABLE_NAME_LEN); @@ -2298,6 +2303,90 @@ end: return code; } +static int32_t tmqWriteRawRawDataImpl(TAOS* taos, void* data, uint32_t dataLen) { + if (taos == NULL || data == NULL) { + uError("invalid parameter in %s", __func__); + return TSDB_CODE_INVALID_PARA; + } + int32_t code = TSDB_CODE_SUCCESS; + SQuery* pQuery = NULL; + SHashObj* pVgroupHash = NULL; + SMqRspObj rspObj = {0}; + SDecoder decoder = {0}; + + SRequestObj* pRequest = NULL; + SCatalog* pCatalog = NULL; + SRequestConnInfo conn = {0}; + + RAW_RETURN_CHECK(buildRawRequest(taos, &pRequest, &pCatalog, &conn)); + uDebug(LOG_ID_TAG " write raw rawdata, data:%p, dataLen:%d", LOG_ID_VALUE, data, dataLen); + RAW_RETURN_CHECK(decodeRawData(&decoder, data, dataLen, tDecodeMqDataRsp, &rspObj)); + + SHashObj* pVgHash = NULL; + SHashObj* pNameHash = NULL; + SHashObj* pMetaHash = NULL; + RAW_RETURN_CHECK(getRawCache(&pVgHash, &pNameHash, &pMetaHash, taos)); + int retry = 0; + while (1) { + RAW_RETURN_CHECK(smlInitHandle(&pQuery)); + uDebug(LOG_ID_TAG " write raw rawdata block num:%d", LOG_ID_VALUE, rspObj.dataRsp.blockNum); + SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)(pQuery)->pRoot; + pVgroupHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + RAW_NULL_CHECK(pVgroupHash); + pStmt->pVgDataBlocks = taosArrayInit(8, POINTER_BYTES); + RAW_NULL_CHECK(pStmt->pVgDataBlocks); + + while (++rspObj.resIter < rspObj.dataRsp.blockNum) { + const char* tbName = (const char*)taosArrayGetP(rspObj.dataRsp.blockTbName, rspObj.resIter); + RAW_NULL_CHECK(tbName); + void* pRetrieve = taosArrayGetP(rspObj.dataRsp.blockData, rspObj.resIter); + RAW_NULL_CHECK(pRetrieve); + void* rawData = getRawDataFromRes(pRetrieve); + RAW_NULL_CHECK(rawData); + + uTrace(LOG_ID_TAG " write raw data block tbname:%s", LOG_ID_VALUE, tbName); + SName pName = {TSDB_TABLE_NAME_T, pRequest->pTscObj->acctId, {0}, {0}}; + tstrncpy(pName.dbname, pRequest->pDb, TSDB_DB_NAME_LEN); + tstrncpy(pName.tname, tbName, TSDB_TABLE_NAME_LEN); + + // find schema data info + STableMeta* pTableMeta = NULL; + RAW_RETURN_CHECK(processCacheMeta(pVgHash, pNameHash, pMetaHash, NULL, pCatalog, &conn, &pName, + &pTableMeta, NULL, NULL, retry)); + char err[ERR_MSG_LEN] = {0}; + code = rawBlockBindRawData(pVgroupHash, pStmt->pVgDataBlocks, pTableMeta, rawData); + if (code != TSDB_CODE_SUCCESS) { + SET_ERROR_MSG("table:%s, err:%s", pName.tname, err); + goto end; + } + } + taosHashCleanup(pVgroupHash); + pVgroupHash = NULL; + + RAW_RETURN_CHECK(smlBuildOutputRaw(pQuery, pVgHash)); + launchQueryImpl(pRequest, pQuery, true, NULL); + code = pRequest->code; + + if (NEED_CLIENT_HANDLE_ERROR(code) && retry++ < 3) { + uInfo("write raw retry:%d/3 end code:%d, msg:%s", retry, code, tstrerror(code)); + qDestroyQuery(pQuery); + pQuery = NULL; + rspObj.resIter = -1; + continue; + } + break; + } + + end: + uDebug(LOG_ID_TAG " write raw rawdata return, msg:%s", LOG_ID_VALUE, tstrerror(code)); + tDeleteMqDataRsp(&rspObj.dataRsp); + tDecoderClear(&decoder); + qDestroyQuery(pQuery); + taosHashCleanup(pVgroupHash); + destroyRequest(pRequest); + return code; +} + static void processSimpleMeta(SMqMetaRsp* pMetaRsp, cJSON** meta) { if (pMetaRsp == NULL || meta == NULL) { uError("invalid parameter in %s", __func__); @@ -2472,6 +2561,7 @@ int32_t tmq_get_raw(TAOS_RES* res, tmq_raw_data* raw) { uError("invalid parameter in %s", __func__); return TSDB_CODE_INVALID_PARA; } + *raw = (tmq_raw_data){0}; SMqRspObj* rspObj = ((SMqRspObj*)res); if (TD_RES_TMQ_META(res)) { raw->raw = rspObj->metaRsp.metaRsp; @@ -2499,6 +2589,12 @@ int32_t tmq_get_raw(TAOS_RES* res, tmq_raw_data* raw) { raw->raw_len = rspObj->batchMetaRsp.metaBuffLen; raw->raw_type = rspObj->resType; uDebug("tmq get raw batch meta:%p", raw); + } else if (TD_RES_TMQ_RAW(res)) { + raw->raw = rspObj->dataRsp.rawData; + rspObj->dataRsp.rawData = NULL; + raw->raw_len = rspObj->dataRsp.len; + raw->raw_type = rspObj->resType; + uDebug("tmq get raw raw:%p", raw); } else { uError("tmq get raw error type:%d", *(int8_t*)res); return TSDB_CODE_TMQ_INVALID_MSG; @@ -2508,8 +2604,11 @@ int32_t tmq_get_raw(TAOS_RES* res, tmq_raw_data* raw) { void tmq_free_raw(tmq_raw_data raw) { uDebug("tmq free raw data type:%d", raw.raw_type); - if (raw.raw_type == RES_TYPE__TMQ || raw.raw_type == RES_TYPE__TMQ_METADATA) { + if (raw.raw_type == RES_TYPE__TMQ || + raw.raw_type == RES_TYPE__TMQ_METADATA) { taosMemoryFree(raw.raw); + } else if(raw.raw_type == RES_TYPE__TMQ_RAWDATA && raw.raw != NULL){ + taosMemoryFree(POINTER_SHIFT(raw.raw, - sizeof(SMqRspHead))); } (void)memset(terrMsg, 0, ERR_MSG_LEN); } @@ -2559,6 +2658,8 @@ static int32_t writeRawImpl(TAOS* taos, void* buf, uint32_t len, uint16_t type) return taosDeleteData(taos, buf, len); } else if (type == RES_TYPE__TMQ_METADATA) { return tmqWriteRawMetaDataImpl(taos, buf, len); + } else if (type == RES_TYPE__TMQ_RAWDATA) { + return tmqWriteRawRawDataImpl(taos, buf, len); } else if (type == RES_TYPE__TMQ) { return tmqWriteRawDataImpl(taos, buf, len); } else if (type == RES_TYPE__TMQ_BATCH_META) { diff --git a/source/client/src/clientStmt2.c b/source/client/src/clientStmt2.c index fe303bc314..67066e1fdd 100644 --- a/source/client/src/clientStmt2.c +++ b/source/client/src/clientStmt2.c @@ -752,6 +752,14 @@ static int32_t stmtInitQueue(STscStmt2* pStmt) { return TSDB_CODE_SUCCESS; } +static int32_t stmtIniAsyncBind(STscStmt2* pStmt) { + (void)taosThreadCondInit(&pStmt->asyncBindParam.waitCond, NULL); + (void)taosThreadMutexInit(&pStmt->asyncBindParam.mutex, NULL); + pStmt->asyncBindParam.asyncBindNum = 0; + + return TSDB_CODE_SUCCESS; +} + static int32_t stmtInitTableBuf(STableBufInfo* pTblBuf) { pTblBuf->buffUnit = sizeof(SStmtQNode); pTblBuf->buffSize = pTblBuf->buffUnit * 1000; @@ -812,13 +820,13 @@ TAOS_STMT2* stmtInit2(STscObj* taos, TAOS_STMT2_OPTION* pOptions) { pStmt->sql.siInfo.mgmtEpSet = getEpSet_s(&pStmt->taos->pAppInfo->mgmtEp); pStmt->sql.siInfo.pTableHash = tSimpleHashInit(100, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY)); if (NULL == pStmt->sql.siInfo.pTableHash) { - (void)stmtClose(pStmt); + (void)stmtClose2(pStmt); return NULL; } pStmt->sql.siInfo.pTableCols = taosArrayInit(STMT_TABLE_COLS_NUM, POINTER_BYTES); if (NULL == pStmt->sql.siInfo.pTableCols) { terrno = terrno; - (void)stmtClose(pStmt); + (void)stmtClose2(pStmt); return NULL; } @@ -831,20 +839,27 @@ TAOS_STMT2* stmtInit2(STscObj* taos, TAOS_STMT2_OPTION* pOptions) { } if (TSDB_CODE_SUCCESS != code) { terrno = code; - (void)stmtClose(pStmt); + (void)stmtClose2(pStmt); return NULL; } } pStmt->sql.siInfo.tableColsReady = true; if (pStmt->options.asyncExecFn) { - if (tsem_init(&pStmt->asyncQuerySem, 0, 1) != 0) { + if (tsem_init(&pStmt->asyncExecSem, 0, 1) != 0) { terrno = TAOS_SYSTEM_ERROR(errno); - (void)stmtClose(pStmt); + (void)stmtClose2(pStmt); return NULL; } } - pStmt->semWaited = false; + code = stmtIniAsyncBind(pStmt); + if (TSDB_CODE_SUCCESS != code) { + terrno = code; + (void)stmtClose2(pStmt); + return NULL; + } + + pStmt->execSemWaited = false; STMT_LOG_SEQ(STMT_INIT); @@ -1659,8 +1674,8 @@ static void asyncQueryCb(void* userdata, TAOS_RES* res, int code) { (void)stmtCleanExecInfo(pStmt, (code ? false : true), false); ++pStmt->sql.runTimes; - if (tsem_post(&pStmt->asyncQuerySem) != 0) { - tscError("failed to post asyncQuerySem"); + if (tsem_post(&pStmt->asyncExecSem) != 0) { + tscError("failed to post asyncExecSem"); } } @@ -1675,6 +1690,12 @@ int stmtExec2(TAOS_STMT2* stmt, int* affected_rows) { return pStmt->errCode; } + taosThreadMutexLock(&pStmt->asyncBindParam.mutex); + if (atomic_load_8((int8_t*)&pStmt->asyncBindParam.asyncBindNum) > 0) { + (void)taosThreadCondWait(&pStmt->asyncBindParam.waitCond, &pStmt->asyncBindParam.mutex); + } + taosThreadMutexUnlock(&pStmt->asyncBindParam.mutex); + if (pStmt->sql.stbInterlaceMode) { STMT_ERR_RET(stmtAddBatch2(pStmt)); } @@ -1749,7 +1770,7 @@ int stmtExec2(TAOS_STMT2* stmt, int* affected_rows) { pRequest->body.queryFp = asyncQueryCb; ((SSyncQueryParam*)(pRequest)->body.interParam)->userParam = pStmt; - pStmt->semWaited = false; + pStmt->execSemWaited = false; launchAsyncQuery(pRequest, pStmt->sql.pQuery, NULL, pWrapper); } @@ -1775,12 +1796,20 @@ int stmtClose2(TAOS_STMT2* stmt) { pStmt->bindThreadInUse = false; } + taosThreadMutexLock(&pStmt->asyncBindParam.mutex); + if (atomic_load_8((int8_t*)&pStmt->asyncBindParam.asyncBindNum) > 0) { + (void)taosThreadCondWait(&pStmt->asyncBindParam.waitCond, &pStmt->asyncBindParam.mutex); + } + taosThreadMutexUnlock(&pStmt->asyncBindParam.mutex); (void)taosThreadCondDestroy(&pStmt->queue.waitCond); (void)taosThreadMutexDestroy(&pStmt->queue.mutex); - if (pStmt->options.asyncExecFn && !pStmt->semWaited) { - if (tsem_wait(&pStmt->asyncQuerySem) != 0) { - tscError("failed to wait asyncQuerySem"); + (void)taosThreadCondDestroy(&pStmt->asyncBindParam.waitCond); + (void)taosThreadMutexDestroy(&pStmt->asyncBindParam.mutex); + + if (pStmt->options.asyncExecFn && !pStmt->execSemWaited) { + if (tsem_wait(&pStmt->asyncExecSem) != 0) { + tscError("failed to wait asyncExecSem"); } } @@ -1798,8 +1827,8 @@ int stmtClose2(TAOS_STMT2* stmt) { STMT_ERR_RET(stmtCleanSQLInfo(pStmt)); if (pStmt->options.asyncExecFn) { - if (tsem_destroy(&pStmt->asyncQuerySem) != 0) { - tscError("failed to destroy asyncQuerySem"); + if (tsem_destroy(&pStmt->asyncExecSem) != 0) { + tscError("failed to destroy asyncExecSem"); } } taosMemoryFree(stmt); @@ -1928,3 +1957,22 @@ TAOS_RES* stmtUseResult2(TAOS_STMT2* stmt) { return pStmt->exec.pRequest; } + +int32_t stmtAsyncBindThreadFunc(void* args) { + qInfo("async stmt bind thread started"); + + ThreadArgs* targs = (ThreadArgs*)args; + STscStmt2* pStmt = (STscStmt2*)targs->stmt; + + int code = taos_stmt2_bind_param(targs->stmt, targs->bindv, targs->col_idx); + targs->fp(targs->param, NULL, code); + (void)taosThreadMutexLock(&(pStmt->asyncBindParam.mutex)); + (void)atomic_sub_fetch_8(&pStmt->asyncBindParam.asyncBindNum, 1); + (void)taosThreadCondSignal(&(pStmt->asyncBindParam.waitCond)); + (void)taosThreadMutexUnlock(&(pStmt->asyncBindParam.mutex)); + taosMemoryFree(args); + + qInfo("async stmt bind thread stopped"); + + return code; +} \ No newline at end of file diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 1d461d5a49..743b94b78e 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -42,18 +42,19 @@ #define PROCESS_POLL_RSP(FUNC,DATA) \ SDecoder decoder = {0}; \ - tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead)); \ + tDecoderInit(&decoder, POINTER_SHIFT(pRspWrapper->pollRsp.data, sizeof(SMqRspHead)), pRspWrapper->pollRsp.len - sizeof(SMqRspHead)); \ if (FUNC(&decoder, DATA) < 0) { \ tDecoderClear(&decoder); \ code = terrno; \ goto END;\ }\ tDecoderClear(&decoder);\ - (void)memcpy(DATA, pMsg->pData, sizeof(SMqRspHead)); + (void)memcpy(DATA, pRspWrapper->pollRsp.data, sizeof(SMqRspHead)); #define DELETE_POLL_RSP(FUNC,DATA) \ SMqPollRspWrapper* pRsp = &rspWrapper->pollRsp;\ - taosMemoryFreeClear(pRsp->pEpset);\ + taosMemoryFreeClear(pRsp->pEpset); \ + taosMemoryFreeClear(pRsp->data); \ FUNC(DATA); enum { @@ -92,6 +93,7 @@ struct tmq_conf_t { int8_t snapEnable; int8_t replayEnable; int8_t sourceExcluded; // do not consume, bit + int8_t rawData; // fetch raw data uint16_t port; int32_t autoCommitInterval; int32_t sessionTimeoutMs; @@ -121,6 +123,7 @@ struct tmq_t { int8_t resetOffsetCfg; int8_t replayEnable; int8_t sourceExcluded; // do not consume, bit + int8_t rawData; // fetch raw data int64_t consumerId; tmq_commit_cb* commitCb; void* commitCbUserParam; @@ -188,6 +191,8 @@ typedef struct { SMqClientTopic* topicHandle; uint64_t reqId; SEpSet* pEpset; + void* data; + uint32_t len; union { struct{ SMqRspHead head; @@ -491,11 +496,17 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value } } if (strcasecmp(key, "msg.consume.excluded") == 0) { - int64_t tmp; + int64_t tmp = 0; code = taosStr2int64(value, &tmp); conf->sourceExcluded = (0 == code && tmp != 0) ? TD_REQ_FROM_TAOX : 0; return TMQ_CONF_OK; } + if (strcasecmp(key, "msg.consume.rawdata") == 0) { + int64_t tmp = 0; + code = taosStr2int64(value, &tmp); + conf->rawData = (0 == code && tmp != 0) ? 1 : 0; + return TMQ_CONF_OK; + } if (strcasecmp(key, "td.connect.db") == 0) { return TMQ_CONF_OK; @@ -812,7 +823,7 @@ static void asyncCommitFromResult(tmq_t* tmq, const TAOS_RES* pRes, tmq_commit_c goto end; } - if (TD_RES_TMQ(pRes) || TD_RES_TMQ_META(pRes) || + if (TD_RES_TMQ(pRes) || TD_RES_TMQ_RAW(pRes) || TD_RES_TMQ_META(pRes) || TD_RES_TMQ_METADATA(pRes) || TD_RES_TMQ_BATCH_META(pRes)) { SMqRspObj* pRspObj = (SMqRspObj*)pRes; pTopicName = pRspObj->topic; @@ -1082,6 +1093,7 @@ void tmqSendHbReq(void* param, void* tmrId) { sendInfo->fp = tmqHbCb; sendInfo->msgType = TDMT_MND_TMQ_HB; + SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp); int32_t code = asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &epSet, NULL, sendInfo); @@ -1122,6 +1134,8 @@ static void tmqFreeRspWrapper(SMqRspWrapper* rspWrapper) { DELETE_POLL_RSP(tDeleteMqMetaRsp,&pRsp->metaRsp) } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_BATCH_META_RSP) { DELETE_POLL_RSP(tDeleteMqBatchMetaRsp,&pRsp->batchMetaRsp) + } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_RAW_DATA_RSP) { + DELETE_POLL_RSP(tDeleteMqRawDataRsp, &pRsp->dataRsp) } } @@ -1733,6 +1747,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { pTmq->resetOffsetCfg = conf->resetOffset; pTmq->replayEnable = conf->replayEnable; pTmq->sourceExcluded = conf->sourceExcluded; + pTmq->rawData = conf->rawData; pTmq->enableBatchMeta = conf->enableBatchMeta; tstrncpy(pTmq->user, user, TSDB_USER_LEN); if (taosGetFqdn(pTmq->fqdn) != 0) { @@ -2057,23 +2072,14 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { goto END; } rspType = ((SMqRspHead*)pMsg->pData)->mqMsgType; - tqDebugC("consumer:0x%" PRIx64 " recv poll rsp, vgId:%d, type %d,QID:0x%" PRIx64, tmq->consumerId, vgId, rspType, requestId); - if (rspType == TMQ_MSG_TYPE__POLL_DATA_RSP) { - PROCESS_POLL_RSP(tDecodeMqDataRsp, &pRspWrapper->pollRsp.dataRsp) - } else if (rspType == TMQ_MSG_TYPE__POLL_META_RSP) { - PROCESS_POLL_RSP(tDecodeMqMetaRsp, &pRspWrapper->pollRsp.metaRsp) - } else if (rspType == TMQ_MSG_TYPE__POLL_DATA_META_RSP) { - PROCESS_POLL_RSP(tDecodeSTaosxRsp, &pRspWrapper->pollRsp.dataRsp) - } else if (rspType == TMQ_MSG_TYPE__POLL_BATCH_META_RSP) { - PROCESS_POLL_RSP(tSemiDecodeMqBatchMetaRsp, &pRspWrapper->pollRsp.batchMetaRsp) - } else { // invalid rspType - tqErrorC("consumer:0x%" PRIx64 " invalid rsp msg received, type:%d ignored", tmq->consumerId, rspType); - code = TSDB_CODE_TSC_INTERNAL_ERROR; - goto END; - } + tqDebugC("consumer:0x%" PRIx64 " recv poll rsp, vgId:%d, type %d(%s),QID:0x%" PRIx64, tmq->consumerId, vgId, rspType, tmqMsgTypeStr[rspType], requestId); + pRspWrapper->tmqRspType = rspType; pRspWrapper->pollRsp.reqId = requestId; pRspWrapper->pollRsp.pEpset = pMsg->pEpSet; + pRspWrapper->pollRsp.data = pMsg->pData; + pRspWrapper->pollRsp.len = pMsg->len; + pMsg->pData = NULL; pMsg->pEpSet = NULL; END: @@ -2087,8 +2093,8 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { taosFreeQitem(pRspWrapper); tqErrorC("consumer:0x%" PRIx64 " put poll res into mqueue failed, code:%d", tmq->consumerId, code); } else { - tqDebugC("consumer:0x%" PRIx64 " put poll res into mqueue, type:%d, vgId:%d, total in queue:%d,QID:0x%" PRIx64, - tmq ? tmq->consumerId : 0, rspType, vgId, taosQueueItemSize(tmq->mqueue), requestId); + tqDebugC("consumer:0x%" PRIx64 " put poll res into mqueue, type:%d(%s), vgId:%d, total in queue:%d,QID:0x%" PRIx64, + tmq ? tmq->consumerId : 0, rspType, tmqMsgTypeStr[rspType], vgId, taosQueueItemSize(tmq->mqueue), requestId); } } @@ -2122,6 +2128,7 @@ void tmqBuildConsumeReqImpl(SMqPollReq* pReq, tmq_t* tmq, int64_t timeout, SMqCl pReq->reqId = generateRequestId(); pReq->enableReplay = tmq->replayEnable; pReq->sourceExcluded = tmq->sourceExcluded; + pReq->rawData = tmq->rawData; pReq->enableBatchMeta = tmq->enableBatchMeta; } @@ -2415,6 +2422,35 @@ static int32_t processMqRspError(tmq_t* tmq, SMqRspWrapper* pRspWrapper){ return code; } + +static int32_t processWrapperData(SMqRspWrapper* pRspWrapper){ + int32_t code = 0; + if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_RSP) { + PROCESS_POLL_RSP(tDecodeMqDataRsp, &pRspWrapper->pollRsp.dataRsp) + pRspWrapper->pollRsp.dataRsp.data = pRspWrapper->pollRsp.data; + pRspWrapper->pollRsp.data = NULL; + } else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_META_RSP) { + PROCESS_POLL_RSP(tDecodeMqMetaRsp, &pRspWrapper->pollRsp.metaRsp) + } else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_META_RSP) { + PROCESS_POLL_RSP(tDecodeSTaosxRsp, &pRspWrapper->pollRsp.dataRsp) + pRspWrapper->pollRsp.dataRsp.data = pRspWrapper->pollRsp.data; + pRspWrapper->pollRsp.data = NULL; + } else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_BATCH_META_RSP) { + PROCESS_POLL_RSP(tSemiDecodeMqBatchMetaRsp, &pRspWrapper->pollRsp.batchMetaRsp) + } else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_RAW_DATA_RSP) { + PROCESS_POLL_RSP(tDecodeMqRawDataRsp, &pRspWrapper->pollRsp.dataRsp) + pRspWrapper->pollRsp.dataRsp.len = pRspWrapper->pollRsp.len - sizeof(SMqRspHead); + pRspWrapper->pollRsp.dataRsp.rawData = POINTER_SHIFT(pRspWrapper->pollRsp.data, sizeof(SMqRspHead)); + pRspWrapper->pollRsp.data = NULL; + } else { + tqErrorC("invalid rsp msg received, type:%d ignored", pRspWrapper->tmqRspType); + code = TSDB_CODE_TSC_INTERNAL_ERROR; + goto END; + } + END: + return code; +} + static SMqRspObj* processMqRsp(tmq_t* tmq, SMqRspWrapper* pRspWrapper){ int32_t code = 0; SMqRspObj* pRspObj = NULL; @@ -2427,6 +2463,10 @@ static SMqRspObj* processMqRsp(tmq_t* tmq, SMqRspWrapper* pRspWrapper){ return pRspObj; } + code = processWrapperData(pRspWrapper); + if (code != 0) { + goto END; + } SMqPollRspWrapper* pollRspWrapper = &pRspWrapper->pollRsp; taosWLockLatch(&tmq->lock); SMqClientVg* pVg = NULL; @@ -2442,7 +2482,9 @@ static SMqRspObj* processMqRsp(tmq_t* tmq, SMqRspWrapper* pRspWrapper){ pVg->epSet = *pollRspWrapper->pEpset; } - if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_RSP || pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_META_RSP) { + if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_RSP || + pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_META_RSP || + pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_RAW_DATA_RSP) { updateVgInfo(pVg, &pollRspWrapper->dataRsp.reqOffset, &pollRspWrapper->dataRsp.rspOffset, pollRspWrapper->head.walsver, pollRspWrapper->head.walever, tmq->consumerId, pollRspWrapper->dataRsp.blockNum != 0); @@ -2459,12 +2501,15 @@ static SMqRspObj* processMqRsp(tmq_t* tmq, SMqRspWrapper* pRspWrapper){ tqErrorC("consumer:0x%" PRIx64 " failed to allocate memory for meta rsp", tmq->consumerId); goto END; } - pRspObj->resType = pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_RSP ? RES_TYPE__TMQ : RES_TYPE__TMQ_METADATA; + pRspObj->resType = pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_RAW_DATA_RSP ? RES_TYPE__TMQ_RAWDATA : + (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_RSP ? RES_TYPE__TMQ : RES_TYPE__TMQ_METADATA); int64_t numOfRows = 0; - tmqBuildRspFromWrapperInner(pollRspWrapper, pVg, &numOfRows, pRspObj); - tmq->totalRows += numOfRows; + if (pRspWrapper->tmqRspType != TMQ_MSG_TYPE__POLL_RAW_DATA_RSP){ + tmqBuildRspFromWrapperInner(pollRspWrapper, pVg, &numOfRows, pRspObj); + tmq->totalRows += numOfRows; + } pVg->emptyBlockReceiveTs = 0; - if (tmq->replayEnable) { + if (tmq->replayEnable && pRspWrapper->tmqRspType != TMQ_MSG_TYPE__POLL_RAW_DATA_RSP) { pVg->blockReceiveTs = taosGetTimestampMs(); pVg->blockSleepForReplay = pRspObj->dataRsp.sleepTime; if (pVg->blockSleepForReplay > 0) { @@ -2523,6 +2568,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq) { returnVal = processMqRsp(tmq, pRspWrapper); code = terrno; } + tmqFreeRspWrapper(pRspWrapper); taosFreeQitem(pRspWrapper); if(returnVal != NULL || code != 0){ @@ -2575,7 +2621,7 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { END: terrno = code; - if (tmq != NULL) { + if (tmq != NULL && terrno != 0) { tqErrorC("consumer:0x%" PRIx64 " poll error at line:%d, msg:%s", tmq->consumerId, lino, tstrerror(terrno)); } return NULL; @@ -2688,6 +2734,8 @@ tmq_res_t tmq_get_res_type(TAOS_RES* res) { return TMQ_RES_METADATA; } else if (TD_RES_TMQ_BATCH_META(res)) { return TMQ_RES_TABLE_META; + } else if (TD_RES_TMQ_RAW(res)) { + return TMQ_RES_RAWDATA; } else { return TMQ_RES_INVALID; } @@ -2697,7 +2745,7 @@ const char* tmq_get_topic_name(TAOS_RES* res) { if (res == NULL) { return NULL; } - if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res) || + if (TD_RES_TMQ_RAW(res) || TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_META(res) || TD_RES_TMQ_BATCH_META(res)) { char* tmp = strchr(((SMqRspObj*)res)->topic, '.'); if (tmp == NULL) { @@ -2714,7 +2762,7 @@ const char* tmq_get_db_name(TAOS_RES* res) { return NULL; } - if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res) || + if (TD_RES_TMQ_RAW(res) || TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res) || TD_RES_TMQ_META(res)) { char* tmp = strchr(((SMqRspObj*)res)->db, '.'); if (tmp == NULL) { @@ -2730,7 +2778,7 @@ int32_t tmq_get_vgroup_id(TAOS_RES* res) { if (res == NULL) { return TSDB_CODE_INVALID_PARA; } - if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res) || + if (TD_RES_TMQ_RAW(res) || TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res) || TD_RES_TMQ_BATCH_META(res) || TD_RES_TMQ_META(res)) { return ((SMqRspObj*)res)->vgId; } else { @@ -2742,7 +2790,7 @@ int64_t tmq_get_vgroup_offset(TAOS_RES* res) { if (res == NULL) { return TSDB_CODE_INVALID_PARA; } - if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) { + if (TD_RES_TMQ_RAW(res) || TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) { SMqRspObj* pRspObj = (SMqRspObj*)res; STqOffsetVal* pOffset = &pRspObj->dataRsp.reqOffset; if (pOffset->type == TMQ_OFFSET__LOG) { @@ -2767,7 +2815,7 @@ const char* tmq_get_table_name(TAOS_RES* res) { if (res == NULL) { return NULL; } - if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) { + if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res) ) { SMqRspObj* pRspObj = (SMqRspObj*)res; SMqDataRsp* data = &pRspObj->dataRsp; if (!data->withTbName || data->blockTbName == NULL || pRspObj->resIter < 0 || @@ -3670,4 +3718,4 @@ TAOS* tmq_get_connect(tmq_t* tmq) { return (TAOS*)(&(tmq->pTscObj->id)); } return NULL; -} \ No newline at end of file +} diff --git a/source/client/test/stmt2Test.cpp b/source/client/test/stmt2Test.cpp index 3a648042a6..407c8745a5 100644 --- a/source/client/test/stmt2Test.cpp +++ b/source/client/test/stmt2Test.cpp @@ -37,7 +37,7 @@ namespace { void checkError(TAOS_STMT2* stmt, int code) { if (code != TSDB_CODE_SUCCESS) { STscStmt2* pStmt = (STscStmt2*)stmt; - if (pStmt == nullptr || pStmt->sql.sqlStr == nullptr) { + if (pStmt == nullptr || pStmt->sql.sqlStr == nullptr || pStmt->exec.pRequest == nullptr) { printf("stmt api error\n stats : %d\n errstr : %s\n", pStmt->sql.status, taos_stmt_errstr(stmt)); } else { printf("stmt api error\n sql : %s\n stats : %d\n errstr : %s\n", pStmt->sql.sqlStr, pStmt->sql.status, @@ -255,48 +255,41 @@ TEST(stmt2Case, stmt2_test_limit) { do_query(taos, "create database IF NOT EXISTS stmt2_testdb_7"); do_query(taos, "create stable stmt2_testdb_7.stb (ts timestamp, b binary(10)) tags(t1 int, t2 binary(10))"); do_query(taos, - "insert into stmt2_testdb_7.tb2 using stmt2_testdb_7.stb tags(2,'xyz') values(1591060628000, " - "'abc'),(1591060628001,'def'),(1591060628004, 'hij')"); + "insert into stmt2_testdb_7.tb2 using stmt2_testdb_7.stb tags(2,'xyz') values(1591060628000, " + "'abc'),(1591060628001,'def'),(1591060628004, 'hij')"); do_query(taos, "use stmt2_testdb_7"); - TAOS_STMT2_OPTION option = {0, true, true, NULL, NULL}; - TAOS_STMT2* stmt = taos_stmt2_init(taos, &option); ASSERT_NE(stmt, nullptr); - const char* sql = "select * from stmt2_testdb_7.tb2 where ts > ? and ts < ? limit ?"; - int code = taos_stmt2_prepare(stmt, sql, 0); + int code = taos_stmt2_prepare(stmt, sql, 0); checkError(stmt, code); - - int t64_len[1] = {sizeof(int64_t)}; - int b_len[1] = {3}; - int x = 2; - int x_len = sizeof(int); - int64_t ts[2] = {1591060627000, 1591060628005}; - TAOS_STMT2_BIND params[3] = {{TSDB_DATA_TYPE_TIMESTAMP, &ts[0], t64_len, NULL, 1}, - {TSDB_DATA_TYPE_TIMESTAMP, &ts[1], t64_len, NULL, 1}, - {TSDB_DATA_TYPE_INT, &x, &x_len, NULL, 1}}; + int t64_len[1] = {sizeof(int64_t)}; + int b_len[1] = {3}; + int x = 2; + int x_len = sizeof(int); + int64_t ts[2] = {1591060627000, 1591060628005}; + TAOS_STMT2_BIND params[3] = {{TSDB_DATA_TYPE_TIMESTAMP, &ts[0], t64_len, NULL, 1}, + {TSDB_DATA_TYPE_TIMESTAMP, &ts[1], t64_len, NULL, 1}, + {TSDB_DATA_TYPE_INT, &x, &x_len, NULL, 1}}; TAOS_STMT2_BIND* paramv = ¶ms[0]; TAOS_STMT2_BINDV bindv = {1, NULL, NULL, ¶mv}; code = taos_stmt2_bind_param(stmt, &bindv, -1); checkError(stmt, code); - taos_stmt2_exec(stmt, NULL); checkError(stmt, code); - TAOS_RES* pRes = taos_stmt2_result(stmt); ASSERT_NE(pRes, nullptr); - int getRecordCounts = 0; while ((taos_fetch_row(pRes))) { - getRecordCounts++; + getRecordCounts++; } ASSERT_EQ(getRecordCounts, 2); taos_stmt2_close(stmt); @@ -304,7 +297,6 @@ TEST(stmt2Case, stmt2_test_limit) { taos_close(taos); } - TEST(stmt2Case, insert_stb_get_fields_Test) { TAOS* taos = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(taos, nullptr); @@ -1617,4 +1609,233 @@ TEST(stmt2Case, errcode) { code = taos_stmt_prepare(stmt, sql, 0); checkError(stmt, code); } + +void stmtAsyncBindCb(void* param, TAOS_RES* pRes, int code) { + bool* finish = (bool*)param; + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + taosMsleep(500); + *finish = true; + return; +} + +void stmtAsyncQueryCb2(void* param, TAOS_RES* pRes, int code) { + ASSERT_EQ(code, TSDB_CODE_SUCCESS); + taosMsleep(500); + return; +} + +void stmtAsyncBindCb2(void* param, TAOS_RES* pRes, int code) { + bool* finish = (bool*)param; + taosMsleep(500); + *finish = true; + return; +} + +TEST(stmt2Case, async_order) { + int CTB_NUMS = 2; + int ROW_NUMS = 2; + int CYC_NUMS = 2; + + TAOS* taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + TAOS_STMT2_OPTION option = {0, true, true, stmtAsyncQueryCb2, NULL}; + char* sql = "insert into ? values(?,?)"; + + do_query(taos, "drop database if exists stmt2_testdb_15"); + do_query(taos, "create database IF NOT EXISTS stmt2_testdb_15"); + do_query(taos, "create stable stmt2_testdb_15.stb (ts timestamp, b binary(10)) tags(t1 int, t2 binary(10))"); + do_query(taos, "use stmt2_testdb_15"); + + TAOS_STMT2* stmt = taos_stmt2_init(taos, &option); + ASSERT_NE(stmt, nullptr); + int code = taos_stmt2_prepare(stmt, sql, 0); + checkError(stmt, code); + int total_affected = 0; + + // tbname + char** tbs = (char**)taosMemoryMalloc(CTB_NUMS * sizeof(char*)); + for (int i = 0; i < CTB_NUMS; i++) { + tbs[i] = (char*)taosMemoryMalloc(sizeof(char) * 20); + sprintf(tbs[i], "ctb_%d", i); + char* tmp = (char*)taosMemoryMalloc(sizeof(char) * 100); + sprintf(tmp, "create table stmt2_testdb_15.%s using stmt2_testdb_15.stb tags(0, 'after')", tbs[i]); + do_query(taos, tmp); + } + // params + TAOS_STMT2_BIND** paramv = (TAOS_STMT2_BIND**)taosMemoryMalloc(CTB_NUMS * sizeof(TAOS_STMT2_BIND*)); + // col params + int64_t** ts = (int64_t**)taosMemoryMalloc(CTB_NUMS * sizeof(int64_t*)); + char** b = (char**)taosMemoryMalloc(CTB_NUMS * sizeof(char*)); + int* ts_len = (int*)taosMemoryMalloc(ROW_NUMS * sizeof(int)); + int* b_len = (int*)taosMemoryMalloc(ROW_NUMS * sizeof(int)); + for (int i = 0; i < ROW_NUMS; i++) { + ts_len[i] = sizeof(int64_t); + b_len[i] = 1; + } + for (int i = 0; i < CTB_NUMS; i++) { + ts[i] = (int64_t*)taosMemoryMalloc(ROW_NUMS * sizeof(int64_t)); + b[i] = (char*)taosMemoryMalloc(ROW_NUMS * sizeof(char)); + for (int j = 0; j < ROW_NUMS; j++) { + ts[i][j] = 1591060628000 + 100000 + j; + b[i][j] = 'a' + j; + } + } + // bind params + for (int i = 0; i < CTB_NUMS; i++) { + // create col params + paramv[i] = (TAOS_STMT2_BIND*)taosMemoryMalloc(2 * sizeof(TAOS_STMT2_BIND)); + paramv[i][0] = {TSDB_DATA_TYPE_TIMESTAMP, &ts[i][0], &ts_len[0], NULL, ROW_NUMS}; + paramv[i][1] = {TSDB_DATA_TYPE_BINARY, &b[i][0], &b_len[0], NULL, ROW_NUMS}; + } + + // case 1 : bind_a->exec_a->bind_a->exec_a->... + { + printf("case 1 : bind_a->exec_a->bind_a->exec_a->...\n"); + for (int r = 0; r < CYC_NUMS; r++) { + // bind + TAOS_STMT2_BINDV bindv = {CTB_NUMS, tbs, NULL, paramv}; + bool finish = false; + code = taos_stmt2_bind_param_a(stmt, &bindv, -1, stmtAsyncBindCb, (void*)&finish); + + checkError(stmt, code); + + // exec + code = taos_stmt2_exec(stmt, NULL); + checkError(stmt, code); + } + } + + // case 2 : bind_a->bind_a->bind_a->exec_a->... + { + printf("case 2 : bind_a->bind_a->bind_a->exec_a->...\n"); + for (int r = 0; r < CYC_NUMS; r++) { + // bind params + TAOS_STMT2_BIND** paramv = (TAOS_STMT2_BIND**)taosMemoryMalloc(CTB_NUMS * sizeof(TAOS_STMT2_BIND*)); + for (int i = 0; i < CTB_NUMS; i++) { + // create col params + paramv[i] = (TAOS_STMT2_BIND*)taosMemoryMalloc(2 * sizeof(TAOS_STMT2_BIND)); + paramv[i][0] = {TSDB_DATA_TYPE_TIMESTAMP, &ts[i][0], &ts_len[0], NULL, ROW_NUMS}; + paramv[i][1] = {TSDB_DATA_TYPE_BINARY, &b[i][0], &b_len[0], NULL, ROW_NUMS}; + } + // bind + TAOS_STMT2_BINDV bindv = {CTB_NUMS, tbs, NULL, paramv}; + bool finish = false; + code = taos_stmt2_bind_param_a(stmt, &bindv, -1, stmtAsyncBindCb, (void*)&finish); + while (!finish) { + taosMsleep(100); + } + checkError(stmt, code); + } + // exec + code = taos_stmt2_exec(stmt, NULL); + checkError(stmt, code); + } + + // case 3 : bind->exec_a->bind->exec_a->... + { + printf("case 3 : bind->exec_a->bind->exec_a->...\n"); + for (int r = 0; r < CYC_NUMS; r++) { + // bind + TAOS_STMT2_BINDV bindv = {CTB_NUMS, tbs, NULL, paramv}; + bool finish = false; + code = taos_stmt2_bind_param(stmt, &bindv, -1); + + checkError(stmt, code); + + // exec + code = taos_stmt2_exec(stmt, NULL); + checkError(stmt, code); + } + } + + // case 4 : bind_a->close + { + printf("case 4 : bind_a->close\n"); + // bind + TAOS_STMT2_BINDV bindv = {CTB_NUMS, tbs, NULL, paramv}; + bool finish = false; + code = taos_stmt2_bind_param_a(stmt, &bindv, -1, stmtAsyncBindCb, (void*)&finish); + checkError(stmt, code); + taos_stmt2_close(stmt); + checkError(stmt, code); + } + + // case 5 : bind_a->exec_a->close + { + printf("case 5 : bind_a->exec_a->close\n"); + // init + TAOS_STMT2* stmt = taos_stmt2_init(taos, &option); + ASSERT_NE(stmt, nullptr); + int code = taos_stmt2_prepare(stmt, sql, 0); + checkError(stmt, code); + // bind + TAOS_STMT2_BINDV bindv = {CTB_NUMS, tbs, NULL, paramv}; + bool finish = false; + code = taos_stmt2_bind_param_a(stmt, &bindv, -1, stmtAsyncBindCb, (void*)&finish); + checkError(stmt, code); + // exec + code = taos_stmt2_exec(stmt, NULL); + checkError(stmt, code); + // close + taos_stmt2_close(stmt); + checkError(stmt, code); + } + + option = {0, false, false, NULL, NULL}; + stmt = taos_stmt2_init(taos, &option); + ASSERT_NE(stmt, nullptr); + code = taos_stmt2_prepare(stmt, sql, 0); + checkError(stmt, code); + + // case 6 : bind_a->exec->bind_a->exec->... + { + printf("case 6 : bind_a->exec->bind_a->exec->...\n"); + // init + + checkError(stmt, code); + for (int r = 0; r < CYC_NUMS; r++) { + // bind + TAOS_STMT2_BINDV bindv = {CTB_NUMS, tbs, NULL, paramv}; + bool finish = false; + code = taos_stmt2_bind_param_a(stmt, &bindv, -1, stmtAsyncBindCb, (void*)&finish); + checkError(stmt, code); + // exec + code = taos_stmt2_exec(stmt, NULL); + checkError(stmt, code); + } + } + + // case 7 (error:no wait error) : bind_a->bind_a + { + printf("case 7 (error:no wait error) : bind_a->bind_a\n"); + // bind + TAOS_STMT2_BINDV bindv = {CTB_NUMS, tbs, NULL, paramv}; + bool finish = false; + code = taos_stmt2_bind_param_a(stmt, &bindv, -1, stmtAsyncBindCb2, (void*)&finish); + checkError(stmt, code); + taosMsleep(200); + code = taos_stmt2_bind_param_a(stmt, &bindv, -1, stmtAsyncBindCb2, (void*)&finish); + ASSERT_EQ(code, TSDB_CODE_TSC_STMT_API_ERROR); + while (!finish) { + taosMsleep(100); + } + } + // close + taos_stmt2_close(stmt); + + // free memory + for (int i = 0; i < CTB_NUMS; i++) { + taosMemoryFree(paramv[i]); + taosMemoryFree(ts[i]); + taosMemoryFree(b[i]); + } + taosMemoryFree(ts); + taosMemoryFree(b); + taosMemoryFree(ts_len); + taosMemoryFree(b_len); + taosMemoryFree(paramv); + for (int i = 0; i < CTB_NUMS; i++) { + taosMemoryFree(tbs[i]); + } + taosMemoryFree(tbs); +} #pragma GCC diagnostic pop diff --git a/source/common/src/msg/tmsg.c b/source/common/src/msg/tmsg.c index 052fcbe2ba..a2e08d7c6b 100644 --- a/source/common/src/msg/tmsg.c +++ b/source/common/src/msg/tmsg.c @@ -14,8 +14,8 @@ */ #define _DEFAULT_SOURCE -#include "tmsg.h" #include "tglobal.h" +#include "tmsg.h" #undef TD_MSG_NUMBER_ #undef TD_MSG_DICT_ @@ -1471,6 +1471,12 @@ int32_t tSerializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) { TAOS_CHECK_EXIT(tEncodeI64(&encoder, pReq->analVer)); TAOS_CHECK_EXIT(tSerializeSMonitorParas(&encoder, &pReq->clusterCfg.monitorParas)); + for (int32_t i = 0; i < vlen; ++i) { + SVnodeLoad *pload = taosArrayGet(pReq->pVloads, i); + TAOS_CHECK_EXIT(tEncodeI64(&encoder, pload->syncAppliedIndex)); + TAOS_CHECK_EXIT(tEncodeI64(&encoder, pload->syncCommitIndex)); + } + tEndEncode(&encoder); _exit: @@ -1600,6 +1606,14 @@ int32_t tDeserializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) { TAOS_CHECK_EXIT(tDeserializeSMonitorParas(&decoder, &pReq->clusterCfg.monitorParas)); } + if (!tDecodeIsEnd(&decoder)) { + for (int32_t i = 0; i < vlen; ++i) { + SVnodeLoad *pLoad = taosArrayGet(pReq->pVloads, i); + TAOS_CHECK_EXIT(tDecodeI64(&decoder, &pLoad->syncAppliedIndex)); + TAOS_CHECK_EXIT(tDecodeI64(&decoder, &pLoad->syncCommitIndex)); + } + } + tEndDecode(&decoder); _exit: @@ -9219,6 +9233,7 @@ int32_t tSerializeSMqPollReq(void *buf, int32_t bufLen, SMqPollReq *pReq) { TAOS_CHECK_EXIT(tEncodeI8(&encoder, pReq->enableReplay)); TAOS_CHECK_EXIT(tEncodeI8(&encoder, pReq->sourceExcluded)); TAOS_CHECK_EXIT(tEncodeI8(&encoder, pReq->enableBatchMeta)); + TAOS_CHECK_EXIT(tEncodeI8(&encoder, pReq->rawData)); tEndEncode(&encoder); @@ -9272,6 +9287,10 @@ int32_t tDeserializeSMqPollReq(void *buf, int32_t bufLen, SMqPollReq *pReq) { pReq->enableBatchMeta = false; } + if (!tDecodeIsEnd(&decoder)) { + TAOS_CHECK_EXIT(tDecodeI8(&decoder, &pReq->rawData)); + } + tEndDecode(&decoder); _exit: @@ -9279,7 +9298,13 @@ _exit: return code; } -void tDestroySMqPollReq(SMqPollReq *pReq) { tOffsetDestroy(&pReq->reqOffset); } +void tDestroySMqPollReq(SMqPollReq *pReq) { + tOffsetDestroy(&pReq->reqOffset); + if (pReq->uidHash != NULL) { + taosHashCleanup(pReq->uidHash); + pReq->uidHash = NULL; + } +} int32_t tSerializeSTaskDropReq(void *buf, int32_t bufLen, STaskDropReq *pReq) { int32_t code = 0; int32_t lino; @@ -11448,12 +11473,14 @@ int32_t tDecodeMqDataRspCommon(SDecoder *pDecoder, SMqDataRsp *pRsp) { } for (int32_t i = 0; i < pRsp->blockNum; i++) { - void *data; - uint64_t bLen; - TAOS_CHECK_EXIT(tDecodeBinaryAlloc(pDecoder, &data, &bLen)); + void *data = NULL; + uint32_t bLen = 0; + TAOS_CHECK_EXIT(tDecodeBinary(pDecoder, (uint8_t**)&data, &bLen)); if (taosArrayPush(pRsp->blockData, &data) == NULL) { TAOS_CHECK_EXIT(terrno); } + pRsp->blockDataElementFree = false; + int32_t len = bLen; if (taosArrayPush(pRsp->blockDataLen, &len) == NULL) { TAOS_CHECK_EXIT(terrno); @@ -11499,10 +11526,25 @@ int32_t tDecodeMqDataRsp(SDecoder *pDecoder, SMqDataRsp *pRsp) { return 0; } +int32_t tDecodeMqRawDataRsp(SDecoder *pDecoder, SMqDataRsp *pRsp) { + int32_t code = 0; + int32_t lino; + + TAOS_CHECK_EXIT(tDecodeSTqOffsetVal(pDecoder, &pRsp->reqOffset)); + TAOS_CHECK_EXIT(tDecodeSTqOffsetVal(pDecoder, &pRsp->rspOffset)); + TAOS_CHECK_EXIT(tDecodeI32(pDecoder, &pRsp->blockNum)); +_exit: + return code; +} + static void tDeleteMqDataRspCommon(SMqDataRsp *pRsp) { taosArrayDestroy(pRsp->blockDataLen); pRsp->blockDataLen = NULL; - taosArrayDestroyP(pRsp->blockData, NULL); + if (pRsp->blockDataElementFree){ + taosArrayDestroyP(pRsp->blockData, NULL); + } else { + taosArrayDestroy(pRsp->blockData); + } pRsp->blockData = NULL; taosArrayDestroyP(pRsp->blockSchema, (FDelete)tDeleteSchemaWrapper); pRsp->blockSchema = NULL; @@ -11510,6 +11552,7 @@ static void tDeleteMqDataRspCommon(SMqDataRsp *pRsp) { pRsp->blockTbName = NULL; tOffsetDestroy(&pRsp->reqOffset); tOffsetDestroy(&pRsp->rspOffset); + taosMemoryFreeClear(pRsp->data); } void tDeleteMqDataRsp(SMqDataRsp *rsp) { tDeleteMqDataRspCommon(rsp); } @@ -11572,6 +11615,14 @@ void tDeleteSTaosxRsp(SMqDataRsp *pRsp) { pRsp->createTableReq = NULL; } +void tDeleteMqRawDataRsp(SMqDataRsp *pRsp) { + tOffsetDestroy(&pRsp->reqOffset); + tOffsetDestroy(&pRsp->rspOffset); + if (pRsp->rawData != NULL){ + taosMemoryFree(POINTER_SHIFT(pRsp->rawData, - sizeof(SMqRspHead))); + } +} + int32_t tEncodeSSingleDeleteReq(SEncoder *pEncoder, const SSingleDeleteReq *pReq) { TAOS_CHECK_RETURN(tEncodeCStr(pEncoder, pReq->tbname)); TAOS_CHECK_RETURN(tEncodeI64(pEncoder, pReq->startTs)); @@ -11658,6 +11709,26 @@ int32_t tDecodeSBatchDeleteReqSetCtime(SDecoder *pDecoder, SBatchDeleteReq *pReq _exit: return code; } +int32_t transformRawSSubmitTbData(void* data, int64_t suid, int64_t uid, int32_t sver){ + int32_t code = 0; + int32_t lino = 0; + SDecoder decoder = {0}; + tDecoderInit(&decoder, (uint8_t *)POINTER_SHIFT(data, INT_BYTES), *(uint32_t*)data); + + int32_t flags = 0; + TAOS_CHECK_EXIT(tDecodeI32v(&decoder, &flags)); + flags |= TD_REQ_FROM_TAOX; + flags &= ~SUBMIT_REQ_AUTO_CREATE_TABLE; + + SEncoder encoder = {0}; + tEncoderInit(&encoder, (uint8_t *)POINTER_SHIFT(data, INT_BYTES), *(uint32_t*)data); + TAOS_CHECK_EXIT(tEncodeI32v(&encoder, flags)); + TAOS_CHECK_EXIT(tEncodeI64(&encoder, suid)); + TAOS_CHECK_EXIT(tEncodeI64(&encoder, uid)); + TAOS_CHECK_EXIT(tEncodeI32v(&encoder, sver)); + _exit: + return code; +} static int32_t tEncodeSSubmitTbData(SEncoder *pCoder, const SSubmitTbData *pSubmitTbData) { int32_t code = 0; @@ -11705,14 +11776,20 @@ _exit: return code; } -static int32_t tDecodeSSubmitTbData(SDecoder *pCoder, SSubmitTbData *pSubmitTbData) { +static int32_t tDecodeSSubmitTbData(SDecoder *pCoder, SSubmitTbData *pSubmitTbData, void* rawData) { int32_t code = 0; int32_t lino; int32_t flags; uint8_t version; + uint8_t* dataAfterCreate = NULL; + uint8_t* dataStart = pCoder->data + pCoder->pos; + uint32_t posAfterCreate = 0; + TAOS_CHECK_EXIT(tStartDecode(pCoder)); + uint32_t pos = pCoder->pos; TAOS_CHECK_EXIT(tDecodeI32v(pCoder, &flags)); + uint32_t flagsLen = pCoder->pos - pos; pSubmitTbData->flags = flags & 0xff; version = (flags >> 8) & 0xff; @@ -11724,6 +11801,8 @@ static int32_t tDecodeSSubmitTbData(SDecoder *pCoder, SSubmitTbData *pSubmitTbDa } TAOS_CHECK_EXIT(tDecodeSVCreateTbReq(pCoder, pSubmitTbData->pCreateTbReq)); + dataAfterCreate = pCoder->data + pCoder->pos; + posAfterCreate = pCoder->pos; } // submit data @@ -11732,7 +11811,7 @@ static int32_t tDecodeSSubmitTbData(SDecoder *pCoder, SSubmitTbData *pSubmitTbDa TAOS_CHECK_EXIT(tDecodeI32v(pCoder, &pSubmitTbData->sver)); if (pSubmitTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { - uint64_t nColData; + uint64_t nColData = 0; TAOS_CHECK_EXIT(tDecodeU64v(pCoder, &nColData)); @@ -11745,7 +11824,7 @@ static int32_t tDecodeSSubmitTbData(SDecoder *pCoder, SSubmitTbData *pSubmitTbDa TAOS_CHECK_EXIT(tDecodeColData(version, pCoder, taosArrayReserve(pSubmitTbData->aCol, 1))); } } else { - uint64_t nRow; + uint64_t nRow = 0; TAOS_CHECK_EXIT(tDecodeU64v(pCoder, &nRow)); pSubmitTbData->aRowP = taosArrayInit(nRow, sizeof(SRow *)); @@ -11768,6 +11847,15 @@ static int32_t tDecodeSSubmitTbData(SDecoder *pCoder, SSubmitTbData *pSubmitTbDa TAOS_CHECK_EXIT(tDecodeI64(pCoder, &pSubmitTbData->ctimeMs)); } + if (rawData != NULL){ + if (dataAfterCreate != NULL){ + TAOS_MEMCPY(dataAfterCreate - INT_BYTES - flagsLen, dataStart, INT_BYTES + flagsLen); + *(int32_t*)(dataAfterCreate - INT_BYTES - flagsLen) = pCoder->pos - posAfterCreate + flagsLen; + *(void**)rawData = dataAfterCreate - INT_BYTES - flagsLen; + }else{ + *(void**)rawData = dataStart; + } + } tEndDecode(pCoder); _exit: @@ -11780,15 +11868,27 @@ int32_t tEncodeSubmitReq(SEncoder *pCoder, const SSubmitReq2 *pReq) { TAOS_CHECK_EXIT(tStartEncode(pCoder)); TAOS_CHECK_EXIT(tEncodeU64v(pCoder, taosArrayGetSize(pReq->aSubmitTbData))); - for (uint64_t i = 0; i < taosArrayGetSize(pReq->aSubmitTbData); i++) { - TAOS_CHECK_EXIT(tEncodeSSubmitTbData(pCoder, taosArrayGet(pReq->aSubmitTbData, i))); + if (pReq->raw){ + for (uint64_t i = 0; i < taosArrayGetSize(pReq->aSubmitTbData); i++) { + void* data = taosArrayGetP(pReq->aSubmitTbData, i); + if (pCoder->data != NULL){ + TAOS_MEMCPY(pCoder->data + pCoder->pos, data, *(uint32_t*)data + INT_BYTES); + + } + pCoder->pos += *(uint32_t*)data + INT_BYTES; + } + } else{ + for (uint64_t i = 0; i < taosArrayGetSize(pReq->aSubmitTbData); i++) { + TAOS_CHECK_EXIT(tEncodeSSubmitTbData(pCoder, taosArrayGet(pReq->aSubmitTbData, i))); + } } + tEndEncode(pCoder); _exit: return code; } -int32_t tDecodeSubmitReq(SDecoder *pCoder, SSubmitReq2 *pReq) { +int32_t tDecodeSubmitReq(SDecoder *pCoder, SSubmitReq2 *pReq, SArray* rawList) { int32_t code = 0; memset(pReq, 0, sizeof(*pReq)); @@ -11812,7 +11912,8 @@ int32_t tDecodeSubmitReq(SDecoder *pCoder, SSubmitReq2 *pReq) { } for (uint64_t i = 0; i < nSubmitTbData; i++) { - if (tDecodeSSubmitTbData(pCoder, taosArrayReserve(pReq->aSubmitTbData, 1)) < 0) { + SSubmitTbData* data = taosArrayReserve(pReq->aSubmitTbData, 1); + if (tDecodeSSubmitTbData(pCoder, data, rawList != NULL ? taosArrayReserve(rawList, 1) : NULL) < 0) { code = TSDB_CODE_INVALID_MSG; goto _exit; } @@ -11821,13 +11922,6 @@ int32_t tDecodeSubmitReq(SDecoder *pCoder, SSubmitReq2 *pReq) { tEndDecode(pCoder); _exit: - if (code) { - if (pReq->aSubmitTbData) { - // todo - taosArrayDestroy(pReq->aSubmitTbData); - pReq->aSubmitTbData = NULL; - } - } return code; } @@ -11883,12 +11977,15 @@ void tDestroySubmitTbData(SSubmitTbData *pTbData, int32_t flag) { void tDestroySubmitReq(SSubmitReq2 *pReq, int32_t flag) { if (pReq->aSubmitTbData == NULL) return; - int32_t nSubmitTbData = TARRAY_SIZE(pReq->aSubmitTbData); - SSubmitTbData *aSubmitTbData = (SSubmitTbData *)TARRAY_DATA(pReq->aSubmitTbData); + if (!pReq->raw){ + int32_t nSubmitTbData = TARRAY_SIZE(pReq->aSubmitTbData); + SSubmitTbData *aSubmitTbData = (SSubmitTbData *)TARRAY_DATA(pReq->aSubmitTbData); - for (int32_t i = 0; i < nSubmitTbData; i++) { - tDestroySubmitTbData(&aSubmitTbData[i], flag); + for (int32_t i = 0; i < nSubmitTbData; i++) { + tDestroySubmitTbData(&aSubmitTbData[i], flag); + } } + taosArrayDestroy(pReq->aSubmitTbData); pReq->aSubmitTbData = NULL; } diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 3699c05713..10c5eb71c1 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -293,12 +293,16 @@ static const SSysDbTableSchema vgroupsSchema[] = { {.name = "tables", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, {.name = "v1_dnode", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true}, {.name = "v1_status", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "v1_applied/committed", .bytes = TSDB_SYNC_APPLY_COMMIT_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "v2_dnode", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true}, {.name = "v2_status", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "v2_applied/committed", .bytes = TSDB_SYNC_APPLY_COMMIT_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "v3_dnode", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true}, {.name = "v3_status", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "v3_applied/committed", .bytes = TSDB_SYNC_APPLY_COMMIT_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "v4_dnode", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true}, {.name = "v4_status", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "v4_applied/committed", .bytes = TSDB_SYNC_APPLY_COMMIT_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "cacheload", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, {.name = "cacheelements", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, {.name = "tsma", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true}, @@ -369,6 +373,8 @@ static const SSysDbTableSchema vnodesSchema[] = { {.name = "role_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true}, {.name = "start_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true}, {.name = "restored", .bytes = 1, .type = TSDB_DATA_TYPE_BOOL, .sysInfo = true}, + {.name = "apply_finish_time", .bytes = TSDB_SYNC_RESTORE_lEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "unapplied", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, }; static const SSysDbTableSchema userUserPrivilegesSchema[] = { diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index cb305cb6c7..4a9a5ac274 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -2731,7 +2731,7 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq2** ppReq, const SSDataBlock* pDat terrno = 0; if (NULL == pReq) { - if (!(pReq = taosMemoryMalloc(sizeof(SSubmitReq2)))) { + if (!(pReq = taosMemoryCalloc(1, sizeof(SSubmitReq2)))) { code = terrno; goto _end; } diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index cdbda5dce3..2fd16b4f67 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -356,6 +356,9 @@ int32_t tsMaxTsmaCalcDelay = 600; int64_t tsmaDataDeleteMark = 1000 * 60 * 60 * 24; // in ms, default to 1d void *pTimezoneNameMap = NULL; +int32_t tsStreamNotifyMessageSize = 8 * 1024; // KB, default 8MB +int32_t tsStreamNotifyFrameSize = 256; // KB, default 256KB + int32_t taosCheckCfgStrValueLen(const char *name, const char *value, int32_t len); #define TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, pName) \ @@ -965,6 +968,9 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { TAOS_CHECK_RETURN(cfgAddInt64(pCfg, "minDiskFreeSize", tsMinDiskFreeSize, TFS_MIN_DISK_FREE_SIZE, 1024 * 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER,CFG_CATEGORY_LOCAL)); TAOS_CHECK_RETURN(cfgAddBool(pCfg, "enableWhiteList", tsEnableWhiteList, CFG_SCOPE_SERVER, CFG_DYN_SERVER,CFG_CATEGORY_GLOBAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "streamNotifyMessageSize", tsStreamNotifyMessageSize, 8, 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_LOCAL)); + TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "streamNotifyFrameSize", tsStreamNotifyFrameSize, 8, 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE,CFG_CATEGORY_LOCAL)); + // clang-format on // GRANT_CFG_ADD; @@ -1850,6 +1856,12 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "minReservedMemorySize"); tsMinReservedMemorySize = pItem->i32; + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "streamNotifyMessageSize"); + tsStreamNotifyMessageSize = pItem->i32; + + TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "streamNotifyFrameSize"); + tsStreamNotifyFrameSize = pItem->i32; + // GRANT_CFG_GET; TAOS_RETURN(TSDB_CODE_SUCCESS); } @@ -3241,4 +3253,4 @@ int32_t taosCheckCfgStrValueLen(const char *name, const char *value, int32_t len TAOS_RETURN(TSDB_CODE_INVALID_CFG_VALUE); } TAOS_RETURN(TSDB_CODE_SUCCESS); -} \ No newline at end of file +} diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 71da8beed9..050c722167 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -136,7 +136,7 @@ typedef enum { typedef enum { TRN_KILL_MODE_SKIP = 0, TRN_KILL_MODE_INTERUPT = 1, - //TRN_KILL_MODE_ROLLBACK = 2, + // TRN_KILL_MODE_ROLLBACK = 2, } ETrnKillMode; typedef enum { @@ -478,6 +478,10 @@ typedef struct { int32_t dnodeId; ESyncState syncState; int64_t syncTerm; + int64_t syncAppliedIndex; + int64_t lastSyncAppliedIndexUpdateTime; + double appliedRate; + int64_t syncCommitIndex; bool syncRestore; bool syncCanRead; int64_t roleTimeMs; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index ca119191eb..1a74573490 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -535,11 +535,37 @@ static int32_t mndCheckClusterCfgPara(SMnode *pMnode, SDnodeObj *pDnode, const S return DND_REASON_ONLINE; } +double calcAppliedRate(int64_t currentCount, int64_t lastCount, int64_t currentTimeMs, int64_t lastTimeMs) { + if ((currentTimeMs <= lastTimeMs) || (currentCount <= lastCount)) { + return 0.0; + } + + int64_t deltaCount = currentCount - lastCount; + int64_t deltaMs = currentTimeMs - lastTimeMs; + double rate = (double)deltaCount / (double)deltaMs; + return rate; +} + static bool mndUpdateVnodeState(int32_t vgId, SVnodeGid *pGid, SVnodeLoad *pVload) { bool stateChanged = false; bool roleChanged = pGid->syncState != pVload->syncState || (pVload->syncTerm != -1 && pGid->syncTerm != pVload->syncTerm) || pGid->roleTimeMs != pVload->roleTimeMs; + + if (pVload->syncCommitIndex > pVload->syncAppliedIndex) { + if (pGid->lastSyncAppliedIndexUpdateTime == 0) { + pGid->lastSyncAppliedIndexUpdateTime = taosGetTimestampMs(); + } else if (pGid->syncAppliedIndex != pVload->syncAppliedIndex) { + int64_t currentTimeMs = taosGetTimestampMs(); + pGid->appliedRate = calcAppliedRate(pVload->syncAppliedIndex, pGid->syncAppliedIndex, currentTimeMs, + pGid->lastSyncAppliedIndexUpdateTime); + + pGid->lastSyncAppliedIndexUpdateTime = currentTimeMs; + } + } + + pGid->syncAppliedIndex = pVload->syncAppliedIndex; + pGid->syncCommitIndex = pVload->syncCommitIndex; if (roleChanged || pGid->syncRestore != pVload->syncRestore || pGid->syncCanRead != pVload->syncCanRead || pGid->startTimeMs != pVload->startTimeMs) { mInfo( diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index e20afb7201..311beb0daa 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -244,6 +244,8 @@ static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOld, SVgObj *pNew) { pNewGid->syncState = pOldGid->syncState; pNewGid->syncRestore = pOldGid->syncRestore; pNewGid->syncCanRead = pOldGid->syncCanRead; + pNewGid->syncAppliedIndex = pOldGid->syncAppliedIndex; + pNewGid->syncCommitIndex = pOldGid->syncCommitIndex; } } } @@ -1122,10 +1124,25 @@ static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p mError("vgId:%d, failed to set role, since %s", pVgroup->vgId, tstrerror(code)); return code; } + + char applyStr[TSDB_SYNC_APPLY_COMMIT_LEN + 1] = {0}; + char buf[TSDB_SYNC_APPLY_COMMIT_LEN + VARSTR_HEADER_SIZE + 1] = {0}; + snprintf(applyStr, sizeof(applyStr), "%" PRId64 "/%" PRId64, pVgroup->vnodeGid[i].syncAppliedIndex, + pVgroup->vnodeGid[i].syncCommitIndex); + STR_WITH_MAXSIZE_TO_VARSTR(buf, applyStr, pShow->pMeta->pSchemas[cols].bytes); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + code = colDataSetVal(pColInfo, numOfRows, (const char *)&buf, false); + if (code != 0) { + mError("vgId:%d, failed to set role, since %s", pVgroup->vgId, tstrerror(code)); + return code; + } } else { colDataSetNULL(pColInfo, numOfRows); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetNULL(pColInfo, numOfRows); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetNULL(pColInfo, numOfRows); } } @@ -1232,6 +1249,21 @@ int64_t mndGetVnodesMemory(SMnode *pMnode, int32_t dnodeId) { return vnodeMemory; } +void calculateRstoreFinishTime(double rate, int64_t applyCount, char *restoreStr, size_t restoreStrSize) { + if (rate == 0) { + snprintf(restoreStr, restoreStrSize, "0:0:0"); + return; + } + + int64_t costTime = applyCount / rate; + int64_t totalSeconds = costTime / 1000; + int64_t hours = totalSeconds / 3600; + totalSeconds %= 3600; + int64_t minutes = totalSeconds / 60; + int64_t seconds = totalSeconds % 60; + snprintf(restoreStr, restoreStrSize, "%" PRId64 ":%" PRId64 ":%" PRId64, hours, minutes, seconds); +} + static int32_t mndRetrieveVnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { SMnode *pMnode = pReq->info.node; SSdb *pSdb = pMnode->pSdb; @@ -1319,6 +1351,26 @@ static int32_t mndRetrieveVnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB return code; } + int64_t unappliedCount = pGid->syncCommitIndex - pGid->syncAppliedIndex; + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + char restoreStr[20] = {0}; + if (unappliedCount > 0) { + calculateRstoreFinishTime(pGid->appliedRate, unappliedCount, restoreStr, sizeof(restoreStr)); + } + STR_TO_VARSTR(buf, restoreStr); + code = colDataSetVal(pColInfo, numOfRows, (const char *)&buf, false); + if (code != 0) { + mError("vgId:%d, failed to set syncRestore finish time, since %s", pVgroup->vgId, tstrerror(code)); + return code; + } + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + code = colDataSetVal(pColInfo, numOfRows, (const char *)&unappliedCount, false); + if (code != 0) { + mError("vgId:%d, failed to set syncRestore, since %s", pVgroup->vgId, tstrerror(code)); + return code; + } + numOfRows++; sdbRelease(pSdb, pDnode); } @@ -2771,7 +2823,7 @@ int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb pVgroup->vnodeGid[0].dnodeId); // add second - if (pNewVgroup->replica == 1){ + if (pNewVgroup->replica == 1) { TAOS_CHECK_RETURN(mndAddVnodeToVgroup(pMnode, pTrans, pNewVgroup, pArray)); } @@ -2792,8 +2844,8 @@ int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb TAOS_CHECK_RETURN(mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, pNewVgroup)); // add third - if (pNewVgroup->replica == 2){ - TAOS_CHECK_RETURN (mndAddVnodeToVgroup(pMnode, pTrans, pNewVgroup, pArray)); + if (pNewVgroup->replica == 2) { + TAOS_CHECK_RETURN(mndAddVnodeToVgroup(pMnode, pTrans, pNewVgroup, pArray)); } pNewVgroup->vnodeGid[0].nodeRole = TAOS_SYNC_ROLE_VOTER; @@ -2823,7 +2875,7 @@ int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb TAOS_CHECK_RETURN(mndRemoveVnodeFromVgroup(pMnode, pTrans, pNewVgroup, pArray, &del2)); TAOS_CHECK_RETURN(mndAddDropVnodeAction(pMnode, pTrans, pNewDb, pNewVgroup, &del2, true)); TAOS_CHECK_RETURN( - mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, pNewVgroup, pNewVgroup->vnodeGid[0].dnodeId)); + mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, pNewVgroup, pNewVgroup->vnodeGid[0].dnodeId)); TAOS_CHECK_RETURN(mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, pNewVgroup)); } else if (pNewDb->cfg.replications == 2) { mInfo("db:%s, vgId:%d, will add 1 vnode, vn:0 dnode:%d", pVgroup->dbName, pVgroup->vgId, diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 50d75c4838..f0e7af50f3 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -242,11 +242,11 @@ SSDataBlock *tqGetResultBlock(STqReader *pReader); int64_t tqGetResultBlockTime(STqReader *pReader); int32_t extractMsgFromWal(SWalReader *pReader, void **pItem, int64_t maxVer, const char *id); -int32_t tqReaderSetSubmitMsg(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver); +int32_t tqReaderSetSubmitMsg(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver, SArray* rawList); +void tqReaderClearSubmitMsg(STqReader *pReader); bool tqNextDataBlockFilterOut(STqReader *pReader, SHashObj *filterOutUids); int32_t tqRetrieveDataBlock(STqReader *pReader, SSDataBlock **pRes, const char *idstr); -int32_t tqRetrieveTaosxBlock(STqReader *pReader, SArray *blocks, SArray *schemas, SSubmitTbData **pSubmitTbDataRet, - int64_t *createTime); +int32_t tqRetrieveTaosxBlock(STqReader *pReader, SMqDataRsp* pRsp, SArray *blocks, SArray *schemas, SSubmitTbData **pSubmitTbDataRet, SArray* rawList, int8_t fetchMeta); int32_t tqGetStreamExecInfo(SVnode *pVnode, int64_t streamId, int64_t *pDelay, bool *fhFinished); // sma diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index e0bf51b333..50d6eb0090 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -90,6 +90,7 @@ typedef struct { // for replay SSDataBlock* block; int64_t blockTime; + SHashObj* tableCreateTimeHash; // for process create table msg in submit if fetch raw data } STqHandle; struct STQ { @@ -117,8 +118,8 @@ int32_t tqScanData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal* int32_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, uint64_t reqId); // tqExec -int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, SMqDataRsp* pRsp, int32_t* totalRows, int8_t sourceExcluded); -int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp, +int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, SMqDataRsp* pRsp, int32_t* totalRows, const SMqPollReq* pRequest); +int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, SMqDataRsp* pRsp, int32_t type, int32_t vgId); void tqPushEmptyDataRsp(STqHandle* pHandle, int32_t vgId); @@ -147,7 +148,7 @@ int32_t tqOffsetRestoreFromFile(STQ* pTq, char* name); // tq util int32_t tqExtractDelDataBlock(const void* pData, int32_t len, int64_t ver, void** pRefBlock, int32_t type, EStreamType blockType); int32_t tqExtractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg); -int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp* pRsp, int32_t epoch, int64_t consumerId, +int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, SMqDataRsp* pRsp, int32_t epoch, int64_t consumerId, int32_t type, int64_t sver, int64_t ever); int32_t tqInitDataRsp(SMqDataRsp* pRsp, STqOffsetVal pOffset); void tqUpdateNodeStage(STQ* pTq, bool isLeader); @@ -183,6 +184,7 @@ int32_t tqSendAllNotifyEvents(const SArray* pBlocks, SStreamTask* pTask, SVnode* #define TQ_SUBSCRIBE_NAME "subscribe" #define TQ_OFFSET_NAME "offset-ver0" #define TQ_POLL_MAX_TIME 1000 +#define TQ_POLL_MAX_BYTES 1048576 #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 5a61c1c124..402ab8da7e 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -167,7 +167,8 @@ int32_t metaDropMultipleTables(SMeta* pMeta, int64_t version, SArray* tb int metaTtlFindExpired(SMeta* pMeta, int64_t timePointMs, SArray* tbUids, int32_t ttlDropMaxCount); int metaAlterTable(SMeta* pMeta, int64_t version, SVAlterTbReq* pReq, STableMetaRsp* pMetaRsp); int metaUpdateChangeTimeWithLock(SMeta* pMeta, tb_uid_t uid, int64_t changeTimeMs); -SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, int lock, int64_t* createTime); +SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, int lock); +int64_t metaGetTableCreateTime(SMeta *pMeta, tb_uid_t uid, int lock); int32_t metaGetTbTSchemaNotNull(SMeta* pMeta, tb_uid_t uid, int32_t sver, int lock, STSchema** ppTSchema); int32_t metaGetTbTSchemaMaybeNull(SMeta* pMeta, tb_uid_t uid, int32_t sver, int lock, STSchema** ppTSchema); STSchema* metaGetTbTSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, int lock); diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index b367232e2d..169adf219f 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -378,7 +378,7 @@ int32_t metaTbCursorPrev(SMTbCursor *pTbCur, ETableType jumpTableType) { return 0; } -SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, int lock, int64_t *createTime) { +SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, int lock) { void *pData = NULL; int nData = 0; int64_t version; @@ -414,9 +414,6 @@ _query: } } else if (me.type == TSDB_CHILD_TABLE) { uid = me.ctbEntry.suid; - if (createTime != NULL){ - *createTime = me.ctbEntry.btime; - } tDecoderClear(&dc); goto _query; } else { @@ -455,6 +452,46 @@ _err: return NULL; } +int64_t metaGetTableCreateTime(SMeta *pMeta, tb_uid_t uid, int lock) { + void *pData = NULL; + int nData = 0; + int64_t version = 0; + SDecoder dc = {0}; + int64_t createTime = INT64_MAX; + if (lock) { + metaRLock(pMeta); + } + + if (tdbTbGet(pMeta->pUidIdx, &uid, sizeof(uid), &pData, &nData) < 0) { + goto _exit; + } + + version = ((SUidIdxVal *)pData)[0].version; + + if (tdbTbGet(pMeta->pTbDb, &(STbDbKey){.uid = uid, .version = version}, sizeof(STbDbKey), &pData, &nData) != 0) { + goto _exit; + } + + SMetaEntry me = {0}; + tDecoderInit(&dc, pData, nData); + int32_t code = metaDecodeEntry(&dc, &me); + if (code) { + tDecoderClear(&dc); + goto _exit; + } + if (me.type == TSDB_CHILD_TABLE) { + createTime = me.ctbEntry.btime; + } + tDecoderClear(&dc); + + _exit: + if (lock) { + metaULock(pMeta); + } + tdbFree(pData); + return createTime; +} + SMCtbCursor *metaOpenCtbCursor(void *pVnode, tb_uid_t uid, int lock) { SMeta *pMeta = ((SVnode *)pVnode)->pMeta; SMCtbCursor *pCtbCur = NULL; @@ -627,7 +664,7 @@ STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, int lock) { STSchema *pTSchema = NULL; SSchemaWrapper *pSW = NULL; - pSW = metaGetTableSchema(pMeta, uid, sver, lock, NULL); + pSW = metaGetTableSchema(pMeta, uid, sver, lock); if (!pSW) return NULL; pTSchema = tBuildTSchema(pSW->pSchema, pSW->nCols, pSW->version); diff --git a/source/dnode/vnode/src/meta/metaSnapshot.c b/source/dnode/vnode/src/meta/metaSnapshot.c index 7374b9ceb5..b227653e5e 100644 --- a/source/dnode/vnode/src/meta/metaSnapshot.c +++ b/source/dnode/vnode/src/meta/metaSnapshot.c @@ -548,7 +548,7 @@ int32_t setForSnapShot(SSnapContext* ctx, int64_t uid) { void taosXSetTablePrimaryKey(SSnapContext* ctx, int64_t uid) { bool ret = false; - SSchemaWrapper* schema = metaGetTableSchema(ctx->pMeta, uid, -1, 1, NULL); + SSchemaWrapper* schema = metaGetTableSchema(ctx->pMeta, uid, -1, 1); if (schema && schema->nCols >= 2 && schema->pSchema[1].flags & COL_IS_KEY) { ret = true; } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 1c0f73249c..66e64b91d9 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -57,6 +57,7 @@ void tqDestroyTqHandle(void* data) { if (pData->pRef) { walCloseRef(pData->pRef->pWal, pData->pRef->refId); } + taosHashCleanup(pData->tableCreateTimeHash); } static bool tqOffsetEqual(const STqOffset* pLeft, const STqOffset* pRight) { @@ -211,7 +212,7 @@ void tqPushEmptyDataRsp(STqHandle* pHandle, int32_t vgId) { tDeleteMqDataRsp(&dataRsp); } -int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp, int32_t type, +int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, SMqDataRsp* pRsp, int32_t type, int32_t vgId) { if (pHandle == NULL || pMsg == NULL || pReq == NULL || pRsp == NULL) { return TSDB_CODE_INVALID_PARA; @@ -414,7 +415,14 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { terrno = TSDB_CODE_INVALID_MSG; goto END; } - + if (req.rawData == 1){ + req.uidHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); + if (req.uidHash == NULL) { + tqError("tq poll rawData taosHashInit failed"); + code = terrno; + goto END; + } + } int64_t consumerId = req.consumerId; int32_t reqEpoch = req.epoch; STqOffsetVal reqOffset = req.reqOffset; diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index 580828b089..b84dcc4703 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -343,6 +343,7 @@ static int tqMetaInitHandle(STQ* pTq, STqHandle* handle) { tqReaderSetTbUidList(handle->execHandle.pTqReader, tbUidList, NULL); taosArrayDestroy(tbUidList); } + handle->tableCreateTimeHash = (SHashObj*)taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK); END: return code; diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 937ca80bcc..d650ae9751 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -283,7 +283,7 @@ void tqSetTablePrimaryKey(STqReader* pReader, int64_t uid) { return; } bool ret = false; - SSchemaWrapper* schema = metaGetTableSchema(pReader->pVnodeMeta, uid, -1, 1, NULL); + SSchemaWrapper* schema = metaGetTableSchema(pReader->pVnodeMeta, uid, -1, 1); if (schema && schema->nCols >= 2 && schema->pSchema[1].flags & COL_IS_KEY) { ret = true; } @@ -352,7 +352,7 @@ int32_t tqReaderSeek(STqReader* pReader, int64_t ver, const char* id) { return TSDB_CODE_INVALID_PARA; } if (walReaderSeekVer(pReader->pWalReader, ver) < 0) { - return -1; + return terrno; } tqDebug("wal reader seek to ver:%" PRId64 " %s", ver, id); return 0; @@ -485,14 +485,14 @@ bool tqNextBlockInWal(STqReader* pReader, const char* id, int sourceExcluded) { void* pBody = POINTER_SHIFT(pWalReader->pHead->head.body, sizeof(SSubmitReq2Msg)); int32_t bodyLen = pWalReader->pHead->head.bodyLen - sizeof(SSubmitReq2Msg); int64_t ver = pWalReader->pHead->head.version; - if (tqReaderSetSubmitMsg(pReader, pBody, bodyLen, ver) != 0) { + if (tqReaderSetSubmitMsg(pReader, pBody, bodyLen, ver, NULL) != 0) { return false; } pReader->nextBlk = 0; } } -int32_t tqReaderSetSubmitMsg(STqReader* pReader, void* msgStr, int32_t msgLen, int64_t ver) { +int32_t tqReaderSetSubmitMsg(STqReader* pReader, void* msgStr, int32_t msgLen, int64_t ver, SArray* rawList) { if (pReader == NULL) { return TSDB_CODE_INVALID_PARA; } @@ -500,11 +500,11 @@ int32_t tqReaderSetSubmitMsg(STqReader* pReader, void* msgStr, int32_t msgLen, i pReader->msg.msgLen = msgLen; pReader->msg.ver = ver; - tqDebug("tq reader set msg pointer:%p, msg len:%d", msgStr, msgLen); + tqTrace("tq reader set msg pointer:%p, msg len:%d", msgStr, msgLen); SDecoder decoder = {0}; tDecoderInit(&decoder, pReader->msg.msgStr, pReader->msg.msgLen); - int32_t code = tDecodeSubmitReq(&decoder, &pReader->submit); + int32_t code = tDecodeSubmitReq(&decoder, &pReader->submit, rawList); tDecoderClear(&decoder); if (code != 0) { @@ -514,6 +514,13 @@ int32_t tqReaderSetSubmitMsg(STqReader* pReader, void* msgStr, int32_t msgLen, i return code; } +void tqReaderClearSubmitMsg(STqReader *pReader) { + tDestroySubmitReq(&pReader->submit, TSDB_MSG_FLG_DECODE); + pReader->nextBlk = 0; + pReader->msg.msgStr = NULL; +} + + SWalReader* tqGetWalReader(STqReader* pReader) { if (pReader == NULL) { return NULL; @@ -551,17 +558,15 @@ bool tqNextBlockImpl(STqReader* pReader, const char* idstr) { void* ret = taosHashGet(pReader->tbIdHash, &pSubmitTbData->uid, sizeof(int64_t)); TSDB_CHECK_CONDITION(ret == NULL, code, lino, END, true); - tqDebug("iterator data block in hash continue, progress:%d/%d, total queried tables:%d, uid:%"PRId64, pReader->nextBlk, blockSz, taosHashGetSize(pReader->tbIdHash), uid); + tqTrace("iterator data block in hash jump block, progress:%d/%d, uid:%" PRId64 "", pReader->nextBlk, blockSz, uid); pReader->nextBlk++; } - tDestroySubmitReq(&pReader->submit, TSDB_MSG_FLG_DECODE); - pReader->nextBlk = 0; - pReader->msg.msgStr = NULL; - tqDebug("iterator data block end, block progress:%d/%d, uid:%"PRId64, pReader->nextBlk, blockSz, uid); + tqReaderClearSubmitMsg(pReader); + tqTrace("iterator data block end, total block num:%d, uid:%"PRId64, blockSz, uid); END: - tqDebug("%s:%d return:%s, uid:%"PRId64, __FUNCTION__, lino, code?"true":"false", uid); + tqTrace("%s:%d return:%s, uid:%"PRId64, __FUNCTION__, lino, code?"true":"false", uid); return code; } @@ -581,17 +586,14 @@ bool tqNextDataBlockFilterOut(STqReader* pReader, SHashObj* filterOutUids) { uid = pSubmitTbData->uid; void* ret = taosHashGet(filterOutUids, &pSubmitTbData->uid, sizeof(int64_t)); TSDB_CHECK_NULL(ret, code, lino, END, true); - tqDebug("iterator data block in hash continue, progress:%d/%d, uid:%" PRId64 "", pReader->nextBlk, blockSz, uid); + tqTrace("iterator data block in hash jump block, progress:%d/%d, uid:%" PRId64 "", pReader->nextBlk, blockSz, uid); pReader->nextBlk++; } - - tDestroySubmitReq(&pReader->submit, TSDB_MSG_FLG_DECODE); - pReader->nextBlk = 0; - pReader->msg.msgStr = NULL; - tqDebug("iterator data block end, block progress:%d/%d, uid:%"PRId64, pReader->nextBlk, blockSz, uid); + tqReaderClearSubmitMsg(pReader); + tqTrace("iterator data block end, total block num:%d, uid:%"PRId64, blockSz, uid); END: - tqDebug("%s:%d return:%s, uid:%"PRId64, __FUNCTION__, lino, code?"true":"false", uid); + tqTrace("%s:%d get data:%s, uid:%"PRId64, __FUNCTION__, lino, code?"true":"false", uid); return code; } @@ -740,7 +742,7 @@ int32_t tqRetrieveDataBlock(STqReader* pReader, SSDataBlock** pRes, const char* (pReader->cachedSchemaVer != sversion)) { tDeleteSchemaWrapper(pReader->pSchemaWrapper); - pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, uid, sversion, 1, NULL); + pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, uid, sversion, 1); if (pReader->pSchemaWrapper == NULL) { tqWarn("vgId:%d, cannot found schema wrapper for table: suid:%" PRId64 ", uid:%" PRId64 "version %d, possibly dropped table", @@ -934,7 +936,7 @@ static int32_t tqProcessColData(STqReader* pReader, SSubmitTbData* pSubmitTbData TQ_NULL_GO_TO_END(pCol); int32_t numOfRows = pCol->nVal; int32_t numOfCols = taosArrayGetSize(pCols); - tqDebug("vgId:%d, tqProcessColData start, col num: %d, rows:%d", pReader->pWalReader->pWal->cfg.vgId, numOfCols, numOfRows); + tqTrace("vgId:%d, tqProcessColData start, col num: %d, rows:%d", pReader->pWalReader->pWal->cfg.vgId, numOfCols, numOfRows); for (int32_t i = 0; i < numOfRows; i++) { bool buildNew = false; @@ -974,7 +976,7 @@ static int32_t tqProcessColData(STqReader* pReader, SSubmitTbData* pSubmitTbData } SSDataBlock* pLastBlock = taosArrayGetLast(blocks); pLastBlock->info.rows = curRow - lastRow; - tqDebug("vgId:%d, tqProcessColData end, col num: %d, rows:%d, block num:%d", pReader->pWalReader->pWal->cfg.vgId, numOfCols, numOfRows, (int)taosArrayGetSize(blocks)); + tqTrace("vgId:%d, tqProcessColData end, col num: %d, rows:%d, block num:%d", pReader->pWalReader->pWal->cfg.vgId, numOfCols, numOfRows, (int)taosArrayGetSize(blocks)); END: if (code != TSDB_CODE_SUCCESS) { tqError("vgId:%d, process col data failed, code:%d", pReader->pWalReader->pWal->cfg.vgId, code); @@ -997,7 +999,7 @@ int32_t tqProcessRowData(STqReader* pReader, SSubmitTbData* pSubmitTbData, SArra int32_t numOfRows = taosArrayGetSize(pRows); pTSchema = tBuildTSchema(pSchemaWrapper->pSchema, pSchemaWrapper->nCols, pSchemaWrapper->version); TQ_NULL_GO_TO_END(pTSchema); - tqDebug("vgId:%d, tqProcessRowData start, rows:%d", pReader->pWalReader->pWal->cfg.vgId, numOfRows); + tqTrace("vgId:%d, tqProcessRowData start, rows:%d", pReader->pWalReader->pWal->cfg.vgId, numOfRows); for (int32_t i = 0; i < numOfRows; i++) { bool buildNew = false; @@ -1036,7 +1038,7 @@ int32_t tqProcessRowData(STqReader* pReader, SSubmitTbData* pSubmitTbData, SArra SSDataBlock* pLastBlock = taosArrayGetLast(blocks); pLastBlock->info.rows = curRow - lastRow; - tqDebug("vgId:%d, tqProcessRowData end, rows:%d, block num:%d", pReader->pWalReader->pWal->cfg.vgId, numOfRows, (int)taosArrayGetSize(blocks)); + tqTrace("vgId:%d, tqProcessRowData end, rows:%d, block num:%d", pReader->pWalReader->pWal->cfg.vgId, numOfRows, (int)taosArrayGetSize(blocks)); END: if (code != TSDB_CODE_SUCCESS) { tqError("vgId:%d, process row data failed, code:%d", pReader->pWalReader->pWal->cfg.vgId, code); @@ -1046,8 +1048,46 @@ END: return code; } -int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas, SSubmitTbData** pSubmitTbDataRet, int64_t *createTime) { - tqDebug("tq reader retrieve data block msg pointer:%p, index:%d", pReader->msg.msgStr, pReader->nextBlk); +static int32_t buildCreateTbInfo(SMqDataRsp* pRsp, SVCreateTbReq* pCreateTbReq){ + int32_t code = 0; + int32_t lino = 0; + void* createReq = NULL; + TSDB_CHECK_NULL(pRsp, code, lino, END, TSDB_CODE_INVALID_PARA); + TSDB_CHECK_NULL(pCreateTbReq, code, lino, END, TSDB_CODE_INVALID_PARA); + + if (pRsp->createTableNum == 0) { + pRsp->createTableLen = taosArrayInit(0, sizeof(int32_t)); + TSDB_CHECK_NULL(pRsp->createTableLen, code, lino, END, terrno); + pRsp->createTableReq = taosArrayInit(0, sizeof(void*)); + TSDB_CHECK_NULL(pRsp->createTableReq, code, lino, END, terrno); + } + + uint32_t len = 0; + tEncodeSize(tEncodeSVCreateTbReq, pCreateTbReq, len, code); + TSDB_CHECK_CODE(code, lino, END); + createReq = taosMemoryCalloc(1, len); + TSDB_CHECK_NULL(createReq, code, lino, END, terrno); + + SEncoder encoder = {0}; + tEncoderInit(&encoder, createReq, len); + code = tEncodeSVCreateTbReq(&encoder, pCreateTbReq); + tEncoderClear(&encoder); + TSDB_CHECK_CODE(code, lino, END); + TSDB_CHECK_NULL(taosArrayPush(pRsp->createTableLen, &len), code, lino, END, terrno); + TSDB_CHECK_NULL(taosArrayPush(pRsp->createTableReq, &createReq), code, lino, END, terrno); + pRsp->createTableNum++; + tqTrace("build create table info msg success"); + + END: + if (code != 0){ + tqError("%s failed at %d, failed to build create table info msg:%s", __FUNCTION__, lino, tstrerror(code)); + taosMemoryFree(createReq); + } + return code; +} + +int32_t tqRetrieveTaosxBlock(STqReader* pReader, SMqDataRsp* pRsp, SArray* blocks, SArray* schemas, SSubmitTbData** pSubmitTbDataRet, SArray* rawList, int8_t fetchMeta) { + tqTrace("tq reader retrieve data block msg pointer:%p, index:%d", pReader->msg.msgStr, pReader->nextBlk); SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); if (pSubmitTbData == NULL) { return terrno; @@ -1063,7 +1103,7 @@ int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas pReader->lastBlkUid = uid; tDeleteSchemaWrapper(pReader->pSchemaWrapper); - pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, uid, sversion, 1, createTime); + pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, uid, sversion, 1); if (pReader->pSchemaWrapper == NULL) { tqWarn("vgId:%d, cannot found schema wrapper for table: suid:%" PRId64 ", version %d, possibly dropped table", pReader->pWalReader->pWal->cfg.vgId, uid, pReader->cachedSchemaVer); @@ -1071,6 +1111,19 @@ int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas return TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; } + if (pSubmitTbData->pCreateTbReq != NULL) { + int32_t code = buildCreateTbInfo(pRsp, pSubmitTbData->pCreateTbReq); + if (code != 0) { + return code; + } + } else if (rawList != NULL) { + if (taosArrayPush(schemas, &pReader->pSchemaWrapper) == NULL){ + return terrno; + } + pReader->pSchemaWrapper = NULL; + return 0; + } + if (pSubmitTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { return tqProcessColData(pReader, pSubmitTbData, blocks, schemas); } else { diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c index a65b118aea..b109be5626 100644 --- a/source/dnode/vnode/src/tq/tqScan.c +++ b/source/dnode/vnode/src/tq/tqScan.c @@ -14,6 +14,33 @@ */ #include "tq.h" +static int32_t tqAddRawDataToRsp(const void* rawData, SMqDataRsp* pRsp, int8_t precision) { + int32_t code = TDB_CODE_SUCCESS; + int32_t lino = 0; + void* buf = NULL; + + int32_t dataStrLen = sizeof(SRetrieveTableRspForTmq) + *(uint32_t *)rawData + INT_BYTES; + buf = taosMemoryCalloc(1, dataStrLen); + TSDB_CHECK_NULL(buf, code, lino, END, terrno); + + SRetrieveTableRspForTmq* pRetrieve = (SRetrieveTableRspForTmq*)buf; + pRetrieve->version = RETRIEVE_TABLE_RSP_TMQ_RAW_VERSION; + pRetrieve->precision = precision; + pRetrieve->compressed = 0; + + memcpy(pRetrieve->data, rawData, *(uint32_t *)rawData + INT_BYTES); + TSDB_CHECK_NULL(taosArrayPush(pRsp->blockDataLen, &dataStrLen), code, lino, END, terrno); + TSDB_CHECK_NULL(taosArrayPush(pRsp->blockData, &buf), code, lino, END, terrno); + pRsp->blockDataElementFree = true; + + tqTrace("tqAddRawDataToRsp add block data to block array, blockDataLen:%d, blockData:%p", dataStrLen, buf); + END: + if (code != TSDB_CODE_SUCCESS) { + taosMemoryFree(buf); + tqError("%s failed at %d, failed to add block data to response:%s", __FUNCTION__, lino, tstrerror(code)); + } + return code; +} static int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t numOfCols, int8_t precision) { int32_t code = 0; @@ -25,7 +52,7 @@ static int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, TSDB_CHECK_NULL(buf, code, lino, END, terrno); SRetrieveTableRspForTmq* pRetrieve = (SRetrieveTableRspForTmq*)buf; - pRetrieve->version = 1; + pRetrieve->version = RETRIEVE_TABLE_RSP_TMQ_VERSION; pRetrieve->precision = precision; pRetrieve->compressed = 0; pRetrieve->numOfRows = htobe64((int64_t)pBlock->info.rows); @@ -36,13 +63,14 @@ static int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, actualLen += sizeof(SRetrieveTableRspForTmq); TSDB_CHECK_NULL(taosArrayPush(pRsp->blockDataLen, &actualLen), code, lino, END, terrno); TSDB_CHECK_NULL(taosArrayPush(pRsp->blockData, &buf), code, lino, END, terrno); + pRsp->blockDataElementFree = true; + tqTrace("tqAddBlockDataToRsp add block data to block array, blockDataLen:%d, blockData:%p", dataStrLen, buf); - buf = NULL; END: - if (code != 0){ + if (code != TSDB_CODE_SUCCESS){ + taosMemoryFree(buf); tqError("%s failed at line %d with msg:%s", __func__, lino, tstrerror(code)); } - taosMemoryFree(buf); return code; } @@ -66,7 +94,7 @@ static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, SMqDataRsp* pRsp, i tqError("failed to push tbName to blockTbName:%s, uid:%"PRId64, tbName, uid); continue; } - tqDebug("add tbName to response success tbname:%s, uid:%"PRId64, tbName, uid); + tqTrace("add tbName to response success tbname:%s, uid:%"PRId64, tbName, uid); } END: @@ -234,7 +262,7 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, SMqBat tbName = NULL; } if (pRsp->withSchema) { - SSchemaWrapper* pSW = tCloneSSchemaWrapper(qExtractSchemaFromTask(task)); + pSW = tCloneSSchemaWrapper(qExtractSchemaFromTask(task)); TSDB_CHECK_NULL(pSW, code, lino, END, terrno); TSDB_CHECK_NULL(taosArrayPush(pRsp->blockSchema, &pSW), code, lino, END, terrno); pSW = NULL; @@ -290,45 +318,8 @@ END: return code; } -static int32_t buildCreateTbInfo(SMqDataRsp* pRsp, SVCreateTbReq* pCreateTbReq){ - int32_t code = 0; - int32_t lino = 0; - void* createReq = NULL; - TSDB_CHECK_NULL(pRsp, code, lino, END, TSDB_CODE_INVALID_PARA); - TSDB_CHECK_NULL(pCreateTbReq, code, lino, END, TSDB_CODE_INVALID_PARA); - - if (pRsp->createTableNum == 0) { - pRsp->createTableLen = taosArrayInit(0, sizeof(int32_t)); - TSDB_CHECK_NULL(pRsp->createTableLen, code, lino, END, terrno); - pRsp->createTableReq = taosArrayInit(0, sizeof(void*)); - TSDB_CHECK_NULL(pRsp->createTableReq, code, lino, END, terrno); - } - - uint32_t len = 0; - tEncodeSize(tEncodeSVCreateTbReq, pCreateTbReq, len, code); - TSDB_CHECK_CODE(code, lino, END); - createReq = taosMemoryCalloc(1, len); - TSDB_CHECK_NULL(createReq, code, lino, END, terrno); - - SEncoder encoder = {0}; - tEncoderInit(&encoder, createReq, len); - code = tEncodeSVCreateTbReq(&encoder, pCreateTbReq); - tEncoderClear(&encoder); - TSDB_CHECK_CODE(code, lino, END); - TSDB_CHECK_NULL(taosArrayPush(pRsp->createTableLen, &len), code, lino, END, terrno); - TSDB_CHECK_NULL(taosArrayPush(pRsp->createTableReq, &createReq), code, lino, END, terrno); - pRsp->createTableNum++; - tqDebug("build create table info msg success"); - -END: - if (code != 0){ - tqError("%s failed at %d, failed to build create table info msg:%s", __FUNCTION__, lino, tstrerror(code)); - taosMemoryFree(createReq); - } - return code; -} - -static void tqProcessSubData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, int32_t* totalRows, int8_t sourceExcluded){ +static void tqProcessSubData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, int32_t* totalRows, + const SMqPollReq* pRequest, SArray* rawList){ int32_t code = 0; int32_t lino = 0; SArray* pBlocks = NULL; @@ -342,78 +333,161 @@ static void tqProcessSubData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, int pSchemas = taosArrayInit(0, sizeof(void*)); TSDB_CHECK_NULL(pSchemas, code, lino, END, terrno); - SSubmitTbData* pSubmitTbDataRet = NULL; - int64_t createTime = INT64_MAX; - code = tqRetrieveTaosxBlock(pReader, pBlocks, pSchemas, &pSubmitTbDataRet, &createTime); + SSubmitTbData* pSubmitTbData = NULL; + code = tqRetrieveTaosxBlock(pReader, pRsp, pBlocks, pSchemas, &pSubmitTbData, rawList, pHandle->fetchMeta); TSDB_CHECK_CODE(code, lino, END); - bool tmp = (pSubmitTbDataRet->flags & sourceExcluded) != 0; + bool tmp = (pSubmitTbData->flags & pRequest->sourceExcluded) != 0; TSDB_CHECK_CONDITION(!tmp, code, lino, END, TSDB_CODE_SUCCESS); + + + int32_t blockNum = taosArrayGetSize(pBlocks) == 0 ? 1 : taosArrayGetSize(pBlocks); if (pRsp->withTbName) { int64_t uid = pExec->pTqReader->lastBlkUid; - code = tqAddTbNameToRsp(pTq, uid, pRsp, taosArrayGetSize(pBlocks)); + code = tqAddTbNameToRsp(pTq, uid, pRsp, blockNum); TSDB_CHECK_CODE(code, lino, END); } - if (pHandle->fetchMeta != WITH_DATA && pSubmitTbDataRet->pCreateTbReq != NULL) { - if (pSubmitTbDataRet->ctimeMs - createTime <= 1000) { // judge if table is already created to avoid sending crateTbReq - code = buildCreateTbInfo(pRsp, pSubmitTbDataRet->pCreateTbReq); - TSDB_CHECK_CODE(code, lino, END); - } - } - tmp = (pHandle->fetchMeta == ONLY_META && pSubmitTbDataRet->pCreateTbReq == NULL); + + tmp = (pHandle->fetchMeta == ONLY_META && pSubmitTbData->pCreateTbReq == NULL); TSDB_CHECK_CONDITION(!tmp, code, lino, END, TSDB_CODE_SUCCESS); - for (int32_t i = 0; i < taosArrayGetSize(pBlocks); i++) { - SSDataBlock* pBlock = taosArrayGet(pBlocks, i); - if (pBlock == NULL) { - continue; + for (int32_t i = 0; i < blockNum; i++) { + if (taosArrayGetSize(pBlocks) == 0){ + void* rawData = taosArrayGetP(rawList, pReader->nextBlk - 1); + if (rawData == NULL) { + continue; + } + if (tqAddRawDataToRsp(rawData, pRsp, pTq->pVnode->config.tsdbCfg.precision) != 0){ + tqError("vgId:%d, failed to add block to rsp msg", pTq->pVnode->config.vgId); + continue; + } + *totalRows += *(uint32_t *)rawData + INT_BYTES; // bytes actually + } else { + SSDataBlock* pBlock = taosArrayGet(pBlocks, i); + if (pBlock == NULL) { + continue; + } + + if (tqAddBlockDataToRsp(pBlock, pRsp, taosArrayGetSize(pBlock->pDataBlock), pTq->pVnode->config.tsdbCfg.precision) != 0){ + tqError("vgId:%d, failed to add block to rsp msg", pTq->pVnode->config.vgId); + continue; + } + *totalRows += pBlock->info.rows; } - if (tqAddBlockDataToRsp(pBlock, pRsp, taosArrayGetSize(pBlock->pDataBlock), pTq->pVnode->config.tsdbCfg.precision) != 0){ - tqError("vgId:%d, failed to add block to rsp msg", pTq->pVnode->config.vgId); - continue; - } - *totalRows += pBlock->info.rows; - blockDataFreeRes(pBlock); - SSchemaWrapper* pSW = taosArrayGetP(pSchemas, i); - if (taosArrayPush(pRsp->blockSchema, &pSW) == NULL){ + + void** pSW = taosArrayGet(pSchemas, i); + if (taosArrayPush(pRsp->blockSchema, pSW) == NULL){ tqError("vgId:%d, failed to add schema to rsp msg", pTq->pVnode->config.vgId); continue; } + *pSW = NULL; pRsp->blockNum++; } - tqDebug("vgId:%d, process sub data success, response blocknum:%d, rows:%d", pTq->pVnode->config.vgId, pRsp->blockNum, *totalRows); + tqTrace("vgId:%d, process sub data success, response blocknum:%d, rows:%d", pTq->pVnode->config.vgId, pRsp->blockNum, *totalRows); END: - if (code != 0){ + if (code != 0) { tqError("%s failed at %d, failed to process sub data:%s", __FUNCTION__, lino, tstrerror(code)); - taosArrayDestroyEx(pBlocks, (FDelete)blockDataFreeRes); - taosArrayDestroyP(pSchemas, (FDelete)tDeleteSchemaWrapper); - } else { - taosArrayDestroy(pBlocks); - taosArrayDestroy(pSchemas); + } + taosArrayDestroyEx(pBlocks, (FDelete)blockDataFreeRes); + taosArrayDestroyP(pSchemas, (FDelete)tDeleteSchemaWrapper); +} + +static void preProcessSubmitMsg(STqHandle* pHandle, const SMqPollReq* pRequest, SArray** rawList){ + STqExecHandle* pExec = &pHandle->execHandle; + STqReader* pReader = pExec->pTqReader; + int32_t blockSz = taosArrayGetSize(pReader->submit.aSubmitTbData); + for (int32_t i = 0; i < blockSz; i++){ + SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, i); + if (pSubmitTbData== NULL){ + taosArrayDestroy(*rawList); + *rawList = NULL; + return; + } + + int64_t uid = pSubmitTbData->uid; + if (taosHashGet(pRequest->uidHash, &uid, LONG_BYTES) != NULL) { + tqDebug("poll rawdata split,uid:%" PRId64 " is already exists", uid); + terrno = TSDB_CODE_TMQ_RAW_DATA_SPLIT; + return; + } else { + int32_t code = taosHashPut(pRequest->uidHash, &uid, LONG_BYTES, &uid, LONG_BYTES); + if (code != 0){ + tqError("failed to add table uid to hash, code:%d, uid:%"PRId64, code, uid); + } + } + + if (pSubmitTbData->pCreateTbReq == NULL){ + continue; + } + + int64_t createTime = INT64_MAX; + int64_t *cTime = (int64_t*)taosHashGet(pHandle->tableCreateTimeHash, &uid, LONG_BYTES); + if (cTime != NULL){ + createTime = *cTime; + } else{ + createTime = metaGetTableCreateTime(pReader->pVnodeMeta, uid, 1); + if (createTime != INT64_MAX){ + int32_t code = taosHashPut(pHandle->tableCreateTimeHash, &uid, LONG_BYTES, &createTime, LONG_BYTES); + if (code != 0){ + tqError("failed to add table create time to hash,code:%d, uid:%"PRId64, code, uid); + } + } + } + if (pHandle->fetchMeta == WITH_DATA || pSubmitTbData->ctimeMs > createTime){ + tDestroySVCreateTbReq(pSubmitTbData->pCreateTbReq, TSDB_MSG_FLG_DECODE); + pSubmitTbData->pCreateTbReq = NULL; + } else{ + taosArrayDestroy(*rawList); + *rawList = NULL; + } } } -int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, SMqDataRsp* pRsp, int32_t* totalRows, int8_t sourceExcluded) { +int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, SMqDataRsp* pRsp, int32_t* totalRows, const SMqPollReq* pRequest) { int32_t code = 0; int32_t lino = 0; - TSDB_CHECK_NULL(pRsp, code, lino, END, TSDB_CODE_INVALID_PARA); - TSDB_CHECK_NULL(pTq, code, lino, END, TSDB_CODE_INVALID_PARA); - TSDB_CHECK_NULL(pHandle, code, lino, END, TSDB_CODE_INVALID_PARA); - TSDB_CHECK_NULL(totalRows, code, lino, END, TSDB_CODE_INVALID_PARA); STqExecHandle* pExec = &pHandle->execHandle; STqReader* pReader = pExec->pTqReader; - code = tqReaderSetSubmitMsg(pReader, submit.msgStr, submit.msgLen, submit.ver); + SArray *rawList = NULL; + if (pRequest->rawData){ + rawList = taosArrayInit(0, POINTER_BYTES); + TSDB_CHECK_NULL(rawList, code, lino, END, terrno); + } + code = tqReaderSetSubmitMsg(pReader, submit.msgStr, submit.msgLen, submit.ver, rawList); TSDB_CHECK_CODE(code, lino, END); + if (pRequest->rawData) { + preProcessSubmitMsg(pHandle, pRequest, &rawList); + } + // data could not contains same uid data in rawdata mode + if (pRequest->rawData != 0 && terrno == TSDB_CODE_TMQ_RAW_DATA_SPLIT){ + goto END; + } + + // this submit data is metadata and previous data is rawdata + if (pRequest->rawData != 0 && *totalRows > 0 && pRsp->createTableNum == 0 && rawList == NULL){ + tqDebug("poll rawdata split,vgId:%d, this wal submit data contains metadata and previous data is data", pTq->pVnode->config.vgId); + terrno = TSDB_CODE_TMQ_RAW_DATA_SPLIT; + goto END; + } + + // this submit data is rawdata and previous data is metadata + if (pRequest->rawData != 0 && pRsp->createTableNum > 0 && rawList != NULL){ + tqDebug("poll rawdata split,vgId:%d, this wal submit data is data and previous data is metadata", pTq->pVnode->config.vgId); + terrno = TSDB_CODE_TMQ_RAW_DATA_SPLIT; + goto END; + } if (pExec->subType == TOPIC_SUB_TYPE__TABLE) { while (tqNextBlockImpl(pReader, NULL)) { - tqProcessSubData(pTq, pHandle, pRsp, totalRows, sourceExcluded); + tqProcessSubData(pTq, pHandle, pRsp, totalRows, pRequest, rawList); } } else if (pExec->subType == TOPIC_SUB_TYPE__DB) { while (tqNextDataBlockFilterOut(pReader, pExec->execDb.pFilterOutTbUid)) { - tqProcessSubData(pTq, pHandle, pRsp, totalRows, sourceExcluded); + tqProcessSubData(pTq, pHandle, pRsp, totalRows, pRequest, rawList); } } END: + tqReaderClearSubmitMsg(pReader); + taosArrayDestroy(rawList); if (code != 0){ tqError("%s failed at %d, failed to scan log:%s", __FUNCTION__, lino, tstrerror(code)); } diff --git a/source/dnode/vnode/src/tq/tqStreamNotify.c b/source/dnode/vnode/src/tq/tqStreamNotify.c index 61200b189b..1c64c404e6 100644 --- a/source/dnode/vnode/src/tq/tqStreamNotify.c +++ b/source/dnode/vnode/src/tq/tqStreamNotify.c @@ -21,9 +21,6 @@ #endif #define STREAM_EVENT_NOTIFY_RETRY_MS 50 // 50 ms -#define STREAM_EVENT_NOTIFY_MESSAAGE_SIZE_KB 8 * 1024 // 8 MB -#define STREAM_EVENT_NOTIFY_FRAME_SIZE 256 * 1024 // 256 KB - typedef struct SStreamNotifyHandle { TdThreadMutex mutex; #ifndef WINDOWS @@ -296,7 +293,7 @@ static int32_t packupStreamNotifyEvent(const char* streamName, const SArray* pBl msgLen += varDataLen(val) + 1; } *nNotifyEvents += pDataBlock->info.rows; - if (msgLen >= STREAM_EVENT_NOTIFY_MESSAAGE_SIZE_KB * 1024) { + if (msgLen >= tsStreamNotifyMessageSize * 1024) { break; } } @@ -381,7 +378,7 @@ static int32_t sendSingleStreamNotify(SStreamNotifyHandle* pHandle, char* msg) { } sentLen = 0; while (sentLen < totalLen) { - size_t chunkSize = TMIN(totalLen - sentLen, STREAM_EVENT_NOTIFY_FRAME_SIZE); + size_t chunkSize = TMIN(totalLen - sentLen, tsStreamNotifyFrameSize * 1024); if (sentLen == 0) { res = curl_ws_send(pHandle->curl, msg, chunkSize, &nbytes, totalLen, CURLWS_TEXT | CURLWS_OFFSET); TSDB_CHECK_CONDITION(res == CURLE_OK, code, lino, _end, TSDB_CODE_FAILED); diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 197a45cdb9..44d9ffda83 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -131,7 +131,7 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand } tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, (latest) offset reset to %" PRId64, consumerId, pHandle->subKey, vgId, dataRsp.rspOffset.version); - code = tqSendDataRsp(pHandle, pMsg, pRequest, &dataRsp, TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); + code = tqSendDataRsp(pHandle, pMsg, pRequest, &dataRsp, (pRequest->rawData == 1) ? TMQ_MSG_TYPE__POLL_RAW_DATA_RSP : TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); tDeleteMqDataRsp(&dataRsp); *pBlockReturned = true; @@ -222,6 +222,10 @@ end: static void tDeleteCommon(void* parm) {} +#define POLL_RSP_TYPE(pRequest,taosxRsp) \ +taosxRsp.createTableNum > 0 ? TMQ_MSG_TYPE__POLL_DATA_META_RSP : \ +(pRequest->rawData == 1 ? TMQ_MSG_TYPE__POLL_RAW_DATA_RSP : TMQ_MSG_TYPE__POLL_DATA_RSP) + static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg, STqOffsetVal* offset) { int32_t vgId = TD_VID(pTq->pVnode); @@ -274,7 +278,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, } tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer); code = tqSendDataRsp(pHandle, pMsg, pRequest, &taosxRsp, - taosxRsp.createTableNum > 0 ? TMQ_MSG_TYPE__POLL_DATA_META_RSP : TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); + POLL_RSP_TYPE(pRequest, taosxRsp), vgId); goto END; } @@ -287,7 +291,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, if (totalRows > 0) { tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer); code = tqSendDataRsp(pHandle, pMsg, pRequest, &taosxRsp, - taosxRsp.createTableNum > 0 ? TMQ_MSG_TYPE__POLL_DATA_META_RSP : TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); + POLL_RSP_TYPE(pRequest, taosxRsp), vgId); goto END; } @@ -370,12 +374,19 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, .ver = pHead->version, }; - TQ_ERR_GO_TO_END(tqTaosxScanLog(pTq, pHandle, submit, &taosxRsp, &totalRows, pRequest->sourceExcluded)); + TQ_ERR_GO_TO_END(tqTaosxScanLog(pTq, pHandle, submit, &taosxRsp, &totalRows, pRequest)); - if (totalRows >= tmqRowSize || (taosGetTimestampMs() - st > TMIN(TQ_POLL_MAX_TIME, pRequest->timeout))) { - tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer + 1); + if ((pRequest->rawData == 0 && totalRows >= tmqRowSize) || + (taosGetTimestampMs() - st > TMIN(TQ_POLL_MAX_TIME, pRequest->timeout)) || + (pRequest->rawData != 0 && (totalRows >= TQ_POLL_MAX_BYTES || + taosArrayGetSize(taosxRsp.blockData) > tmqRowSize || + terrno == TSDB_CODE_TMQ_RAW_DATA_SPLIT))) { +// tqDebug("start to send rsp, block num:%d, totalRows:%d, createTableNum:%d, terrno:%d", +// (int)taosArrayGetSize(taosxRsp.blockData), totalRows, taosxRsp.createTableNum, terrno); + tqOffsetResetToLog(&taosxRsp.rspOffset, terrno == TSDB_CODE_TMQ_RAW_DATA_SPLIT ? fetchVer : fetchVer + 1); code = tqSendDataRsp(pHandle, pMsg, pRequest, &taosxRsp, - taosxRsp.createTableNum > 0 ? TMQ_MSG_TYPE__POLL_DATA_META_RSP : TMQ_MSG_TYPE__POLL_DATA_RSP, vgId); + POLL_RSP_TYPE(pRequest, taosxRsp), vgId); + if (terrno == TSDB_CODE_TMQ_RAW_DATA_SPLIT){terrno = 0;} goto END; } else { fetchVer++; @@ -526,7 +537,7 @@ int32_t tqSendMetaPollRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPoll return 0; } -int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp* pRsp, int32_t epoch, int64_t consumerId, +int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, SMqDataRsp* pRsp, int32_t epoch, int64_t consumerId, int32_t type, int64_t sver, int64_t ever) { if (pRpcHandleInfo == NULL || pRsp == NULL) { return TSDB_CODE_TMQ_INVALID_MSG; @@ -534,7 +545,12 @@ int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp* int32_t len = 0; int32_t code = 0; - if (type == TMQ_MSG_TYPE__POLL_DATA_RSP || type == TMQ_MSG_TYPE__WALINFO_RSP) { + if (type == TMQ_MSG_TYPE__POLL_RAW_DATA_RSP){ + pRsp->withSchema = 0; + } + if (type == TMQ_MSG_TYPE__POLL_DATA_RSP || + type == TMQ_MSG_TYPE__WALINFO_RSP || + type == TMQ_MSG_TYPE__POLL_RAW_DATA_RSP) { tEncodeSize(tEncodeMqDataRsp, pRsp, len, code); } else if (type == TMQ_MSG_TYPE__POLL_DATA_META_RSP) { tEncodeSize(tEncodeSTaosxRsp, pRsp, len, code); @@ -558,7 +574,9 @@ int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp* SEncoder encoder = {0}; tEncoderInit(&encoder, abuf, len); - if (type == TMQ_MSG_TYPE__POLL_DATA_RSP || type == TMQ_MSG_TYPE__WALINFO_RSP) { + if (type == TMQ_MSG_TYPE__POLL_DATA_RSP || + type == TMQ_MSG_TYPE__WALINFO_RSP || + type == TMQ_MSG_TYPE__POLL_RAW_DATA_RSP) { code = tEncodeMqDataRsp(&encoder, pRsp); } else if (type == TMQ_MSG_TYPE__POLL_DATA_META_RSP) { code = tEncodeSTaosxRsp(&encoder, pRsp); diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index c7b1a816cd..d765861381 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -530,6 +530,8 @@ _exit: int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad) { SSyncState state = syncGetState(pVnode->sync); + pLoad->syncAppliedIndex = pVnode->state.applied; + syncGetCommitIndex(pVnode->sync, &pLoad->syncCommitIndex); pLoad->vgId = TD_VID(pVnode); pLoad->syncState = state.state; @@ -739,7 +741,7 @@ int32_t vnodeGetCtbNum(SVnode *pVnode, int64_t suid, int64_t *num) { } int32_t vnodeGetStbColumnNum(SVnode *pVnode, tb_uid_t suid, int *num) { - SSchemaWrapper *pSW = metaGetTableSchema(pVnode->pMeta, suid, -1, 0, NULL); + SSchemaWrapper *pSW = metaGetTableSchema(pVnode->pMeta, suid, -1, 0); if (pSW) { *num = pSW->nCols; tDeleteSchemaWrapper(pSW); diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index a6208871f6..abaa61744d 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -314,25 +314,25 @@ static int32_t vnodePreProcessSubmitTbData(SVnode *pVnode, SDecoder *pCoder, int uint64_t nColData; if (tDecodeU64v(pCoder, &nColData) < 0) { code = TSDB_CODE_INVALID_MSG; - goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); } SColData colData = {0}; code = tDecodeColData(version, pCoder, &colData); if (code) { code = TSDB_CODE_INVALID_MSG; - goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); } if (colData.flag != HAS_VALUE) { code = TSDB_CODE_INVALID_MSG; - goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); } for (int32_t iRow = 0; iRow < colData.nVal; iRow++) { if (((TSKEY *)colData.pData)[iRow] < minKey || ((TSKEY *)colData.pData)[iRow] > maxKey) { code = TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE; - goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); } } @@ -340,14 +340,14 @@ static int32_t vnodePreProcessSubmitTbData(SVnode *pVnode, SDecoder *pCoder, int code = tDecodeColData(version, pCoder, &colData); if (code) { code = TSDB_CODE_INVALID_MSG; - goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); } } } else { uint64_t nRow; if (tDecodeU64v(pCoder, &nRow) < 0) { code = TSDB_CODE_INVALID_MSG; - goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); } for (int32_t iRow = 0; iRow < nRow; ++iRow) { @@ -356,7 +356,7 @@ static int32_t vnodePreProcessSubmitTbData(SVnode *pVnode, SDecoder *pCoder, int if (pRow->ts < minKey || pRow->ts > maxKey) { code = TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE; - goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); } } } @@ -369,6 +369,9 @@ static int32_t vnodePreProcessSubmitTbData(SVnode *pVnode, SDecoder *pCoder, int tEndDecode(pCoder); _exit: + if (code) { + vError("vgId:%d, %s:%d failed to vnodePreProcessSubmitTbData submit request since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); + } return code; } static int32_t vnodePreProcessSubmitMsg(SVnode *pVnode, SRpcMsg *pMsg) { @@ -1889,7 +1892,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, in len -= sizeof(SSubmitReq2Msg); SDecoder dc = {0}; tDecoderInit(&dc, pReq, len); - if (tDecodeSubmitReq(&dc, pSubmitReq) < 0) { + if (tDecodeSubmitReq(&dc, pSubmitReq, NULL) < 0) { code = TSDB_CODE_INVALID_MSG; goto _exit; } diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index 649a7a4524..7349feb70f 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -398,16 +398,16 @@ static int32_t doScanCacheNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) { // check for tag values if (pInfo->pRes->info.rows > 0) { - if (pInfo->pseudoExprSup.numOfExprs > 0) { - SExprSupp* pSup = &pInfo->pseudoExprSup; + SExprSupp* pSup = &pInfo->pseudoExprSup; - STableKeyInfo* pKeyInfo = &((STableKeyInfo*)pList)[0]; - pInfo->pRes->info.id.groupId = pKeyInfo->groupId; + STableKeyInfo* pKeyInfo = &((STableKeyInfo*)pList)[0]; + pInfo->pRes->info.id.groupId = pKeyInfo->groupId; - if (taosArrayGetSize(pInfo->pUidList) > 0) { - void* pUid = taosArrayGet(pInfo->pUidList, 0); - QUERY_CHECK_NULL(pUid, code, lino, _end, terrno); - pInfo->pRes->info.id.uid = *(tb_uid_t*)pUid; + if (taosArrayGetSize(pInfo->pUidList) > 0) { + void* pUid = taosArrayGet(pInfo->pUidList, 0); + QUERY_CHECK_NULL(pUid, code, lino, _end, terrno); + pInfo->pRes->info.id.uid = *(tb_uid_t*)pUid; + if (pInfo->pseudoExprSup.numOfExprs > 0) { code = addTagPseudoColumnData(&pInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pInfo->pRes, pInfo->pRes->info.rows, pTaskInfo, NULL); QUERY_CHECK_CODE(code, lino, _end); diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c index ec041cba70..3bd6f4e64a 100644 --- a/source/libs/executor/src/dataInserter.c +++ b/source/libs/executor/src/dataInserter.c @@ -187,7 +187,7 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp terrno = TSDB_CODE_SUCCESS; if (NULL == pReq) { - if (!(pReq = taosMemoryMalloc(sizeof(SSubmitReq2)))) { + if (!(pReq = taosMemoryCalloc(1, sizeof(SSubmitReq2)))) { goto _end; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 1060cbcffe..4f647a2e9c 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -3953,7 +3953,7 @@ FETCH_NEXT_BLOCK: QUERY_CHECK_NULL(pSubmit, code, lino, _end, terrno); qDebug("set %d/%d as the input submit block, %s", current + 1, totalBlocks, id); - if (pAPI->tqReaderFn.tqReaderSetSubmitMsg(pInfo->tqReader, pSubmit->msgStr, pSubmit->msgLen, pSubmit->ver) < + if (pAPI->tqReaderFn.tqReaderSetSubmitMsg(pInfo->tqReader, pSubmit->msgStr, pSubmit->msgLen, pSubmit->ver, NULL) < 0) { qError("submit msg messed up when initializing stream submit block %p, current %d/%d, %s", pSubmit, current, totalBlocks, id); diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index 3b799eea23..fbb55301cd 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -2977,7 +2977,6 @@ static int32_t rebuildSessionWindow(SOperatorInfo* pOperator, SArray* pWinArray, if (winCode == TSDB_CODE_SUCCESS && inWinRange(&pWinKey->win, &childWin.sessionWin.win)) { if (num == 0) { - int32_t winCode = TSDB_CODE_SUCCESS; code = setSessionOutputBuf(pAggSup, pWinKey->win.skey, pWinKey->win.ekey, pWinKey->groupId, &parentWin, &winCode); QUERY_CHECK_CODE(code, lino, _end); diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 161c5f7ca7..0e7a719c2c 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -510,6 +510,7 @@ static int32_t logicScanCopy(const SScanLogicNode* pSrc, SScanLogicNode* pDst) { COPY_SCALAR_FIELD(paraTablesSort); COPY_SCALAR_FIELD(smallDataTsSort); COPY_SCALAR_FIELD(needSplit); + COPY_SCALAR_FIELD(noPseudoRefAfterGrp); return TSDB_CODE_SUCCESS; } @@ -849,11 +850,6 @@ static int32_t physiDispatchCopy(const SDataDispatcherNode* pSrc, SDataDispatche return TSDB_CODE_SUCCESS; } -static int32_t physiInserterCopy(const SDataInserterNode* pSrc, SDataInserterNode* pDst) { - COPY_BASE_OBJECT_FIELD(sink, dataSinkNodeCopy); - return TSDB_CODE_SUCCESS; -} - static int32_t physiQueryInserterCopy(const SQueryInserterNode* pSrc, SQueryInserterNode* pDst) { COPY_BASE_OBJECT_FIELD(sink, dataSinkNodeCopy); CLONE_NODE_LIST_FIELD(pCols); @@ -1135,9 +1131,6 @@ int32_t nodesCloneNode(const SNode* pNode, SNode** ppNode) { case QUERY_NODE_PHYSICAL_PLAN_DISPATCH: code = physiDispatchCopy((const SDataDispatcherNode*)pNode, (SDataDispatcherNode*)pDst); break; - //case QUERY_NODE_PHYSICAL_PLAN_INSERT: - // code = physiInserterCopy((const SDataInserterNode*)pNode, (SDataInserterNode*)pDst); - // break; case QUERY_NODE_PHYSICAL_PLAN_QUERY_INSERT: code = physiQueryInserterCopy((const SQueryInserterNode*)pNode, (SQueryInserterNode*)pDst); break; diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 7006979216..9473e75642 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -2000,7 +2000,7 @@ void nodesDestroyNode(SNode* pNode) { case QUERY_NODE_PHYSICAL_PLAN_INSERT: { SDataInserterNode* pSink = (SDataInserterNode*)pNode; destroyDataSinkNode((SDataSinkNode*)pSink); - taosMemoryFreeClear(pSink->pData); + taosMemFreeClear(pSink->pData); break; } case QUERY_NODE_PHYSICAL_PLAN_QUERY_INSERT: { diff --git a/source/libs/parser/src/parInsertSml.c b/source/libs/parser/src/parInsertSml.c index 676aed4464..bd463bfd9d 100644 --- a/source/libs/parser/src/parInsertSml.c +++ b/source/libs/parser/src/parInsertSml.c @@ -403,3 +403,18 @@ end: } return code; } + +int32_t smlBuildOutputRaw(SQuery* handle, SHashObj* pVgHash) { + int32_t lino = 0; + int32_t code = 0; + + SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)(handle)->pRoot; + code = insBuildVgDataBlocks(pVgHash, pStmt->pVgDataBlocks, &pStmt->pDataBlocks, false); + TSDB_CHECK_CODE(code, lino, end); + + end: + if (code != 0) { + uError("%s failed at %d since %s", __func__, lino, tstrerror(code)); + } + return code; +} diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index ed1f498a32..1931f0803d 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -483,7 +483,7 @@ static int32_t fillVgroupDataCxt(STableDataCxt* pTableCxt, SVgroupDataCxt* pVgCx return code; } -static int32_t createVgroupDataCxt(STableDataCxt* pTableCxt, SHashObj* pVgroupHash, SArray* pVgroupList, +static int32_t createVgroupDataCxt(int32_t vgId, SHashObj* pVgroupHash, SArray* pVgroupList, SVgroupDataCxt** pOutput) { SVgroupDataCxt* pVgCxt = taosMemoryCalloc(1, sizeof(SVgroupDataCxt)); if (NULL == pVgCxt) { @@ -495,7 +495,7 @@ static int32_t createVgroupDataCxt(STableDataCxt* pTableCxt, SHashObj* pVgroupHa return terrno; } - pVgCxt->vgId = pTableCxt->pMeta->vgId; + pVgCxt->vgId = vgId; int32_t code = taosHashPut(pVgroupHash, &pVgCxt->vgId, sizeof(pVgCxt->vgId), &pVgCxt, POINTER_BYTES); if (TSDB_CODE_SUCCESS == code) { if (NULL == taosArrayPush(pVgroupList, &pVgCxt)) { @@ -642,7 +642,7 @@ int32_t insAppendStmtTableDataCxt(SHashObj* pAllVgHash, STableColsData* pTbData, if (NULL == pp) { pp = taosHashGet(pBuildInfo->pVgroupHash, &vgId, sizeof(vgId)); if (NULL == pp) { - code = createVgroupDataCxt(pTbCtx, pBuildInfo->pVgroupHash, pBuildInfo->pVgroupList, &pVgCxt); + code = createVgroupDataCxt(vgId, pBuildInfo->pVgroupHash, pBuildInfo->pVgroupList, &pVgCxt); } else { pVgCxt = *(SVgroupDataCxt**)pp; } @@ -723,7 +723,7 @@ int32_t tbNum) { SHashObj* pVgroupHash = taosHashInit(128, taosGetDefaultHashFun */ int32_t insMergeTableDataCxt(SHashObj* pTableHash, SArray** pVgDataBlocks, bool isRebuild) { - SHashObj* pVgroupHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false); + SHashObj* pVgroupHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); SArray* pVgroupList = taosArrayInit(8, POINTER_BYTES); if (NULL == pVgroupHash || NULL == pVgroupList) { taosHashCleanup(pVgroupHash); @@ -777,7 +777,7 @@ int32_t insMergeTableDataCxt(SHashObj* pTableHash, SArray** pVgDataBlocks, bool int32_t vgId = pTableCxt->pMeta->vgId; void** pp = taosHashGet(pVgroupHash, &vgId, sizeof(vgId)); if (NULL == pp) { - code = createVgroupDataCxt(pTableCxt, pVgroupHash, pVgroupList, &pVgCxt); + code = createVgroupDataCxt(vgId, pVgroupHash, pVgroupList, &pVgCxt); } else { pVgCxt = *(SVgroupDataCxt**)pp; } @@ -830,6 +830,7 @@ static int32_t buildSubmitReq(int32_t vgId, SSubmitReq2* pReq, void** pData, uin } static void destroyVgDataBlocks(void* p) { + if (p == NULL) return; SVgDataBlocks* pVg = p; taosMemoryFree(pVg->pData); taosMemoryFree(pVg); @@ -855,7 +856,6 @@ int32_t insBuildVgDataBlocks(SHashObj* pVgroupsHashObj, SArray* pVgDataCxtList, if (TSDB_CODE_SUCCESS == code) { dst->numOfTables = taosArrayGetSize(src->pData->aSubmitTbData); code = taosHashGetDup(pVgroupsHashObj, (const char*)&src->vgId, sizeof(src->vgId), &dst->vg); - // uError("td23101 3vgId:%d, numEps:%d", src->vgId, dst->vg.epSet.numOfEps); } if (TSDB_CODE_SUCCESS == code) { code = buildSubmitReq(src->vgId, src->pData, &dst->pData, &dst->size); @@ -863,6 +863,9 @@ int32_t insBuildVgDataBlocks(SHashObj* pVgroupsHashObj, SArray* pVgDataCxtList, if (TSDB_CODE_SUCCESS == code) { code = (NULL == taosArrayPush(pDataBlocks, &dst) ? terrno : TSDB_CODE_SUCCESS); } + if (TSDB_CODE_SUCCESS != code) { + destroyVgDataBlocks(dst); + } } if (append) { @@ -1074,3 +1077,33 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate end: return ret; } + +int rawBlockBindRawData(SHashObj* pVgroupHash, SArray* pVgroupList, STableMeta* pTableMeta, void* data) { + transformRawSSubmitTbData(data, pTableMeta->suid, pTableMeta->uid, pTableMeta->sversion); + SVgroupDataCxt* pVgCxt = NULL; + void** pp = taosHashGet(pVgroupHash, &pTableMeta->vgId, sizeof(pTableMeta->vgId)); + if (NULL == pp) { + int code = createVgroupDataCxt(pTableMeta->vgId, pVgroupHash, pVgroupList, &pVgCxt); + if (code != 0){ + return code; + } + } else { + pVgCxt = *(SVgroupDataCxt**)pp; + } + if (NULL == pVgCxt->pData->aSubmitTbData) { + pVgCxt->pData->aSubmitTbData = taosArrayInit(0, POINTER_BYTES); + pVgCxt->pData->raw = true; + if (NULL == pVgCxt->pData->aSubmitTbData) { + return terrno; + } + } + + // push data to submit, rebuild empty data for next submit + if (NULL == taosArrayPush(pVgCxt->pData->aSubmitTbData, &data)) { + return terrno; + } + + uTrace("add raw data to vgId:%d, len:%d", pTableMeta->vgId, *(int32_t*)data); + + return 0; +} diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index c3fd9cdcf2..3f064f2b66 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -406,6 +406,47 @@ static int32_t makeScanLogicNode(SLogicPlanContext* pCxt, SRealTableNode* pRealT static bool needScanDefaultCol(EScanType scanType) { return SCAN_TYPE_TABLE_COUNT != scanType; } +static int32_t updateScanNoPseudoRefAfterGrp(SSelectStmt* pSelect, SScanLogicNode* pScan, SRealTableNode* pRealTable) { + if (NULL == pScan->pScanPseudoCols || pScan->pScanPseudoCols->length <= 0 || NULL != pSelect->pTags || NULL != pSelect->pSubtable) { + return TSDB_CODE_SUCCESS; + } + + SNodeList* pList = NULL; + int32_t code = 0; + if (NULL == pSelect->pPartitionByList || pSelect->pPartitionByList->length <= 0) { + if (NULL == pSelect->pGroupByList || pSelect->pGroupByList->length <= 0) { + return TSDB_CODE_SUCCESS; + } + + code = nodesCollectColumns(pSelect, SQL_CLAUSE_GROUP_BY, pRealTable->table.tableAlias, COLLECT_COL_TYPE_TAG, + &pList); + if (TSDB_CODE_SUCCESS == code) { + code = nodesCollectFuncs(pSelect, SQL_CLAUSE_GROUP_BY, pRealTable->table.tableAlias, fmIsScanPseudoColumnFunc, + &pList); + } + if (TSDB_CODE_SUCCESS == code && (NULL == pList || pList->length <= 0)) { + pScan->noPseudoRefAfterGrp = true; + } + goto _return; + } + + code = nodesCollectColumns(pSelect, SQL_CLAUSE_PARTITION_BY, pRealTable->table.tableAlias, COLLECT_COL_TYPE_TAG, + &pList); + if (TSDB_CODE_SUCCESS == code) { + code = nodesCollectFuncs(pSelect, SQL_CLAUSE_PARTITION_BY, pRealTable->table.tableAlias, fmIsScanPseudoColumnFunc, + &pList); + } + + if (TSDB_CODE_SUCCESS == code && (NULL == pList || pList->length <= 0)) { + pScan->noPseudoRefAfterGrp = true; + } + +_return: + + nodesDestroyList(pList); + return code; +} + static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SRealTableNode* pRealTable, SLogicNode** pLogicNode) { SScanLogicNode* pScan = NULL; @@ -437,7 +478,13 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect &pScan->pScanPseudoCols); } - pScan->scanType = getScanType(pCxt, pScan->pScanPseudoCols, pScan->pScanCols, pScan->tableType, pSelect->tagScan); + if (TSDB_CODE_SUCCESS == code) { + code = updateScanNoPseudoRefAfterGrp(pSelect, pScan, pRealTable); + } + + if (TSDB_CODE_SUCCESS == code) { + pScan->scanType = getScanType(pCxt, pScan->pScanPseudoCols, pScan->pScanCols, pScan->tableType, pSelect->tagScan); + } // rewrite the expression in subsequent clauses if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 38f2e5024f..e8a8e6889a 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -93,6 +93,12 @@ typedef struct SCpdCollectTableColCxt { int32_t errCode; } SCpdCollectTableColCxt; +typedef struct SCollectColsCxt { + SHashObj* pColHash; + int32_t errCode; +} SCollectColsCxt; + + typedef enum ECondAction { COND_ACTION_STAY = 1, COND_ACTION_PUSH_JOIN, @@ -3302,6 +3308,70 @@ static int32_t partTagsRewriteGroupTagsToFuncs(SNodeList* pGroupTags, int32_t st return code; } +static EDealRes partTagsCollectColsNodes(SNode* pNode, void* pContext) { + SCollectColsCxt* pCxt = pContext; + if (QUERY_NODE_COLUMN == nodeType(pNode)) { + SColumnNode* pCol = (SColumnNode*)pNode; + if (NULL == taosHashGet(pCxt->pColHash, pCol->colName, strlen(pCol->colName))) { + pCxt->errCode = taosHashPut(pCxt->pColHash, pCol->colName, strlen(pCol->colName), NULL, 0); + } + } + + return (TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_CONTINUE : DEAL_RES_ERROR); +} + + +static bool partTagsIsScanPseudoColsInConds(SScanLogicNode* pScan) { + SCollectColsCxt cxt = { + .errCode = TSDB_CODE_SUCCESS, + .pColHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK)}; + if (NULL == cxt.pColHash) { + return true; + } + + nodesWalkExpr(pScan->node.pConditions, partTagsCollectColsNodes, &cxt); + if (cxt.errCode) { + taosHashCleanup(cxt.pColHash); + return true; + } + + SNode* pNode = NULL; + FOREACH(pNode, pScan->pScanPseudoCols) { + if (taosHashGet(cxt.pColHash, ((SExprNode*)pNode)->aliasName, strlen(((SExprNode*)pNode)->aliasName))) { + taosHashCleanup(cxt.pColHash); + return true; + } + } + + taosHashCleanup(cxt.pColHash); + return false; +} + +static int32_t partTagsOptRemovePseudoCols(SScanLogicNode* pScan) { + if (!pScan->noPseudoRefAfterGrp || NULL == pScan->pScanPseudoCols || pScan->pScanPseudoCols->length <= 0) { + return TSDB_CODE_SUCCESS; + } + + if (pScan->node.pConditions && partTagsIsScanPseudoColsInConds(pScan)) { + return TSDB_CODE_SUCCESS; + } + + SNode* pNode = NULL, *pTarget = NULL; + FOREACH(pNode, pScan->pScanPseudoCols) { + FOREACH(pTarget, pScan->node.pTargets) { + if (0 == strcmp(((SExprNode*)pNode)->aliasName, ((SColumnNode*)pTarget)->colName)) { + ERASE_NODE(pScan->node.pTargets); + break; + } + } + } + + nodesDestroyList(pScan->pScanPseudoCols); + pScan->pScanPseudoCols = NULL; + + return TSDB_CODE_SUCCESS; +} + static int32_t partTagsOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { SLogicNode* pNode = optFindPossibleNode(pLogicSubplan->pNode, partTagsOptMayBeOptimized, NULL); if (NULL == pNode) { @@ -3362,6 +3432,9 @@ static int32_t partTagsOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSub code = partTagsRewriteGroupTagsToFuncs(pScan->pGroupTags, start, pAgg); } } + if (TSDB_CODE_SUCCESS == code) { + code = partTagsOptRemovePseudoCols(pScan); + } if (TSDB_CODE_SUCCESS == code) { code = partTagsOptRebuildTbanme(pScan->pGroupTags); } diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 9513e90c50..316699bba1 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -2839,7 +2839,8 @@ static int32_t createDataInserter(SPhysiPlanContext* pCxt, SVgDataBlocks* pBlock pInserter->numOfTables = pBlocks->numOfTables; pInserter->size = pBlocks->size; - TSWAP(pInserter->pData, pBlocks->pData); + pInserter->pData = pBlocks->pData; + pBlocks->pData = NULL; *pSink = (SDataSinkNode*)pInserter; return TSDB_CODE_SUCCESS; diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index 85b1c543c0..35f258c554 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -202,6 +202,18 @@ int32_t taosAsyncRecover() { return taskQueue.wrokrerPool.pCb->afterRecoverFromBlocking(&taskQueue.wrokrerPool); } +int32_t taosStmt2AsyncBind(__async_exec_fn_t bindFn, void* bindParam) { + SSchedMsg* pSchedMsg; + int32_t rc = taosAllocateQitem(sizeof(SSchedMsg), DEF_QITEM, 0, (void **)&pSchedMsg); + if (rc) return rc; + pSchedMsg->fp = NULL; + pSchedMsg->ahandle = bindFn; + pSchedMsg->thandle = bindParam; + // pSchedMsg->msg = code; + + return taosWriteQitem(taskQueue.pTaskQueue, pSchedMsg); +} + void destroySendMsgInfo(SMsgSendInfo* pMsgBody) { if (NULL == pMsgBody) { return; diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index a031bc08de..ec9913d6ac 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -1110,7 +1110,7 @@ _return: } int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, int32_t msgType, void* param) { - int32_t msgSize = 0; + int32_t msgSize = 0; void *msg = NULL; int32_t code = 0; bool isCandidateAddr = false; @@ -1136,13 +1136,8 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, case TDMT_VND_SUBMIT: case TDMT_VND_COMMIT: { msgSize = pTask->msgLen; - msg = taosMemoryCalloc(1, msgSize); - if (NULL == msg) { - SCH_TASK_ELOG("calloc %d failed", msgSize); - SCH_ERR_RET(terrno); - } - - TAOS_MEMCPY(msg, pTask->msg, msgSize); + msg = pTask->msg; + pTask->msg = NULL; break; } diff --git a/source/libs/scheduler/src/schStatus.c b/source/libs/scheduler/src/schStatus.c index f24ee74101..9792af22f6 100644 --- a/source/libs/scheduler/src/schStatus.c +++ b/source/libs/scheduler/src/schStatus.c @@ -22,9 +22,11 @@ #include "trpc.h" int32_t schSwitchJobStatus(SSchJob* pJob, int32_t status, void* param) { - int32_t code = 0; - SCH_ERR_JRET(schUpdateJobStatus(pJob, status)); - + int32_t code = schUpdateJobStatus(pJob, status); + if (TSDB_CODE_SUCCESS != code) { + SCH_ERR_JRET((param && *(int32_t*)param) ? *(int32_t*)param : code); + } + switch (status) { case JOB_TASK_STATUS_INIT: break; diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index d1209d08ce..f9bb48e1e0 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -679,6 +679,14 @@ SSyncState syncGetState(int64_t rid) { return state; } +void syncGetCommitIndex(int64_t rid, int64_t* syncCommitIndex) { + SSyncNode* pSyncNode = syncNodeAcquire(rid); + if (pSyncNode != NULL) { + *syncCommitIndex = pSyncNode->commitIndex; + syncNodeRelease(pSyncNode); + } +} + int32_t syncGetArbToken(int64_t rid, char* outToken) { int32_t code = 0; SSyncNode* pSyncNode = syncNodeAcquire(rid); diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index d7576005d7..b3b69d81c0 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -163,7 +163,7 @@ void* rpcMallocCont(int64_t contLen) { tTrace("malloc mem:%p size:%" PRId64, start, size); } - return start + sizeof(STransMsgHead); + return start + TRANS_MSG_OVERHEAD; } void rpcFreeCont(void* cont) { transFreeMsg(cont); } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index ba2d471ccf..f34b00bec5 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -859,6 +859,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_REPLAY_NOT_SUPPORT, "Replay is disabled TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_NO_TABLE_QUALIFIED, "No table qualified for query") TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_NO_NEED_REBALANCE, "No need rebalance") TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_INVALID_STATUS, "Invalid status, please subscribe topic first") +TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_INVALID_DATA, "Invalid data use here") +TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_RAW_DATA_SPLIT, "Split submit data for rawdata") // stream TAOS_DEFINE_ERROR(TSDB_CODE_STREAM_TASK_NOT_EXIST, "Stream task not exist") diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 0e3e236b36..2460c7c650 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -1502,31 +1502,46 @@ bool taosAssertRelease(bool condition) { } #endif +#define NUM_BASE 100 +#define DIGIT_LENGTH 2 +#define MAX_DIGITS 24 + char* u64toaFastLut(uint64_t val, char* buf) { + // Look-up table for 2-digit numbers static const char* lut = "0001020304050607080910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455" "5657585960616263646566676869707172737475767778798081828384858687888990919293949596979899"; - char temp[24]; - char* p = temp; + char temp[MAX_DIGITS]; + char* p = temp + tListLen(temp); - while (val >= 100) { - strncpy(p, lut + (val % 100) * 2, 2); - val /= 100; - p += 2; + // Process the digits greater than or equal to 100 + while (val >= NUM_BASE) { + // Get the last 2 digits from the look-up table and add to the buffer + p -= DIGIT_LENGTH; + strncpy(p, lut + (val % NUM_BASE) * DIGIT_LENGTH, DIGIT_LENGTH); + val /= NUM_BASE; } + // Process the remaining 1 or 2 digits if (val >= 10) { - strncpy(p, lut + val * 2, 2); - p += 2; + // If the number is 10 or more, get the 2 digits from the look-up table + p -= DIGIT_LENGTH; + strncpy(p, lut + val * DIGIT_LENGTH, DIGIT_LENGTH); } else if (val > 0 || p == temp) { - *(p++) = val + '0'; + // If the number is less than 10, add the single digit to the buffer + p -= 1; + *p = val + '0'; } - while (p != temp) { - *buf++ = *--p; + int64_t len = temp + tListLen(temp) - p; + if (len > 0) { + memcpy(buf, p, len); + } else { + buf[0] = '0'; + len = 1; } + buf[len] = '\0'; - *buf = '\0'; - return buf; + return buf + len; } diff --git a/source/util/test/log.cpp b/source/util/test/log.cpp index 1899aac2c4..ae1be94e40 100644 --- a/source/util/test/log.cpp +++ b/source/util/test/log.cpp @@ -139,3 +139,28 @@ TEST(log, misc) { taosCloseLog(); } + +TEST(log, test_u64toa) { + char buf[64] = {0}; + char *p = buf; + + p = u64toaFastLut(0, buf); + EXPECT_EQ(p, buf + 1); + EXPECT_EQ(strcmp(buf, "0"), 0); + + p = u64toaFastLut(1, buf); + EXPECT_EQ(p, buf + 1); + EXPECT_EQ(strcmp(buf, "1"), 0); + + p = u64toaFastLut(12, buf); + EXPECT_EQ(p, buf + 2); + EXPECT_EQ(strcmp(buf, "12"), 0); + + p = u64toaFastLut(12345, buf); + EXPECT_EQ(p, buf + 5); + EXPECT_EQ(strcmp(buf, "12345"), 0); + + p = u64toaFastLut(1234567890, buf); + EXPECT_EQ(p, buf + 10); + EXPECT_EQ(strcmp(buf, "1234567890"), 0); +} diff --git a/tests/army/cluster/arbitrator.py b/tests/army/cluster/arbitrator.py index 98881c5fc4..766181635f 100644 --- a/tests/army/cluster/arbitrator.py +++ b/tests/army/cluster/arbitrator.py @@ -61,11 +61,11 @@ class TDTestCase(TBase): tdSql.query("show db.vgroups;") - if(tdSql.getData(0, 4) == "follower") and (tdSql.getData(0, 6) == "leader"): + if(tdSql.getData(0, 4) == "follower") and (tdSql.getData(0, 7) == "leader"): tdLog.info("stop dnode2") sc.dnodeStop(2) - if(tdSql.getData(0, 6) == "follower") and (tdSql.getData(0, 4) == "leader"): + if(tdSql.getData(0, 7) == "follower") and (tdSql.getData(0, 4) == "leader"): tdLog.info("stop dnode 3") sc.dnodeStop(3) @@ -74,7 +74,7 @@ class TDTestCase(TBase): while count < 100: tdSql.query("show db.vgroups;") - if(tdSql.getData(0, 4) == "assigned ") or (tdSql.getData(0, 6) == "assigned "): + if(tdSql.getData(0, 4) == "assigned ") or (tdSql.getData(0, 7) == "assigned "): break tdLog.info("wait %d seconds for set assigned"%count) diff --git a/tests/army/frame/clusterCommonCheck.py b/tests/army/frame/clusterCommonCheck.py index 9cbac776b9..75b82d2101 100644 --- a/tests/army/frame/clusterCommonCheck.py +++ b/tests/army/frame/clusterCommonCheck.py @@ -291,9 +291,9 @@ class ClusterComCheck: return True elif self.db_replica == 3 : - vgroup_status_first=[tdSql.res[0][4],tdSql.res[0][6],tdSql.res[0][8]] + vgroup_status_first=[tdSql.res[0][4],tdSql.res[0][7],tdSql.res[0][10]] - vgroup_status_last=[tdSql.res[last_number][4],tdSql.res[last_number][6],tdSql.res[last_number][8]] + vgroup_status_last=[tdSql.res[last_number][4],tdSql.res[last_number][7],tdSql.res[last_number][10]] if vgroup_status_first.count('leader') == 1 and vgroup_status_first.count('follower') == 2: if vgroup_status_last.count('leader') == 1 and vgroup_status_last.count('follower') == 2: tdSql.query(f"select `replica` from information_schema.ins_databases where `name`='{db_name}';") @@ -301,6 +301,18 @@ class ClusterComCheck: if tdSql.res[0][0] == db_replica: tdLog.success(f"elections of {db_name}.vgroups with replica {self.db_replica} are ready in {count} s") return True + + vgruop_apply_commit_first = [tdSql.res[0][5],tdSql.res[0][8],tdSql.res[0][11]] + vgruop_apply_commit_end = [tdSql.res[last_number][5],tdSql.res[last_number][8],tdSql.res[last_number][11]] + for i in range(3): + v = vgruop_apply_commit_first[i].split('/') + assert (int(v[0]) <= int(v[1])) ,f"apply {v[0]} > commit {v[1]}" + + v = vgruop_apply_commit_end[i].split('/') + assert (int(v[0]) <= int(v[1])) ,f"apply {v[0]} > commit {v[1]}" + + + else: tdLog.debug(tdSql.res) tdLog.notice(f"elections of {db_name} all vgroups with replica {self.db_replica} are failed in {count} s ") diff --git a/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165561.0.avro b/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165561.0.avro deleted file mode 100644 index bd6659545a..0000000000 Binary files a/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165561.0.avro and /dev/null differ diff --git a/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165563.2.avro b/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165563.2.avro deleted file mode 100644 index 1dc4853c13..0000000000 Binary files a/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165563.2.avro and /dev/null differ diff --git a/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165566.4.avro b/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165566.4.avro deleted file mode 100644 index 6b9825f77e..0000000000 Binary files a/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165566.4.avro and /dev/null differ diff --git a/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165569.5.avro b/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165569.5.avro deleted file mode 100644 index 7465371600..0000000000 Binary files a/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165569.5.avro and /dev/null differ diff --git a/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165572.6.avro b/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165572.6.avro deleted file mode 100644 index f239d68fd8..0000000000 Binary files a/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165572.6.avro and /dev/null differ diff --git a/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165574.7.avro b/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165574.7.avro deleted file mode 100644 index 0d29e52cc5..0000000000 Binary files a/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165574.7.avro and /dev/null differ diff --git a/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165576.8.avro b/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165576.8.avro deleted file mode 100644 index 1112bc8610..0000000000 Binary files a/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165576.8.avro and /dev/null differ diff --git a/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165578.9.avro b/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165578.9.avro deleted file mode 100644 index e9adc51dd0..0000000000 Binary files a/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165578.9.avro and /dev/null differ diff --git a/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165610.3.avro b/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165610.3.avro deleted file mode 100644 index 4fd5995027..0000000000 Binary files a/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165610.3.avro and /dev/null differ diff --git a/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165620.1.avro b/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165620.1.avro deleted file mode 100644 index 59dafef4e6..0000000000 Binary files a/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/data0/test.3479004165620.1.avro and /dev/null differ diff --git a/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/dbs.sql b/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/dbs.sql deleted file mode 100644 index 4123f8c954..0000000000 --- a/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/dbs.sql +++ /dev/null @@ -1,10 +0,0 @@ -#!server_ver: ver:3.1.0.0 -#!taosdump_ver: 2.5.2_cf16c4d -#!os_id: LINUX -#!escape_char: true -#!loose_mode: false -#!charset: UTF-8 -CREATE DATABASE IF NOT EXISTS test REPLICA 1 DURATION 14400m KEEP 5256000m,5256000m,5256000m PRECISION 'ms' MINROWS 100 MAXROWS 4096 COMP 2 ; - -CREATE TABLE IF NOT EXISTS test.`meters`(`ts` timestamp,`current` float,`voltage` int,`phase` float) TAGS(`groupid` int,`location` binary(24)); - diff --git a/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/test.3479004165475.avro-tbtags b/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/test.3479004165475.avro-tbtags deleted file mode 100644 index 96392475d9..0000000000 Binary files a/tests/army/tools/taosdump/native/compa/taosdump.3479004165464/test.3479004165475.avro-tbtags and /dev/null differ diff --git a/tests/army/tools/taosdump/native/compa/taosdump.3479698875513/data0/test.3479698875538.0.avro b/tests/army/tools/taosdump/native/compa/taosdump.3479698875513/data0/test.3479698875538.0.avro new file mode 100644 index 0000000000..3c2ecc1e03 Binary files /dev/null and b/tests/army/tools/taosdump/native/compa/taosdump.3479698875513/data0/test.3479698875538.0.avro differ diff --git a/tests/army/tools/taosdump/native/compa/taosdump.3479698875513/dbs.sql b/tests/army/tools/taosdump/native/compa/taosdump.3479698875513/dbs.sql new file mode 100644 index 0000000000..c7e7296979 --- /dev/null +++ b/tests/army/tools/taosdump/native/compa/taosdump.3479698875513/dbs.sql @@ -0,0 +1,10 @@ +#!server_ver: ver:3.1.0.0 +#!taosdump_ver: 2.5.2_cf16c4d +#!os_id: LINUX +#!escape_char: true +#!loose_mode: false +#!charset: UTF-8 +CREATE DATABASE IF NOT EXISTS test REPLICA 1 DURATION 14400m KEEP 5256000m,5256000m,5256000m PRECISION 'ms' MINROWS 100 MAXROWS 4096 COMP 2 ; + +CREATE TABLE IF NOT EXISTS test.`meters`(`ts` timestamp,`bc` bool,`fc` float,`dc` double,`ti` tinyint,`si` smallint,`ic` int,`bi` bigint,`uti` tinyint unsigned,`usi` smallint unsigned,`ui` int unsigned,`ubi` bigint unsigned,`bin` binary(4),`nch` nchar(8)) TAGS(`tbc` bool,`tfc` float,`tdc` double,`tti` tinyint,`tsi` smallint,`tic` int,`tbi` bigint,`tuti` tinyint unsigned,`tusi` smallint unsigned,`tui` int unsigned,`tubi` bigint unsigned,`tbin` binary(4),`tnch` nchar(8)); + diff --git a/tests/army/tools/taosdump/native/compa/taosdump.3479698875513/test.3479698875523.avro-tbtags b/tests/army/tools/taosdump/native/compa/taosdump.3479698875513/test.3479698875523.avro-tbtags new file mode 100644 index 0000000000..eb1c6ac080 Binary files /dev/null and b/tests/army/tools/taosdump/native/compa/taosdump.3479698875513/test.3479698875523.avro-tbtags differ diff --git a/tests/army/tools/taosdump/native/taosdumpCompa.py b/tests/army/tools/taosdump/native/taosdumpCompa.py index 30088a1f6e..780cb303e7 100644 --- a/tests/army/tools/taosdump/native/taosdumpCompa.py +++ b/tests/army/tools/taosdump/native/taosdumpCompa.py @@ -59,21 +59,40 @@ class TDTestCase(TBase): # sum pk db sql = f"select {aggfun} from {db}.{stb}" tdSql.query(sql) - value = tdSql.getData(0,0) - - if value == expect: - tdLog.info(f"{aggfun} not equal. real={value} expect={expect}") - else: - tdLog.info(f"{aggfun} equal. real={value} expect={expect}") - + tdSql.checkData(0, 0, expect, show=True) def verifyResult(self, db): # compare sum(pk) stb = "meters" - self.checkSame(db, stb, "count(ts)", 100000) - self.checkSame(db, stb, "sum(current)", 1005767.2491703) - self.checkSame(db, stb, "avg(voltage)", 208.58818) + self.checkSame(db, stb, "count(ts)", 5000) + self.checkSame(db, stb, "last(ts)", "2023-11-15 07:36:39") + self.checkSame(db, stb, "last(bc)", False) + self.checkSame(db, stb, "sum(fc)", 2468.910999777726829) + self.checkSame(db, stb, "sum(dc)", 24811.172123999996984) + self.checkSame(db, stb, "sum(ti)", -411) + self.checkSame(db, stb, "sum(si)", 117073) + self.checkSame(db, stb, "sum(ic)", -39181) + self.checkSame(db, stb, "sum(bi)", -2231976) + self.checkSame(db, stb, "sum(uti)", 248825) + self.checkSame(db, stb, "sum(usi)", 248333) + self.checkSame(db, stb, "sum(ui)", 2484501) + self.checkSame(db, stb, "sum(ubi)", 25051956) + self.checkSame(db, stb, "last(bin)", "kwax") + self.checkSame(db, stb, "last(nch)", "0cYzPVcV") + + self.checkSame(db, stb, "sum(tfc)", 3420.000076293945312) + self.checkSame(db, stb, "sum(tdc)", 3020.234999999780030) + self.checkSame(db, stb, "sum(tti)", -100000) + self.checkSame(db, stb, "sum(tsi)", -85000) + self.checkSame(db, stb, "sum(tic)", -4795000) + self.checkSame(db, stb, "sum(tbi)", -1125000) + self.checkSame(db, stb, "sum(tuti)", 475000) + self.checkSame(db, stb, "sum(tusi)", 460000) + self.checkSame(db, stb, "sum(tui)", 520000) + self.checkSame(db, stb, "sum(tubi)", 43155000) + self.checkSame(db, stb, "last(tbin)", "ywkc") + self.checkSame(db, stb, "last(tnch)", "kEoWzCBj") def run(self): # database @@ -89,7 +108,6 @@ class TDTestCase(TBase): # verify db self.verifyResult(db) - def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/army/whole/checkErrorCode.py b/tests/army/whole/checkErrorCode.py index b2aa1fce1b..1a14269f54 100644 --- a/tests/army/whole/checkErrorCode.py +++ b/tests/army/whole/checkErrorCode.py @@ -61,7 +61,8 @@ ignoreCodes = [ '0x80003107', '0x80003108', '0x80003109', '0x80003110', '0x80003111', '0x80003112', '0x80003250', '0x80004003', '0x80004004', '0x80004005', '0x80004006', '0x80004007', '0x80004008', '0x80004009', '0x80004010', '0x80004011', '0x80004012', '0x80004013', '0x80004014', '0x80004015', '0x80004016', '0x80004102', '0x80004103', '0x80004104', '0x80004105', '0x80004106', '0x80004107', '0x80004108', '0x80004109', '0x80005100', - '0x80005101', '0x80006000', '0x80006100', '0x80006101', '0x80006102', '0x80000019', '0x80002639', '0x80002666', '0x80000237'] + '0x80005101', '0x80006000', '0x80006100', '0x80006101', '0x80006102', '0x80000019', '0x80002639', '0x80002666', '0x80000237', '0x80004018', + '0x80004019'] class TDTestCase(TBase): diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index accd57d739..26cc122021 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -455,6 +455,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_ts5466.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_td33504.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_ts-5473.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_ts-5776.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_ts5906.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/td-32187.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/td-33225.py diff --git a/tests/script/tsim/dnode/balance3.sim b/tests/script/tsim/dnode/balance3.sim index d0235e096e..f2363fb0f3 100644 --- a/tests/script/tsim/dnode/balance3.sim +++ b/tests/script/tsim/dnode/balance3.sim @@ -70,10 +70,10 @@ endi if $data(2)[4] == leader then $leaderExist = 1 endi -if $data(2)[6] == leader then +if $data(2)[7] == leader then $leaderExist = 1 endi -if $data(2)[8] == leader then +if $data(2)[10] == leader then $leaderExist = 1 endi if $leaderExist != 1 then @@ -98,10 +98,10 @@ endi if $data(3)[4] == leader then $leaderExist = 1 endi -if $data(3)[6] == leader then +if $data(3)[7] == leader then $leaderExist = 1 endi -if $data(3)[8] == leader then +if $data(3)[10] == leader then $leaderExist = 1 endi if $leaderExist != 1 then diff --git a/tests/script/tsim/dnode/balance_replica3.sim b/tests/script/tsim/dnode/balance_replica3.sim index afd7603b16..37b4a6beca 100644 --- a/tests/script/tsim/dnode/balance_replica3.sim +++ b/tests/script/tsim/dnode/balance_replica3.sim @@ -75,10 +75,10 @@ endi if $data(2)[4] == leader then $leaderExist = 1 endi -if $data(2)[6] == leader then +if $data(2)[7] == leader then $leaderExist = 1 endi -if $data(2)[8] == leader then +if $data(2)[10] == leader then $leaderExist = 1 endi if $leaderExist != 1 then @@ -105,10 +105,10 @@ endi if $data(3)[4] == leader then $leaderExist = 1 endi -if $data(3)[6] == leader then +if $data(3)[7] == leader then $leaderExist = 1 endi -if $data(3)[8] == leader then +if $data(3)[10] == leader then $leaderExist = 1 endi if $leaderExist != 1 then @@ -135,10 +135,10 @@ endi if $data(4)[4] == leader then $leaderExist = 1 endi -if $data(4)[6] == leader then +if $data(4)[7] == leader then $leaderExist = 1 endi -if $data(4)[8] == leader then +if $data(4)[10] == leader then $leaderExist = 1 endi if $leaderExist != 1 then @@ -165,10 +165,10 @@ endi if $data(4)[4] == leader then $leaderExist = 1 endi -if $data(4)[6] == leader then +if $data(4)[7] == leader then $leaderExist = 1 endi -if $data(4)[8] == leader then +if $data(4)[10] == leader then $leaderExist = 1 endi if $leaderExist != 1 then @@ -247,10 +247,10 @@ endi if $data(2)[4] == leader then $leaderExist = 1 endi -if $data(2)[6] == leader then +if $data(2)[7] == leader then $leaderExist = 1 endi -if $data(2)[8] == leader then +if $data(2)[10] == leader then $leaderExist = 1 endi if $leaderExist != 1 then @@ -307,10 +307,10 @@ endi if $data(4)[4] == leader then $leaderExist = 1 endi -if $data(4)[6] == leader then +if $data(4)[7] == leader then $leaderExist = 1 endi -if $data(4)[8] == leader then +if $data(4)[10] == leader then $leaderExist = 1 endi if $leaderExist != 1 then @@ -337,10 +337,10 @@ endi if $data(4)[4] == leader then $leaderExist = 1 endi -if $data(4)[6] == leader then +if $data(4)[7] == leader then $leaderExist = 1 endi -if $data(4)[8] == leader then +if $data(4)[10] == leader then $leaderExist = 1 endi if $leaderExist != 1 then diff --git a/tests/script/tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim b/tests/script/tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim index ef5001dcee..4425865bba 100644 --- a/tests/script/tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim +++ b/tests/script/tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim @@ -70,10 +70,10 @@ endi if $data(2)[4] == leader then $leaderExist = 1 endi -if $data(2)[6] == leader then +if $data(2)[7] == leader then $leaderExist = 1 endi -if $data(2)[8] == leader then +if $data(2)[10] == leader then $leaderExist = 1 endi if $leaderExist != 1 then @@ -100,10 +100,10 @@ endi if $data(3)[4] == leader then $leaderExist = 1 endi -if $data(3)[6] == leader then +if $data(3)[7] == leader then $leaderExist = 1 endi -if $data(3)[8] == leader then +if $data(3)[10] == leader then $leaderExist = 1 endi if $leaderExist != 1 then @@ -130,10 +130,10 @@ endi if $data(4)[4] == leader then $leaderExist = 1 endi -if $data(4)[6] == leader then +if $data(4)[7] == leader then $leaderExist = 1 endi -if $data(4)[8] == leader then +if $data(4)[10] == leader then $leaderExist = 1 endi if $leaderExist != 1 then @@ -160,10 +160,10 @@ endi if $data(4)[4] == leader then $leaderExist = 1 endi -if $data(4)[6] == leader then +if $data(4)[7] == leader then $leaderExist = 1 endi -if $data(4)[8] == leader then +if $data(4)[10] == leader then $leaderExist = 1 endi if $leaderExist != 1 then @@ -256,10 +256,10 @@ endi if $data(2)[4] == leader then $leaderExist = 1 endi -if $data(2)[6] == leader then +if $data(2)[7] == leader then $leaderExist = 1 endi -if $data(2)[8] == leader then +if $data(2)[10] == leader then $leaderExist = 1 endi if $leaderExist != 1 then @@ -286,10 +286,10 @@ endi if $data(3)[4] == leader then $leaderExist = 1 endi -if $data(3)[6] == leader then +if $data(3)[7] == leader then $leaderExist = 1 endi -if $data(3)[8] == leader then +if $data(3)[10] == leader then $leaderExist = 1 endi if $leaderExist != 1 then @@ -316,10 +316,10 @@ endi if $data(4)[4] == leader then $leaderExist = 1 endi -if $data(4)[6] == leader then +if $data(4)[7] == leader then $leaderExist = 1 endi -if $data(4)[8] == leader then +if $data(4)[10] == leader then $leaderExist = 1 endi if $leaderExist != 1 then @@ -346,10 +346,10 @@ endi if $data(4)[4] == leader then $leaderExist = 1 endi -if $data(4)[6] == leader then +if $data(4)[7] == leader then $leaderExist = 1 endi -if $data(4)[8] == leader then +if $data(4)[10] == leader then $leaderExist = 1 endi if $leaderExist != 1 then diff --git a/tests/script/tsim/dnode/drop_dnode_has_vnode_replica3.sim b/tests/script/tsim/dnode/drop_dnode_has_vnode_replica3.sim index 2510692846..9c6abe536b 100644 --- a/tests/script/tsim/dnode/drop_dnode_has_vnode_replica3.sim +++ b/tests/script/tsim/dnode/drop_dnode_has_vnode_replica3.sim @@ -73,13 +73,13 @@ if $data(2)[4] == leader then $follower1 = 3 $follower2 = 4 endi -if $data(2)[6] == leader then +if $data(2)[7] == leader then $leaderExist = 1 $leaderVnode = 3 $follower1 = 2 $follower2 = 4 endi -if $data(2)[8] == leader then +if $data(2)[10] == leader then $leaderExist = 1 $leaderVnode = 4 $follower1 = 2 @@ -109,10 +109,10 @@ endi if $data(2)[3] != 2 then return -1 endi -if $data(2)[5] != 3 then +if $data(2)[6] != 3 then return -1 endi -if $data(2)[7] != 4 then +if $data(2)[9] != 4 then return -1 endi @@ -199,10 +199,10 @@ endi if $data(2)[4] == leader then $leaderExist = 1 endi -if $data(2)[6] == leader then +if $data(2)[7] == leader then $leaderExist = 1 endi -if $data(2)[8] == leader then +if $data(2)[10] == leader then $leaderExist = 1 endi if $leaderExist != 1 then diff --git a/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim index a576969697..ff34a8884e 100644 --- a/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim +++ b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim @@ -137,13 +137,13 @@ if $data(2)[4] == leader then $follower1 = 3 $follower2 = 4 endi -if $data(2)[6] == leader then +if $data(2)[7] == leader then $leaderExist = 1 $leaderVnode = 3 $follower1 = 2 $follower2 = 4 endi -if $data(2)[8] == leader then +if $data(2)[10] == leader then $leaderExist = 1 $leaderVnode = 4 $follower1 = 2 diff --git a/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim index 739e3f2984..c6ee0dfeea 100644 --- a/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim +++ b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim @@ -116,13 +116,13 @@ if $data(2)[4] == leader then $follower1 = 3 $follower2 = 4 endi -if $data(2)[6] == leader then +if $data(2)[7] == leader then $leaderExist = 1 $leaderVnode = 3 $follower1 = 2 $follower2 = 4 endi -if $data(2)[8] == leader then +if $data(2)[10] == leader then $leaderExist = 1 $leaderVnode = 4 $follower1 = 2 diff --git a/tests/script/tsim/dnode/redistribute_vgroup_replica3_v2.sim b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v2.sim index 652f0c14b4..ae0b60c943 100644 --- a/tests/script/tsim/dnode/redistribute_vgroup_replica3_v2.sim +++ b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v2.sim @@ -122,13 +122,13 @@ if $data(2)[4] == leader then $follower1 = 3 $follower2 = 4 endi -if $data(2)[6] == leader then +if $data(2)[7] == leader then $leaderExist = 1 $leaderVnode = 3 $follower1 = 2 $follower2 = 4 endi -if $data(2)[8] == leader then +if $data(2)[10] == leader then $leaderExist = 1 $leaderVnode = 4 $follower1 = 2 diff --git a/tests/script/tsim/dnode/redistribute_vgroup_replica3_v3.sim b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v3.sim index 30e644f170..b3746dfcf4 100644 --- a/tests/script/tsim/dnode/redistribute_vgroup_replica3_v3.sim +++ b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v3.sim @@ -136,13 +136,13 @@ if $data(2)[4] == leader then $follower1 = 3 $follower2 = 4 endi -if $data(2)[6] == leader then +if $data(2)[7] == leader then $leaderExist = 1 $leaderVnode = 3 $follower1 = 2 $follower2 = 4 endi -if $data(2)[8] == leader then +if $data(2)[10] == leader then $leaderExist = 1 $leaderVnode = 4 $follower1 = 2 diff --git a/tests/script/tsim/sync/3Replica1VgElect.sim b/tests/script/tsim/sync/3Replica1VgElect.sim index aae1b25636..6ebee885a8 100644 --- a/tests/script/tsim/sync/3Replica1VgElect.sim +++ b/tests/script/tsim/sync/3Replica1VgElect.sim @@ -89,20 +89,20 @@ if $rows != $vgroups then endi if $data[0][4] == leader then - if $data[0][6] == follower then - if $data[0][8] == follower then + if $data[0][7] == follower then + if $data[0][10] == follower then print ---- vgroup $data[0][0] leader locate on dnode $data[0][3] endi endi elif $data[0][6] == leader then - if $data[0][4] == follower then - if $data[0][8] == follower then + if $data[0][7] == follower then + if $data[0][10] == follower then print ---- vgroup $data[0][0] leader locate on dnode $data[0][5] endi endi -elif $data[0][8] == leader then +elif $data[0][10] == leader then if $data[0][4] == follower then - if $data[0][6] == follower then + if $data[0][7] == follower then print ---- vgroup $data[0][0] leader locate on dnode $data[0][7] endi endi @@ -225,20 +225,20 @@ if $rows != $vgroups then endi if $data[0][4] == leader then - if $data[0][6] == follower then - if $data[0][8] == follower then + if $data[0][7] == follower then + if $data[0][10] == follower then print ---- vgroup $data[0][0] leader locate on dnode $data[0][3] endi endi -elif $data[0][6] == leader then +elif $data[0][7] == leader then if $data[0][4] == follower then - if $data[0][8] == follower then + if $data[0][10] == follower then print ---- vgroup $data[0][0] leader locate on dnode $data[0][5] endi endi -elif $data[0][8] == leader then +elif $data[0][10] == leader then if $data[0][4] == follower then - if $data[0][6] == follower then + if $data[0][7] == follower then print ---- vgroup $data[0][0] leader locate on dnode $data[0][7] endi endi @@ -437,20 +437,20 @@ if $rows != $vgroups then endi if $data[0][4] == leader then - if $data[0][6] == follower then - if $data[0][8] == follower then + if $data[0][7] == follower then + if $data[0][10] == follower then print ---- vgroup $data[0][0] leader locate on dnode $data[0][3] endi endi -elif $data[0][6] == leader then +elif $data[0][7] == leader then if $data[0][4] == follower then - if $data[0][8] == follower then + if $data[0][10] == follower then print ---- vgroup $data[0][0] leader locate on dnode $data[0][5] endi endi -elif $data[0][8] == leader then +elif $data[0][10] == leader then if $data[0][4] == follower then - if $data[0][6] == follower then + if $data[0][7] == follower then print ---- vgroup $data[0][0] leader locate on dnode $data[0][7] endi endi diff --git a/tests/script/tsim/sync/3Replica5VgElect.sim b/tests/script/tsim/sync/3Replica5VgElect.sim index 37e4199e23..01bd41af42 100644 --- a/tests/script/tsim/sync/3Replica5VgElect.sim +++ b/tests/script/tsim/sync/3Replica5VgElect.sim @@ -92,20 +92,20 @@ if $rows != $vgroups then endi if $data[0][4] == leader then - if $data[0][6] == follower then - if $data[0][8] == follower then + if $data[0][7] == follower then + if $data[0][10] == follower then print ---- vgroup $data[0][0] leader locate on dnode $data[0][3] endi endi -elif $data[0][6] == leader then +elif $data[0][7] == leader then if $data[0][4] == follower then - if $data[0][8] == follower then + if $data[0][10] == follower then print ---- vgroup $data[0][0] leader locate on dnode $data[0][5] endi endi -elif $data[0][8] == leader then +elif $data[0][10] == leader then if $data[0][4] == follower then - if $data[0][6] == follower then + if $data[0][7] == follower then print ---- vgroup $data[0][0] leader locate on dnode $data[0][7] endi endi @@ -114,20 +114,20 @@ else endi if $data[1][4] == leader then - if $data[1][6] == follower then - if $data[1][8] == follower then + if $data[1][7] == follower then + if $data[1][10] == follower then print ---- vgroup $data[1][0] leader locate on dnode $data[1][3] endi endi -elif $data[1][6] == leader then +elif $data[1][7] == leader then if $data[1][4] == follower then - if $data[1][8] == follower then + if $data[1][10] == follower then print ---- vgroup $data[1][0] leader locate on dnode $data[1][5] endi endi -elif $data[1][8] == leader then +elif $data[1][10] == leader then if $data[1][4] == follower then - if $data[1][6] == follower then + if $data[1][7] == follower then print ---- vgroup $data[1][0] leader locate on dnode $data[1][7] endi endi @@ -136,20 +136,20 @@ else endi if $data[2][4] == leader then - if $data[2][6] == follower then - if $data[2][8] == follower then + if $data[2][7] == follower then + if $data[2][10] == follower then print ---- vgroup $data[2][0] leader locate on dnode $data[2][3] endi endi -elif $data[2][6] == leader then +elif $data[2][7] == leader then if $data[2][4] == follower then - if $data[2][8] == follower then + if $data[2][10] == follower then print ---- vgroup $data[2][0] leader locate on dnode $data[2][5] endi endi -elif $data[2][8] == leader then +elif $data[2][10] == leader then if $data[2][4] == follower then - if $data[2][6] == follower then + if $data[2][7] == follower then print ---- vgroup $data[2][0] leader locate on dnode $data[2][7] endi endi @@ -158,20 +158,20 @@ else endi if $data[3][4] == leader then - if $data[3][6] == follower then - if $data[3][8] == follower then + if $data[3][7] == follower then + if $data[3][10] == follower then print ---- vgroup $data[3][0] leader locate on dnode $data[3][3] endi endi -elif $data[3][6] == leader then +elif $data[3][7] == leader then if $data[3][4] == follower then - if $data[3][8] == follower then + if $data[3][10] == follower then print ---- vgroup $data[3][0] leader locate on dnode $data[3][5] endi endi -elif $data[3][8] == leader then +elif $data[3][10] == leader then if $data[3][4] == follower then - if $data[3][6] == follower then + if $data[3][7] == follower then print ---- vgroup $data[3][0] leader locate on dnode $data[3][7] endi endi @@ -180,20 +180,20 @@ else endi if $data[4][4] == leader then - if $data[4][6] == follower then - if $data[4][8] == follower then + if $data[4][7] == follower then + if $data[4][10] == follower then print ---- vgroup $data[4][0] leader locate on dnode $data[4][3] endi endi -elif $data[4][6] == leader then +elif $data[4][7] == leader then if $data[4][4] == follower then - if $data[4][8] == follower then + if $data[4][10] == follower then print ---- vgroup $data[4][0] leader locate on dnode $data[4][5] endi endi -elif $data[4][8] == leader then +elif $data[4][10] == leader then if $data[4][4] == follower then - if $data[4][6] == follower then + if $data[4][7] == follower then print ---- vgroup $data[4][0] leader locate on dnode $data[4][7] endi endi @@ -305,20 +305,20 @@ if $rows != $vgroups then endi if $data[0][4] == leader then - if $data[0][6] == follower then - if $data[0][8] == follower then + if $data[0][7] == follower then + if $data[0][10] == follower then print ---- vgroup $data[0][0] leader locate on dnode $data[0][3] endi endi -elif $data[0][6] == leader then +elif $data[0][7] == leader then if $data[0][4] == follower then - if $data[0][8] == follower then + if $data[0][10] == follower then print ---- vgroup $data[0][0] leader locate on dnode $data[0][5] endi endi -elif $data[0][8] == leader then +elif $data[0][10] == leader then if $data[0][4] == follower then - if $data[0][6] == follower then + if $data[0][7] == follower then print ---- vgroup $data[0][0] leader locate on dnode $data[0][7] endi endi @@ -327,20 +327,20 @@ else endi if $data[1][4] == leader then - if $data[1][6] == follower then - if $data[1][8] == follower then + if $data[1][7] == follower then + if $data[1][10] == follower then print ---- vgroup $data[1][0] leader locate on dnode $data[1][3] endi endi -elif $data[1][6] == leader then +elif $data[1][7] == leader then if $data[1][4] == follower then - if $data[1][8] == follower then + if $data[1][10] == follower then print ---- vgroup $data[1][0] leader locate on dnode $data[1][5] endi endi -elif $data[1][8] == leader then +elif $data[1][10] == leader then if $data[1][4] == follower then - if $data[1][6] == follower then + if $data[1][7] == follower then print ---- vgroup $data[1][0] leader locate on dnode $data[1][7] endi endi @@ -349,20 +349,20 @@ else endi if $data[2][4] == leader then - if $data[2][6] == follower then - if $data[2][8] == follower then + if $data[2][7] == follower then + if $data[2][10] == follower then print ---- vgroup $data[2][0] leader locate on dnode $data[2][3] endi endi -elif $data[2][6] == leader then +elif $data[2][7] == leader then if $data[2][4] == follower then - if $data[2][8] == follower then + if $data[2][10] == follower then print ---- vgroup $data[2][0] leader locate on dnode $data[2][5] endi endi -elif $data[2][8] == leader then +elif $data[2][10] == leader then if $data[2][4] == follower then - if $data[2][6] == follower then + if $data[2][7] == follower then print ---- vgroup $data[2][0] leader locate on dnode $data[2][7] endi endi @@ -371,20 +371,20 @@ else endi if $data[3][4] == leader then - if $data[3][6] == follower then - if $data[3][8] == follower then + if $data[3][7] == follower then + if $data[3][10] == follower then print ---- vgroup $data[3][0] leader locate on dnode $data[3][3] endi endi -elif $data[3][6] == leader then +elif $data[3][7] == leader then if $data[3][4] == follower then - if $data[3][8] == follower then + if $data[3][10] == follower then print ---- vgroup $data[3][0] leader locate on dnode $data[3][5] endi endi -elif $data[3][8] == leader then +elif $data[3][10] == leader then if $data[3][4] == follower then - if $data[3][6] == follower then + if $data[3][7] == follower then print ---- vgroup $data[3][0] leader locate on dnode $data[3][7] endi endi @@ -393,20 +393,20 @@ else endi if $data[4][4] == leader then - if $data[4][6] == follower then - if $data[4][8] == follower then + if $data[4][7] == follower then + if $data[4][10] == follower then print ---- vgroup $data[4][0] leader locate on dnode $data[4][3] endi endi -elif $data[4][6] == leader then +elif $data[4][7] == leader then if $data[4][4] == follower then - if $data[4][8] == follower then + if $data[4][10] == follower then print ---- vgroup $data[4][0] leader locate on dnode $data[4][5] endi endi -elif $data[4][8] == leader then +elif $data[4][10] == leader then if $data[4][4] == follower then - if $data[4][6] == follower then + if $data[4][7] == follower then print ---- vgroup $data[4][0] leader locate on dnode $data[4][7] endi endi @@ -607,20 +607,20 @@ if $rows != $vgroups then endi if $data[0][4] == leader then - if $data[0][6] == follower then - if $data[0][8] == follower then + if $data[0][7] == follower then + if $data[0][10] == follower then print ---- vgroup $data[0][0] leader locate on dnode $data[0][3] endi endi -elif $data[0][6] == leader then +elif $data[0][7] == leader then if $data[0][4] == follower then - if $data[0][8] == follower then + if $data[0][10] == follower then print ---- vgroup $data[0][0] leader locate on dnode $data[0][5] endi endi -elif $data[0][8] == leader then +elif $data[0][10] == leader then if $data[0][4] == follower then - if $data[0][6] == follower then + if $data[0][7] == follower then print ---- vgroup $data[0][0] leader locate on dnode $data[0][7] endi endi @@ -629,20 +629,20 @@ else endi if $data[1][4] == leader then - if $data[1][6] == follower then - if $data[1][8] == follower then + if $data[1][7] == follower then + if $data[1][10] == follower then print ---- vgroup $data[1][0] leader locate on dnode $data[1][3] endi endi -elif $data[1][6] == leader then +elif $data[1][7] == leader then if $data[1][4] == follower then - if $data[1][8] == follower then + if $data[1][10] == follower then print ---- vgroup $data[1][0] leader locate on dnode $data[1][5] endi endi -elif $data[1][8] == leader then +elif $data[1][10] == leader then if $data[1][4] == follower then - if $data[1][6] == follower then + if $data[1][7] == follower then print ---- vgroup $data[1][0] leader locate on dnode $data[1][7] endi endi @@ -651,20 +651,20 @@ else endi if $data[2][4] == leader then - if $data[2][6] == follower then - if $data[2][8] == follower then + if $data[2][7] == follower then + if $data[2][10] == follower then print ---- vgroup $data[2][0] leader locate on dnode $data[2][3] endi endi -elif $data[2][6] == leader then +elif $data[2][7] == leader then if $data[2][4] == follower then - if $data[2][8] == follower then + if $data[2][10] == follower then print ---- vgroup $data[2][0] leader locate on dnode $data[2][5] endi endi -elif $data[2][8] == leader then +elif $data[2][10] == leader then if $data[2][4] == follower then - if $data[2][6] == follower then + if $data[2][7] == follower then print ---- vgroup $data[2][0] leader locate on dnode $data[2][7] endi endi @@ -673,20 +673,20 @@ else endi if $data[3][4] == leader then - if $data[3][6] == follower then - if $data[3][8] == follower then + if $data[3][7] == follower then + if $data[3][10] == follower then print ---- vgroup $data[3][0] leader locate on dnode $data[3][3] endi endi -elif $data[3][6] == leader then +elif $data[3][7] == leader then if $data[3][4] == follower then - if $data[3][8] == follower then + if $data[3][10] == follower then print ---- vgroup $data[3][0] leader locate on dnode $data[3][5] endi endi -elif $data[3][8] == leader then +elif $data[3][10] == leader then if $data[3][4] == follower then - if $data[3][6] == follower then + if $data[3][7] == follower then print ---- vgroup $data[3][0] leader locate on dnode $data[3][7] endi endi @@ -695,20 +695,20 @@ else endi if $data[4][4] == leader then - if $data[4][6] == follower then - if $data[4][8] == follower then + if $data[4][7] == follower then + if $data[4][10] == follower then print ---- vgroup $data[4][0] leader locate on dnode $data[4][3] endi endi -elif $data[4][6] == leader then +elif $data[4][7] == leader then if $data[4][4] == follower then - if $data[4][8] == follower then + if $data[4][10] == follower then print ---- vgroup $data[4][0] leader locate on dnode $data[4][5] endi endi -elif $data[4][8] == leader then +elif $data[4][10] == leader then if $data[4][4] == follower then - if $data[4][6] == follower then + if $data[4][7] == follower then print ---- vgroup $data[4][0] leader locate on dnode $data[4][7] endi endi diff --git a/tests/script/tsim/sync/vnodesnapshot-rsma-test.sim b/tests/script/tsim/sync/vnodesnapshot-rsma-test.sim index 8b1720d213..d07c66bc53 100644 --- a/tests/script/tsim/sync/vnodesnapshot-rsma-test.sim +++ b/tests/script/tsim/sync/vnodesnapshot-rsma-test.sim @@ -90,20 +90,20 @@ if $rows != $vgroups then endi if $data[0][4] == leader then - if $data[0][6] == follower then - if $data[0][8] == follower then + if $data[0][7] == follower then + if $data[0][10] == follower then print ---- vgroup $data[0][0] leader locate on dnode $data[0][3] endi endi -elif $data[0][6] == leader then +elif $data[0][7] == leader then if $data[0][4] == follower then - if $data[0][8] == follower then + if $data[0][10] == follower then print ---- vgroup $data[0][0] leader locate on dnode $data[0][5] endi endi -elif $data[0][8] == leader then +elif $data[0][10] == leader then if $data[0][4] == follower then - if $data[0][6] == follower then + if $data[0][7] == follower then print ---- vgroup $data[0][0] leader locate on dnode $data[0][7] endi endi diff --git a/tests/script/tsim/vnode/replica3_basic.sim b/tests/script/tsim/vnode/replica3_basic.sim index 473e53e84b..5da9fe6b38 100644 --- a/tests/script/tsim/vnode/replica3_basic.sim +++ b/tests/script/tsim/vnode/replica3_basic.sim @@ -96,10 +96,10 @@ $leaderExist = 0 if $data(2)[4] == leader then $leaderExist = 1 endi -if $data(2)[6] == leader then +if $data(2)[7] == leader then $leaderExist = 1 endi -if $data(2)[8] == leader then +if $data(2)[10] == leader then $leaderExist = 1 endi if $leaderExist != 1 then @@ -141,10 +141,10 @@ $leaderExist = 0 if $data(2)[4] == leader then $leaderExist = 1 endi -if $data(2)[6] == leader then +if $data(2)[7] == leader then $leaderExist = 1 endi -if $data(2)[8] == leader then +if $data(2)[10] == leader then $leaderExist = 1 endi if $leaderExist != 1 then @@ -180,10 +180,10 @@ $leaderExist = 0 if $data(2)[4] == leader then $leaderExist = 1 endi -if $data(2)[6] == leader then +if $data(2)[7] == leader then $leaderExist = 1 endi -if $data(2)[8] == leader then +if $data(2)[10] == leader then $leaderExist = 1 endi if $leaderExist != 1 then @@ -220,10 +220,10 @@ $leaderExist = 0 if $data(2)[4] == leader then $leaderExist = 1 endi -if $data(2)[6] == leader then +if $data(2)[7] == leader then $leaderExist = 1 endi -if $data(2)[8] == leader then +if $data(2)[10] == leader then $leaderExist = 1 endi if $leaderExist != 1 then @@ -260,10 +260,10 @@ $leaderExist = 0 if $data(2)[4] == leader then $leaderExist = 1 endi -if $data(2)[6] == leader then +if $data(2)[7] == leader then $leaderExist = 1 endi -if $data(2)[8] == leader then +if $data(2)[10] == leader then $leaderExist = 1 endi if $leaderExist != 1 then @@ -300,10 +300,10 @@ $leaderExist = 0 if $data(2)[4] == leader then $leaderExist = 1 endi -if $data(2)[6] == leader then +if $data(2)[7] == leader then $leaderExist = 1 endi -if $data(2)[8] == leader then +if $data(2)[10] == leader then $leaderExist = 1 endi if $leaderExist != 1 then @@ -340,10 +340,10 @@ $leaderExist = 0 if $data(2)[4] == leader then $leaderExist = 1 endi -if $data(2)[6] == leader then +if $data(2)[7] == leader then $leaderExist = 1 endi -if $data(2)[8] == leader then +if $data(2)[10] == leader then $leaderExist = 1 endi if $leaderExist != 1 then diff --git a/tests/script/tsim/vnode/replica3_import.sim b/tests/script/tsim/vnode/replica3_import.sim index 99608c72e2..2840fd45a1 100644 --- a/tests/script/tsim/vnode/replica3_import.sim +++ b/tests/script/tsim/vnode/replica3_import.sim @@ -71,10 +71,10 @@ endi if $data(2)[4] == leader then $leaderExist = 1 endi -if $data(2)[6] == leader then +if $data(2)[7] == leader then $leaderExist = 1 endi -if $data(2)[8] == leader then +if $data(2)[10] == leader then $leaderExist = 1 endi if $leaderExist != 1 then @@ -157,10 +157,10 @@ $leaderExist = 0 if $data(2)[4] == leader then $leaderExist = 1 endi -if $data(2)[6] == leader then +if $data(2)[7] == leader then $leaderExist = 1 endi -if $data(2)[8] == leader then +if $data(2)[10] == leader then $leaderExist = 1 endi if $leaderExist != 1 then @@ -227,10 +227,10 @@ $leaderExist = 0 if $data(2)[4] == leader then $leaderExist = 1 endi -if $data(2)[6] == leader then +if $data(2)[7] == leader then $leaderExist = 1 endi -if $data(2)[8] == leader then +if $data(2)[10] == leader then $leaderExist = 1 endi if $leaderExist != 1 then @@ -275,10 +275,10 @@ $leaderExist = 0 if $data(2)[4] == leader then $leaderExist = 1 endi -if $data(2)[6] == leader then +if $data(2)[7] == leader then $leaderExist = 1 endi -if $data(2)[8] == leader then +if $data(2)[10] == leader then $leaderExist = 1 endi if $leaderExist != 1 then @@ -313,10 +313,10 @@ $leaderExist = 0 if $data(2)[4] == leader then $leaderExist = 1 endi -if $data(2)[6] == leader then +if $data(2)[7] == leader then $leaderExist = 1 endi -if $data(2)[8] == leader then +if $data(2)[10] == leader then $leaderExist = 1 endi if $leaderExist != 1 then diff --git a/tests/script/tsim/vnode/replica3_many.sim b/tests/script/tsim/vnode/replica3_many.sim index e3c73b2018..930eebc688 100644 --- a/tests/script/tsim/vnode/replica3_many.sim +++ b/tests/script/tsim/vnode/replica3_many.sim @@ -68,10 +68,10 @@ endi if $data(2)[4] == leader then $leaderExist = 1 endi -if $data(2)[6] == leader then +if $data(2)[7] == leader then $leaderExist = 1 endi -if $data(2)[8] == leader then +if $data(2)[10] == leader then $leaderExist = 1 endi if $leaderExist != 1 then @@ -96,10 +96,10 @@ endi if $data(2)[4] == leader then $leaderExist = 1 endi -if $data(2)[6] == leader then +if $data(2)[7] == leader then $leaderExist = 1 endi -if $data(2)[8] == leader then +if $data(2)[10] == leader then $leaderExist = 1 endi if $leaderExist != 1 then @@ -121,10 +121,10 @@ $leaderExist = 0 if $data(2)[4] == leader then $leaderExist = 1 endi -if $data(2)[6] == leader then +if $data(2)[7] == leader then $leaderExist = 1 endi -if $data(2)[8] == leader then +if $data(2)[10] == leader then $leaderExist = 1 endi if $leaderExist != 1 then @@ -146,10 +146,10 @@ $leaderExist = 0 if $data(2)[4] == leader then $leaderExist = 1 endi -if $data(2)[6] == leader then +if $data(2)[7] == leader then $leaderExist = 1 endi -if $data(2)[8] == leader then +if $data(2)[10] == leader then $leaderExist = 1 endi if $leaderExist != 1 then diff --git a/tests/script/tsim/vnode/replica3_repeat.sim b/tests/script/tsim/vnode/replica3_repeat.sim index 10e18d01d0..defee0fe0a 100644 --- a/tests/script/tsim/vnode/replica3_repeat.sim +++ b/tests/script/tsim/vnode/replica3_repeat.sim @@ -62,10 +62,10 @@ endi if $data(2)[4] == leader then $leaderExist = 1 endi -if $data(2)[6] == leader then +if $data(2)[7] == leader then $leaderExist = 1 endi -if $data(2)[8] == leader then +if $data(2)[10] == leader then $leaderExist = 1 endi if $leaderExist != 1 then diff --git a/tests/script/tsim/vnode/replica3_vgroup.sim b/tests/script/tsim/vnode/replica3_vgroup.sim index 6643966344..a7a8b725b9 100644 --- a/tests/script/tsim/vnode/replica3_vgroup.sim +++ b/tests/script/tsim/vnode/replica3_vgroup.sim @@ -70,10 +70,10 @@ $leaderExist = 0 if $data(2)[4] == leader then $leaderExist = 1 endi -if $data(2)[6] == leader then +if $data(2)[7] == leader then $leaderExist = 1 endi -if $data(2)[8] == leader then +if $data(2)[10] == leader then $leaderExist = 1 endi if $leaderExist != 1 then @@ -83,10 +83,10 @@ $leaderExist = 0 if $data(3)[4] == leader then $leaderExist = 1 endi -if $data(3)[6] == leader then +if $data(3)[7] == leader then $leaderExist = 1 endi -if $data(3)[8] == leader then +if $data(3)[10] == leader then $leaderExist = 1 endi if $leaderExist != 1 then diff --git a/tests/system-test/0-others/information_schema.py b/tests/system-test/0-others/information_schema.py index 4874174f38..96cd596a20 100644 --- a/tests/system-test/0-others/information_schema.py +++ b/tests/system-test/0-others/information_schema.py @@ -222,8 +222,7 @@ class TDTestCase: tdSql.query("select * from information_schema.ins_columns where db_name ='information_schema'") tdLog.info(len(tdSql.queryResult)) - tdSql.checkEqual(True, len(tdSql.queryResult) in range(314, 315)) - + tdSql.checkEqual(True, len(tdSql.queryResult) in range(320, 321)) tdSql.query("select * from information_schema.ins_columns where db_name ='performance_schema'") tdSql.checkEqual(61, len(tdSql.queryResult)) diff --git a/tests/system-test/1-insert/alter_replica.py b/tests/system-test/1-insert/alter_replica.py index 900b64d943..6d364618ee 100644 --- a/tests/system-test/1-insert/alter_replica.py +++ b/tests/system-test/1-insert/alter_replica.py @@ -27,8 +27,8 @@ class TDTestCase: flag = 0 for vgid in range (vgNum) : v1_status = tdSql.queryResult[vgid][4] - v2_status = tdSql.queryResult[vgid][6] - v3_status = tdSql.queryResult[vgid][8] + v2_status = tdSql.queryResult[vgid][7] + v3_status = tdSql.queryResult[vgid][10] if ((v1_status == 'leader') and (v2_status == 'follower') and (v3_status == 'follower')) \ or ((v2_status == 'leader') and (v1_status == 'follower') and (v3_status == 'follower')) \ or ((v3_status == 'leader') and (v2_status == 'follower') and (v1_status == 'follower')): diff --git a/tests/system-test/3-enterprise/restore/restoreBasic.py b/tests/system-test/3-enterprise/restore/restoreBasic.py index 74cf572018..9ca48ed7e8 100644 --- a/tests/system-test/3-enterprise/restore/restoreBasic.py +++ b/tests/system-test/3-enterprise/restore/restoreBasic.py @@ -80,7 +80,7 @@ class RestoreBasic: for i in range(8): leader = False for j in range(3): - status = tdSql.getData(i, 4 + j*2) + status = tdSql.getData(i, 4 + j*3) if status == "leader": leader = True elif status == "follower": diff --git a/tests/system-test/6-cluster/clusterCommonCheck.py b/tests/system-test/6-cluster/clusterCommonCheck.py index f3e2b5d5bc..c42e3d918d 100644 --- a/tests/system-test/6-cluster/clusterCommonCheck.py +++ b/tests/system-test/6-cluster/clusterCommonCheck.py @@ -251,9 +251,9 @@ class ClusterComCheck: return True elif self.db_replica == 3 : - vgroup_status_first=[tdSql.queryResult[0][4],tdSql.queryResult[0][6],tdSql.queryResult[0][8]] + vgroup_status_first=[tdSql.queryResult[0][4],tdSql.queryResult[0][7],tdSql.queryResult[0][10]] - vgroup_status_last=[tdSql.queryResult[last_number][4],tdSql.queryResult[last_number][6],tdSql.queryResult[last_number][8]] + vgroup_status_last=[tdSql.queryResult[last_number][4],tdSql.queryResult[last_number][7],tdSql.queryResult[last_number][10]] if vgroup_status_first.count('leader') == 1 and vgroup_status_first.count('follower') == 2: if vgroup_status_last.count('leader') == 1 and vgroup_status_last.count('follower') == 2: tdSql.query(f"select `replica` from information_schema.ins_databases where `name`='{db_name}';") diff --git a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py index 9eea490eba..cdc1b1d607 100644 --- a/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py +++ b/tests/system-test/6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py @@ -135,11 +135,9 @@ class TDTestCase: vgroup_id = vgroup_info[0] vgroup_status = [] for ind , role in enumerate(vgroup_info[3:-4]): - - if ind%2==0: - continue - else: + if role in ['leader', 'leader*', 'leader**', 'follower']: vgroup_status.append(role) + if vgroup_status.count("leader")!=1 or vgroup_status.count("follower")!=2: status = False return status diff --git a/tests/system-test/7-tmq/taosx-performance.py b/tests/system-test/7-tmq/taosx-performance.py new file mode 100755 index 0000000000..5f7c6c3e07 --- /dev/null +++ b/tests/system-test/7-tmq/taosx-performance.py @@ -0,0 +1,280 @@ +import taos +import sys +import time +import socket +import os +import threading + +sys.path.append("../../pytest") + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +from taos.tmq import * + +sys.path.append("./7-tmq") +from tmqCommon import * + +tdDnodes1 = TDDnodes() +tdDnodes2 = TDDnodes() +vgroups = 1 +tables = 1 +rows_of_each_table = 1 +path = "./taosx_perf" +dnode1 = "./dnode1" +dnode2 = "./dnode2" +cfg = "sim/dnode1/cfg" +taosx = "taosx" +taosd = "taosd" +taosxLog = "taosx.log" +taosxTimeout = 2 +timeCost = [] +speedupStr = [] +insertJson = '''{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "connection_pool_size": 20, + "thread_bind_vgroup": "yes", + "thread_count": 20, + "create_table_thread_count": 16, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "num_of_records_per_req": 10, + "prepared_rand": 10000, + "chinese": "no", + "escape_character": "yes", + "continue_if_fail": "no", + "databases": [ + { + "dbinfo": { + "name": "test", + "drop": "yes", + "vgroups": 8, + "precision": "ms", + "WAL_RETENTION_PERIOD": 864000 + }, + "super_tables": [ + { + "name": "meters", + "child_table_exists": "no", + "childtable_count": 10000, + "childtable_prefix": "d", + "auto_create_table": "yes", + "batch_create_tbl_num": 500, + "data_source": "rand", + "insert_mode": "taosc", + "non_stop_mode": "no", + "line_protocol": "line", + "insert_rows": 1000, + "childtable_limit": 0, + "childtable_offset": 0, + "interlace_rows": 1, + "insert_interval": 0, + "partial_col_num": 0, + "timestamp_step": 10, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "use_sample_ts": "no", + "tags_file": "", + "generate_row_rule": 2, + "columns": [ + {"type": "TINYINT", "name": "current", "max": 128, "min": 1 }, + { "type": "BOOL", "name": "phaseewe" }, + { "type": "BINARY", "name": "str", "len":16374, + "values": [ + "{kkey00000k: kvalue00000k, kkey00001k: kvalue00001k, kkey00002k: kvalue00002k, kkey00003k: kvalue00003k, kkey00004k: kvalue00004k, kkey00005k: kvalue00005k, kkey00006k: kvalue00006k, kkey00007k: kvalue00007k, kkey00008k: kvalue00008k, kkey00009k: kvalue00009k, kkey00010k: kvalue00010k, kkey00011k: kvalue00011k, kkey00012k: kvalue00012k, kkey00013k: kvalue00013k, kkey00014k: kvalue00014k, kkey00015k: kvalue00015k, kkey00016k: kvalue00016k, kkey00017k: kvalue00017k, kkey00018k: kvalue00018k, kkey00019k: kvalue00019k, kkey00020k: kvalue00020k, kkey00021k: kvalue00021k, kkey00022k: kvalue00022k, kkey00023k: kvalue00023k, kkey00024k: kvalue00024k, kkey00025k: kvalue00025k, kkey00026k: kvalue00026k, kkey00027k: kvalue00027k, kkey00028k: kvalue00028k, kkey00029k: kvalue00029k, kkey00030k: kvalue00030k, kkey00031k: kvalue00031k, kkey00032k: kvalue00032k, kkey00033k: kvalue00033k, kkey00034k: kvalue00034k, kkey00035k: kvalue00035k, kkey00036k: kvalue00036k, kkey00037k: kvalue00037k, kkey00038k: kvalue00038k, kkey00039k: kvalue00039k, kkey00040k: kvalue00040k, kkey00041k: kvalue00041k, kkey00042k: kvalue00042k, kkey00043k: kvalue00043k, kkey00044k: kvalue00044k, kkey00045k: kvalue00045k, kkey00046k: kvalue00046k, kkey00047k: kvalue00047k, kkey00048k: kvalue00048k, kkey00049k: kvalue00049k, kkey00050k: kvalue00050k, kkey00051k: kvalue00051k, kkey00052k: kvalue00052k, kkey00053k: kvalue00053k, kkey00054k: kvalue00054k, kkey00055k: kvalue00055k, kkey00056k: kvalue00056k, kkey00057k: kvalue00057k, kkey00058k: kvalue00058k, kkey00059k: kvalue00059k, kkey00060k: kvalue00060k, kkey00061k: kvalue00061k, kkey00062k: kvalue00062k, kkey00063k: kvalue00063k, kkey00064k: kvalue00064k, kkey00065k: kvalue00065k, kkey00066k: kvalue00066k, kkey00067k: kvalue00067k, kkey00068k: kvalue00068k, kkey00069k: kvalue00069k, kkey00070k: kvalue00070k, kkey00071k: kvalue00071k, kkey00072k: kvalue00072k, kkey00073k: kvalue00073k, kkey00074k: kvalue00074k, kkey00075k: kvalue00075k, kkey00076k: kvalue00076k, kkey00077k: kvalue00077k, kkey00078k: kvalue00078k, kkey00079k: kvalue00079k, kkey00080k: kvalue00080k, kkey00081k: kvalue00081k, kkey00082k: kvalue00082k, kkey00083k: kvalue00083k, kkey00084k: kvalue00084k, kkey00085k: kvalue00085k, kkey00086k: kvalue00086k, kkey00087k: kvalue00087k, kkey00088k: kvalue00088k, kkey00089k: kvalue00089k, kkey00090k: kvalue00090k, kkey00091k: kvalue00091k, kkey00092k: kvalue00092k, kkey00093k: kvalue00093k, kkey00094k: kvalue00094k, kkey00095k: kvalue00095k, kkey00096k: kvalue00096k, kkey00097k: kvalue00097k, kkey00098k: kvalue00098k, kkey00099k: kvalue00099k, kkey00100k: kvalue00100k, kkey00101k: kvalue00101k, kkey00102k: kvalue00102k, kkey00103k: kvalue00103k, kkey00104k: kvalue00104k, kkey00105k: kvalue00105k, kkey00106k: kvalue00106k, kkey00107k: kvalue00107k, kkey00108k: kvalue00108k, kkey00109k: kvalue00109k, kkey00110k: kvalue00110k, kkey00111k: kvalue00111k, kkey00112k: kvalue00112k, kkey00113k: kvalue00113k, kkey00114k: kvalue00114k, kkey00115k: kvalue00115k, kkey00116k: kvalue00116k, kkey00117k: kvalue00117k, kkey00118k: kvalue00118k, kkey00119k: kvalue00119k, kkey00120k: kvalue00120k, kkey00121k: kvalue00121k, kkey00122k: kvalue00122k, kkey00123k: kvalue00123k, kkey00124k: kvalue00124k, kkey00125k: kvalue00125k, kkey00126k: kvalue00126k, kkey00127k: kvalue00127k, kkey00128k: kvalue00128k, kkey00129k: kvalue00129k, kkey00130k: kvalue00130k, kkey00131k: kvalue00131k, kkey00132k: kvalue00132k, kkey00133k: kvalue00133k, kkey00134k: kvalue00134k, kkey00135k: kvalue00135k, kkey00136k: kvalue00136k, kkey00137k: kvalue00137k, kkey00138k: kvalue00138k, kkey00139k: kvalue00139k, kkey00140k: kvalue00140k, kkey00141k: kvalue00141k, kkey00142k: kvalue00142k, kkey00143k: kvalue00143k, kkey00144k: kvalue00144k, kkey00145k: kvalue00145k, kkey00146k: kvalue00146k, kkey00147k: kvalue00147k, kkey00148k: kvalue00148k, kkey00149k: kvalue00149k, kkey00150k: kvalue00150k, kkey00151k: kvalue00151k, kkey00152k: kvalue00152k, kkey00153k: kvalue00153k, kkey00154k: kvalue00154k, kkey00155k: kvalue00155k, kkey00156k: kvalue00156k, kkey00157k: kvalue00157k, kkey00158k: kvalue00158k, kkey00159k: kvalue00159k, kkey00160k: kvalue00160k, kkey00161k: kvalue00161k, kkey00162k: kvalue00162k, kkey00163k: kvalue00163k, kkey00164k: kvalue00164k, kkey00165k: kvalue00165k, kkey00166k: kvalue00166k, kkey00167k: kvalue00167k, kkey00168k: kvalue00168k, kkey00169k: kvalue00169k, kkey00170k: kvalue00170k, kkey00171k: kvalue00171k, kkey00172k: kvalue00172k, kkey00173k: kvalue00173k, kkey00174k: kvalue00174k, kkey00175k: kvalue00175k, kkey00176k: kvalue00176k, kkey00177k: kvalue00177k, kkey00178k: kvalue00178k, kkey00179k: kvalue00179k, kkey00180k: kvalue00180k, kkey00181k: kvalue00181k, kkey00182k: kvalue00182k, kkey00183k: kvalue00183k, kkey00184k: kvalue00184k, kkey00185k: kvalue00185k, kkey00186k: kvalue00186k, kkey00187k: kvalue00187k, kkey00188k: kvalue00188k, kkey00189k: kvalue00189k, kkey00190k: kvalue00190k, kkey00191k: kvalue00191k, kkey00192k: kvalue00192k, kkey00193k: kvalue00193k, kkey00194k: kvalue00194k, kkey00195k: kvalue00195k, kkey00196k: kvalue00196k, kkey00197k: kvalue00197k, kkey00198k: kvalue00198k, kkey00199k: kvalue00199k}", + "NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL","NULL", + "NULerL","NdrerULL","NU232LL","NUL23L","NU23LL","NU23LL","NUL23L","NUL32L" + ] + }, + { "type": "BIGINT", "name": "cnt", "max" : 2563332323232, "min":1 }, + { "type": "DOUBLE", "name": "phase", "max": 1000, "min": 0 } + ], + "tags": [ + {"type": "TINYINT", "name": "groupid", "max": 10, "min": 1}, + {"type": "BINARY", "name": "location", "len": 16, + "values": ["San Francisco", "Los Angles", "San Diego", + "San Jose", "Palo Alto", "Campbell", "Mountain View", + "Sunnyvale", "Santa Clara", "Cupertino"] + } + ] + } + ] + } + ] +}''' + + +def initEnv(): + updatecfgDict2 = {'debugFlag': 131, 'serverPort': 7030} + tdDnodes2.init(dnode2) + tdDnodes2.deploy(1, updatecfgDict2) + tdDnodes2.start(1) + + updatecfgDict1 = {'debugFlag': 131, 'serverPort': 6030} + tdDnodes1.init(dnode1) + tdDnodes1.deploy(1, updatecfgDict1) + tdDnodes1.start(1) + + with open('taosx-performance.json', 'w') as file: + file.write(insertJson) + + changeVgroups = f"sed -i '/vgroups/c\ \"vgroups\": {vgroups},' taosx-performance.json" + os.system(changeVgroups) + + changeTables = f"sed -i '/childtable_count/c\ \"childtable_count\": {tables},' taosx-performance.json" + os.system(changeTables) + + changeRows = f"sed -i '/insert_rows/c\ \"insert_rows\": {rows_of_each_table},' taosx-performance.json" + os.system(changeRows) + os.system("taosBenchmark -f taosx-performance.json") + +def stopTaosd(str): + psCmd = f"ps -ef | grep -w taosd | grep -v grep | grep {str}" + " | awk '{print $2}' | xargs" + processID = subprocess.check_output(psCmd, shell=True).decode("utf-8").strip() + print(f"kill taosd pid={processID}") + if processID: + cmd = f"kill -9 {processID}" + os.system(cmd) + +def cleanDb(): + dropTopic = f"{taosd}/bin/taos -c {dnode1}/{cfg} -s \"drop topic if exists test\"" + print("dropTopic:%s" % dropTopic) + os.system(dropTopic) + + dropDb = f"{taosd}/bin/taos -c {dnode2}/{cfg} -s \"drop database if exists test\"" + print("dropDb:%s" % dropDb) + os.system(dropDb) + + createDb = f"{taosd}/bin/taos -c {dnode2}/{cfg} -s \"create database test vgroups {vgroups}\"" + print("createDb:%s" % createDb) + os.system(createDb) + + +def restartTaosd(): + cmd1 = f"{taosd}/bin/taosd -c {dnode1}/{cfg} > /dev/null 2>&1 &" + cmd2 = f"{taosd}/bin/taosd -c {dnode2}/{cfg} > /dev/null 2>&1 &" + print("start taosd1 :%s" % cmd1) + print("start taosd2 :%s" % cmd2) + os.system(cmd1) + os.system(cmd2) + + +def runTaosx(): + cmd = f"{taosx} run -f \"tmq://root:taosdata@localhost:6030/test?group.id=taosx-new-`date +%s`&timeout={taosxTimeout}s&experimental.snapshot.enable=false&auto.offset.reset=earliest&prefer=raw&libraryPath={taosd}/lib/libtaos.so\" -t \"taos://root:taosdata@localhost:7030/test?libraryPath={taosd}/lib/libtaos.so\" > {taosxLog}" + print("run taosx:%s" % cmd) + os.system(cmd) + + +def parseArgs(argv): + if len(argv) < 6: + print( + "Usage: python3 taosx-performance.py path_to_taosx path_to_taosd init vgroups tables rows_of_each_table [path]") + sys.exit(1) + + global taosx + global taosd + global init + global vgroups + global tables + global rows_of_each_table + global path + global dnode1 + global dnode2 + taosx = argv[0] + taosd = argv[1] + init = argv[2] + vgroups = argv[3] + tables = argv[4] + rows_of_each_table = argv[5] + if len(argv) == 7: + path = argv[6] + + if not os.path.isdir(path): + mkCmd = f"mkdir {path}" + os.system(mkCmd) + # os.system("git clone https://github.com/brendangregg/FlameGraph.git taosx-perf") +def getCost(): + costCmd = f"cat {taosxLog}| grep 'time cost'" + "| awk '{print $3}' | xargs" + cost = subprocess.check_output(costCmd, shell=True).decode("utf-8").strip() + return cost + +def runOnce(argv): + parseArgs(argv) + + os.chdir(path) + print("current dir:" + os.getcwd()) + + stopTaosd("dnode2") + stopTaosd("dnode1") + time.sleep(2) + + if init == "true": + initEnv() + else: + restartTaosd() + + cleanDb() + runTaosx() + + cost = getCost() + print("cost:%s" % cost) + timeCost.append(cost) + +''' +python3 taosx-performance.py once path_to_taosx path_to_taosd init vgroups tables rows_of_each_table [path] +''' +paras = [ + ("/root/taosx/taosx/target/release/taosx", "~/TDinternal/community/debug/build/bin/taosd", "true", 1, 5000, 200, "/tmp/taosx_perf"), + ("/root/taosx/taosx/target/release/taosx", "~/TDengine/debug/build/bin/taosd", "false", 1, 5000, 200, "/tmp/taosx_perf"), + + ("/root/taosx/taosx/target/release/taosx", "~/TDinternal/community/debug/build/bin/taosd", "true", 1, 5000, 1000, "/tmp/taosx_perf"), + ("/root/taosx/taosx/target/release/taosx", "~/TDengine/debug/build/bin/taosd", "false", 1, 5000, 1000, "/tmp/taosx_perf"), + + ("/root/taosx/taosx/target/release/taosx", "~/TDinternal/community/debug/build/bin/taosd", "true", 1, 5000, 2000, "/tmp/taosx_perf"), + ("/root/taosx/taosx/target/release/taosx", "~/TDengine/debug/build/bin/taosd", "false", 1, 5000, 2000, "/tmp/taosx_perf"), + + ("/root/taosx/taosx/target/release/taosx", "~/TDinternal/community/debug/build/bin/taosd", "true", 2, 10000, 100, "/tmp/taosx_perf"), + ("/root/taosx/taosx/target/release/taosx", "~/TDengine/debug/build/bin/taosd", "false", 2, 10000, 100, "/tmp/taosx_perf"), + + ("/root/taosx/taosx/target/release/taosx", "~/TDinternal/community/debug/build/bin/taosd", "true", 2, 10000, 500, "/tmp/taosx_perf"), + ("/root/taosx/taosx/target/release/taosx", "~/TDengine/debug/build/bin/taosd", "false", 2, 10000, 500, "/tmp/taosx_perf"), + + ("/root/taosx/taosx/target/release/taosx", "~/TDinternal/community/debug/build/bin/taosd", "true", 2, 10000, 1000, "/tmp/taosx_perf"), + ("/root/taosx/taosx/target/release/taosx", "~/TDengine/debug/build/bin/taosd", "false", 2, 10000, 1000, "/tmp/taosx_perf"), + + ("/root/taosx/taosx/target/release/taosx", "~/TDinternal/community/debug/build/bin/taosd", "true", 4, 20000, 500, "/tmp/taosx_perf"), + ("/root/taosx/taosx/target/release/taosx", "~/TDengine/debug/build/bin/taosd", "false", 4, 20000, 500, "/tmp/taosx_perf"), + + ("/root/taosx/taosx/target/release/taosx", "~/TDinternal/community/debug/build/bin/taosd", "true", 8, 40000, 250, "/tmp/taosx_perf"), + ("/root/taosx/taosx/target/release/taosx", "~/TDengine/debug/build/bin/taosd", "false", 8, 40000, 250, "/tmp/taosx_perf"), + + ("/root/taosx/taosx/target/release/taosx", "~/TDinternal/community/debug/build/bin/taosd", "true", 16, 100000, 100, "/tmp/taosx_perf"), + ("/root/taosx/taosx/target/release/taosx", "~/TDengine/debug/build/bin/taosd", "false", 16, 100000, 100, "/tmp/taosx_perf"), +] +if __name__ == "__main__": + print("run performance start") + + once = sys.argv[1] + if once == "once": + runOnce(sys.argv[2:]) + else: + for i in range(len(paras)): + runOnce(paras[i]) + if i % 2 == 1 : + print(f"opti cost:{float(timeCost[0]) - taosxTimeout}") + print(f"old cost:{float(timeCost[1]) - taosxTimeout}") + tmp = str(paras[i]) + f" speedup:{(float(timeCost[1]) - taosxTimeout)/(float(timeCost[0]) - taosxTimeout)}" + speedupStr.append(tmp) + print(tmp + "\n\n\n") + timeCost.clear() + + print("performance result:\n" + str(speedupStr)) + tdLog.info("run performance end") diff --git a/tests/system-test/7-tmq/tmq_taosx.py b/tests/system-test/7-tmq/tmq_taosx.py index 5047ada1d1..845e7229a8 100644 --- a/tests/system-test/7-tmq/tmq_taosx.py +++ b/tests/system-test/7-tmq/tmq_taosx.py @@ -17,6 +17,8 @@ from tmqCommon import * class TDTestCase: updatecfgDict = {'debugFlag': 135, 'asynclog': 0} + clientCfgDict = {'debugFlag': 135, 'asynclog': 0} + updatecfgDict["clientCfg"] = clientCfgDict def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug(f"start to excute {__file__}") @@ -65,7 +67,7 @@ class TDTestCase: tdSql.checkData(1, 5, "sttb4") tdSql.query("select * from stt order by ts") - tdSql.checkRows(3) + tdSql.checkRows(5) tdSql.checkData(0, 1, 1) tdSql.checkData(2, 1, 21) tdSql.checkData(0, 2, 2) @@ -96,7 +98,7 @@ class TDTestCase: tdSql.checkData(1, 5, "sttb4") tdSql.query("select * from stt order by ts") - tdSql.checkRows(3) + tdSql.checkRows(5) tdSql.checkData(0, 1, 1) tdSql.checkData(2, 1, 21) tdSql.checkData(0, 2, 2) @@ -256,6 +258,17 @@ class TDTestCase: return + def checkWalMultiVgroupsRawData(self): + buildPath = tdCom.getBuildPath() + cmdStr = '%s/build/bin/tmq_taosx_ci -sv 3 -dv 5 -raw'%(buildPath) + tdLog.info(cmdStr) + os.system(cmdStr) + + self.checkData() + self.checkDropData(False) + + return + def checkWalMultiVgroupsWithDropTable(self): buildPath = tdCom.getBuildPath() cmdStr = '%s/build/bin/tmq_taosx_ci -sv 3 -dv 5 -d'%(buildPath) @@ -681,6 +694,7 @@ class TDTestCase: self.checkSnapshot1VgroupTable() self.checkWalMultiVgroups() + self.checkWalMultiVgroupsRawData() self.checkSnapshotMultiVgroups() self.checkWalMultiVgroupsWithDropTable() diff --git a/tests/system-test/7-tmq/tmq_ts-5776.py b/tests/system-test/7-tmq/tmq_ts-5776.py new file mode 100644 index 0000000000..738d69701f --- /dev/null +++ b/tests/system-test/7-tmq/tmq_ts-5776.py @@ -0,0 +1,39 @@ + +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +from taos.tmq import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + updatecfgDict = {'debugFlag': 135, 'asynclog': 0} + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + #tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def run(self): + buildPath = tdCom.getBuildPath() + cmdStr = '%s/build/bin/tmq_ts5776'%(buildPath) + tdLog.info(cmdStr) + os.system(cmdStr) + + return + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 86b13ca723..1ee2bc4ce6 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -16,10 +16,6 @@ IF(TD_WEBSOCKET) DEPENDS ${TAOS_LIB} BUILD_IN_SOURCE 1 CONFIGURE_COMMAND cmake -E echo "taosws-rs no need cmake to config" - UPDATE_COMMAND - COMMAND echo "Starting to switch branch" - COMMAND git fetch origin main && git checkout -f origin/main - COMMAND echo "Switched to the main branch successfully" PATCH_COMMAND COMMAND git clean -f -d BUILD_COMMAND @@ -39,10 +35,6 @@ IF(TD_WEBSOCKET) DEPENDS ${TAOS_LIB} BUILD_IN_SOURCE 1 CONFIGURE_COMMAND cmake -E echo "taosws-rs no need cmake to config" - UPDATE_COMMAND - COMMAND echo "Starting to switch branch" - COMMAND git fetch origin main && git checkout -f origin/main - COMMAND echo "Switched to the main branch successfully" PATCH_COMMAND COMMAND git clean -f -d BUILD_COMMAND @@ -63,10 +55,6 @@ IF(TD_WEBSOCKET) DEPENDS ${TAOS_LIB} BUILD_IN_SOURCE 1 CONFIGURE_COMMAND cmake -E echo "taosws-rs no need cmake to config" - UPDATE_COMMAND - COMMAND echo "Starting to switch branch" - COMMAND git fetch origin main && git checkout -f origin/main - COMMAND echo "Switched to the main branch successfully" PATCH_COMMAND COMMAND git clean -f -d BUILD_COMMAND diff --git a/utils/test/c/CMakeLists.txt b/utils/test/c/CMakeLists.txt index d1c049ef1e..d73f91aad3 100644 --- a/utils/test/c/CMakeLists.txt +++ b/utils/test/c/CMakeLists.txt @@ -6,6 +6,7 @@ add_executable(tmq_taosx_ci tmq_taosx_ci.c) add_executable(tmq_ts5466 tmq_ts5466.c) add_executable(tmq_td32526 tmq_td32526.c) add_executable(tmq_td32187 tmq_td32187.c) +add_executable(tmq_ts5776 tmq_ts5776.c) add_executable(tmq_td32471 tmq_td32471.c) add_executable(tmq_write_raw_test tmq_write_raw_test.c) add_executable(write_raw_block_test write_raw_block_test.c) @@ -87,6 +88,13 @@ target_link_libraries( PUBLIC common PUBLIC os ) +target_link_libraries( + tmq_ts5776 + PUBLIC ${TAOS_LIB} + PUBLIC util + PUBLIC common + PUBLIC os +) target_link_libraries( tmq_taosx_ci PUBLIC ${TAOS_LIB} diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c index 117f9fa2e1..bc4a79bf12 100644 --- a/utils/test/c/tmq_taosx_ci.c +++ b/utils/test/c/tmq_taosx_ci.c @@ -34,6 +34,7 @@ typedef struct { int dstVgroups; char dir[256]; bool btMeta; + bool rawData; } Config; Config g_conf = {0}; @@ -424,6 +425,15 @@ int buildDatabase(TAOS* pConn, TAOS_RES* pRes) { } taos_free_result(pRes); + pRes = + taos_query(pConn, + "insert into stt1 values(now + 322s, 3, 2, 'stt1')"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table stt1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + pRes = taos_query(pConn, "insert into stt1 values(now + 2s, 3, 2, 'stt1') stt3 using stt tags(23, \"stt3\", true) values(now + " @@ -436,6 +446,15 @@ int buildDatabase(TAOS* pConn, TAOS_RES* pRes) { } taos_free_result(pRes); + pRes = + taos_query(pConn, + "insert into stt1 values(now + 442s, 3, 2, 'stt1')"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table stt1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + return 0; } @@ -633,6 +652,9 @@ tmq_t* build_consumer() { tmq_conf_set(conf, "enable.auto.commit", "true"); tmq_conf_set(conf, "auto.offset.reset", "earliest"); tmq_conf_set(conf, "msg.consume.excluded", "1"); + if (g_conf.rawData) { + tmq_conf_set(conf, "msg.consume.rawdata", "1"); + } // tmq_conf_set(conf, "session.timeout.ms", "1000000"); // tmq_conf_set(conf, "max.poll.interval.ms", "20000"); @@ -1238,6 +1260,8 @@ int main(int argc, char* argv[]) { g_conf.meta = 1; } else if (strcmp(argv[i], "-bt") == 0) { g_conf.btMeta = true; + } else if (strcmp(argv[i], "-raw") == 0) { + g_conf.rawData = true; } } diff --git a/utils/test/c/tmq_ts5776.c b/utils/test/c/tmq_ts5776.c new file mode 100644 index 0000000000..c50809bccd --- /dev/null +++ b/utils/test/c/tmq_ts5776.c @@ -0,0 +1,255 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include +#include +#include "cJSON.h" +#include "taos.h" +#include "tmsg.h" +#include "types.h" + +static TAOS* use_db() { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + if (pConn == NULL) { + return NULL; + } + + TAOS_RES* pRes = taos_query(pConn, "use db_dst"); + if (taos_errno(pRes) != 0) { + printf("error in use db_taosx, reason:%s\n", taos_errstr(pRes)); + return NULL; + } + taos_free_result(pRes); + return pConn; +} + +static void msg_process(TAOS_RES* msg) { + printf("-----------topic-------------: %s\n", tmq_get_topic_name(msg)); + printf("db: %s\n", tmq_get_db_name(msg)); + printf("vg: %d\n", tmq_get_vgroup_id(msg)); + TAOS* pConn = use_db(); + if (tmq_get_res_type(msg) == TMQ_RES_TABLE_META || tmq_get_res_type(msg) == TMQ_RES_METADATA) { + char* result = tmq_get_json_meta(msg); + printf("meta result: %s\n", result); + tmq_free_json_meta(result); + } + + tmq_raw_data raw = {0}; + tmq_get_raw(msg, &raw); + printf("write raw data type: %d\n", raw.raw_type); + int32_t ret = tmq_write_raw(pConn, raw); + printf("write raw data: %s\n", tmq_err2str(ret)); + ASSERT(ret == 0); + + tmq_free_raw(raw); + taos_close(pConn); +} + +int buildDatabase(TAOS* pConn, TAOS_RES* pRes) { + pRes = taos_query(pConn, + "create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(16)) tags(t1 int, t3 " + "nchar(8), t4 bool)"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query( + pConn, + "insert into ct3 using st1(t1) tags(3000) values(1626006833600, 5, 6, 'c') ct1 using st1(t1) tags(2000) values(1626006833601, 2, 3, 'sds') (1626006833602, 4, 5, " + "'ddd') ct0 using st1 tags(1000, \"ttt\", true) values(1626006833603, 4, 3, 'hwj') ct1 using st1(t1) tags(2000) values(now+5s, 23, 32, 's21ds')"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into ct3, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query( + pConn, + "insert into ct3 values(1626006839602, 5, 6, 'c')"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into ct3, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query( + pConn, + "insert into ct3 using st1(t1) tags(3000) values(1626006833602, 5, 6, 'c') ct1 values(1626006833611, 2, 3, 'sds') (1626006833612, 4, 5, " + "'ddd') ct10 using st1 tags(1000, \"ttt\", true) values(1626006833603, 4, 3, 'hwj')"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into ct3, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + return 0; +} + +int32_t init_env() { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + if (pConn == NULL) { + return -1; + } + + TAOS_RES* pRes = taos_query(pConn, "drop database if exists db_dst"); + if (taos_errno(pRes) != 0) { + printf("error in drop db_taosx, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create database if not exists db_dst vgroups 1 wal_retention_period 3600"); + if (taos_errno(pRes) != 0) { + printf("error in create db_taosx, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "drop topic if exists topic_db"); + if (taos_errno(pRes) != 0) { + printf("error in drop topic, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "drop database if exists db_src"); + if (taos_errno(pRes) != 0) { + printf("error in drop db, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create database if not exists db_src vgroups 1 wal_retention_period 3600"); + if (taos_errno(pRes) != 0) { + printf("error in create db, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "use db_src"); + if (taos_errno(pRes) != 0) { + printf("error in use db, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + buildDatabase(pConn, pRes); + + taos_close(pConn); + return 0; +} + +int32_t create_topic() { + printf("create topic\n"); + TAOS_RES* pRes; + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + if (pConn == NULL) { + return -1; + } + + pRes = taos_query(pConn, "create topic topic_db with meta as database db_src"); + if (taos_errno(pRes) != 0) { + printf("failed to create topic topic_db, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + taos_close(pConn); + return 0; +} + +void tmq_commit_cb_print(tmq_t* tmq, int32_t code, void* param) { + printf("commit %d tmq %p param %p\n", code, tmq, param); +} + +tmq_t* build_consumer() { + tmq_conf_t* conf = tmq_conf_new(); + tmq_conf_set(conf, "group.id", "tg2"); + tmq_conf_set(conf, "client.id", "my app 1"); + tmq_conf_set(conf, "td.connect.user", "root"); + tmq_conf_set(conf, "td.connect.pass", "taosdata"); + tmq_conf_set(conf, "msg.with.table.name", "true"); + tmq_conf_set(conf, "enable.auto.commit", "true"); + tmq_conf_set(conf, "auto.offset.reset", "earliest"); + tmq_conf_set(conf, "msg.consume.rawdata", "1"); + + tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL); + tmq_t* tmq = tmq_consumer_new(conf, NULL, 0); + assert(tmq); + tmq_conf_destroy(conf); + return tmq; +} + +tmq_list_t* build_topic_list() { + tmq_list_t* topic_list = tmq_list_new(); + tmq_list_append(topic_list, "topic_db"); + return topic_list; +} + +void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) { + int32_t code; + + if ((code = tmq_subscribe(tmq, topics))) { + fprintf(stderr, "%% Failed to start consuming topics: %s\n", tmq_err2str(code)); + printf("subscribe err\n"); + return; + } + int32_t cnt = 0; + while (1) { + TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 5000); + if (tmqmessage) { + cnt++; + msg_process(tmqmessage); + taos_free_result(tmqmessage); + } else { + break; + } + } + + code = tmq_consumer_close(tmq); + if (code) + fprintf(stderr, "%% Failed to close consumer: %s\n", tmq_err2str(code)); + else + fprintf(stderr, "%% Consumer closed\n"); +} + +void check_result(){ + TAOS* taos = use_db(); + TAOS_RES *res = taos_query(taos, "select * from st1"); + int code = taos_errno(res); + assert(code == 0); + int affectedRows = taos_affected_rows(res); + printf("affected rows %d\n", affectedRows); + assert(affectedRows == 10); + taos_free_result(res); + taos_close(taos); + +} +int main(int argc, char* argv[]) { + if (init_env() < 0) { + return -1; + } + create_topic(); + + tmq_t* tmq = build_consumer(); + tmq_list_t* topic_list = build_topic_list(); + basic_consume_loop(tmq, topic_list); + tmq_list_destroy(topic_list); +}